diff --git a/cranelift/codegen/src/binemit/memorysink.rs b/cranelift/codegen/src/binemit/memorysink.rs index 18aa578f09..8e07db01f6 100644 --- a/cranelift/codegen/src/binemit/memorysink.rs +++ b/cranelift/codegen/src/binemit/memorysink.rs @@ -14,7 +14,7 @@ //! relocations to a `RelocSink` trait object. Relocations are less frequent than the //! `CodeSink::put*` methods, so the performance impact of the virtual callbacks is less severe. -use super::{Addend, CodeOffset, CodeSink, Reloc}; +use super::{Addend, CodeInfo, CodeOffset, CodeSink, Reloc}; use crate::ir::{ExternalName, JumpTable, SourceLoc, TrapCode}; use core::ptr::write_unaligned; @@ -30,12 +30,14 @@ use core::ptr::write_unaligned; /// Note that `MemoryCodeSink` writes multi-byte values in the native byte order of the host. This /// is not the right thing to do for cross compilation. pub struct MemoryCodeSink<'a> { + /// Pointer to start of sink's preallocated memory. data: *mut u8, + /// Offset is isize because its major consumer needs it in that form. offset: isize, - /// Size of the machine code portion of output - pub code_size: isize, relocs: &'a mut RelocSink, traps: &'a mut TrapSink, + /// Information about the generated code and read-only data. + pub info: CodeInfo, } impl<'a> MemoryCodeSink<'a> { @@ -47,7 +49,12 @@ impl<'a> MemoryCodeSink<'a> { Self { data, offset: 0, - code_size: 0, + info: CodeInfo { + code_size: 0, + jumptables_size: 0, + rodata_size: 0, + total_size: 0, + }, relocs, traps, } @@ -75,40 +82,35 @@ pub trait TrapSink { fn trap(&mut self, _: CodeOffset, _: SourceLoc, _: TrapCode); } +impl<'a> MemoryCodeSink<'a> { + fn write(&mut self, x: T) { + unsafe { + #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] + write_unaligned(self.data.offset(self.offset) as *mut T, x); + self.offset += std::mem::size_of::() as isize; + } + } +} + impl<'a> CodeSink for MemoryCodeSink<'a> { fn offset(&self) -> CodeOffset { self.offset as CodeOffset } fn put1(&mut self, x: u8) { - unsafe { - write_unaligned(self.data.offset(self.offset), x); - } - self.offset += 1; + self.write(x); } fn put2(&mut self, x: u16) { - unsafe { - #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] - write_unaligned(self.data.offset(self.offset) as *mut u16, x); - } - self.offset += 2; + self.write(x); } fn put4(&mut self, x: u32) { - unsafe { - #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] - write_unaligned(self.data.offset(self.offset) as *mut u32, x); - } - self.offset += 4; + self.write(x); } fn put8(&mut self, x: u64) { - unsafe { - #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] - write_unaligned(self.data.offset(self.offset) as *mut u64, x); - } - self.offset += 8; + self.write(x); } fn reloc_ebb(&mut self, rel: Reloc, ebb_offset: CodeOffset) { @@ -131,8 +133,17 @@ impl<'a> CodeSink for MemoryCodeSink<'a> { self.traps.trap(ofs, srcloc, code); } + fn begin_jumptables(&mut self) { + self.info.code_size = self.offset(); + } + fn begin_rodata(&mut self) { - self.code_size = self.offset; + self.info.jumptables_size = self.offset() - self.info.code_size; + } + + fn end_codegen(&mut self) { + self.info.rodata_size = self.offset() - self.info.jumptables_size; + self.info.total_size = self.offset(); } } diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index 63fa7ef40b..21b07587ed 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -33,6 +33,8 @@ pub enum Reloc { Abs8, /// x86 PC-relative 4-byte X86PCRel4, + /// x86 PC-relative 4-byte offset to trailing rodata + X86PCRelRodata4, /// x86 call to PC-relative 4-byte X86CallPCRel4, /// x86 call to PLT-relative 4-byte @@ -55,6 +57,7 @@ impl fmt::Display for Reloc { Reloc::Abs4 => write!(f, "Abs4"), Reloc::Abs8 => write!(f, "Abs8"), Reloc::X86PCRel4 => write!(f, "PCRel4"), + Reloc::X86PCRelRodata4 => write!(f, "PCRelRodata4"), Reloc::X86CallPCRel4 => write!(f, "CallPCRel4"), Reloc::X86CallPLTRel4 => write!(f, "CallPLTRel4"), Reloc::X86GOTPCRel4 => write!(f, "GOTPCRel4"), @@ -63,6 +66,38 @@ impl fmt::Display for Reloc { } } +/// Container for information about a vector of compiled code and its supporting read-only data. +/// +/// The code starts at offset 0 and is followed optionally by relocatable jump tables and copyable +/// (raw binary) read-only data. Any padding between sections is always part of the section that +/// precedes the boundary between the sections. +#[derive(PartialEq)] +pub struct CodeInfo { + /// Number of bytes of machine code (the code starts at offset 0). + pub code_size: CodeOffset, + + /// Number of bytes of jumptables. + pub jumptables_size: CodeOffset, + + /// Number of bytes of rodata. + pub rodata_size: CodeOffset, + + /// Number of bytes in total. + pub total_size: CodeOffset, +} + +impl CodeInfo { + /// Offset of any relocatable jump tables, or equal to rodata if there are no jump tables. + pub fn jumptables(&self) -> CodeOffset { + self.code_size + } + + /// Offset of any copyable read-only data, or equal to total_size if there are no rodata. + pub fn rodata(&self) -> CodeOffset { + self.code_size + self.jumptables_size + } +} + /// Abstract interface for adding bytes to the code segment. /// /// A `CodeSink` will receive all of the machine code for a function. It also accepts relocations @@ -95,8 +130,14 @@ pub trait CodeSink { /// Add trap information for the current offset. fn trap(&mut self, _: TrapCode, _: SourceLoc); - /// Code output is complete, read-only data may follow. + /// Machine code output is complete, jump table data may follow. + fn begin_jumptables(&mut self); + + /// Jump table output is complete, raw read-only data may follow. fn begin_rodata(&mut self); + + /// Read-only data output is complete, we're done. + fn end_codegen(&mut self); } /// Report a bad encoding error. @@ -127,7 +168,7 @@ where } } - sink.begin_rodata(); + sink.begin_jumptables(); // output jump tables for (jt, jt_data) in func.jump_tables.iter() { @@ -137,4 +178,9 @@ where sink.put4(rel_offset as u32) } } + + sink.begin_rodata(); + // TODO: No read-only data (constant pools) at this time. + + sink.end_codegen(); } diff --git a/cranelift/codegen/src/binemit/relaxation.rs b/cranelift/codegen/src/binemit/relaxation.rs index c354a763cd..d1b5815e7c 100644 --- a/cranelift/codegen/src/binemit/relaxation.rs +++ b/cranelift/codegen/src/binemit/relaxation.rs @@ -27,7 +27,7 @@ //! ebb23: //! ``` -use crate::binemit::CodeOffset; +use crate::binemit::{CodeInfo, CodeOffset}; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{Function, InstructionData, Opcode}; use crate::isa::{EncInfo, TargetIsa}; @@ -40,7 +40,7 @@ use log::debug; /// Relax branches and compute the final layout of EBB headers in `func`. /// /// Fill in the `func.offsets` table so the function is ready for binary emission. -pub fn relax_branches(func: &mut Function, isa: &TargetIsa) -> CodegenResult { +pub fn relax_branches(func: &mut Function, isa: &TargetIsa) -> CodegenResult { let _tt = timing::relax_branches(); let encinfo = isa.encoding_info(); @@ -109,6 +109,9 @@ pub fn relax_branches(func: &mut Function, isa: &TargetIsa) -> CodegenResult CodegenResult, relocs: &mut RelocSink, traps: &mut TrapSink, - ) -> CodegenResult<(CodeOffset, CodeOffset)> { - let total_size = self.compile(isa)?; + ) -> CodegenResult { + let info = self.compile(isa)?; let old_len = mem.len(); - mem.resize(old_len + total_size as usize, 0); - let code_size = + mem.resize(old_len + info.total_size as usize, 0); + let new_info = unsafe { self.emit_to_memory(isa, mem.as_mut_ptr().add(old_len), relocs, traps) }; - Ok((code_size, total_size - code_size)) + debug_assert!(new_info == info); + Ok(info) } /// Compile the function. @@ -115,8 +116,8 @@ impl Context { /// represented by `isa`. This does not include the final step of emitting machine code into a /// code sink. /// - /// Returns the size of the function's code and read-only data. - pub fn compile(&mut self, isa: &TargetIsa) -> CodegenResult { + /// Returns information about the function's code and read-only data. + pub fn compile(&mut self, isa: &TargetIsa) -> CodegenResult { let _tt = timing::compile(); self.verify_if(isa)?; @@ -160,18 +161,18 @@ impl Context { /// This function is unsafe since it does not perform bounds checking on the memory buffer, /// and it can't guarantee that the `mem` pointer is valid. /// - /// Returns the size of the function's code. + /// Returns information about the emitted code and data. pub unsafe fn emit_to_memory( &self, isa: &TargetIsa, mem: *mut u8, relocs: &mut RelocSink, traps: &mut TrapSink, - ) -> CodeOffset { + ) -> CodeInfo { let _tt = timing::binemit(); let mut sink = MemoryCodeSink::new(mem, relocs, traps); isa.emit_function_to_memory(&self.func, &mut sink); - sink.code_size as CodeOffset + sink.info } /// Run the verifier on the function. @@ -325,12 +326,13 @@ impl Context { Ok(()) } - /// Run the branch relaxation pass and return the final code size. - pub fn relax_branches(&mut self, isa: &TargetIsa) -> CodegenResult { - let code_size = relax_branches(&mut self.func, isa)?; + /// Run the branch relaxation pass and return information about the function's code and + /// read-only data. + pub fn relax_branches(&mut self, isa: &TargetIsa) -> CodegenResult { + let info = relax_branches(&mut self.func, isa)?; self.verify_if(isa)?; self.verify_locations_if(isa)?; - Ok(code_size) + Ok(info) } /// Builds ranges and location for specified value labels. diff --git a/cranelift/codegen/src/isa/x86/binemit.rs b/cranelift/codegen/src/isa/x86/binemit.rs index 513d6347fa..ef44766d0b 100644 --- a/cranelift/codegen/src/isa/x86/binemit.rs +++ b/cranelift/codegen/src/isa/x86/binemit.rs @@ -339,4 +339,5 @@ fn disp4(destination: Ebb, func: &Function, sink: &mut CS fn jt_disp4(jt: JumpTable, func: &Function, sink: &mut CS) { let delta = func.jt_offsets[jt].wrapping_sub(sink.offset() + 4); sink.put4(delta); + sink.reloc_jt(Reloc::X86PCRelRodata4, jt); } diff --git a/cranelift/faerie/src/backend.rs b/cranelift/faerie/src/backend.rs index 047ddeb87d..9313ddaae9 100644 --- a/cranelift/faerie/src/backend.rs +++ b/cranelift/faerie/src/backend.rs @@ -138,9 +138,9 @@ impl Backend for FaerieBackend { name: &str, ctx: &cranelift_codegen::Context, namespace: &ModuleNamespace, - code_size: u32, + total_size: u32, ) -> ModuleResult { - let mut code: Vec = vec![0; code_size as usize]; + let mut code: Vec = vec![0; total_size as usize]; // Non-lexical lifetimes would obviate the braces here. { @@ -153,7 +153,7 @@ impl Backend for FaerieBackend { }; if let Some(ref mut trap_manifest) = self.trap_manifest { - let mut trap_sink = FaerieTrapSink::new(name, code_size); + let mut trap_sink = FaerieTrapSink::new(name, total_size); unsafe { ctx.emit_to_memory( &*self.isa, diff --git a/cranelift/filetests/filetests/isa/x86/binary64.clif b/cranelift/filetests/filetests/isa/x86/binary64.clif index c2a042ac01..ed1316ad24 100644 --- a/cranelift/filetests/filetests/isa/x86/binary64.clif +++ b/cranelift/filetests/filetests/isa/x86/binary64.clif @@ -1424,8 +1424,8 @@ function %I64_JT(i64 [%rdi]) { ebb0(v0: i64 [%rdi]): ; Note: The next two lines will need to change whenever instructions are ; added or removed from this test. - [-, %rax] v1 = jump_table_base.i64 jt0 ; bin: 48 8d 05 00000039 - [-, %r10] v2 = jump_table_base.i64 jt0 ; bin: 4c 8d 15 00000032 + [-, %rax] v1 = jump_table_base.i64 jt0 ; bin: 48 8d 05 00000039 PCRelRodata4(jt0) + [-, %r10] v2 = jump_table_base.i64 jt0 ; bin: 4c 8d 15 00000032 PCRelRodata4(jt0) [-, %rbx] v10 = iconst.i64 1 [-, %r13] v11 = iconst.i64 2 diff --git a/cranelift/filetests/src/test_binemit.rs b/cranelift/filetests/src/test_binemit.rs index 3c959a09bc..a80b3aefe6 100644 --- a/cranelift/filetests/src/test_binemit.rs +++ b/cranelift/filetests/src/test_binemit.rs @@ -5,8 +5,7 @@ use crate::match_directive::match_directive; use crate::subtest::{Context, SubTest, SubtestResult}; -use cranelift_codegen::binemit; -use cranelift_codegen::binemit::{CodeSink, RegDiversions}; +use cranelift_codegen::binemit::{self, CodeInfo, CodeSink, RegDiversions}; use cranelift_codegen::dbg::DisplayList; use cranelift_codegen::ir; use cranelift_codegen::ir::entities::AnyEntity; @@ -96,9 +95,12 @@ impl binemit::CodeSink for TextSink { write!(self.text, "{} ", code).unwrap(); } - fn begin_rodata(&mut self) { + fn begin_jumptables(&mut self) { self.code_size = self.offset } + + fn begin_rodata(&mut self) {} + fn end_codegen(&mut self) {} } impl SubTest for TestBinEmit { @@ -164,7 +166,7 @@ impl SubTest for TestBinEmit { } // Relax branches and compute EBB offsets based on the encodings. - let code_size = binemit::relax_branches(&mut func, isa) + let CodeInfo { total_size, .. } = binemit::relax_branches(&mut func, isa) .map_err(|e| pretty_error(&func, context.isa, e))?; // Collect all of the 'bin:' directives on instructions. @@ -288,7 +290,7 @@ impl SubTest for TestBinEmit { } } - sink.begin_rodata(); + sink.begin_jumptables(); for (jt, jt_data) in func.jump_tables.iter() { let jt_offset = func.jt_offsets[jt]; @@ -298,10 +300,15 @@ impl SubTest for TestBinEmit { } } - if sink.offset != code_size { + sink.begin_rodata(); + // TODO: Read-only (constant pool) data. + + sink.end_codegen(); + + if sink.offset != total_size { return Err(format!( "Expected code size {}, got {}", - code_size, sink.offset + total_size, sink.offset )); } diff --git a/cranelift/filetests/src/test_compile.rs b/cranelift/filetests/src/test_compile.rs index 7cb9c702a4..fee681d609 100644 --- a/cranelift/filetests/src/test_compile.rs +++ b/cranelift/filetests/src/test_compile.rs @@ -4,8 +4,9 @@ use crate::subtest::{run_filecheck, Context, SubTest, SubtestResult}; use cranelift_codegen; +use cranelift_codegen::binemit::{self, CodeInfo}; +use cranelift_codegen::ir; use cranelift_codegen::print_errors::pretty_error; -use cranelift_codegen::{binemit, ir}; use cranelift_reader::TestCommand; use log::info; use std::borrow::Cow; @@ -38,13 +39,13 @@ impl SubTest for TestCompile { let isa = context.isa.expect("compile needs an ISA"); let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned()); - let code_size = comp_ctx + let CodeInfo { total_size, .. } = comp_ctx .compile(isa) .map_err(|e| pretty_error(&comp_ctx.func, context.isa, e))?; info!( "Generated {} bytes of code:\n{}", - code_size, + total_size, comp_ctx.func.display(isa) ); @@ -56,10 +57,10 @@ impl SubTest for TestCompile { &mut sink, ); - if sink.offset != code_size { + if sink.offset != total_size { return Err(format!( "Expected code size {}, got {}", - code_size, sink.offset + total_size, sink.offset )); } @@ -105,5 +106,7 @@ impl binemit::CodeSink for SizeSink { } fn reloc_jt(&mut self, _reloc: binemit::Reloc, _jt: ir::JumpTable) {} fn trap(&mut self, _code: ir::TrapCode, _srcloc: ir::SourceLoc) {} + fn begin_jumptables(&mut self) {} fn begin_rodata(&mut self) {} + fn end_codegen(&mut self) {} } diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index 3da6c37810..5d8f3c5f60 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -8,8 +8,9 @@ use super::HashMap; use crate::data_context::DataContext; use crate::Backend; +use cranelift_codegen::binemit::{self, CodeInfo}; use cranelift_codegen::entity::{entity_impl, PrimaryMap}; -use cranelift_codegen::{binemit, ir, isa, CodegenError, Context}; +use cranelift_codegen::{ir, isa, CodegenError, Context}; use failure::Fail; use log::info; use std::borrow::ToOwned; @@ -519,7 +520,7 @@ where /// Define a function, producing the function body from the given `Context`. /// - /// Returns the size of the function's code. + /// Returns the size of the function's code and constant data. /// /// Note: After calling this function the given `Context` will contain the compiled function. pub fn define_function( @@ -527,7 +528,7 @@ where func: FuncId, ctx: &mut Context, ) -> ModuleResult { - let code_size = ctx.compile(self.backend.isa()).map_err(|e| { + let CodeInfo { total_size, .. } = ctx.compile(self.backend.isa()).map_err(|e| { info!( "defining function {}: {}", func, @@ -550,12 +551,12 @@ where &ModuleNamespace:: { contents: &self.contents, }, - code_size, + total_size, )?); self.contents.functions[func].compiled = compiled; self.functions_to_finalize.push(func); - Ok(code_size) + Ok(total_size) } /// Define a function, producing the data contents from the given `DataContext`. diff --git a/cranelift/src/compile.rs b/cranelift/src/compile.rs index f5e34c6417..4c663323b1 100644 --- a/cranelift/src/compile.rs +++ b/cranelift/src/compile.rs @@ -65,7 +65,7 @@ fn handle_module( let mut mem = vec![]; // Compile and encode the result to machine code. - let (code_size, rodata_size) = context + let code_info = context .compile_and_emit(isa, &mut mem, &mut relocs, &mut traps) .map_err(|err| pretty_error(&context.func, Some(isa), err))?; @@ -74,7 +74,14 @@ fn handle_module( } if flag_disasm { - print_all(isa, &mem, code_size, rodata_size, &relocs, &traps)?; + print_all( + isa, + &mem, + code_info.code_size, + code_info.jumptables_size + code_info.rodata_size, + &relocs, + &traps, + )?; } } diff --git a/cranelift/src/wasm.rs b/cranelift/src/wasm.rs index 61a8d8b1a9..dadb13389f 100644 --- a/cranelift/src/wasm.rs +++ b/cranelift/src/wasm.rs @@ -176,16 +176,16 @@ fn handle_module( return Err(pretty_verifier_error(&context.func, fisa.isa, None, errors)); } } else { - let (code_size, rodata_size) = context + let code_info = context .compile_and_emit(isa, &mut mem, &mut relocs, &mut traps) .map_err(|err| pretty_error(&context.func, fisa.isa, err))?; if flag_print_size { println!( "Function #{} code size: {} bytes", - func_index, code_size + rodata_size + func_index, code_info.total_size, ); - total_module_code_size += code_size + rodata_size; + total_module_code_size += code_info.total_size; println!( "Function #{} bytecode size: {} bytes", func_index, @@ -194,7 +194,7 @@ fn handle_module( } if flag_print_disasm { - saved_sizes = Some((code_size, rodata_size)); + saved_sizes = Some((code_info.code_size, code_info.jumptables_size + code_info.rodata_size)); } }