diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index 0ab260d946..1afbc25e89 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -1,13 +1,13 @@ use crate::builder::LinkOptions; -use crate::debug::ModuleMemoryOffset; +use crate::debug::{DwarfSectionRelocTarget, ModuleMemoryOffset}; use crate::func_environ::{get_func_name, FuncEnvironment}; -use crate::obj::ObjectBuilder; +use crate::obj::ModuleTextBuilder; use crate::{ blank_sig, func_signature, indirect_signature, value_type, wasmtime_call_conv, - CompiledFunction, FunctionAddressMap, Relocation, RelocationTarget, + CompiledFunction, CompiledFunctions, FunctionAddressMap, Relocation, RelocationTarget, }; use anyhow::{Context as _, Result}; -use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags}; +use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags, Value}; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::Context; @@ -19,10 +19,12 @@ use cranelift_wasm::{ DefinedFuncIndex, DefinedMemoryIndex, FuncIndex, FuncTranslator, MemoryIndex, SignatureIndex, WasmFuncType, }; -use object::write::Object; +use object::write::{Object, StandardSegment, SymbolId}; +use object::{RelocationEncoding, RelocationKind, SectionKind}; use std::any::Any; use std::cmp; use std::collections::BTreeMap; +use std::collections::HashMap; use std::convert::TryFrom; use std::mem; use std::sync::Mutex; @@ -272,14 +274,14 @@ impl wasmtime_environ::Compiler for Compiler { tunables: &Tunables, obj: &mut Object<'static>, ) -> Result<(PrimaryMap, Vec)> { - let funcs: crate::CompiledFunctions = funcs + let funcs: CompiledFunctions = funcs .into_iter() .map(|(_i, f)| *f.downcast().unwrap()) .collect(); - let mut builder = ObjectBuilder::new(obj, &translation.module, &*self.isa); + let mut builder = ModuleTextBuilder::new(obj, &translation.module, &*self.isa); if self.linkopts.force_jump_veneers { - builder.text.force_veneers(); + builder.force_veneers(); } let mut addrs = AddressMapSection::default(); let mut traps = TrapEncodingBuilder::default(); @@ -297,13 +299,7 @@ impl wasmtime_environ::Compiler for Compiler { } traps.push(range.clone(), &func.traps); func_starts.push(range.start); - if self.linkopts.padding_between_functions > 0 { - builder.text.append( - false, - &vec![0; self.linkopts.padding_between_functions], - Some(1), - ); - } + builder.append_padding(self.linkopts.padding_between_functions); } // Build trampolines for every signature that can be used by this module. @@ -316,40 +312,9 @@ impl wasmtime_environ::Compiler for Compiler { trampolines.push(builder.trampoline(*i, &func)); } - builder.unwind_info(); - - if tunables.generate_native_debuginfo && funcs.len() > 0 { - let ofs = VMOffsets::new( - self.isa - .triple() - .architecture - .pointer_width() - .unwrap() - .bytes(), - &translation.module, - ); - - let memory_offset = if ofs.num_imported_memories > 0 { - ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) - } else if ofs.num_defined_memories > 0 { - ModuleMemoryOffset::Defined( - ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)), - ) - } else { - ModuleMemoryOffset::None - }; - let dwarf_sections = crate::debug::emit_dwarf( - &*self.isa, - &translation.debuginfo, - &funcs, - &memory_offset, - ) - .with_context(|| "failed to emit DWARF debug information")?; - builder.dwarf_sections(&dwarf_sections)?; - } - - builder.finish()?; + let symbols = builder.finish()?; + self.append_dwarf(obj, translation, &funcs, tunables, &symbols)?; if tunables.generate_address_map { addrs.append_to(obj); } @@ -377,10 +342,9 @@ impl wasmtime_environ::Compiler for Compiler { let host_to_wasm = self.host_to_wasm_trampoline(ty)?; let wasm_to_host = self.wasm_to_host_trampoline(ty, host_fn)?; let module = Module::new(); - let mut builder = ObjectBuilder::new(obj, &module, &*self.isa); + let mut builder = ModuleTextBuilder::new(obj, &module, &*self.isa); let a = builder.trampoline(SignatureIndex::new(0), &host_to_wasm); let b = builder.trampoline(SignatureIndex::new(1), &wasm_to_host); - builder.unwind_info(); builder.finish()?; Ok((a, b)) } @@ -508,6 +472,28 @@ impl Compiler { Ok(func) } + /// Creates a trampoline for WebAssembly calling into the host where all the + /// arguments are spilled to the stack and results are loaded from the + /// stack. + /// + /// This style of trampoline is currently only used with the + /// `Func::new`-style created functions in the Wasmtime embedding API. The + /// generated trampoline has a function signature appropriate to the `ty` + /// specified (e.g. a System-V ABI) and will call a `host_fn` that has a + /// type signature of: + /// + /// ```ignore + /// extern "C" fn(*mut VMContext, *mut VMContext, *mut ValRaw, usize) + /// ``` + /// + /// where the first two arguments are forwarded from the trampoline + /// generated here itself, and the second two arguments are a pointer/length + /// into stack-space of this trampoline with storage for both the arguments + /// to the function and the results. + /// + /// Note that `host_fn` is an immediate which is an actual function pointer + /// in this process. As such this compiled trampoline is not suitable for + /// serialization. fn wasm_to_host_trampoline( &self, ty: &WasmFuncType, @@ -523,12 +509,6 @@ impl Compiler { host_signature.params.push(ir::AbiParam::new(pointer_type)); host_signature.params.push(ir::AbiParam::new(pointer_type)); - // Compute the size of the values vector. The vmctx and caller vmctx are passed separately. - let value_size = mem::size_of::(); - let values_vec_len = cmp::max(ty.params().len(), ty.returns().len()); - let values_vec_byte_size = u32::try_from(value_size * values_vec_len).unwrap(); - let values_vec_len = u32::try_from(values_vec_len).unwrap(); - let CompilerContext { mut func_translator, codegen_context: mut context, @@ -537,35 +517,16 @@ impl Compiler { context.func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wasm_signature); - let ss = context.func.create_stack_slot(ir::StackSlotData::new( - ir::StackSlotKind::ExplicitSlot, - values_vec_byte_size, - )); - let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context()); let block0 = builder.create_block(); - builder.append_block_params_for_function_params(block0); - builder.switch_to_block(block0); - builder.seal_block(block0); - - let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0); - let mut mflags = MemFlags::trusted(); - mflags.set_endianness(ir::Endianness::Little); - for i in 0..ty.params().len() { - let val = builder.func.dfg.block_params(block0)[i + 2]; - builder - .ins() - .store(mflags, val, values_vec_ptr_val, (i * value_size) as i32); - } + let (values_vec_ptr_val, values_vec_len) = + self.wasm_to_host_spill_args(ty, &mut builder, block0); let block_params = builder.func.dfg.block_params(block0); - let vmctx_ptr_val = block_params[0]; - let caller_vmctx_ptr_val = block_params[1]; - - let callee_args = vec![ - vmctx_ptr_val, - caller_vmctx_ptr_val, + let callee_args = [ + block_params[0], + block_params[1], values_vec_ptr_val, builder .ins() @@ -573,12 +534,94 @@ impl Compiler { ]; let new_sig = builder.import_signature(host_signature); - let callee_value = builder.ins().iconst(pointer_type, host_fn as i64); builder .ins() .call_indirect(new_sig, callee_value, &callee_args); + self.wasm_to_host_load_results(ty, &mut builder, values_vec_ptr_val); + + let func = self.finish_trampoline(&mut context, isa)?; + self.save_context(CompilerContext { + func_translator, + codegen_context: context, + }); + Ok(func) + } + + /// Used for spilling arguments in wasm-to-host trampolines into the stack + /// of the function of `builder` specified. + /// + /// The `block0` is the entry block of the function and `ty` is the wasm + /// signature of the trampoline generated. This function will allocate a + /// stack slot suitable for storing both the arguments and return values of + /// the function, and then the arguments will all be stored in this block. + /// + /// The stack slot pointer is returned in addition to the size, in units of + /// `ValRaw`, of the stack slot. + fn wasm_to_host_spill_args( + &self, + ty: &WasmFuncType, + builder: &mut FunctionBuilder, + block0: ir::Block, + ) -> (Value, u32) { + let isa = &*self.isa; + let pointer_type = isa.pointer_type(); + + // Compute the size of the values vector. + let value_size = mem::size_of::(); + let values_vec_len = cmp::max(ty.params().len(), ty.returns().len()); + let values_vec_byte_size = u32::try_from(value_size * values_vec_len).unwrap(); + let values_vec_len = u32::try_from(values_vec_len).unwrap(); + + let ss = builder.func.create_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + values_vec_byte_size, + )); + + builder.append_block_params_for_function_params(block0); + builder.switch_to_block(block0); + builder.seal_block(block0); + + // Note that loads and stores are unconditionally done in the + // little-endian format rather than the host's native-endianness, + // despite this load/store being unrelated to execution in wasm itself. + // For more details on this see the `ValRaw` type in the + // `wasmtime-runtime` crate. + let mut mflags = MemFlags::trusted(); + mflags.set_endianness(ir::Endianness::Little); + + let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0); + for i in 0..ty.params().len() { + let val = builder.func.dfg.block_params(block0)[i + 2]; + builder + .ins() + .store(mflags, val, values_vec_ptr_val, (i * value_size) as i32); + } + (values_vec_ptr_val, values_vec_len) + } + + /// Use for loading the results of a host call from a trampoline's stack + /// space. + /// + /// This is intended to be used with the stack space allocated by + /// `wasm_to_host_spill_args` above. This is called after the function call + /// is made which will load results from the stack space and then return + /// them with the appropriate ABI (e.g. System-V). + fn wasm_to_host_load_results( + &self, + ty: &WasmFuncType, + builder: &mut FunctionBuilder, + values_vec_ptr_val: Value, + ) { + let isa = &*self.isa; + let value_size = mem::size_of::(); + + // Note that this is little-endian like `wasm_to_host_spill_args` above, + // see notes there for more information. + let mut mflags = MemFlags::trusted(); + mflags.set_endianness(ir::Endianness::Little); + let mut results = Vec::new(); for (i, r) in ty.returns().iter().enumerate() { let load = builder.ins().load( @@ -591,13 +634,6 @@ impl Compiler { } builder.ins().return_(&results); builder.finalize(); - - let func = self.finish_trampoline(&mut context, isa)?; - self.save_context(CompilerContext { - func_translator, - codegen_context: context, - }); - Ok(func) } fn finish_trampoline( @@ -642,6 +678,81 @@ impl Compiler { traps: Vec::new(), }) } + + pub fn append_dwarf( + &self, + obj: &mut Object<'_>, + translation: &ModuleTranslation<'_>, + funcs: &CompiledFunctions, + tunables: &Tunables, + func_symbols: &PrimaryMap, + ) -> Result<()> { + if !tunables.generate_native_debuginfo || funcs.len() == 0 { + return Ok(()); + } + let ofs = VMOffsets::new( + self.isa + .triple() + .architecture + .pointer_width() + .unwrap() + .bytes(), + &translation.module, + ); + + let memory_offset = if ofs.num_imported_memories > 0 { + ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) + } else if ofs.num_defined_memories > 0 { + ModuleMemoryOffset::Defined( + ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)), + ) + } else { + ModuleMemoryOffset::None + }; + let dwarf_sections = + crate::debug::emit_dwarf(&*self.isa, &translation.debuginfo, &funcs, &memory_offset) + .with_context(|| "failed to emit DWARF debug information")?; + + let (debug_bodies, debug_relocs): (Vec<_>, Vec<_>) = dwarf_sections + .iter() + .map(|s| ((s.name, &s.body), (s.name, &s.relocs))) + .unzip(); + let mut dwarf_sections_ids = HashMap::new(); + for (name, body) in debug_bodies { + let segment = obj.segment_name(StandardSegment::Debug).to_vec(); + let section_id = obj.add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug); + dwarf_sections_ids.insert(name, section_id); + obj.append_section_data(section_id, &body, 1); + } + + // Write all debug data relocations. + for (name, relocs) in debug_relocs { + let section_id = *dwarf_sections_ids.get(name).unwrap(); + for reloc in relocs { + let target_symbol = match reloc.target { + DwarfSectionRelocTarget::Func(index) => { + func_symbols[DefinedFuncIndex::new(index)] + } + DwarfSectionRelocTarget::Section(name) => { + obj.section_symbol(dwarf_sections_ids[name]) + } + }; + obj.add_relocation( + section_id, + object::write::Relocation { + offset: u64::from(reloc.offset), + size: reloc.size << 3, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: target_symbol, + addend: i64::from(reloc.addend), + }, + )?; + } + } + + Ok(()) + } } // Collects an iterator of `InstructionAddressMap` into a `Vec` for insertion diff --git a/crates/cranelift/src/obj.rs b/crates/cranelift/src/obj.rs index 08373bb2cf..063afd2393 100644 --- a/crates/cranelift/src/obj.rs +++ b/crates/cranelift/src/obj.rs @@ -13,7 +13,6 @@ //! function body, the imported wasm function do not. The trampolines symbol //! names have format "_trampoline_N", where N is `SignatureIndex`. -use crate::debug::{DwarfSection, DwarfSectionRelocTarget}; use crate::{CompiledFunction, RelocationTarget}; use anyhow::Result; use cranelift_codegen::isa::{ @@ -23,56 +22,22 @@ use cranelift_codegen::isa::{ use cranelift_codegen::TextSectionBuilder; use gimli::write::{Address, EhFrame, EndianVec, FrameTable, Writer}; use gimli::RunTimeEndian; -use object::write::{ - Object, Relocation as ObjectRelocation, SectionId, StandardSegment, Symbol, SymbolId, - SymbolSection, -}; -use object::{ - Architecture, RelocationEncoding, RelocationKind, SectionKind, SymbolFlags, SymbolKind, - SymbolScope, -}; -use std::collections::HashMap; +use object::write::{Object, SectionId, StandardSegment, Symbol, SymbolId, SymbolSection}; +use object::{Architecture, SectionKind, SymbolFlags, SymbolKind, SymbolScope}; use std::convert::TryFrom; use std::ops::Range; use wasmtime_environ::obj; -use wasmtime_environ::{ - DefinedFuncIndex, EntityRef, FuncIndex, Module, PrimaryMap, SignatureIndex, Trampoline, -}; +use wasmtime_environ::{DefinedFuncIndex, Module, PrimaryMap, SignatureIndex, Trampoline}; const TEXT_SECTION_NAME: &[u8] = b".text"; -/// Iterates through all `LibCall` members and all runtime exported functions. -#[macro_export] -macro_rules! for_each_libcall { - ($op:ident) => { - $op![ - (UdivI64, wasmtime_i64_udiv), - (UdivI64, wasmtime_i64_udiv), - (SdivI64, wasmtime_i64_sdiv), - (UremI64, wasmtime_i64_urem), - (SremI64, wasmtime_i64_srem), - (IshlI64, wasmtime_i64_ishl), - (UshrI64, wasmtime_i64_ushr), - (SshrI64, wasmtime_i64_sshr), - (CeilF32, wasmtime_f32_ceil), - (FloorF32, wasmtime_f32_floor), - (TruncF32, wasmtime_f32_trunc), - (NearestF32, wasmtime_f32_nearest), - (CeilF64, wasmtime_f64_ceil), - (FloorF64, wasmtime_f64_floor), - (TruncF64, wasmtime_f64_trunc), - (NearestF64, wasmtime_f64_nearest) - ]; - }; -} - /// A helper structure used to assemble the final text section of an exectuable, /// plus unwinding information and other related details. /// /// This builder relies on Cranelift-specific internals but assembles into a /// generic `Object` which will get further appended to in a compiler-agnostic /// fashion later. -pub struct ObjectBuilder<'a> { +pub struct ModuleTextBuilder<'a> { /// The target that we're compiling for, used to query target-specific /// information as necessary. isa: &'a dyn TargetIsa, @@ -83,49 +48,21 @@ pub struct ObjectBuilder<'a> { /// The WebAssembly module we're generating code for. module: &'a Module, - windows_unwind_info_id: Option, + text_section: SectionId, - /// Packed form of windows unwind tables which, if present, will get emitted - /// to a windows-specific unwind info section. - windows_unwind_info: Vec, - - systemv_unwind_info_id: Option, - - /// Pending unwinding information for DWARF-based platforms. This is used to - /// build a `.eh_frame` lookalike at the very end of object building. - systemv_unwind_info: Vec<(u64, &'a systemv::UnwindInfo)>, + unwind_info: UnwindInfoBuilder<'a>, /// The corresponding symbol for each function, inserted as they're defined. /// /// If an index isn't here yet then it hasn't been defined yet. - func_symbols: PrimaryMap, - - /// `object`-crate identifier for the text section. - text_section: SectionId, + func_symbols: PrimaryMap, /// In-progress text section that we're using cranelift's `MachBuffer` to /// build to resolve relocations (calls) between functions. - pub text: Box, - - /// The unwind info _must_ come directly after the text section. Our FDE's - /// instructions are encoded to rely on this placement. We use this `bool` - /// for debug assertions to ensure that we get the ordering correct. - added_unwind_info: bool, + text: Box, } -// This is a mirror of `RUNTIME_FUNCTION` in the Windows API, but defined here -// to ensure everything is always `u32` and to have it available on all -// platforms. Note that all of these specifiers here are relative to a "base -// address" which we define as the base of where the text section is eventually -// loaded. -#[allow(non_camel_case_types)] -struct RUNTIME_FUNCTION { - begin: u32, - end: u32, - unwind_address: u32, -} - -impl<'a> ObjectBuilder<'a> { +impl<'a> ModuleTextBuilder<'a> { pub fn new(obj: &'a mut Object<'static>, module: &'a Module, isa: &'a dyn TargetIsa) -> Self { // Entire code (functions and trampolines) will be placed // in the ".text" section. @@ -135,37 +72,15 @@ impl<'a> ObjectBuilder<'a> { SectionKind::Text, ); - // Create symbols for imports -- needed during linking. - let mut func_symbols = PrimaryMap::with_capacity(module.functions.len()); - for index in 0..module.num_imported_funcs { - let symbol_id = obj.add_symbol(Symbol { - name: obj::func_symbol_name(FuncIndex::new(index)) - .as_bytes() - .to_vec(), - value: 0, - size: 0, - kind: SymbolKind::Text, - scope: SymbolScope::Linkage, - weak: false, - section: SymbolSection::Undefined, - flags: SymbolFlags::None, - }); - func_symbols.push(symbol_id); - } - + let num_defined = module.functions.len() - module.num_imported_funcs; Self { isa, obj, module, text_section, - func_symbols, - windows_unwind_info_id: None, - windows_unwind_info: Vec::new(), - systemv_unwind_info_id: None, - systemv_unwind_info: Vec::new(), - text: isa - .text_section_builder((module.functions.len() - module.num_imported_funcs) as u32), - added_unwind_info: false, + func_symbols: PrimaryMap::with_capacity(num_defined), + unwind_info: Default::default(), + text: isa.text_section_builder(num_defined as u32), } } @@ -193,35 +108,10 @@ impl<'a> ObjectBuilder<'a> { flags: SymbolFlags::None, }); - match &func.unwind_info { - // Windows unwind information is preferred to come after the code - // itself. The information is appended here just after the function, - // aligned to 4-bytes as required by Windows. - // - // The location of the unwind info, and the function it describes, - // is then recorded in an unwind info table to get embedded into the - // object at the end of compilation. - Some(UnwindInfo::WindowsX64(info)) => { - // Windows prefers Unwind info after the code -- writing it here. - let unwind_size = info.emit_size(); - let mut unwind_info = vec![0; unwind_size]; - info.emit(&mut unwind_info); - let unwind_off = self.text.append(false, &unwind_info, Some(4)); - self.windows_unwind_info.push(RUNTIME_FUNCTION { - begin: u32::try_from(off).unwrap(), - end: u32::try_from(off + body_len).unwrap(), - unwind_address: u32::try_from(unwind_off).unwrap(), - }); - } - - // System-V is different enough that we just record the unwinding - // information to get processed at a later time. - Some(UnwindInfo::SystemV(info)) => { - self.systemv_unwind_info.push((off, info)); - } - - Some(_) => panic!("some unwind info isn't handled here"), - None => {} + if let Some(info) = &func.unwind_info { + self.unwind_info.push(off, body_len, info, |data, align| { + self.text.append(false, data, Some(align)) + }); } for r in func.relocations.iter() { @@ -272,16 +162,13 @@ impl<'a> ObjectBuilder<'a> { /// /// This is expected to be called in-order for ascending `index` values. pub fn func(&mut self, index: DefinedFuncIndex, func: &'a CompiledFunction) -> Range { - assert!(!self.added_unwind_info); - let index = self.module.func_index(index); - let name = obj::func_symbol_name(index); + let name = obj::func_symbol_name(self.module.func_index(index)); let (symbol_id, range) = self.append_func(true, name.into_bytes(), func); assert_eq!(self.func_symbols.push(symbol_id), index); range } pub fn trampoline(&mut self, sig: SignatureIndex, func: &'a CompiledFunction) -> Trampoline { - assert!(!self.added_unwind_info); let name = obj::trampoline_symbol_name(sig); let (_, range) = self.append_func(false, name.into_bytes(), func); Trampoline { @@ -291,117 +178,164 @@ impl<'a> ObjectBuilder<'a> { } } - pub fn dwarf_sections(&mut self, sections: &[DwarfSection]) -> Result<()> { - assert!( - self.added_unwind_info, - "can't add dwarf yet; unwind info must directly follow the text section" - ); - - // If we have DWARF data, write it in the object file. - let (debug_bodies, debug_relocs): (Vec<_>, Vec<_>) = sections - .iter() - .map(|s| ((s.name, &s.body), (s.name, &s.relocs))) - .unzip(); - let mut dwarf_sections_ids = HashMap::new(); - for (name, body) in debug_bodies { - let segment = self.obj.segment_name(StandardSegment::Debug).to_vec(); - let section_id = - self.obj - .add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug); - dwarf_sections_ids.insert(name, section_id); - self.obj.append_section_data(section_id, &body, 1); - } - - // Write all debug data relocations. - for (name, relocs) in debug_relocs { - let section_id = *dwarf_sections_ids.get(name).unwrap(); - for reloc in relocs { - let target_symbol = match reloc.target { - DwarfSectionRelocTarget::Func(index) => { - self.func_symbols[self.module.func_index(DefinedFuncIndex::new(index))] - } - DwarfSectionRelocTarget::Section(name) => { - self.obj.section_symbol(dwarf_sections_ids[name]) - } - }; - self.obj.add_relocation( - section_id, - ObjectRelocation { - offset: u64::from(reloc.offset), - size: reloc.size << 3, - kind: RelocationKind::Absolute, - encoding: RelocationEncoding::Generic, - symbol: target_symbol, - addend: i64::from(reloc.addend), - }, - )?; - } - } - - Ok(()) + /// Forces "veneers" to be used for inter-function calls in the text + /// section which means that in-bounds optimized addresses are never used. + /// + /// This is only useful for debugging cranelift itself and typically this + /// option is disabled. + pub fn force_veneers(&mut self) { + self.text.force_veneers(); } - pub fn unwind_info(&mut self) { - assert!(!self.added_unwind_info); - - if self.windows_unwind_info.len() > 0 { - let segment = self.obj.segment_name(StandardSegment::Data).to_vec(); - self.windows_unwind_info_id = Some(self.obj.add_section( - segment, - b"_wasmtime_winx64_unwind".to_vec(), - SectionKind::ReadOnlyData, - )); + /// Appends the specified amount of bytes of padding into the text section. + /// + /// This is only useful when fuzzing and/or debugging cranelift itself and + /// for production scenarios `padding` is 0 and this function does nothing. + pub fn append_padding(&mut self, padding: usize) { + if padding == 0 { + return; } - if self.systemv_unwind_info.len() > 0 { - let segment = self.obj.segment_name(StandardSegment::Data).to_vec(); - self.systemv_unwind_info_id = Some(self.obj.add_section( - segment, - b".eh_frame".to_vec(), - SectionKind::ReadOnlyData, - )); - } - - self.added_unwind_info = true; + self.text.append(false, &vec![0; padding], Some(1)); } - pub fn finish(&mut self) -> Result<()> { + /// Indicates that the text section has been written completely and this + /// will finish appending it to the original object. + /// + /// Note that this will also write out the unwind information sections if + /// necessary. + pub fn finish(mut self) -> Result> { // Finish up the text section now that we're done adding functions. let text = self.text.finish(); self.obj .section_mut(self.text_section) .set_data(text, self.isa.code_section_alignment()); - // With all functions added we can also emit the fully-formed unwinding - // information sections. - if self.windows_unwind_info.len() > 0 { - self.append_windows_unwind_info(); + // Append the unwind information for all our functions, if necessary. + self.unwind_info + .append_section(self.isa, self.obj, self.text_section); + + Ok(self.func_symbols) + } +} + +/// Builder used to create unwind information for a set of functions added to a +/// text section. +#[derive(Default)] +struct UnwindInfoBuilder<'a> { + windows_unwind_info: Vec, + systemv_unwind_info: Vec<(u64, &'a systemv::UnwindInfo)>, +} + +// This is a mirror of `RUNTIME_FUNCTION` in the Windows API, but defined here +// to ensure everything is always `u32` and to have it available on all +// platforms. Note that all of these specifiers here are relative to a "base +// address" which we define as the base of where the text section is eventually +// loaded. +#[allow(non_camel_case_types)] +struct RUNTIME_FUNCTION { + begin: u32, + end: u32, + unwind_address: u32, +} + +impl<'a> UnwindInfoBuilder<'a> { + /// Pushes the unwind information for a function into this builder. + /// + /// The function being described must be located at `function_offset` within + /// the text section itself, and the function's size is specified by + /// `function_len`. + /// + /// The `info` should come from Cranelift itself and this function may + /// append more data to the text section in which case the `append_data` + /// callback will be invoked. The `append_data` callback receives the data + /// to append to the text section as well as the alignment it needs to be + /// written at. The return value of `append_data` should be the offset + /// within the text section for where the data was written. + fn push( + &mut self, + function_offset: u64, + function_len: u64, + info: &'a UnwindInfo, + append_data: impl FnOnce(&[u8], u32) -> u64, + ) { + match info { + // Windows unwind information is preferred to come after the code + // itself. The information is appended here just after the function, + // aligned to 4-bytes as required by Windows. + // + // The location of the unwind info, and the function it describes, + // is then recorded in an unwind info table to get embedded into the + // object at the end of compilation. + UnwindInfo::WindowsX64(info) => { + // Windows prefers Unwind info after the code -- writing it here. + let unwind_size = info.emit_size(); + let mut unwind_info = vec![0; unwind_size]; + info.emit(&mut unwind_info); + let unwind_off = append_data(&unwind_info, 4); + self.windows_unwind_info.push(RUNTIME_FUNCTION { + begin: u32::try_from(function_offset).unwrap(), + end: u32::try_from(function_offset + function_len).unwrap(), + unwind_address: u32::try_from(unwind_off).unwrap(), + }); + } + + // System-V is different enough that we just record the unwinding + // information to get processed at a later time. + UnwindInfo::SystemV(info) => { + self.systemv_unwind_info.push((function_offset, info)); + } + + _ => panic!("some unwind info isn't handled here"), } - if self.systemv_unwind_info.len() > 0 { - self.append_systemv_unwind_info(); + } + + /// Appends the unwind information section, if any, to the `obj` specified. + /// + /// This function must be called immediately after the text section was + /// added to a builder. The unwind information section must trail the text + /// section immediately. + /// + /// The `text_section`'s section identifier is passed into this function. + fn append_section(&self, isa: &dyn TargetIsa, obj: &mut Object<'_>, text_section: SectionId) { + // This write will align the text section to a page boundary and then + // return the offset at that point. This gives us the full size of the + // text section at that point, after alignment. + let text_section_size = + obj.append_section_data(text_section, &[], isa.code_section_alignment()); + + if self.windows_unwind_info.len() > 0 { + assert!(self.systemv_unwind_info.len() == 0); + let segment = obj.segment_name(StandardSegment::Data).to_vec(); + let section_id = obj.add_section( + segment, + b"_wasmtime_winx64_unwind".to_vec(), + SectionKind::ReadOnlyData, + ); + self.write_windows_unwind_info(obj, section_id); } - Ok(()) + if self.systemv_unwind_info.len() > 0 { + let segment = obj.segment_name(StandardSegment::Data).to_vec(); + let section_id = + obj.add_section(segment, b".eh_frame".to_vec(), SectionKind::ReadOnlyData); + self.write_systemv_unwind_info(isa, obj, section_id, text_section_size) + } } /// This function appends a nonstandard section to the object which is only - /// used during `CodeMemory::allocate_for_object`. + /// used during `CodeMemory::publish`. /// /// This custom section effectively stores a `[RUNTIME_FUNCTION; N]` into /// the object file itself. This way registration of unwind info can simply /// pass this slice to the OS itself and there's no need to recalculate /// anything on the other end of loading a module from a precompiled object. - fn append_windows_unwind_info(&mut self) { + /// + /// Support for reading this is in `crates/jit/src/unwind/winx64.rs`. + fn write_windows_unwind_info(&self, obj: &mut Object<'_>, section_id: SectionId) { // Currently the binary format supported here only supports // little-endian for x86_64, or at least that's all where it's tested. // This may need updates for other platforms. - assert_eq!(self.obj.architecture(), Architecture::X86_64); - - let section_id = self.windows_unwind_info_id.unwrap(); - - // Page-align the text section so the unwind info can reside on a - // separate page that doesn't need executable permissions. - self.obj - .append_section_data(self.text_section, &[], self.isa.code_section_alignment()); + assert_eq!(obj.architecture(), Architecture::X86_64); let mut unwind_info = Vec::with_capacity(self.windows_unwind_info.len() * 3 * 4); for info in self.windows_unwind_info.iter() { @@ -409,11 +343,11 @@ impl<'a> ObjectBuilder<'a> { unwind_info.extend_from_slice(&info.end.to_le_bytes()); unwind_info.extend_from_slice(&info.unwind_address.to_le_bytes()); } - self.obj.append_section_data(section_id, &unwind_info, 4); + obj.append_section_data(section_id, &unwind_info, 4); } /// This function appends a nonstandard section to the object which is only - /// used during `CodeMemory::allocate_for_object`. + /// used during `CodeMemory::publish`. /// /// This will generate a `.eh_frame` section, but not one that can be /// naively loaded. The goal of this section is that we can create the @@ -450,22 +384,20 @@ impl<'a> ObjectBuilder<'a> { /// This allows `.eh_frame` to have different virtual memory permissions, /// such as being purely read-only instead of read/execute like the code /// bits. - fn append_systemv_unwind_info(&mut self) { - let section_id = self.systemv_unwind_info_id.unwrap(); - let mut cie = self - .isa + fn write_systemv_unwind_info( + &self, + isa: &dyn TargetIsa, + obj: &mut Object<'_>, + section_id: SectionId, + text_section_size: u64, + ) { + let mut cie = isa .create_systemv_cie() .expect("must be able to create a CIE for system-v unwind info"); let mut table = FrameTable::default(); cie.fde_address_encoding = gimli::constants::DW_EH_PE_pcrel; let cie_id = table.add_cie(cie); - // This write will align the text section to a page boundary - // and then return the offset at that point. This gives us the full size - // of the text section at that point, after alignment. - let text_section_size = - self.obj - .append_section_data(self.text_section, &[], self.isa.code_section_alignment()); for (text_section_off, unwind_info) in self.systemv_unwind_info.iter() { let backwards_off = text_section_size - text_section_off; let actual_offset = -i64::try_from(backwards_off).unwrap(); @@ -476,7 +408,7 @@ impl<'a> ObjectBuilder<'a> { let fde = unwind_info.to_fde(Address::Constant(actual_offset as u64)); table.add_fde(cie_id, fde); } - let endian = match self.isa.triple().endianness().unwrap() { + let endian = match isa.triple().endianness().unwrap() { target_lexicon::Endianness::Little => RunTimeEndian::Little, target_lexicon::Endianness::Big => RunTimeEndian::Big, }; @@ -487,8 +419,7 @@ impl<'a> ObjectBuilder<'a> { // a 0 is written at the end of the table for those implementations. let mut endian_vec = (eh_frame.0).0; endian_vec.write_u32(0).unwrap(); - self.obj - .append_section_data(section_id, endian_vec.slice(), 1); + obj.append_section_data(section_id, endian_vec.slice(), 1); use gimli::constants; use gimli::write::Error; diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 8102f5e26e..b0d414564b 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -70,28 +70,3 @@ pub const WASM64_MAX_PAGES: u64 = 1 << 48; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); - -/// Iterates through all `LibCall` members and all runtime exported functions. -#[macro_export] -macro_rules! for_each_libcall { - ($op:ident) => { - $op![ - (UdivI64, wasmtime_i64_udiv), - (UdivI64, wasmtime_i64_udiv), - (SdivI64, wasmtime_i64_sdiv), - (UremI64, wasmtime_i64_urem), - (SremI64, wasmtime_i64_srem), - (IshlI64, wasmtime_i64_ishl), - (UshrI64, wasmtime_i64_ushr), - (SshrI64, wasmtime_i64_sshr), - (CeilF32, wasmtime_f32_ceil), - (FloorF32, wasmtime_f32_floor), - (TruncF32, wasmtime_f32_trunc), - (NearestF32, wasmtime_f32_nearest), - (CeilF64, wasmtime_f64_ceil), - (FloorF64, wasmtime_f64_floor), - (TruncF64, wasmtime_f64_trunc), - (NearestF64, wasmtime_f64_nearest) - ]; - }; -}