diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8a3363799f..0c0bfb7351 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -231,6 +231,8 @@ jobs: gcc: aarch64-linux-gnu-gcc qemu: qemu-aarch64 -L /usr/aarch64-linux-gnu qemu_target: aarch64-linux-user + # FIXME(#3183) shouldn't be necessary to specify this + qemu_flags: -cpu cortex-a72 steps: - uses: actions/checkout@v2 with: @@ -263,7 +265,7 @@ jobs: # Configure Cargo for cross compilation and tell it how it can run # cross executables upcase=$(echo ${{ matrix.target }} | awk '{ print toupper($0) }' | sed 's/-/_/g') - echo CARGO_TARGET_${upcase}_RUNNER=${{ runner.tool_cache }}/qemu/bin/${{ matrix.qemu }} >> $GITHUB_ENV + echo CARGO_TARGET_${upcase}_RUNNER=${{ runner.tool_cache }}/qemu/bin/${{ matrix.qemu }} ${{ matrix.qemu_flags }} >> $GITHUB_ENV echo CARGO_TARGET_${upcase}_LINKER=${{ matrix.gcc }} >> $GITHUB_ENV # QEMU emulation is not always the speediest, so total testing time diff --git a/Cargo.lock b/Cargo.lock index 46e9e3f2dc..6a3aac53f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3684,7 +3684,6 @@ dependencies = [ "wasmtime-environ", "wasmtime-fuzzing", "wasmtime-jit", - "wasmtime-obj", "wasmtime-runtime", "wasmtime-wasi", "wasmtime-wasi-crypto", @@ -3706,6 +3705,7 @@ dependencies = [ "cranelift-native", "cranelift-wasm", "gimli", + "object", "target-lexicon", "wasmparser", "wasmtime-debug", @@ -3717,6 +3717,7 @@ name = "wasmtime-debug" version = "0.29.0" dependencies = [ "anyhow", + "cranelift-codegen", "gimli", "more-asserts", "object", @@ -3813,7 +3814,6 @@ dependencies = [ "wasmtime-cranelift", "wasmtime-environ", "wasmtime-lightbeam", - "wasmtime-obj", "wasmtime-profiling", "wasmtime-runtime", "winapi", @@ -3832,17 +3832,6 @@ dependencies = [ "wasmtime-environ", ] -[[package]] -name = "wasmtime-obj" -version = "0.29.0" -dependencies = [ - "anyhow", - "more-asserts", - "object", - "target-lexicon", - "wasmtime-environ", -] - [[package]] name = "wasmtime-profiling" version = "0.29.0" diff --git a/Cargo.toml b/Cargo.toml index 38783c7800..c1d575f685 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ wasmtime-cache = { path = "crates/cache", version = "0.29.0" } wasmtime-debug = { path = "crates/debug", version = "0.29.0" } wasmtime-environ = { path = "crates/environ", version = "0.29.0" } wasmtime-jit = { path = "crates/jit", version = "0.29.0" } -wasmtime-obj = { path = "crates/obj", version = "0.29.0" } wasmtime-wast = { path = "crates/wast", version = "0.29.0" } wasmtime-wasi = { path = "crates/wasi", version = "0.29.0" } wasmtime-wasi-crypto = { path = "crates/wasi-crypto", version = "0.29.0", optional = true } diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index 7b5aa944be..b520a5277a 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -22,6 +22,7 @@ wasmtime-debug = { path = '../debug', version = '0.29.0' } wasmparser = "0.80.0" target-lexicon = "0.12" gimli = "0.25.0" +object = { version = "0.26.0", default-features = false, features = ['write'] } [features] all-arch = ["cranelift-codegen/all-arch"] diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index fe0f4cdb2a..5c960d506b 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -1,24 +1,29 @@ use crate::func_environ::{get_func_name, FuncEnvironment}; +use crate::obj::{ObjectBuilder, ObjectBuilderTarget}; use crate::{blank_sig, func_signature, indirect_signature, value_type, wasmtime_call_conv}; -use anyhow::Result; +use anyhow::{Context as _, Result}; use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags}; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::settings; use cranelift_codegen::MachSrcLoc; use cranelift_codegen::{binemit, Context}; +use cranelift_entity::EntityRef; use cranelift_frontend::FunctionBuilder; -use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator, WasmFuncType}; +use cranelift_wasm::{ + DefinedFuncIndex, DefinedMemoryIndex, FuncIndex, FuncTranslator, MemoryIndex, SignatureIndex, + WasmFuncType, +}; use std::cmp; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::convert::TryFrom; use std::mem; use std::sync::Mutex; use wasmtime_environ::{ - CompileError, CompiledFunction, CompiledFunctions, DebugInfoData, DwarfSection, FlagValue, - FunctionAddressMap, FunctionBodyData, InstructionAddressMap, ModuleMemoryOffset, - ModuleTranslation, Relocation, RelocationTarget, StackMapInformation, TrapInformation, - Tunables, TypeTables, + CompileError, CompiledFunction, CompiledFunctions, FlagValue, FunctionAddressMap, + FunctionBodyData, InstructionAddressMap, Module, ModuleMemoryOffset, ModuleTranslation, + Relocation, RelocationTarget, StackMapInformation, TrapInformation, Tunables, TypeTables, + VMOffsets, }; /// A compiler that compiles a WebAssembly module with Compiler, translating @@ -204,6 +209,117 @@ impl wasmtime_environ::Compiler for Compiler { }) } + fn emit_obj( + &self, + translation: &ModuleTranslation, + types: &TypeTables, + funcs: &CompiledFunctions, + emit_dwarf: bool, + ) -> Result> { + const CODE_SECTION_ALIGNMENT: u64 = 0x1000; + + // Build trampolines for every signature that can be used by this module. + let signatures = translation + .module + .functions + .iter() + .filter_map(|(i, sig)| match translation.module.defined_func_index(i) { + Some(i) if !translation.module.possibly_exported_funcs.contains(&i) => None, + _ => Some(*sig), + }) + .collect::>(); + let mut trampolines = Vec::with_capacity(signatures.len()); + for i in signatures { + let func = self.host_to_wasm_trampoline(&types.wasm_signatures[i])?; + trampolines.push((i, func)); + } + + let target = ObjectBuilderTarget::elf(self.isa.triple().architecture)?; + let mut builder = ObjectBuilder::new(target, &translation.module); + + for (i, func) in funcs.iter() { + builder.func(i, func); + } + for (i, func) in trampolines.iter() { + builder.trampoline(*i, func); + } + builder.align_text_to(CODE_SECTION_ALIGNMENT); + + if emit_dwarf && funcs.len() > 0 { + let ofs = VMOffsets::new( + self.isa + .triple() + .architecture + .pointer_width() + .unwrap() + .bytes(), + &translation.module, + ); + + let memory_offset = if ofs.num_imported_memories > 0 { + ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) + } else if ofs.num_defined_memories > 0 { + ModuleMemoryOffset::Defined( + ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)), + ) + } else { + ModuleMemoryOffset::None + }; + let dwarf_sections = wasmtime_debug::emit_dwarf( + &*self.isa, + &translation.debuginfo, + funcs, + &memory_offset, + ) + .with_context(|| "failed to emit DWARF debug information")?; + builder.dwarf_sections(&dwarf_sections)?; + } + + Ok(builder.finish(&*self.isa)?) + } + + fn emit_trampoline_obj(&self, ty: &WasmFuncType, host_fn: usize) -> Result> { + let host_to_wasm = self.host_to_wasm_trampoline(ty)?; + let wasm_to_host = self.wasm_to_host_trampoline(ty, host_fn)?; + let target = ObjectBuilderTarget::elf(self.isa.triple().architecture)?; + let module = Module::new(); + let mut builder = ObjectBuilder::new(target, &module); + builder.trampoline(SignatureIndex::new(0), &host_to_wasm); + builder.trampoline(SignatureIndex::new(1), &wasm_to_host); + Ok(builder.finish(&*self.isa)?) + } + + fn triple(&self) -> &target_lexicon::Triple { + self.isa.triple() + } + + fn flags(&self) -> HashMap { + self.isa + .flags() + .iter() + .map(|val| (val.name.to_string(), to_flag_value(&val))) + .collect() + } + + fn isa_flags(&self) -> HashMap { + self.isa + .isa_flags() + .iter() + .map(|val| (val.name.to_string(), to_flag_value(val))) + .collect() + } +} + +fn to_flag_value(v: &settings::Value) -> FlagValue { + match v.kind() { + settings::SettingKind::Enum => FlagValue::Enum(v.as_enum().unwrap().into()), + settings::SettingKind::Num => FlagValue::Num(v.as_num().unwrap()), + settings::SettingKind::Bool => FlagValue::Bool(v.as_bool().unwrap()), + settings::SettingKind::Preset => unreachable!(), + } +} + +impl Compiler { fn host_to_wasm_trampoline(&self, ty: &WasmFuncType) -> Result { let isa = &*self.isa; let value_size = mem::size_of::(); @@ -361,50 +477,6 @@ impl wasmtime_environ::Compiler for Compiler { Ok(func) } - fn emit_dwarf( - &self, - debuginfo_data: &DebugInfoData, - funcs: &CompiledFunctions, - memory_offset: &ModuleMemoryOffset, - ) -> Result> { - wasmtime_debug::emit_dwarf(&*self.isa, debuginfo_data, funcs, memory_offset) - } - - fn triple(&self) -> &target_lexicon::Triple { - self.isa.triple() - } - - fn create_systemv_cie(&self) -> Option { - self.isa.create_systemv_cie() - } - - fn flags(&self) -> HashMap { - self.isa - .flags() - .iter() - .map(|val| (val.name.to_string(), to_flag_value(&val))) - .collect() - } - - fn isa_flags(&self) -> HashMap { - self.isa - .isa_flags() - .iter() - .map(|val| (val.name.to_string(), to_flag_value(val))) - .collect() - } -} - -fn to_flag_value(v: &settings::Value) -> FlagValue { - match v.kind() { - settings::SettingKind::Enum => FlagValue::Enum(v.as_enum().unwrap().into()), - settings::SettingKind::Num => FlagValue::Num(v.as_num().unwrap()), - settings::SettingKind::Bool => FlagValue::Bool(v.as_bool().unwrap()), - settings::SettingKind::Preset => unreachable!(), - } -} - -impl Compiler { fn finish_trampoline( &self, mut context: Context, diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 29924451cb..56eadf02e8 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -99,6 +99,7 @@ pub use builder::builder; mod builder; mod compiler; mod func_environ; +mod obj; /// Creates a new cranelift `Signature` with no wasm params/results for the /// given calling convention. diff --git a/crates/cranelift/src/obj.rs b/crates/cranelift/src/obj.rs new file mode 100644 index 0000000000..18fb0f4101 --- /dev/null +++ b/crates/cranelift/src/obj.rs @@ -0,0 +1,549 @@ +//! Object file builder. +//! +//! Creates ELF image based on `Compilation` information. The ELF contains +//! functions and trampolines in the ".text" section. It also contains all +//! relocation records for the linking stage. If DWARF sections exist, their +//! content will be written as well. +//! +//! The object file has symbols for each function and trampoline, as well as +//! symbols that refer to libcalls. +//! +//! The function symbol names have format "_wasm_function_N", where N is +//! `FuncIndex`. The defined wasm function symbols refer to a JIT compiled +//! function body, the imported wasm function do not. The trampolines symbol +//! names have format "_trampoline_N", where N is `SignatureIndex`. + +#![allow(missing_docs)] + +use anyhow::Result; +use cranelift_codegen::binemit::Reloc; +use cranelift_codegen::ir::{JumpTableOffsets, LibCall}; +use cranelift_codegen::isa::{ + unwind::{systemv, UnwindInfo}, + TargetIsa, +}; +use gimli::write::{Address, EhFrame, EndianVec, FrameTable, Writer}; +use gimli::RunTimeEndian; +use object::write::{ + Object, Relocation as ObjectRelocation, SectionId, StandardSegment, Symbol, SymbolId, + SymbolSection, +}; +use object::{ + elf, Architecture, BinaryFormat, Endianness, RelocationEncoding, RelocationKind, SectionKind, + SymbolFlags, SymbolKind, SymbolScope, +}; +use std::collections::HashMap; +use std::convert::TryFrom; +use wasmtime_debug::{DwarfSection, DwarfSectionRelocTarget}; +use wasmtime_environ::entity::{EntityRef, PrimaryMap}; +use wasmtime_environ::obj; +use wasmtime_environ::wasm::{DefinedFuncIndex, FuncIndex, SignatureIndex}; +use wasmtime_environ::{CompiledFunction, Module, Relocation, RelocationTarget}; + +fn to_object_architecture( + arch: target_lexicon::Architecture, +) -> Result { + use target_lexicon::Architecture::*; + Ok(match arch { + X86_32(_) => Architecture::I386, + X86_64 => Architecture::X86_64, + Arm(_) => Architecture::Arm, + Aarch64(_) => Architecture::Aarch64, + S390x => Architecture::S390x, + architecture => { + anyhow::bail!("target architecture {:?} is unsupported", architecture,); + } + }) +} + +const TEXT_SECTION_NAME: &[u8] = b".text"; + +/// Iterates through all `LibCall` members and all runtime exported functions. +#[macro_export] +macro_rules! for_each_libcall { + ($op:ident) => { + $op![ + (UdivI64, wasmtime_i64_udiv), + (UdivI64, wasmtime_i64_udiv), + (SdivI64, wasmtime_i64_sdiv), + (UremI64, wasmtime_i64_urem), + (SremI64, wasmtime_i64_srem), + (IshlI64, wasmtime_i64_ishl), + (UshrI64, wasmtime_i64_ushr), + (SshrI64, wasmtime_i64_sshr), + (CeilF32, wasmtime_f32_ceil), + (FloorF32, wasmtime_f32_floor), + (TruncF32, wasmtime_f32_trunc), + (NearestF32, wasmtime_f32_nearest), + (CeilF64, wasmtime_f64_ceil), + (FloorF64, wasmtime_f64_floor), + (TruncF64, wasmtime_f64_trunc), + (NearestF64, wasmtime_f64_nearest) + ]; + }; +} + +fn write_libcall_symbols(obj: &mut Object) -> HashMap { + let mut libcalls = HashMap::new(); + macro_rules! add_libcall_symbol { + [$(($libcall:ident, $export:ident)),*] => {{ + $( + let symbol_id = obj.add_symbol(Symbol { + name: stringify!($export).as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: true, + section: SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + libcalls.insert(LibCall::$libcall, symbol_id); + )+ + }}; + } + for_each_libcall!(add_libcall_symbol); + + libcalls +} + +pub struct ObjectBuilderTarget { + pub(crate) binary_format: BinaryFormat, + pub(crate) architecture: Architecture, + pub(crate) endianness: Endianness, +} + +impl ObjectBuilderTarget { + pub fn elf(arch: target_lexicon::Architecture) -> Result { + Ok(Self { + binary_format: BinaryFormat::Elf, + architecture: to_object_architecture(arch)?, + endianness: match arch.endianness().unwrap() { + target_lexicon::Endianness::Little => object::Endianness::Little, + target_lexicon::Endianness::Big => object::Endianness::Big, + }, + }) + } +} + +pub struct ObjectBuilder<'a> { + obj: Object, + module: &'a Module, + text_section: SectionId, + func_symbols: PrimaryMap, + jump_tables: PrimaryMap, + libcalls: HashMap, + pending_relocations: Vec<(u64, &'a [Relocation])>, + windows_unwind_info: Vec, + systemv_unwind_info: Vec<(u64, &'a systemv::UnwindInfo)>, +} + +// This is a mirror of `RUNTIME_FUNCTION` in the Windows API, but defined here +// to ensure everything is always `u32` and to have it available on all +// platforms. Note that all of these specifiers here are relative to a "base +// address" which we define as the base of where the text section is eventually +// loaded. +#[allow(non_camel_case_types)] +struct RUNTIME_FUNCTION { + begin: u32, + end: u32, + unwind_address: u32, +} + +impl<'a> ObjectBuilder<'a> { + pub fn new(target: ObjectBuilderTarget, module: &'a Module) -> Self { + let mut obj = Object::new(target.binary_format, target.architecture, target.endianness); + + // Entire code (functions and trampolines) will be placed + // in the ".text" section. + let text_section = obj.add_section( + obj.segment_name(StandardSegment::Text).to_vec(), + TEXT_SECTION_NAME.to_vec(), + SectionKind::Text, + ); + + // Create symbols for imports -- needed during linking. + let mut func_symbols = PrimaryMap::with_capacity(module.functions.len()); + for index in 0..module.num_imported_funcs { + let symbol_id = obj.add_symbol(Symbol { + name: obj::func_symbol_name(FuncIndex::new(index)) + .as_bytes() + .to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + func_symbols.push(symbol_id); + } + + let libcalls = write_libcall_symbols(&mut obj); + + Self { + obj, + module, + text_section, + func_symbols, + libcalls, + pending_relocations: Vec::new(), + jump_tables: PrimaryMap::with_capacity(module.functions.len()), + windows_unwind_info: Vec::new(), + systemv_unwind_info: Vec::new(), + } + } + + fn append_func(&mut self, name: Vec, func: &'a CompiledFunction) -> SymbolId { + let off = self + .obj + .append_section_data(self.text_section, &func.body, 1); + let symbol_id = self.obj.add_symbol(Symbol { + name, + value: off, + size: func.body.len() as u64, + kind: SymbolKind::Text, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Section(self.text_section), + flags: SymbolFlags::None, + }); + + match &func.unwind_info { + // Windows unwind information is preferred to come after the code + // itself. The information is appended here just after the function, + // aligned to 4-bytes as required by Windows. + // + // The location of the unwind info, and the function it describes, + // is then recorded in an unwind info table to get embedded into the + // object at the end of compilation. + Some(UnwindInfo::WindowsX64(info)) => { + // Windows prefers Unwind info after the code -- writing it here. + let unwind_size = info.emit_size(); + let mut unwind_info = vec![0; unwind_size]; + info.emit(&mut unwind_info); + let unwind_off = self + .obj + .append_section_data(self.text_section, &unwind_info, 4); + self.windows_unwind_info.push(RUNTIME_FUNCTION { + begin: u32::try_from(off).unwrap(), + end: u32::try_from(off + func.body.len() as u64).unwrap(), + unwind_address: u32::try_from(unwind_off).unwrap(), + }); + } + + // System-V is different enough that we just record the unwinding + // information to get processed at a later time. + Some(UnwindInfo::SystemV(info)) => { + self.systemv_unwind_info.push((off, info)); + } + + Some(_) => panic!("some unwind info isn't handled here"), + None => {} + } + if !func.relocations.is_empty() { + self.pending_relocations.push((off, &func.relocations)); + } + symbol_id + } + + pub fn func(&mut self, index: DefinedFuncIndex, func: &'a CompiledFunction) { + assert_eq!(self.jump_tables.push(&func.jt_offsets), index); + let index = self.module.func_index(index); + let name = obj::func_symbol_name(index); + let symbol_id = self.append_func(name.into_bytes(), func); + assert_eq!(self.func_symbols.push(symbol_id), index); + } + + pub fn trampoline(&mut self, sig: SignatureIndex, func: &'a CompiledFunction) { + let name = obj::trampoline_symbol_name(sig); + self.append_func(name.into_bytes(), func); + } + + pub fn align_text_to(&mut self, align: u64) { + self.obj.append_section_data(self.text_section, &[], align); + } + + pub fn dwarf_sections(&mut self, sections: &[DwarfSection]) -> Result<()> { + // If we have DWARF data, write it in the object file. + let (debug_bodies, debug_relocs): (Vec<_>, Vec<_>) = sections + .iter() + .map(|s| ((s.name, &s.body), (s.name, &s.relocs))) + .unzip(); + let mut dwarf_sections_ids = HashMap::new(); + for (name, body) in debug_bodies { + let segment = self.obj.segment_name(StandardSegment::Debug).to_vec(); + let section_id = + self.obj + .add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug); + dwarf_sections_ids.insert(name, section_id); + self.obj.append_section_data(section_id, &body, 1); + } + + // Write all debug data relocations. + for (name, relocs) in debug_relocs { + let section_id = *dwarf_sections_ids.get(name).unwrap(); + for reloc in relocs { + let target_symbol = match reloc.target { + DwarfSectionRelocTarget::Func(index) => { + self.func_symbols[self.module.func_index(DefinedFuncIndex::new(index))] + } + DwarfSectionRelocTarget::Section(name) => { + self.obj.section_symbol(dwarf_sections_ids[name]) + } + }; + self.obj.add_relocation( + section_id, + ObjectRelocation { + offset: u64::from(reloc.offset), + size: reloc.size << 3, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: target_symbol, + addend: i64::from(reloc.addend), + }, + )?; + } + } + + Ok(()) + } + + pub fn finish(&mut self, isa: &dyn TargetIsa) -> Result> { + self.append_relocations()?; + if self.windows_unwind_info.len() > 0 { + self.append_windows_unwind_info(); + } + if self.systemv_unwind_info.len() > 0 { + self.append_systemv_unwind_info(isa); + } + Ok(self.obj.write()?) + } + + fn append_relocations(&mut self) -> Result<()> { + for (off, relocations) in self.pending_relocations.iter() { + for r in relocations.iter() { + let (symbol, symbol_offset) = match r.reloc_target { + RelocationTarget::UserFunc(index) => (self.func_symbols[index], 0), + RelocationTarget::LibCall(call) => (self.libcalls[&call], 0), + RelocationTarget::JumpTable(f, jt) => { + let df = self.module.defined_func_index(f).unwrap(); + let offset = *self + .jump_tables + .get(df) + .and_then(|t| t.get(jt)) + .expect("func jump table"); + (self.func_symbols[f], offset) + } + }; + let (kind, encoding, size) = match r.reloc { + Reloc::Abs4 => (RelocationKind::Absolute, RelocationEncoding::Generic, 32), + Reloc::Abs8 => (RelocationKind::Absolute, RelocationEncoding::Generic, 64), + Reloc::X86PCRel4 => (RelocationKind::Relative, RelocationEncoding::Generic, 32), + Reloc::X86CallPCRel4 => { + (RelocationKind::Relative, RelocationEncoding::X86Branch, 32) + } + // TODO: Get Cranelift to tell us when we can use + // R_X86_64_GOTPCRELX/R_X86_64_REX_GOTPCRELX. + Reloc::X86CallPLTRel4 => ( + RelocationKind::PltRelative, + RelocationEncoding::X86Branch, + 32, + ), + Reloc::X86GOTPCRel4 => { + (RelocationKind::GotRelative, RelocationEncoding::Generic, 32) + } + Reloc::ElfX86_64TlsGd => ( + RelocationKind::Elf(elf::R_X86_64_TLSGD), + RelocationEncoding::Generic, + 32, + ), + Reloc::X86PCRelRodata4 => { + continue; + } + Reloc::Arm64Call => ( + RelocationKind::Elf(elf::R_AARCH64_CALL26), + RelocationEncoding::Generic, + 32, + ), + Reloc::S390xPCRel32Dbl => { + (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32) + } + other => unimplemented!("Unimplemented relocation {:?}", other), + }; + self.obj.add_relocation( + self.text_section, + ObjectRelocation { + offset: off + r.offset as u64, + size, + kind, + encoding, + symbol, + addend: r.addend.wrapping_add(symbol_offset as i64), + }, + )?; + } + } + Ok(()) + } + + /// This function appends a nonstandard section to the object which is only + /// used during `CodeMemory::allocate_for_object`. + /// + /// This custom section effectively stores a `[RUNTIME_FUNCTION; N]` into + /// the object file itself. This way registration of unwind info can simply + /// pass this slice to the OS itself and there's no need to recalculate + /// anything on the other end of loading a module from a precompiled object. + fn append_windows_unwind_info(&mut self) { + // Currently the binary format supported here only supports + // little-endian for x86_64, or at least that's all where it's tested. + // This may need updates for other platforms. + assert_eq!(self.obj.architecture(), Architecture::X86_64); + + // Page-align the text section so the unwind info can reside on a + // separate page that doesn't need executable permissions. + self.obj.append_section_data(self.text_section, &[], 0x1000); + + let segment = self.obj.segment_name(StandardSegment::Data).to_vec(); + let section_id = self.obj.add_section( + segment, + b"_wasmtime_winx64_unwind".to_vec(), + SectionKind::ReadOnlyData, + ); + let mut unwind_info = Vec::with_capacity(self.windows_unwind_info.len() * 3 * 4); + for info in self.windows_unwind_info.iter() { + unwind_info.extend_from_slice(&info.begin.to_le_bytes()); + unwind_info.extend_from_slice(&info.end.to_le_bytes()); + unwind_info.extend_from_slice(&info.unwind_address.to_le_bytes()); + } + self.obj.append_section_data(section_id, &unwind_info, 1); + } + + /// This function appends a nonstandard section to the object which is only + /// used during `CodeMemory::allocate_for_object`. + /// + /// This will generate a `.eh_frame` section, but not one that can be + /// naively loaded. The goal of this section is that we can create the + /// section once here and never again does it need to change. To describe + /// dynamically loaded functions though each individual FDE needs to talk + /// about the function's absolute address that it's referencing. Naturally + /// we don't actually know the function's absolute address when we're + /// creating an object here. + /// + /// To solve this problem the FDE address encoding mode is set to + /// `DW_EH_PE_pcrel`. This means that the actual effective address that the + /// FDE describes is a relative to the address of the FDE itself. By + /// leveraging this relative-ness we can assume that the relative distance + /// between the FDE and the function it describes is constant, which should + /// allow us to generate an FDE ahead-of-time here. + /// + /// For now this assumes that all the code of functions will start at a + /// page-aligned address when loaded into memory. The eh_frame encoded here + /// then assumes that the text section is itself page aligned to its size + /// and the eh_frame will follow just after the text section. This means + /// that the relative offsets we're using here is the FDE going backwards + /// into the text section itself. + /// + /// Note that the library we're using to create the FDEs, `gimli`, doesn't + /// actually encode addresses relative to the FDE itself. Instead the + /// addresses are encoded relative to the start of the `.eh_frame` section. + /// This makes it much easier for us where we provide the relative offset + /// from the start of `.eh_frame` to the function in the text section, which + /// given our layout basically means the offset of the function in the text + /// section from the end of the text section. + /// + /// A final note is that the reason we page-align the text section's size is + /// so the .eh_frame lives on a separate page from the text section itself. + /// This allows `.eh_frame` to have different virtual memory permissions, + /// such as being purely read-only instead of read/execute like the code + /// bits. + fn append_systemv_unwind_info(&mut self, isa: &dyn TargetIsa) { + let segment = self.obj.segment_name(StandardSegment::Data).to_vec(); + let section_id = self.obj.add_section( + segment, + b"_wasmtime_eh_frame".to_vec(), + SectionKind::ReadOnlyData, + ); + let mut cie = isa + .create_systemv_cie() + .expect("must be able to create a CIE for system-v unwind info"); + let mut table = FrameTable::default(); + cie.fde_address_encoding = gimli::constants::DW_EH_PE_pcrel; + let cie_id = table.add_cie(cie); + + // This write will align the text section to a page boundary (0x1000) + // and then return the offset at that point. This gives us the full size + // of the text section at that point, after alignment. + let text_section_size = self.obj.append_section_data(self.text_section, &[], 0x1000); + for (text_section_off, unwind_info) in self.systemv_unwind_info.iter() { + let backwards_off = text_section_size - text_section_off; + let actual_offset = -i64::try_from(backwards_off).unwrap(); + // Note that gimli wants an unsigned 64-bit integer here, but + // unwinders just use this constant for a relative addition with the + // address of the FDE, which means that the sign doesn't actually + // matter. + let fde = unwind_info.to_fde(Address::Constant(actual_offset as u64)); + table.add_fde(cie_id, fde); + } + let endian = match isa.triple().endianness().unwrap() { + target_lexicon::Endianness::Little => RunTimeEndian::Little, + target_lexicon::Endianness::Big => RunTimeEndian::Big, + }; + let mut eh_frame = EhFrame(MyVec(EndianVec::new(endian))); + table.write_eh_frame(&mut eh_frame).unwrap(); + + // Some unwinding implementations expect a terminating "empty" length so + // a 0 is written at the end of the table for those implementations. + let mut endian_vec = (eh_frame.0).0; + endian_vec.write_u32(0).unwrap(); + self.obj + .append_section_data(section_id, endian_vec.slice(), 1); + + use gimli::constants; + use gimli::write::Error; + + struct MyVec(EndianVec); + + impl Writer for MyVec { + type Endian = RunTimeEndian; + + fn endian(&self) -> RunTimeEndian { + self.0.endian() + } + + fn len(&self) -> usize { + self.0.len() + } + + fn write(&mut self, buf: &[u8]) -> Result<(), Error> { + self.0.write(buf) + } + + fn write_at(&mut self, pos: usize, buf: &[u8]) -> Result<(), Error> { + self.0.write_at(pos, buf) + } + + // FIXME(gimli-rs/gimli#576) this is the definition we want for + // `write_eh_pointer` but the default implementation, at the time + // of this writing, uses `offset - val` instead of `val - offset`. + // A PR has been merged to fix this but until that's published we + // can't use it. + fn write_eh_pointer( + &mut self, + address: Address, + eh_pe: constants::DwEhPe, + size: u8, + ) -> Result<(), Error> { + let val = match address { + Address::Constant(val) => val, + Address::Symbol { .. } => unreachable!(), + }; + assert_eq!(eh_pe.application(), constants::DW_EH_PE_pcrel); + let offset = self.len() as u64; + let val = val.wrapping_sub(offset); + self.write_eh_pointer_data(val, eh_pe.format(), size) + } + } + } +} diff --git a/crates/debug/Cargo.toml b/crates/debug/Cargo.toml index 4b058eb164..5a29bcb2c3 100644 --- a/crates/debug/Cargo.toml +++ b/crates/debug/Cargo.toml @@ -19,6 +19,7 @@ target-lexicon = { version = "0.12.0", default-features = false } anyhow = "1.0" thiserror = "1.0.4" more-asserts = "0.2.1" +cranelift-codegen = { path = "../../cranelift/codegen", version = "0.76.0" } [badges] maintenance = { status = "actively-developed" } diff --git a/crates/debug/src/transform/address_transform.rs b/crates/debug/src/transform/address_transform.rs index 2ef2e456bb..0277e6455e 100644 --- a/crates/debug/src/transform/address_transform.rs +++ b/crates/debug/src/transform/address_transform.rs @@ -1,9 +1,9 @@ +use cranelift_codegen::ir::SourceLoc; use gimli::write; use more_asserts::assert_le; use std::collections::BTreeMap; use std::iter::FromIterator; use wasmtime_environ::entity::{EntityRef, PrimaryMap}; -use wasmtime_environ::ir::SourceLoc; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::{CompiledFunctions, FunctionAddressMap, WasmFileInfo}; diff --git a/crates/debug/src/transform/attr.rs b/crates/debug/src/transform/attr.rs index b3a018455a..8ae62562d1 100644 --- a/crates/debug/src/transform/attr.rs +++ b/crates/debug/src/transform/attr.rs @@ -4,11 +4,11 @@ use super::range_info_builder::RangeInfoBuilder; use super::refs::{PendingDebugInfoRefs, PendingUnitRefs}; use super::{DebugInputContext, Reader, TransformError}; use anyhow::{bail, Error}; +use cranelift_codegen::isa::TargetIsa; use gimli::{ write, AttributeValue, DebugLineOffset, DebugLineStr, DebugStr, DebugStrOffsets, DebuggingInformationEntry, Unit, }; -use wasmtime_environ::isa::TargetIsa; #[derive(Debug)] pub(crate) enum FileAttributeContext<'a> { diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs index 377b9b6447..e26e3571b2 100644 --- a/crates/debug/src/transform/expression.rs +++ b/crates/debug/src/transform/expression.rs @@ -1,5 +1,8 @@ use super::address_transform::AddressTransform; use anyhow::{Context, Error, Result}; +use cranelift_codegen::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLoc}; +use cranelift_codegen::isa::TargetIsa; +use cranelift_codegen::ValueLabelsRanges; use gimli::{self, write, Expression, Operation, Reader, ReaderOffset, X86_64}; use more_asserts::{assert_le, assert_lt}; use std::cmp::PartialEq; @@ -7,8 +10,6 @@ use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; use std::rc::Rc; use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc}; -use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::{get_vmctx_value_label, DefinedFuncIndex}; use wasmtime_environ::ModuleMemoryOffset; @@ -1219,9 +1220,10 @@ mod tests { } fn create_mock_value_ranges() -> (ValueLabelsRanges, (ValueLabel, ValueLabel, ValueLabel)) { + use cranelift_codegen::ir::{LabelValueLoc, ValueLoc}; + use cranelift_codegen::ValueLocRange; use std::collections::HashMap; use wasmtime_environ::entity::EntityRef; - use wasmtime_environ::ir::{LabelValueLoc, ValueLoc, ValueLocRange}; let mut value_ranges = HashMap::new(); let value_0 = ValueLabel::new(0); let value_1 = ValueLabel::new(1); @@ -1263,8 +1265,8 @@ mod tests { #[test] fn test_debug_value_range_builder() { use super::ValueLabelRangesBuilder; + use cranelift_codegen::ir::StackSlots; use wasmtime_environ::entity::EntityRef; - use wasmtime_environ::ir::StackSlots; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::ModuleMemoryOffset; diff --git a/crates/debug/src/transform/mod.rs b/crates/debug/src/transform/mod.rs index e6b6bb431f..a29a93823c 100644 --- a/crates/debug/src/transform/mod.rs +++ b/crates/debug/src/transform/mod.rs @@ -3,13 +3,13 @@ use self::simulate::generate_simulated_dwarf; use self::unit::clone_unit; use crate::gc::build_dependencies; use anyhow::Error; +use cranelift_codegen::isa::TargetIsa; use gimli::{ write, DebugAddr, DebugLine, DebugLineStr, DebugStr, DebugStrOffsets, LocationLists, RangeLists, UnitSectionOffset, }; use std::collections::HashSet; use thiserror::Error; -use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::{CompiledFunctions, DebugInfoData, ModuleMemoryOffset}; pub use address_transform::AddressTransform; diff --git a/crates/debug/src/transform/simulate.rs b/crates/debug/src/transform/simulate.rs index 0e86087a6e..e0af268d5c 100644 --- a/crates/debug/src/transform/simulate.rs +++ b/crates/debug/src/transform/simulate.rs @@ -2,6 +2,7 @@ use super::expression::{CompiledExpression, FunctionFrameInfo}; use super::utils::{add_internal_types, append_vmctx_info, get_function_frame_info}; use super::AddressTransform; use anyhow::{Context, Error}; +use cranelift_codegen::isa::TargetIsa; use gimli::write; use gimli::{self, LineEncoding}; use std::collections::{HashMap, HashSet}; @@ -9,7 +10,6 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use wasmparser::Type as WasmType; use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::{get_vmctx_value_label, DefinedFuncIndex}; use wasmtime_environ::{ CompiledFunctions, DebugInfoData, FunctionMetadata, ModuleMemoryOffset, WasmFileInfo, diff --git a/crates/debug/src/transform/unit.rs b/crates/debug/src/transform/unit.rs index d46e4085fb..ae7ce94f49 100644 --- a/crates/debug/src/transform/unit.rs +++ b/crates/debug/src/transform/unit.rs @@ -7,11 +7,11 @@ use super::refs::{PendingDebugInfoRefs, PendingUnitRefs, UnitRefsMap}; use super::utils::{add_internal_types, append_vmctx_info, get_function_frame_info}; use super::{DebugInputContext, Reader, TransformError}; use anyhow::{Context, Error}; +use cranelift_codegen::ir::Endianness; +use cranelift_codegen::isa::TargetIsa; use gimli::write; use gimli::{AttributeValue, DebuggingInformationEntry, Unit}; use std::collections::HashSet; -use wasmtime_environ::ir::Endianness; -use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::{CompiledFunctions, ModuleMemoryOffset}; diff --git a/crates/debug/src/transform/utils.rs b/crates/debug/src/transform/utils.rs index 1864f7d40e..740586db4f 100644 --- a/crates/debug/src/transform/utils.rs +++ b/crates/debug/src/transform/utils.rs @@ -1,8 +1,8 @@ use super::address_transform::AddressTransform; use super::expression::{CompiledExpression, FunctionFrameInfo}; use anyhow::Error; +use cranelift_codegen::isa::TargetIsa; use gimli::write; -use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::{CompiledFunctions, ModuleMemoryOffset}; diff --git a/crates/debug/src/write_debuginfo.rs b/crates/debug/src/write_debuginfo.rs index 3d7b1451da..ff47471df4 100644 --- a/crates/debug/src/write_debuginfo.rs +++ b/crates/debug/src/write_debuginfo.rs @@ -1,12 +1,33 @@ pub use crate::transform::transform_dwarf; +use cranelift_codegen::ir::Endianness; +use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; use gimli::write::{Address, Dwarf, EndianVec, FrameTable, Result, Sections, Writer}; use gimli::{RunTimeEndian, SectionId}; use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::ir::Endianness; -use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::{CompiledFunctions, DebugInfoData, ModuleMemoryOffset}; -pub use wasmtime_environ::{DwarfSection, DwarfSectionReloc, DwarfSectionRelocTarget}; +#[allow(missing_docs)] +pub struct DwarfSection { + pub name: &'static str, + pub body: Vec, + pub relocs: Vec, +} + +#[allow(missing_docs)] +#[derive(Clone)] +pub struct DwarfSectionReloc { + pub target: DwarfSectionRelocTarget, + pub offset: u32, + pub addend: i32, + pub size: u8, +} + +#[allow(missing_docs)] +#[derive(Clone)] +pub enum DwarfSectionRelocTarget { + Func(usize), + Section(&'static str), +} fn emit_dwarf_sections( isa: &dyn TargetIsa, diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index 551804aba7..3211f9cc70 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -177,46 +177,25 @@ pub trait Compiler: Send + Sync { types: &TypeTables, ) -> Result; - /// Creates a trampoline which the host can use to enter wasm. + /// Collects the results of compilation and emits an in-memory ELF object + /// which is the serialized representation of all compiler artifacts. /// - /// The generated trampoline will have the type `VMTrampoline` and will - /// call a function of type `ty` specified. - fn host_to_wasm_trampoline(&self, ty: &WasmFuncType) -> Result; - - /// Creates a trampoline suitable for a wasm module to import. - /// - /// The trampoline has the type specified by `ty` and will call the function - /// `host_fn` which has type `VMTrampoline`. Note that `host_fn` is - /// directly embedded into the generated code so this is not suitable for a - /// cached value or if `host_fn` does not live as long as the compiled - /// function. - /// - /// This is primarily used for `Func::new` in `wasmtime`. - fn wasm_to_host_trampoline( + /// Note that ELF is used regardless of the target architecture. + fn emit_obj( &self, - ty: &WasmFuncType, - host_fn: usize, - ) -> Result; - - /// Creates DWARF debugging data for a compilation unit. - /// - /// This function maps DWARF information found in a wasm module to native - /// DWARF debugging information. This is currently implemented by the - /// `wasmtime-debug` crate. - fn emit_dwarf( - &self, - debuginfo_data: &crate::DebugInfoData, + module: &ModuleTranslation, + types: &TypeTables, funcs: &CompiledFunctions, - memory_offset: &crate::ModuleMemoryOffset, - ) -> Result>; + emit_dwarf: bool, + ) -> Result>; + + /// Emits a small ELF object file in-memory which has two functions for the + /// host-to-wasm and wasm-to-host trampolines for the wasm type given. + fn emit_trampoline_obj(&self, ty: &WasmFuncType, host_fn: usize) -> Result>; /// Returns the target triple that this compiler is compiling for. fn triple(&self) -> &target_lexicon::Triple; - /// If supported by the target creates a SystemV CIE used for dwarf - /// unwinding information. - fn create_systemv_cie(&self) -> Option; - /// Returns a list of configured settings for this compiler. fn flags(&self) -> HashMap; @@ -224,29 +203,6 @@ pub trait Compiler: Send + Sync { fn isa_flags(&self) -> HashMap; } -#[allow(missing_docs)] -pub struct DwarfSection { - pub name: &'static str, - pub body: Vec, - pub relocs: Vec, -} - -#[allow(missing_docs)] -#[derive(Clone)] -pub struct DwarfSectionReloc { - pub target: DwarfSectionRelocTarget, - pub offset: u32, - pub addend: i32, - pub size: u8, -} - -#[allow(missing_docs)] -#[derive(Clone)] -pub enum DwarfSectionRelocTarget { - Func(usize), - Section(&'static str), -} - /// Value of a configured setting for a [`Compiler`] #[derive(Serialize, Deserialize, Hash, Eq, PartialEq)] pub enum FlagValue { diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 7f6d23d4e4..8c96ca1606 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -1,18 +1,8 @@ #![doc(hidden)] pub mod ir { - pub use cranelift_codegen::binemit::{Reloc, StackMap}; - pub use cranelift_codegen::ir::{ - types, AbiParam, ArgumentPurpose, Endianness, JumpTableOffsets, LabelValueLoc, LibCall, - Signature, SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc, - }; - pub use cranelift_codegen::{ValueLabelsRanges, ValueLocRange}; -} - -pub mod isa { - pub use cranelift_codegen::isa::{ - lookup, unwind, Builder, CallConv, RegUnit, TargetFrontendConfig, TargetIsa, - }; + pub use cranelift_codegen::binemit::StackMap; + pub use cranelift_codegen::ir::{types, SourceLoc, TrapCode, Type}; } pub mod entity { diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 0cf1961a25..702786f5b0 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -29,6 +29,7 @@ mod compilation; mod data_structures; mod module; mod module_environ; +pub mod obj; mod tunables; mod vmoffsets; diff --git a/crates/environ/src/obj.rs b/crates/environ/src/obj.rs new file mode 100644 index 0000000000..10d8d6f171 --- /dev/null +++ b/crates/environ/src/obj.rs @@ -0,0 +1,34 @@ +//! Utilities for working with object files that operate as Wasmtime's +//! serialization and intermediate format for compiled modules. + +use cranelift_entity::EntityRef; +use cranelift_wasm::{FuncIndex, SignatureIndex}; + +const FUNCTION_PREFIX: &str = "_wasm_function_"; +const TRAMPOLINE_PREFIX: &str = "_trampoline_"; + +/// Returns the symbol name in an object file for the corresponding wasm +/// function index in a module. +pub fn func_symbol_name(index: FuncIndex) -> String { + format!("{}{}", FUNCTION_PREFIX, index.index()) +} + +/// Attempts to extract the corresponding function index from a symbol possibly produced by +/// `func_symbol_name`. +pub fn try_parse_func_name(name: &str) -> Option { + let n = name.strip_prefix(FUNCTION_PREFIX)?.parse().ok()?; + Some(FuncIndex::new(n)) +} + +/// Returns the symbol name in an object file for the corresponding trampoline +/// for the given signature in a module. +pub fn trampoline_symbol_name(index: SignatureIndex) -> String { + format!("{}{}", TRAMPOLINE_PREFIX, index.index()) +} + +/// Attempts to extract the corresponding signature index from a symbol +/// possibly produced by `trampoline_symbol_name`. +pub fn try_parse_trampoline_name(name: &str) -> Option { + let n = name.strip_prefix(TRAMPOLINE_PREFIX)?.parse().ok()?; + Some(SignatureIndex::new(n)) +} diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index c7db67080f..cd1db4f15c 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -16,7 +16,6 @@ wasmtime-runtime = { path = "../runtime", version = "0.29.0" } wasmtime-cranelift = { path = "../cranelift", version = "0.29.0" } wasmtime-lightbeam = { path = "../lightbeam/wasmtime", version = "0.29.0", optional = true } wasmtime-profiling = { path = "../profiling", version = "0.29.0" } -wasmtime-obj = { path = "../obj", version = "0.29.0" } rayon = { version = "1.0", optional = true } region = "2.2.0" thiserror = "1.0.4" @@ -26,8 +25,8 @@ more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "1.0" log = "0.4" -gimli = { version = "0.25.0", default-features = false, features = ["write"] } -object = { version = "0.26.0", default-features = false, features = ["write"] } +gimli = { version = "0.25.0", default-features = false, features = ["std", "read"] } +object = { version = "0.26.0", default-features = false, features = ["std", "read_core", "elf"] } serde = { version = "1.0.94", features = ["derive"] } addr2line = { version = "0.16.0", default-features = false } diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 2b943d8f59..e704247eca 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -1,44 +1,38 @@ //! Memory management for executable code. -use crate::object::{ - utils::{try_parse_func_name, try_parse_trampoline_name}, - ObjectUnwindInfo, -}; -use crate::unwind::UnwindRegistry; -use crate::Compiler; +use crate::unwind::UnwindRegistration; use anyhow::{Context, Result}; use object::read::{File as ObjectFile, Object, ObjectSection, ObjectSymbol}; -use region; use std::collections::BTreeMap; use std::mem::ManuallyDrop; -use std::{cmp, mem}; -use wasmtime_environ::{ - isa::unwind::UnwindInfo, - wasm::{FuncIndex, SignatureIndex}, - CompiledFunction, -}; +use wasmtime_environ::obj::{try_parse_func_name, try_parse_trampoline_name}; +use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; use wasmtime_runtime::{Mmap, VMFunctionBody}; struct CodeMemoryEntry { mmap: ManuallyDrop, - registry: ManuallyDrop, - len: usize, + unwind_registration: ManuallyDrop>, + text_len: usize, + unwind_info_len: usize, } impl CodeMemoryEntry { - fn with_capacity(cap: usize) -> Result { - let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?); - let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize)); + fn new(text_len: usize, unwind_info_len: usize) -> Result { + let mmap = ManuallyDrop::new(Mmap::with_at_least(text_len + unwind_info_len)?); Ok(Self { mmap, - registry, - len: 0, + unwind_registration: ManuallyDrop::new(None), + text_len, + unwind_info_len, }) } + // Note that this intentionally excludes any unwinding information, if + // present, since consumers largely are only interested in code memory + // itself. fn range(&self) -> (usize, usize) { let start = self.mmap.as_ptr() as usize; - let end = start + self.len; + let end = start + self.text_len; (start, end) } } @@ -47,23 +41,20 @@ impl Drop for CodeMemoryEntry { fn drop(&mut self) { unsafe { // The registry needs to be dropped before the mmap - ManuallyDrop::drop(&mut self.registry); + ManuallyDrop::drop(&mut self.unwind_registration); ManuallyDrop::drop(&mut self.mmap); } } } -pub(crate) struct CodeMemoryObjectAllocation<'a> { - buf: &'a mut [u8], +pub struct CodeMemoryObjectAllocation<'a, 'b> { + pub code_range: &'a mut [u8], funcs: BTreeMap, trampolines: BTreeMap, + pub obj: ObjectFile<'b>, } -impl<'a> CodeMemoryObjectAllocation<'a> { - pub fn code_range(self) -> &'a mut [u8] { - self.buf - } - +impl<'a> CodeMemoryObjectAllocation<'a, '_> { pub fn funcs_len(&self) -> usize { self.funcs.len() } @@ -73,7 +64,7 @@ impl<'a> CodeMemoryObjectAllocation<'a> { } pub fn funcs(&'a self) -> impl Iterator + 'a { - let buf = self.buf as *const _ as *mut [u8]; + let buf = self.code_range as *const _ as *mut [u8]; self.funcs.iter().map(move |(i, (start, len))| { (*i, unsafe { CodeMemory::view_as_mut_vmfunc_slice(&mut (*buf)[*start..*start + *len]) @@ -84,7 +75,7 @@ impl<'a> CodeMemoryObjectAllocation<'a> { pub fn trampolines( &'a self, ) -> impl Iterator + 'a { - let buf = self.buf as *const _ as *mut [u8]; + let buf = self.code_range as *const _ as *mut [u8]; self.trampolines.iter().map(move |(i, (start, len))| { (*i, unsafe { CodeMemory::view_as_mut_vmfunc_slice(&mut (*buf)[*start..*start + *len]) @@ -95,7 +86,6 @@ impl<'a> CodeMemoryObjectAllocation<'a> { /// Memory manager for executable code. pub struct CodeMemory { - current: Option, entries: Vec, published: usize, } @@ -109,140 +99,49 @@ impl CodeMemory { /// Create a new `CodeMemory` instance. pub fn new() -> Self { Self { - current: None, entries: Vec::new(), published: 0, } } - /// Allocate a continuous memory block for a single compiled function. - /// TODO: Reorganize the code that calls this to emit code directly into the - /// mmap region rather than into a Vec that we need to copy in. - pub fn allocate_for_function<'a>( - &mut self, - func: &'a CompiledFunction, - ) -> Result<&mut [VMFunctionBody]> { - let size = Self::function_allocation_size(func); - - let (buf, registry, start) = self.allocate(size)?; - - let (_, _, vmfunc) = Self::copy_function(func, start as u32, buf, registry); - - Ok(vmfunc) - } - /// Make all allocated memory executable. - pub fn publish(&mut self, compiler: &Compiler) { - self.push_current(0) - .expect("failed to push current memory map"); + pub fn publish(&mut self) { + for entry in &mut self.entries[self.published..] { + assert!(!entry.mmap.is_empty()); - for CodeMemoryEntry { - mmap: m, - registry: r, - .. - } in &mut self.entries[self.published..] - { - // Remove write access to the pages due to the relocation fixups. - r.publish(compiler) - .expect("failed to publish function unwind registry"); - - if !m.is_empty() { - unsafe { - region::protect(m.as_mut_ptr(), m.len(), region::Protection::READ_EXECUTE) - } + unsafe { + // Switch the executable portion from read/write to + // read/execute, notably not using read/write/execute to prevent + // modifications. + region::protect( + entry.mmap.as_mut_ptr(), + entry.text_len, + region::Protection::READ_EXECUTE, + ) .expect("unable to make memory readonly and executable"); + + if entry.unwind_info_len == 0 { + continue; + } + + // With all our memory setup use the platform-specific + // `UnwindRegistration` implementation to inform the general + // runtime that there's unwinding information available for all + // our just-published JIT functions. + *entry.unwind_registration = Some( + UnwindRegistration::new( + entry.mmap.as_mut_ptr(), + entry.mmap.as_mut_ptr().add(entry.text_len), + entry.unwind_info_len, + ) + .expect("failed to create unwind info registration"), + ); } } self.published = self.entries.len(); } - /// Allocate `size` bytes of memory which can be made executable later by - /// calling `publish()`. Note that we allocate the memory as writeable so - /// that it can be written to and patched, though we make it readonly before - /// actually executing from it. - /// - /// A few values are returned: - /// - /// * A mutable slice which references the allocated memory - /// * A function table instance where unwind information is registered - /// * The offset within the current mmap that the slice starts at - /// - /// TODO: Add an alignment flag. - fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize)> { - assert!(size > 0); - - if match &self.current { - Some(e) => e.mmap.len() - e.len < size, - None => true, - } { - self.push_current(cmp::max(0x10000, size))?; - } - - let e = self.current.as_mut().unwrap(); - let old_position = e.len; - e.len += size; - - Ok(( - &mut e.mmap.as_mut_slice()[old_position..e.len], - &mut e.registry, - old_position, - )) - } - - /// Calculates the allocation size of the given compiled function. - fn function_allocation_size(func: &CompiledFunction) -> usize { - match &func.unwind_info { - Some(UnwindInfo::WindowsX64(info)) => { - // Windows unwind information is required to be emitted into code memory - // This is because it must be a positive relative offset from the start of the memory - // Account for necessary unwind information alignment padding (32-bit alignment) - ((func.body.len() + 3) & !3) + info.emit_size() - } - _ => func.body.len(), - } - } - - /// Copies the data of the compiled function to the given buffer. - /// - /// This will also add the function to the current unwind registry. - fn copy_function<'a>( - func: &CompiledFunction, - func_start: u32, - buf: &'a mut [u8], - registry: &mut UnwindRegistry, - ) -> (u32, &'a mut [u8], &'a mut [VMFunctionBody]) { - let func_len = func.body.len(); - let mut func_end = func_start + (func_len as u32); - - let (body, mut remainder) = buf.split_at_mut(func_len); - body.copy_from_slice(&func.body); - let vmfunc = Self::view_as_mut_vmfunc_slice(body); - - if let Some(UnwindInfo::WindowsX64(info)) = &func.unwind_info { - // Windows unwind information is written following the function body - // Keep unwind information 32-bit aligned (round up to the nearest 4 byte boundary) - let unwind_start = (func_end + 3) & !3; - let unwind_size = info.emit_size(); - let padding = (unwind_start - func_end) as usize; - - let (slice, r) = remainder.split_at_mut(padding + unwind_size); - - info.emit(&mut slice[padding..]); - - func_end = unwind_start + (unwind_size as u32); - remainder = r; - } - - if let Some(info) = &func.unwind_info { - registry - .register(func_start, func_len as u32, info) - .expect("failed to register unwind information"); - } - - (func_end, remainder, vmfunc) - } - /// Convert mut a slice from u8 to VMFunctionBody. fn view_as_mut_vmfunc_slice(slice: &mut [u8]) -> &mut [VMFunctionBody] { let byte_ptr: *mut [u8] = slice; @@ -250,24 +149,6 @@ impl CodeMemory { unsafe { &mut *body_ptr } } - /// Pushes the current entry and allocates a new one with the given size. - fn push_current(&mut self, new_size: usize) -> Result<()> { - let previous = mem::replace( - &mut self.current, - if new_size == 0 { - None - } else { - Some(CodeMemoryEntry::with_capacity(cmp::max(0x10000, new_size))?) - }, - ); - - if let Some(e) = previous { - self.entries.push(e); - } - - Ok(()) - } - /// Returns all published segment ranges. pub fn published_ranges<'a>(&'a self) -> impl Iterator + 'a { self.entries[..self.published] @@ -277,29 +158,52 @@ impl CodeMemory { /// Allocates and copies the ELF image code section into CodeMemory. /// Returns references to functions and trampolines defined there. - pub(crate) fn allocate_for_object<'a>( + pub fn allocate_for_object<'a, 'b>( &'a mut self, - obj: &ObjectFile, - unwind_info: &[ObjectUnwindInfo], - ) -> Result> { + obj: &'b [u8], + ) -> Result> { + let obj = ObjectFile::parse(obj) + .with_context(|| "failed to parse internal ELF compilation artifact")?; let text_section = obj.section_by_name(".text").unwrap(); + let text_section_size = text_section.size() as usize; - if text_section.size() == 0 { + if text_section_size == 0 { // No code in the image. return Ok(CodeMemoryObjectAllocation { - buf: &mut [], + code_range: &mut [], funcs: BTreeMap::new(), trampolines: BTreeMap::new(), + obj, }); } - // Allocate chunk memory that spans entire code section. - let (buf, registry, start) = self.allocate(text_section.size() as usize)?; - buf.copy_from_slice( + // Find the platform-specific unwind section, if present, which contains + // unwinding tables that will be used to load unwinding information + // dynamically at runtime. + let unwind_section = obj.section_by_name(UnwindRegistration::section_name()); + let unwind_section_size = unwind_section + .as_ref() + .map(|s| s.size() as usize) + .unwrap_or(0); + + // Allocate memory for the text section and unwinding information if it + // is present. Then we can copy in all of the code and unwinding memory + // over. + let entry = CodeMemoryEntry::new(text_section_size, unwind_section_size)?; + self.entries.push(entry); + let entry = self.entries.last_mut().unwrap(); + entry.mmap.as_mut_slice()[..text_section_size].copy_from_slice( text_section .data() - .with_context(|| "cannot read section data")?, + .with_context(|| "cannot read text section data")?, ); + if let Some(section) = unwind_section { + entry.mmap.as_mut_slice()[text_section_size..][..unwind_section_size].copy_from_slice( + section + .data() + .with_context(|| "cannot read unwind section data")?, + ); + } // Track locations of all defined functions and trampolines. let mut funcs = BTreeMap::new(); @@ -310,43 +214,21 @@ impl CodeMemory { if let Some(index) = try_parse_func_name(name) { let is_import = sym.section_index().is_none(); if !is_import { - funcs.insert( - index, - (start + sym.address() as usize, sym.size() as usize), - ); + funcs.insert(index, (sym.address() as usize, sym.size() as usize)); } } else if let Some(index) = try_parse_trampoline_name(name) { - trampolines - .insert(index, (start + sym.address() as usize, sym.size() as usize)); + trampolines.insert(index, (sym.address() as usize, sym.size() as usize)); } } Err(_) => (), } } - // Register all unwind entries for functions and trampolines. - // TODO will `u32` type for start/len be enough for large code base. - for i in unwind_info { - match i { - ObjectUnwindInfo::Func(func_index, info) => { - let (start, len) = funcs.get(&func_index).unwrap(); - registry - .register(*start as u32, *len as u32, &info) - .expect("failed to register unwind information"); - } - ObjectUnwindInfo::Trampoline(trampoline_index, info) => { - let (start, len) = trampolines.get(&trampoline_index).unwrap(); - registry - .register(*start as u32, *len as u32, &info) - .expect("failed to register unwind information"); - } - } - } - Ok(CodeMemoryObjectAllocation { - buf: &mut buf[..text_section.size() as usize], + code_range: &mut entry.mmap.as_mut_slice()[..text_section_size], funcs, trampolines, + obj, }) } } diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index a94dc8eae4..8dd013abb3 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -1,8 +1,6 @@ //! JIT compilation. use crate::instantiate::SetupError; -use crate::object::{build_object, ObjectUnwindInfo}; -use object::write::Object; #[cfg(feature = "parallel-compilation")] use rayon::prelude::*; use serde::{Deserialize, Serialize}; @@ -10,11 +8,9 @@ use std::collections::BTreeMap; use std::hash::{Hash, Hasher}; use std::mem; use wasmparser::WasmFeatures; -use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::wasm::{DefinedMemoryIndex, MemoryIndex}; use wasmtime_environ::{ - CompiledFunctions, Compiler as EnvCompiler, CompilerBuilder, ModuleMemoryOffset, - ModuleTranslation, Tunables, TypeTables, VMOffsets, + CompiledFunctions, Compiler as EnvCompiler, CompilerBuilder, ModuleTranslation, Tunables, + TypeTables, }; /// Select which kind of compilation to use. @@ -75,8 +71,7 @@ fn _assert_compiler_send_sync() { #[allow(missing_docs)] pub struct Compilation { - pub obj: Object, - pub unwind_info: Vec, + pub obj: Vec, pub funcs: CompiledFunctions, } @@ -118,40 +113,14 @@ impl Compiler { .into_iter() .collect::(); - let dwarf_sections = if self.tunables.generate_native_debuginfo && !funcs.is_empty() { - let ofs = VMOffsets::new( - self.compiler - .triple() - .architecture - .pointer_width() - .unwrap() - .bytes(), - &translation.module, - ); + let obj = self.compiler.emit_obj( + &translation, + types, + &funcs, + self.tunables.generate_native_debuginfo, + )?; - let memory_offset = if ofs.num_imported_memories > 0 { - ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) - } else if ofs.num_defined_memories > 0 { - ModuleMemoryOffset::Defined( - ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)), - ) - } else { - ModuleMemoryOffset::None - }; - self.compiler - .emit_dwarf(&translation.debuginfo, &funcs, &memory_offset) - .map_err(SetupError::DebugInfo)? - } else { - vec![] - }; - - let (obj, unwind_info) = build_object(self, &translation, types, &funcs, dwarf_sections)?; - - Ok(Compilation { - obj, - unwind_info, - funcs, - }) + Ok(Compilation { obj, funcs }) } /// Run the given closure in parallel if the compiler is configured to do so. diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index f333a0b9e4..3fe55df70c 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -7,9 +7,7 @@ use crate::code_memory::CodeMemory; use crate::compiler::{Compilation, Compiler}; use crate::debug::create_gdbjit_image; use crate::link::link_module; -use crate::object::ObjectUnwindInfo; -use anyhow::{Context, Result}; -use object::File as ObjectFile; +use anyhow::Result; use serde::{Deserialize, Serialize}; use std::ops::Range; use std::sync::Arc; @@ -57,9 +55,6 @@ pub struct CompilationArtifacts { /// ELF image with functions code. obj: Box<[u8]>, - /// Unwind information for function code. - unwind_info: Box<[ObjectUnwindInfo]>, - /// Descriptions of compiled functions funcs: PrimaryMap, @@ -110,11 +105,7 @@ impl CompilationArtifacts { let list = compiler.run_maybe_parallel::<_, _, SetupError, _>( translations, |mut translation| { - let Compilation { - obj, - unwind_info, - funcs, - } = compiler.compile(&mut translation, &types)?; + let Compilation { obj, funcs } = compiler.compile(&mut translation, &types)?; let ModuleTranslation { mut module, @@ -129,16 +120,9 @@ impl CompilationArtifacts { } } - let obj = obj.write().map_err(|_| { - SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!( - "failed to create image memory" - ))) - })?; - Ok(CompilationArtifacts { module: Arc::new(module), obj: obj.into_boxed_slice(), - unwind_info: unwind_info.into_boxed_slice(), funcs: funcs .into_iter() .map(|(_, func)| FunctionInfo { @@ -221,34 +205,26 @@ impl CompiledModule { /// artifacts. pub fn from_artifacts_list( artifacts: Vec, - compiler: &Compiler, profiler: &dyn ProfilingAgent, + compiler: &Compiler, ) -> Result>, SetupError> { - compiler.run_maybe_parallel(artifacts, |a| { - CompiledModule::from_artifacts(a, compiler, profiler) - }) + compiler.run_maybe_parallel(artifacts, |a| CompiledModule::from_artifacts(a, profiler)) } /// Creates `CompiledModule` directly from `CompilationArtifacts`. pub fn from_artifacts( artifacts: CompilationArtifacts, - compiler: &Compiler, profiler: &dyn ProfilingAgent, ) -> Result, SetupError> { // Allocate all of the compiled functions into executable memory, // copying over their contents. - let (code_memory, code_range, finished_functions, trampolines) = build_code_memory( - compiler, - &artifacts.obj, - &artifacts.module, - &artifacts.unwind_info, - ) - .map_err(|message| { - SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!( - "failed to build code memory for functions: {}", - message - ))) - })?; + let (code_memory, code_range, finished_functions, trampolines) = + build_code_memory(&artifacts.obj, &artifacts.module).map_err(|message| { + SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!( + "failed to build code memory for functions: {}", + message + ))) + })?; // Register GDB JIT images; initialize profiler and load the wasm module. let dbg_jit_registration = if artifacts.native_debug_info_present { @@ -475,21 +451,17 @@ fn create_dbg_image( } fn build_code_memory( - compiler: &Compiler, obj: &[u8], module: &Module, - unwind_info: &[ObjectUnwindInfo], ) -> Result<( CodeMemory, (*const u8, usize), PrimaryMap, Vec<(SignatureIndex, VMTrampoline)>, )> { - let obj = ObjectFile::parse(obj).with_context(|| "Unable to read obj")?; - let mut code_memory = CodeMemory::new(); - let allocation = code_memory.allocate_for_object(&obj, unwind_info)?; + let allocation = code_memory.allocate_for_object(obj)?; // Populate the finished functions from the allocation let mut finished_functions = PrimaryMap::with_capacity(allocation.funcs_len()); @@ -519,14 +491,17 @@ fn build_code_memory( trampolines.push((i, fnptr)); } - let code_range = allocation.code_range(); + link_module( + &allocation.obj, + &module, + allocation.code_range, + &finished_functions, + ); - link_module(&obj, &module, code_range, &finished_functions); - - let code_range = (code_range.as_ptr(), code_range.len()); + let code_range = (allocation.code_range.as_ptr(), allocation.code_range.len()); // Make all code compiled thus far executable. - code_memory.publish(compiler); + code_memory.publish(); Ok((code_memory, code_range, finished_functions, trampolines)) } diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index f35010d1b3..1226f76d13 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -25,7 +25,6 @@ mod compiler; mod debug; mod instantiate; mod link; -mod object; mod unwind; pub use crate::code_memory::CodeMemory; diff --git a/crates/jit/src/link.rs b/crates/jit/src/link.rs index fffd9ecca8..25a8c80ce3 100644 --- a/crates/jit/src/link.rs +++ b/crates/jit/src/link.rs @@ -1,10 +1,10 @@ //! Linking for JIT-compiled code. -use crate::object::utils::try_parse_func_name; use object::read::{Object, ObjectSection, Relocation, RelocationTarget}; use object::{elf, File, ObjectSymbol, RelocationEncoding, RelocationKind}; use std::ptr::{read_unaligned, write_unaligned}; use wasmtime_environ::entity::PrimaryMap; +use wasmtime_environ::obj::try_parse_func_name; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; use wasmtime_runtime::libcalls; diff --git a/crates/jit/src/object.rs b/crates/jit/src/object.rs deleted file mode 100644 index 2953490324..0000000000 --- a/crates/jit/src/object.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Object file generation. - -use crate::Compiler; -use object::write::Object; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeSet; -use wasmtime_environ::isa::unwind::UnwindInfo; -use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; -use wasmtime_environ::{CompiledFunctions, DwarfSection, ModuleTranslation, TypeTables}; -use wasmtime_obj::{ObjectBuilder, ObjectBuilderTarget}; - -pub use wasmtime_obj::utils; - -/// Unwind information for object files functions (including trampolines). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum ObjectUnwindInfo { - Func(FuncIndex, UnwindInfo), - Trampoline(SignatureIndex, UnwindInfo), -} - -// Builds ELF image from the module `Compilation`. -pub(crate) fn build_object( - compiler: &Compiler, - translation: &ModuleTranslation, - types: &TypeTables, - funcs: &CompiledFunctions, - dwarf_sections: Vec, -) -> Result<(Object, Vec), anyhow::Error> { - const CODE_SECTION_ALIGNMENT: u64 = 0x1000; - - let mut unwind_info = Vec::new(); - - // Preserve function unwind info. - unwind_info.extend(funcs.iter().filter_map(|(index, func)| { - func.unwind_info - .as_ref() - .map(|info| ObjectUnwindInfo::Func(translation.module.func_index(index), info.clone())) - })); - - // Build trampolines for every signature that can be used by this module. - let signatures = translation - .module - .functions - .iter() - .filter_map(|(i, sig)| match translation.module.defined_func_index(i) { - Some(i) if !translation.module.possibly_exported_funcs.contains(&i) => None, - _ => Some(*sig), - }) - .collect::>(); - let mut trampolines = Vec::with_capacity(signatures.len()); - for i in signatures { - let func = compiler - .compiler() - .host_to_wasm_trampoline(&types.wasm_signatures[i])?; - // Preserve trampoline function unwind info. - if let Some(info) = &func.unwind_info { - unwind_info.push(ObjectUnwindInfo::Trampoline(i, info.clone())) - } - trampolines.push((i, func)); - } - - let target = ObjectBuilderTarget::new(compiler.compiler().triple().architecture)?; - let mut builder = ObjectBuilder::new(target, &translation.module, funcs); - builder - .set_code_alignment(CODE_SECTION_ALIGNMENT) - .set_trampolines(trampolines) - .set_dwarf_sections(dwarf_sections); - let obj = builder.build()?; - - Ok((obj, unwind_info)) -} diff --git a/crates/jit/src/unwind/systemv.rs b/crates/jit/src/unwind/systemv.rs index c8a9142b06..31a34bc5cc 100644 --- a/crates/jit/src/unwind/systemv.rs +++ b/crates/jit/src/unwind/systemv.rs @@ -1,20 +1,10 @@ //! Module for System V ABI unwind registry. -use crate::Compiler; -use anyhow::{bail, Result}; -use gimli::{ - write::{Address, EhFrame, EndianVec, FrameTable, Writer}, - RunTimeEndian, -}; -use wasmtime_environ::isa::unwind::UnwindInfo; +use anyhow::Result; -/// Represents a registry of function unwind information for System V ABI. -pub struct UnwindRegistry { - base_address: usize, - functions: Vec, - frame_table: Vec, +/// Represents a registration of function unwind information for System V ABI. +pub struct UnwindRegistration { registrations: Vec, - published: bool, } extern "C" { @@ -23,100 +13,34 @@ extern "C" { fn __deregister_frame(fde: *const u8); } -impl UnwindRegistry { - /// Creates a new unwind registry with the given base address. - pub fn new(base_address: usize) -> Self { - Self { - base_address, - functions: Vec::new(), - frame_table: Vec::new(), - registrations: Vec::new(), - published: false, - } - } - - /// Registers a function given the start offset, length, and unwind information. - pub fn register(&mut self, func_start: u32, _func_len: u32, info: &UnwindInfo) -> Result<()> { - if self.published { - bail!("unwind registry has already been published"); - } - - match info { - UnwindInfo::SystemV(info) => { - self.functions.push(info.to_fde(Address::Constant( - self.base_address as u64 + func_start as u64, - ))); - } - _ => bail!("unsupported unwind information"), - } - - Ok(()) - } - - /// Publishes all registered functions. - pub fn publish(&mut self, compiler: &Compiler) -> Result<()> { - if self.published { - bail!("unwind registry has already been published"); - } - - if self.functions.is_empty() { - self.published = true; - return Ok(()); - } - - self.set_frame_table(compiler)?; - - unsafe { - self.register_frames(); - } - - self.published = true; - - Ok(()) - } - - fn set_frame_table(&mut self, compiler: &Compiler) -> Result<()> { - let mut table = FrameTable::default(); - let cie_id = table.add_cie(match compiler.compiler().create_systemv_cie() { - Some(cie) => cie, - None => bail!("ISA does not support System V unwind information"), - }); - - let functions = std::mem::replace(&mut self.functions, Vec::new()); - - for func in functions { - table.add_fde(cie_id, func); - } - - let mut eh_frame = EhFrame(EndianVec::new(RunTimeEndian::default())); - table.write_eh_frame(&mut eh_frame).unwrap(); - +impl UnwindRegistration { + /// Registers precompiled unwinding information with the system. + /// + /// The `_base_address` field is ignored here (only used on other + /// platforms), but the `unwind_info` and `unwind_len` parameters should + /// describe an in-memory representation of a `.eh_frame` section. This is + /// typically arranged for by the `wasmtime-obj` crate. + pub unsafe fn new( + _base_address: *mut u8, + unwind_info: *mut u8, + unwind_len: usize, + ) -> Result { + let mut registrations = Vec::new(); if cfg!(any( all(target_os = "linux", target_env = "gnu"), target_os = "freebsd" )) { - // libgcc expects a terminating "empty" length, so write a 0 length at the end of the table. - eh_frame.0.write_u32(0).unwrap(); - } - - self.frame_table = eh_frame.0.into_vec(); - - Ok(()) - } - - unsafe fn register_frames(&mut self) { - if cfg!(any( - all(target_os = "linux", target_env = "gnu"), - target_os = "freebsd" - )) { - // On gnu (libgcc), `__register_frame` will walk the FDEs until an entry of length 0 - let ptr = self.frame_table.as_ptr(); - __register_frame(ptr); - self.registrations.push(ptr as usize); + // On gnu (libgcc), `__register_frame` will walk the FDEs until an + // entry of length 0 + __register_frame(unwind_info); + registrations.push(unwind_info as usize); } else { - // For libunwind, `__register_frame` takes a pointer to a single FDE - let start = self.frame_table.as_ptr(); - let end = start.add(self.frame_table.len()); + // For libunwind, `__register_frame` takes a pointer to a single + // FDE. Note that we subtract 4 from the length of unwind info since + // wasmtime-encode .eh_frame sections always have a trailing 32-bit + // zero for the platforms above. + let start = unwind_info; + let end = start.add(unwind_len - 4); let mut current = start; // Walk all of the entries in the frame table and register them @@ -126,31 +50,36 @@ impl UnwindRegistry { // Skip over the CIE if current != start { __register_frame(current); - self.registrations.push(current as usize); + registrations.push(current as usize); } - // Move to the next table entry (+4 because the length itself is not inclusive) + // Move to the next table entry (+4 because the length itself is + // not inclusive) current = current.add(len + 4); } } + + Ok(UnwindRegistration { registrations }) + } + + pub fn section_name() -> &'static str { + "_wasmtime_eh_frame" } } -impl Drop for UnwindRegistry { +impl Drop for UnwindRegistration { fn drop(&mut self) { - if self.published { - unsafe { - // libgcc stores the frame entries as a linked list in decreasing sort order - // based on the PC value of the registered entry. - // - // As we store the registrations in increasing order, it would be O(N^2) to - // deregister in that order. - // - // To ensure that we just pop off the first element in the list upon every - // deregistration, walk our list of registrations backwards. - for fde in self.registrations.iter().rev() { - __deregister_frame(*fde as *const _); - } + unsafe { + // libgcc stores the frame entries as a linked list in decreasing + // sort order based on the PC value of the registered entry. + // + // As we store the registrations in increasing order, it would be + // O(N^2) to deregister in that order. + // + // To ensure that we just pop off the first element in the list upon + // every deregistration, walk our list of registrations backwards. + for fde in self.registrations.iter().rev() { + __deregister_frame(*fde as *const _); } } } diff --git a/crates/jit/src/unwind/winx64.rs b/crates/jit/src/unwind/winx64.rs index 260a8cf629..b011fadb80 100644 --- a/crates/jit/src/unwind/winx64.rs +++ b/crates/jit/src/unwind/winx64.rs @@ -1,92 +1,46 @@ //! Module for Windows x64 ABI unwind registry. -use crate::Compiler; use anyhow::{bail, Result}; -use wasmtime_environ::isa::unwind::UnwindInfo; +use std::mem; use winapi::um::winnt; /// Represents a registry of function unwind information for Windows x64 ABI. -pub struct UnwindRegistry { - base_address: usize, - functions: Vec, - published: bool, +pub struct UnwindRegistration { + functions: usize, } -impl UnwindRegistry { - /// Creates a new unwind registry with the given base address. - pub fn new(base_address: usize) -> Self { - Self { - base_address, - functions: Vec::new(), - published: false, +impl UnwindRegistration { + pub unsafe fn new( + base_address: *mut u8, + unwind_info: *mut u8, + unwind_len: usize, + ) -> Result { + assert!(unwind_info as usize % 4 == 0); + let unit_len = mem::size_of::(); + assert!(unwind_len % unit_len == 0); + if winnt::RtlAddFunctionTable( + unwind_info as *mut _, + (unwind_len / unit_len) as u32, + base_address as u64, + ) == 0 + { + bail!("failed to register function table"); } + + Ok(UnwindRegistration { + functions: unwind_info as usize, + }) } - /// Registers a function given the start offset, length, and unwind information. - pub fn register(&mut self, func_start: u32, func_len: u32, info: &UnwindInfo) -> Result<()> { - if self.published { - bail!("unwind registry has already been published"); - } - - match info { - UnwindInfo::WindowsX64(_) => { - let mut entry = winnt::RUNTIME_FUNCTION::default(); - - entry.BeginAddress = func_start; - entry.EndAddress = func_start + func_len; - - // The unwind information should be immediately following the function - // with padding for 4 byte alignment - unsafe { - *entry.u.UnwindInfoAddress_mut() = (entry.EndAddress + 3) & !3; - } - - self.functions.push(entry); - - Ok(()) - } - _ => bail!("unsupported unwind information"), - } - } - - /// Publishes all registered functions. - pub fn publish(&mut self, _compiler: &Compiler) -> Result<()> { - if self.published { - bail!("unwind registry has already been published"); - } - - self.published = true; - - if !self.functions.is_empty() { - // Windows heap allocations are 32-bit aligned, but assert just in case - assert_eq!( - (self.functions.as_mut_ptr() as u64) % 4, - 0, - "function table allocation was not aligned" - ); - - unsafe { - if winnt::RtlAddFunctionTable( - self.functions.as_mut_ptr(), - self.functions.len() as u32, - self.base_address as u64, - ) == 0 - { - bail!("failed to register function table"); - } - } - } - - Ok(()) + pub fn section_name() -> &'static str { + "_wasmtime_winx64_unwind" } } -impl Drop for UnwindRegistry { +impl Drop for UnwindRegistration { fn drop(&mut self) { - if self.published { - unsafe { - winnt::RtlDeleteFunctionTable(self.functions.as_mut_ptr()); - } + unsafe { + winnt::RtlDeleteFunctionTable(self.functions as _); } } } diff --git a/crates/lightbeam/wasmtime/src/lib.rs b/crates/lightbeam/wasmtime/src/lib.rs index 3b98f719ac..7df603fbe4 100644 --- a/crates/lightbeam/wasmtime/src/lib.rs +++ b/crates/lightbeam/wasmtime/src/lib.rs @@ -14,10 +14,9 @@ use wasmtime_environ::wasm::{ GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmFuncType, }; use wasmtime_environ::{ - BuiltinFunctionIndex, CompileError, CompiledFunction, CompiledFunctions, Compiler, - DebugInfoData, DwarfSection, FlagValue, FunctionBodyData, Module, ModuleMemoryOffset, - ModuleTranslation, Relocation, RelocationTarget, TrapInformation, Tunables, TypeTables, - VMOffsets, + BuiltinFunctionIndex, CompileError, CompiledFunction, CompiledFunctions, Compiler, FlagValue, + FunctionBodyData, Module, ModuleTranslation, Relocation, RelocationTarget, TrapInformation, + Tunables, TypeTables, VMOffsets, }; /// A compiler that compiles a WebAssembly module with Lightbeam, directly translating the Wasm file. @@ -79,27 +78,17 @@ impl Compiler for Lightbeam { // }) } - fn host_to_wasm_trampoline( + fn emit_obj( &self, - _ty: &WasmFuncType, - ) -> Result { - unimplemented!() - } - - fn wasm_to_host_trampoline( - &self, - _ty: &WasmFuncType, - _host_fn: usize, - ) -> Result { - unimplemented!() - } - - fn emit_dwarf( - &self, - _debuginfo_data: &DebugInfoData, + _module: &ModuleTranslation, + _types: &TypeTables, _funcs: &CompiledFunctions, - _memory_offset: &crate::ModuleMemoryOffset, - ) -> Result> { + _emit_dwarf: bool, + ) -> Result> { + unimplemented!() + } + + fn emit_trampoline_obj(&self, _ty: &WasmFuncType, _host_fn: usize) -> Result> { unimplemented!() } @@ -107,10 +96,6 @@ impl Compiler for Lightbeam { unimplemented!() } - fn create_systemv_cie(&self) -> Option { - unimplemented!() - } - fn flags(&self) -> HashMap { unimplemented!() } diff --git a/crates/obj/.gitignore b/crates/obj/.gitignore deleted file mode 100644 index 4308d82204..0000000000 --- a/crates/obj/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -target/ -**/*.rs.bk -Cargo.lock diff --git a/crates/obj/Cargo.toml b/crates/obj/Cargo.toml deleted file mode 100644 index 5e7a4f4975..0000000000 --- a/crates/obj/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "wasmtime-obj" -version = "0.29.0" -authors = ["The Wasmtime Project Developers"] -description = "Native object file output for WebAsssembly code in Wasmtime" -license = "Apache-2.0 WITH LLVM-exception" -repository = "https://github.com/bytecodealliance/wasmtime" -categories = ["wasm"] -keywords = ["webassembly", "wasm"] -edition = "2018" - -[dependencies] -anyhow = "1.0" -wasmtime-environ = { path = "../environ", version = "0.29.0" } -object = { version = "0.26.0", default-features = false, features = ["write"] } -more-asserts = "0.2.1" -target-lexicon = { version = "0.12.0", default-features = false } - -[badges] -maintenance = { status = "experimental" } diff --git a/crates/obj/LICENSE b/crates/obj/LICENSE deleted file mode 100644 index f9d81955f4..0000000000 --- a/crates/obj/LICENSE +++ /dev/null @@ -1,220 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ---- LLVM Exceptions to the Apache 2.0 License ---- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into an Object form of such source code, you -may redistribute such embedded portions in such Object form without complying -with the conditions of Sections 4(a), 4(b) and 4(d) of the License. - -In addition, if you combine or link compiled forms of this Software with -software that is licensed under the GPLv2 ("Combined Software") and if a -court of competent jurisdiction determines that the patent provision (Section -3), the indemnity provision (Section 9) or other Section of the License -conflicts with the conditions of the GPLv2, you may retroactively and -prospectively choose to deem waived or otherwise exclude such Section(s) of -the License, but only in their entirety and only with respect to the Combined -Software. - diff --git a/crates/obj/src/builder.rs b/crates/obj/src/builder.rs deleted file mode 100644 index becc8264eb..0000000000 --- a/crates/obj/src/builder.rs +++ /dev/null @@ -1,448 +0,0 @@ -//! Object file builder. -//! -//! Creates ELF image based on `Compilation` information. The ELF contains -//! functions and trampolines in the ".text" section. It also contains all -//! relocation records for linking stage. If DWARF sections exist, their -//! content will be written as well. -//! -//! The object file has symbols for each function and trampoline, as well as -//! symbols that refer libcalls. -//! -//! The function symbol names have format "_wasm_function_N", where N is -//! `FuncIndex`. The defined wasm function symbols refer to a JIT compiled -//! function body, the imported wasm function do not. The trampolines symbol -//! names have format "_trampoline_N", where N is `SignatureIndex`. - -#![allow(missing_docs)] - -use anyhow::bail; -use object::write::{ - Object, Relocation as ObjectRelocation, SectionId, StandardSegment, Symbol, SymbolId, - SymbolSection, -}; -use object::{ - elf, Architecture, BinaryFormat, Endianness, RelocationEncoding, RelocationKind, SectionKind, - SymbolFlags, SymbolKind, SymbolScope, -}; -use std::collections::HashMap; -use target_lexicon::Triple; -use wasmtime_environ::entity::{EntityRef, PrimaryMap}; -use wasmtime_environ::ir::{LibCall, Reloc}; -use wasmtime_environ::isa::unwind::UnwindInfo; -use wasmtime_environ::wasm::{DefinedFuncIndex, FuncIndex, SignatureIndex}; -use wasmtime_environ::{ - CompiledFunction, CompiledFunctions, DwarfSection, DwarfSectionRelocTarget, Module, Relocation, - RelocationTarget, -}; - -fn to_object_relocations<'a>( - it: impl Iterator + 'a, - off: u64, - module: &'a Module, - funcs: &'a PrimaryMap, - libcalls: &'a HashMap, - compiled_funcs: &'a CompiledFunctions, -) -> impl Iterator + 'a { - it.filter_map(move |r| { - let (symbol, symbol_offset) = match r.reloc_target { - RelocationTarget::UserFunc(index) => (funcs[index], 0), - RelocationTarget::LibCall(call) => (libcalls[&call], 0), - RelocationTarget::JumpTable(f, jt) => { - let df = module.defined_func_index(f).unwrap(); - let offset = *compiled_funcs - .get(df) - .and_then(|f| f.jt_offsets.get(jt)) - .expect("func jump table"); - (funcs[f], offset) - } - }; - let (kind, encoding, size) = match r.reloc { - Reloc::Abs4 => (RelocationKind::Absolute, RelocationEncoding::Generic, 32), - Reloc::Abs8 => (RelocationKind::Absolute, RelocationEncoding::Generic, 64), - Reloc::X86PCRel4 => (RelocationKind::Relative, RelocationEncoding::Generic, 32), - Reloc::X86CallPCRel4 => (RelocationKind::Relative, RelocationEncoding::X86Branch, 32), - // TODO: Get Cranelift to tell us when we can use - // R_X86_64_GOTPCRELX/R_X86_64_REX_GOTPCRELX. - Reloc::X86CallPLTRel4 => ( - RelocationKind::PltRelative, - RelocationEncoding::X86Branch, - 32, - ), - Reloc::X86GOTPCRel4 => (RelocationKind::GotRelative, RelocationEncoding::Generic, 32), - Reloc::ElfX86_64TlsGd => ( - RelocationKind::Elf(elf::R_X86_64_TLSGD), - RelocationEncoding::Generic, - 32, - ), - Reloc::X86PCRelRodata4 => { - return None; - } - Reloc::Arm64Call => ( - RelocationKind::Elf(elf::R_AARCH64_CALL26), - RelocationEncoding::Generic, - 32, - ), - Reloc::S390xPCRel32Dbl => (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32), - other => unimplemented!("Unimplemented relocation {:?}", other), - }; - Some(ObjectRelocation { - offset: off + r.offset as u64, - size, - kind, - encoding, - symbol, - addend: r.addend.wrapping_add(symbol_offset as i64), - }) - }) -} - -fn to_object_architecture( - arch: target_lexicon::Architecture, -) -> Result { - use target_lexicon::Architecture::*; - Ok(match arch { - X86_32(_) => Architecture::I386, - X86_64 => Architecture::X86_64, - Arm(_) => Architecture::Arm, - Aarch64(_) => Architecture::Aarch64, - S390x => Architecture::S390x, - architecture => { - anyhow::bail!("target architecture {:?} is unsupported", architecture,); - } - }) -} - -const TEXT_SECTION_NAME: &[u8] = b".text"; - -fn process_unwind_info(info: &UnwindInfo, obj: &mut Object, code_section: SectionId) { - if let UnwindInfo::WindowsX64(info) = &info { - // Windows prefers Unwind info after the code -- writing it here. - let unwind_size = info.emit_size(); - let mut unwind_info = vec![0; unwind_size]; - info.emit(&mut unwind_info); - let _off = obj.append_section_data(code_section, &unwind_info, 4); - } -} - -/// Builds ELF image from the module `Compilation`. -// const CODE_SECTION_ALIGNMENT: u64 = 0x1000; - -/// Iterates through all `LibCall` members and all runtime exported functions. -#[macro_export] -macro_rules! for_each_libcall { - ($op:ident) => { - $op![ - (UdivI64, wasmtime_i64_udiv), - (UdivI64, wasmtime_i64_udiv), - (SdivI64, wasmtime_i64_sdiv), - (UremI64, wasmtime_i64_urem), - (SremI64, wasmtime_i64_srem), - (IshlI64, wasmtime_i64_ishl), - (UshrI64, wasmtime_i64_ushr), - (SshrI64, wasmtime_i64_sshr), - (CeilF32, wasmtime_f32_ceil), - (FloorF32, wasmtime_f32_floor), - (TruncF32, wasmtime_f32_trunc), - (NearestF32, wasmtime_f32_nearest), - (CeilF64, wasmtime_f64_ceil), - (FloorF64, wasmtime_f64_floor), - (TruncF64, wasmtime_f64_trunc), - (NearestF64, wasmtime_f64_nearest) - ]; - }; -} - -fn write_libcall_symbols(obj: &mut Object) -> HashMap { - let mut libcalls = HashMap::new(); - macro_rules! add_libcall_symbol { - [$(($libcall:ident, $export:ident)),*] => {{ - $( - let symbol_id = obj.add_symbol(Symbol { - name: stringify!($export).as_bytes().to_vec(), - value: 0, - size: 0, - kind: SymbolKind::Text, - scope: SymbolScope::Linkage, - weak: true, - section: SymbolSection::Undefined, - flags: SymbolFlags::None, - }); - libcalls.insert(LibCall::$libcall, symbol_id); - )+ - }}; - } - for_each_libcall!(add_libcall_symbol); - - libcalls -} - -pub mod utils { - use wasmtime_environ::entity::EntityRef; - use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; - - pub const FUNCTION_PREFIX: &str = "_wasm_function_"; - pub const TRAMPOLINE_PREFIX: &str = "_trampoline_"; - - pub fn func_symbol_name(index: FuncIndex) -> String { - format!("_wasm_function_{}", index.index()) - } - - pub fn try_parse_func_name(name: &str) -> Option { - if !name.starts_with(FUNCTION_PREFIX) { - return None; - } - name[FUNCTION_PREFIX.len()..] - .parse() - .ok() - .map(FuncIndex::new) - } - - pub fn trampoline_symbol_name(index: SignatureIndex) -> String { - format!("_trampoline_{}", index.index()) - } - - pub fn try_parse_trampoline_name(name: &str) -> Option { - if !name.starts_with(TRAMPOLINE_PREFIX) { - return None; - } - name[TRAMPOLINE_PREFIX.len()..] - .parse() - .ok() - .map(SignatureIndex::new) - } -} - -pub struct ObjectBuilderTarget { - pub(crate) binary_format: BinaryFormat, - pub(crate) architecture: Architecture, - pub(crate) endianness: Endianness, -} - -impl ObjectBuilderTarget { - pub fn new(arch: target_lexicon::Architecture) -> Result { - Ok(Self { - binary_format: BinaryFormat::Elf, - architecture: to_object_architecture(arch)?, - endianness: match arch.endianness().unwrap() { - target_lexicon::Endianness::Little => object::Endianness::Little, - target_lexicon::Endianness::Big => object::Endianness::Big, - }, - }) - } - - pub fn from_triple(triple: &Triple) -> Result { - let binary_format = match triple.binary_format { - target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, - target_lexicon::BinaryFormat::Coff => object::BinaryFormat::Coff, - target_lexicon::BinaryFormat::Macho => object::BinaryFormat::MachO, - target_lexicon::BinaryFormat::Wasm => { - bail!("binary format wasm is unsupported"); - } - target_lexicon::BinaryFormat::Unknown => { - bail!("binary format is unknown"); - } - other => bail!("binary format {} is unsupported", other), - }; - let architecture = to_object_architecture(triple.architecture)?; - let endianness = match triple.endianness().unwrap() { - target_lexicon::Endianness::Little => object::Endianness::Little, - target_lexicon::Endianness::Big => object::Endianness::Big, - }; - Ok(Self { - binary_format, - architecture, - endianness, - }) - } -} - -pub struct ObjectBuilder<'a> { - target: ObjectBuilderTarget, - module: &'a Module, - code_alignment: u64, - compilation: &'a CompiledFunctions, - trampolines: Vec<(SignatureIndex, CompiledFunction)>, - dwarf_sections: Vec, -} - -impl<'a> ObjectBuilder<'a> { - pub fn new( - target: ObjectBuilderTarget, - module: &'a Module, - compilation: &'a CompiledFunctions, - ) -> Self { - Self { - target, - module, - code_alignment: 1, - trampolines: Vec::new(), - dwarf_sections: vec![], - compilation, - } - } - - pub fn set_code_alignment(&mut self, code_alignment: u64) -> &mut Self { - self.code_alignment = code_alignment; - self - } - - pub fn set_trampolines( - &mut self, - trampolines: Vec<(SignatureIndex, CompiledFunction)>, - ) -> &mut Self { - self.trampolines = trampolines; - self - } - - pub fn set_dwarf_sections(&mut self, dwarf_sections: Vec) -> &mut Self { - self.dwarf_sections = dwarf_sections; - self - } - - pub fn build(self) -> Result { - let mut obj = Object::new( - self.target.binary_format, - self.target.architecture, - self.target.endianness, - ); - - let module = self.module; - - // Entire code (functions and trampolines) will be placed - // in the ".text" section. - let section_id = obj.add_section( - obj.segment_name(StandardSegment::Text).to_vec(), - TEXT_SECTION_NAME.to_vec(), - SectionKind::Text, - ); - - // Create symbols for imports -- needed during linking. - let mut func_symbols = PrimaryMap::with_capacity(self.compilation.len()); - for index in 0..module.num_imported_funcs { - let symbol_id = obj.add_symbol(Symbol { - name: utils::func_symbol_name(FuncIndex::new(index)) - .as_bytes() - .to_vec(), - value: 0, - size: 0, - kind: SymbolKind::Text, - scope: SymbolScope::Linkage, - weak: false, - section: SymbolSection::Undefined, - flags: SymbolFlags::None, - }); - func_symbols.push(symbol_id); - } - - let mut append_func = |name: Vec, func: &CompiledFunction| { - let off = obj.append_section_data(section_id, &func.body, 1); - let symbol_id = obj.add_symbol(Symbol { - name, - value: off, - size: func.body.len() as u64, - kind: SymbolKind::Text, - scope: SymbolScope::Compilation, - weak: false, - section: SymbolSection::Section(section_id), - flags: SymbolFlags::None, - }); - // Preserve function unwind info. - if let Some(info) = &func.unwind_info { - process_unwind_info(info, &mut obj, section_id); - } - symbol_id - }; - - // Create symbols and section data for the compiled functions. - for (index, func) in self.compilation.iter() { - let name = utils::func_symbol_name(module.func_index(index)) - .as_bytes() - .to_vec(); - let symbol_id = append_func(name, func); - func_symbols.push(symbol_id); - } - let mut trampolines = Vec::new(); - for (i, func) in self.trampolines.iter() { - let name = utils::trampoline_symbol_name(*i).as_bytes().to_vec(); - trampolines.push(append_func(name, func)); - } - - obj.append_section_data(section_id, &[], self.code_alignment); - - // If we have DWARF data, write it in the object file. - let (debug_bodies, debug_relocs) = self - .dwarf_sections - .into_iter() - .map(|s| ((s.name, s.body), (s.name, s.relocs))) - .unzip::<_, _, Vec<_>, Vec<_>>(); - let mut dwarf_sections_ids = HashMap::new(); - for (name, body) in debug_bodies { - let segment = obj.segment_name(StandardSegment::Debug).to_vec(); - let section_id = obj.add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug); - dwarf_sections_ids.insert(name.to_string(), section_id); - obj.append_section_data(section_id, &body, 1); - } - - let libcalls = write_libcall_symbols(&mut obj); - - // Write all functions relocations. - for (index, func) in self.compilation.into_iter() { - let func_index = module.func_index(index); - let (_, off) = obj - .symbol_section_and_offset(func_symbols[func_index]) - .unwrap(); - for r in to_object_relocations( - func.relocations.iter(), - off, - module, - &func_symbols, - &libcalls, - &self.compilation, - ) { - obj.add_relocation(section_id, r)?; - } - } - - for ((_, func), symbol) in self.trampolines.iter().zip(trampolines) { - let (_, off) = obj.symbol_section_and_offset(symbol).unwrap(); - for r in to_object_relocations( - func.relocations.iter(), - off, - module, - &func_symbols, - &libcalls, - &self.compilation, - ) { - obj.add_relocation(section_id, r)?; - } - } - - // Write all debug data relocations. - for (name, relocs) in debug_relocs { - let section_id = *dwarf_sections_ids.get(name).unwrap(); - for reloc in relocs { - let target_symbol = match reloc.target { - DwarfSectionRelocTarget::Func(index) => { - func_symbols[module.func_index(DefinedFuncIndex::new(index))] - } - DwarfSectionRelocTarget::Section(name) => { - obj.section_symbol(*dwarf_sections_ids.get(name).unwrap()) - } - }; - obj.add_relocation( - section_id, - ObjectRelocation { - offset: u64::from(reloc.offset), - size: reloc.size << 3, - kind: RelocationKind::Absolute, - encoding: RelocationEncoding::Generic, - symbol: target_symbol, - addend: i64::from(reloc.addend), - }, - )?; - } - } - - Ok(obj) - } -} diff --git a/crates/obj/src/lib.rs b/crates/obj/src/lib.rs deleted file mode 100644 index fd7c85c5e6..0000000000 --- a/crates/obj/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! Object-file writing library using the wasmtime environment. - -#![deny( - missing_docs, - trivial_numeric_casts, - unused_extern_crates, - unstable_features -)] -#![warn(unused_import_braces)] -#![cfg_attr(feature = "clippy", plugin(clippy(conf_file = "../../clippy.toml")))] -#![cfg_attr(feature = "cargo-clippy", allow(clippy::new_without_default))] -#![cfg_attr( - feature = "cargo-clippy", - warn( - clippy::float_arithmetic, - clippy::mut_mut, - clippy::nonminimal_bool, - clippy::map_unwrap_or, - clippy::clippy::print_stdout, - clippy::unicode_not_nfc, - clippy::use_self - ) -)] - -mod builder; - -pub use crate::builder::{utils, ObjectBuilder, ObjectBuilderTarget}; - -/// Version number of this crate. -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index a862d84244..85e2c04350 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -318,8 +318,8 @@ impl Module { let modules = CompiledModule::from_artifacts_list( artifacts, - engine.compiler(), &*engine.config().profiler, + engine.compiler(), )?; Self::from_parts(engine, modules, main_module, Arc::new(types), &[]) diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs index 507af1fdba..9b68dc58f3 100644 --- a/crates/wasmtime/src/module/serialization.rs +++ b/crates/wasmtime/src/module/serialization.rs @@ -233,8 +233,8 @@ impl<'a> SerializedModule<'a> { .into_iter() .map(|i| i.unwrap_owned()) .collect(), - engine.compiler(), &*engine.config().profiler, + engine.compiler(), )?; assert!(!modules.is_empty()); diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index e47f83d04b..9cf91725da 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -77,23 +77,24 @@ pub fn create_function( func: Box Result<(), Trap> + Send + Sync>, engine: &Engine, ) -> Result<(InstanceHandle, VMTrampoline)> { - let wasm_trampoline = engine + let obj = engine .compiler() .compiler() - .wasm_to_host_trampoline(ft.as_wasm_func_type(), stub_fn as usize)?; - let host_trampoline = engine - .compiler() - .compiler() - .host_to_wasm_trampoline(ft.as_wasm_func_type())?; + .emit_trampoline_obj(ft.as_wasm_func_type(), stub_fn as usize)?; let mut code_memory = CodeMemory::new(); - let host_trampoline = code_memory - .allocate_for_function(&host_trampoline)? - .as_ptr(); - let wasm_trampoline = - code_memory.allocate_for_function(&wasm_trampoline)? as *mut [VMFunctionBody]; + let alloc = code_memory.allocate_for_object(&obj)?; + let mut trampolines = alloc.trampolines(); + let (host_i, host_trampoline) = trampolines.next().unwrap(); + assert_eq!(host_i.as_u32(), 0); + let (wasm_i, wasm_trampoline) = trampolines.next().unwrap(); + assert_eq!(wasm_i.as_u32(), 1); + assert!(trampolines.next().is_none()); + let host_trampoline = host_trampoline.as_ptr(); + let wasm_trampoline = wasm_trampoline as *mut [_]; + drop(trampolines); - code_memory.publish(engine.compiler()); + code_memory.publish(); let sig = engine.signatures().register(ft.as_wasm_func_type()); diff --git a/src/commands/wasm2obj.rs b/src/commands/wasm2obj.rs index b6426ed13a..46fe5fd0c7 100644 --- a/src/commands/wasm2obj.rs +++ b/src/commands/wasm2obj.rs @@ -68,7 +68,7 @@ impl WasmToObjCommand { let mut file = File::create(Path::new(&self.output)).context("failed to create object file")?; - file.write_all(&obj.write()?) + file.write_all(&obj) .context("failed to write object file")?; Ok(()) diff --git a/src/obj.rs b/src/obj.rs index 3ef7498dc2..8059d52f49 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -1,5 +1,4 @@ use anyhow::{bail, Context as _, Result}; -use object::write::Object; use target_lexicon::Triple; use wasmparser::WasmFeatures; use wasmtime::Strategy; @@ -14,7 +13,7 @@ pub fn compile_to_obj( enable_simd: bool, opt_level: wasmtime::OptLevel, debug_info: bool, -) -> Result { +) -> Result> { let strategy = match strategy { Strategy::Auto => wasmtime_jit::CompilationStrategy::Auto, Strategy::Cranelift => wasmtime_jit::CompilationStrategy::Cranelift, diff --git a/tests/all/debug/obj.rs b/tests/all/debug/obj.rs index f4b3d14073..3818572c09 100644 --- a/tests/all/debug/obj.rs +++ b/tests/all/debug/obj.rs @@ -21,7 +21,7 @@ pub fn compile_cranelift( )?; let mut file = File::create(output).context("failed to create object file")?; - file.write_all(&obj.write()?) + file.write_all(&obj) .context("failed to write object file")?; Ok(())