diff --git a/Cargo.lock b/Cargo.lock index 114e632f6a..f9762689cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3775,6 +3775,7 @@ name = "wasmtime-winch" version = "3.0.0" dependencies = [ "anyhow", + "object", "target-lexicon", "wasmtime-environ", "winch-codegen", diff --git a/Cargo.toml b/Cargo.toml index a8b20d49a6..eb2697107f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ libc = "0.2.60" humantime = "2.0.0" once_cell = { workspace = true } listenfd = "1.0.0" +wat = { workspace = true } [target.'cfg(unix)'.dependencies] rustix = { workspace = true, features = ["mm", "param"] } @@ -90,7 +91,7 @@ members = [ "examples/wasi/wasm", "examples/tokio/wasm", "fuzz", - "winch", + "winch", "winch/codegen" ] exclude = [ diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index 1a0a7c9972..bef16944f6 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -184,7 +184,7 @@ impl TargetIsa for AArch64Backend { Some(inst::unwind::systemv::create_cie()) } - fn text_section_builder(&self, num_funcs: u32) -> Box { + fn text_section_builder(&self, num_funcs: usize) -> Box { Box::new(MachTextSectionBuilder::::new(num_funcs)) } diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index ebe1e87c9f..c319433f5d 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -285,7 +285,7 @@ pub trait TargetIsa: fmt::Display + Send + Sync { /// The `num_labeled_funcs` argument here is the number of functions which /// will be "labeled" or might have calls between them, typically the number /// of defined functions in the object file. - fn text_section_builder(&self, num_labeled_funcs: u32) -> Box; + fn text_section_builder(&self, num_labeled_funcs: usize) -> Box; /// The function alignment required by this ISA. fn function_alignment(&self) -> u32; diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs index d47d2ec618..603279c06f 100644 --- a/cranelift/codegen/src/isa/riscv64/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/mod.rs @@ -152,7 +152,7 @@ impl TargetIsa for Riscv64Backend { Some(inst::unwind::systemv::create_cie()) } - fn text_section_builder(&self, num_funcs: u32) -> Box { + fn text_section_builder(&self, num_funcs: usize) -> Box { Box::new(MachTextSectionBuilder::::new(num_funcs)) } diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs index a5b85f6d6f..134cd2165f 100644 --- a/cranelift/codegen/src/isa/s390x/mod.rs +++ b/cranelift/codegen/src/isa/s390x/mod.rs @@ -167,7 +167,7 @@ impl TargetIsa for S390xBackend { inst::unwind::systemv::map_reg(reg).map(|reg| reg.0) } - fn text_section_builder(&self, num_funcs: u32) -> Box { + fn text_section_builder(&self, num_funcs: usize) -> Box { Box::new(MachTextSectionBuilder::::new(num_funcs)) } diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 3d20183fdb..4a6664e628 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -156,7 +156,7 @@ impl TargetIsa for X64Backend { inst::unwind::systemv::map_reg(reg).map(|reg| reg.0) } - fn text_section_builder(&self, num_funcs: u32) -> Box { + fn text_section_builder(&self, num_funcs: usize) -> Box { Box::new(MachTextSectionBuilder::::new(num_funcs)) } diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index c4eb5dd2da..8a71d33cbc 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -1612,9 +1612,9 @@ pub struct MachTextSectionBuilder { } impl MachTextSectionBuilder { - pub fn new(num_funcs: u32) -> MachTextSectionBuilder { + pub fn new(num_funcs: usize) -> MachTextSectionBuilder { let mut buf = MachBuffer::new(); - buf.reserve_labels_for_blocks(num_funcs as usize); + buf.reserve_labels_for_blocks(num_funcs); MachTextSectionBuilder { buf, next_func: 0, @@ -1624,7 +1624,7 @@ impl MachTextSectionBuilder { } impl TextSectionBuilder for MachTextSectionBuilder { - fn append(&mut self, named: bool, func: &[u8], align: u32) -> u64 { + fn append(&mut self, labeled: bool, func: &[u8], align: u32) -> u64 { // Conditionally emit an island if it's necessary to resolve jumps // between functions which are too far away. let size = func.len() as u32; @@ -1634,7 +1634,7 @@ impl TextSectionBuilder for MachTextSectionBuilder { self.buf.align_to(align); let pos = self.buf.cur_offset(); - if named { + if labeled { self.buf .bind_label(MachLabel::from_block(BlockIndex::new(self.next_func))); self.next_func += 1; @@ -1643,8 +1643,8 @@ impl TextSectionBuilder for MachTextSectionBuilder { u64::from(pos) } - fn resolve_reloc(&mut self, offset: u64, reloc: Reloc, addend: Addend, target: u32) -> bool { - let label = MachLabel::from_block(BlockIndex::new(target as usize)); + fn resolve_reloc(&mut self, offset: u64, reloc: Reloc, addend: Addend, target: usize) -> bool { + let label = MachLabel::from_block(BlockIndex::new(target)); let offset = u32::try_from(offset).unwrap(); match I::LabelUse::from_reloc(reloc, addend) { Some(label_use) => { diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index 044eddddc5..fba277b894 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -397,8 +397,10 @@ impl CompiledCode { pub trait TextSectionBuilder { /// Appends `data` to the text section with the `align` specified. /// - /// If `labeled` is `true` then the offset of the final data is used to - /// resolve relocations in `resolve_reloc` in the future. + /// If `labeled` is `true` then this also binds the appended data to the + /// `n`th label for how many times this has been called with `labeled: + /// true`. The label target can be passed as the `target` argument to + /// `resolve_reloc`. /// /// This function returns the offset at which the data was placed in the /// text section. @@ -418,7 +420,7 @@ pub trait TextSectionBuilder { /// If this builder does not know how to handle `reloc` then this function /// will return `false`. Otherwise this function will return `true` and this /// relocation will be resolved in the final bytes returned by `finish`. - fn resolve_reloc(&mut self, offset: u64, reloc: Reloc, addend: Addend, target: u32) -> bool; + fn resolve_reloc(&mut self, offset: u64, reloc: Reloc, addend: Addend, target: usize) -> bool; /// A debug-only option which is used to for fn force_veneers(&mut self); diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index 6c6c10b154..1fa1603620 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -4,7 +4,7 @@ use crate::func_environ::FuncEnvironment; use crate::obj::ModuleTextBuilder; use crate::{ blank_sig, func_signature, indirect_signature, value_type, wasmtime_call_conv, - CompiledFunction, CompiledFunctions, FunctionAddressMap, Relocation, RelocationTarget, + CompiledFunction, FunctionAddressMap, Relocation, RelocationTarget, }; use anyhow::{Context as _, Result}; use cranelift_codegen::ir::{ @@ -18,8 +18,7 @@ use cranelift_codegen::{CompiledCode, MachSrcLoc, MachStackMap}; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_frontend::FunctionBuilder; use cranelift_wasm::{ - DefinedFuncIndex, FuncIndex, FuncTranslator, MemoryIndex, OwnedMemoryIndex, SignatureIndex, - WasmFuncType, + DefinedFuncIndex, FuncIndex, FuncTranslator, MemoryIndex, OwnedMemoryIndex, WasmFuncType, }; use object::write::{Object, StandardSegment, SymbolId}; use object::{RelocationEncoding, RelocationKind, SectionKind}; @@ -32,10 +31,9 @@ use std::mem; use std::sync::{Arc, Mutex}; use wasmparser::{FuncValidatorAllocations, FunctionBody}; use wasmtime_environ::{ - AddressMapSection, CacheStore, CompileError, FilePos, FlagValue, FunctionBodyData, - FunctionInfo, InstructionAddressMap, Module, ModuleTranslation, ModuleTypes, PtrSize, - StackMapInformation, Trampoline, TrapCode, TrapEncodingBuilder, TrapInformation, Tunables, - VMOffsets, + AddressMapSection, CacheStore, CompileError, FilePos, FlagValue, FunctionBodyData, FunctionLoc, + InstructionAddressMap, ModuleTranslation, ModuleTypes, PtrSize, StackMapInformation, TrapCode, + TrapEncodingBuilder, TrapInformation, Tunables, VMOffsets, WasmFunctionInfo, }; #[cfg(feature = "component-model")] @@ -189,7 +187,7 @@ impl wasmtime_environ::Compiler for Compiler { input: FunctionBodyData<'_>, tunables: &Tunables, types: &ModuleTypes, - ) -> Result, CompileError> { + ) -> Result<(WasmFunctionInfo, Box), CompileError> { let isa = &*self.isa; let module = &translation.module; let func_index = module.func_index(func_index); @@ -324,22 +322,22 @@ impl wasmtime_environ::Compiler for Compiler { validator_allocations: validator.into_allocations(), }); - Ok(Box::new(CompiledFunction { - body: code_buf, - relocations: func_relocs, - value_labels_ranges: ranges.unwrap_or(Default::default()), - sized_stack_slots, - unwind_info, - traps, - info: FunctionInfo { + Ok(( + WasmFunctionInfo { start_srcloc: address_transform.start_srcloc, - stack_maps, - start: 0, - length, - alignment, + stack_maps: stack_maps.into(), }, - address_map: address_transform, - })) + Box::new(CompiledFunction { + body: code_buf, + relocations: func_relocs, + value_labels_ranges: ranges.unwrap_or(Default::default()), + sized_stack_slots, + unwind_info, + traps, + alignment, + address_map: address_transform, + }), + )) } fn compile_host_to_wasm_trampoline( @@ -350,75 +348,44 @@ impl wasmtime_environ::Compiler for Compiler { .map(|x| Box::new(x) as Box<_>) } - fn emit_obj( + fn append_code( &self, - translation: &ModuleTranslation, - funcs: PrimaryMap>, - compiled_trampolines: Vec>, - tunables: &Tunables, obj: &mut Object<'static>, - ) -> Result<(PrimaryMap, Vec)> { - let funcs: CompiledFunctions = funcs - .into_iter() - .map(|(_i, f)| *f.downcast().unwrap()) - .collect(); - let compiled_trampolines: Vec = compiled_trampolines - .into_iter() - .map(|f| *f.downcast().unwrap()) - .collect(); - - let mut builder = ModuleTextBuilder::new(obj, &translation.module, &*self.isa); + funcs: &[(String, Box)], + tunables: &Tunables, + resolve_reloc: &dyn Fn(usize, FuncIndex) -> usize, + ) -> Result> { + let mut builder = ModuleTextBuilder::new(obj, &*self.isa, funcs.len()); if self.linkopts.force_jump_veneers { builder.force_veneers(); } let mut addrs = AddressMapSection::default(); let mut traps = TrapEncodingBuilder::default(); - let mut func_starts = Vec::with_capacity(funcs.len()); - for (i, func) in funcs.iter() { - let range = builder.func(i, func); + let mut ret = Vec::with_capacity(funcs.len()); + for (i, (sym, func)) in funcs.iter().enumerate() { + let func = func.downcast_ref().unwrap(); + let (sym, range) = builder.append_func(&sym, func, |idx| resolve_reloc(i, idx)); if tunables.generate_address_map { addrs.push(range.clone(), &func.address_map.instructions); } traps.push(range.clone(), &func.traps); - func_starts.push(range.start); builder.append_padding(self.linkopts.padding_between_functions); + let info = FunctionLoc { + start: u32::try_from(range.start).unwrap(), + length: u32::try_from(range.end - range.start).unwrap(), + }; + ret.push((sym, info)); } - // Build trampolines for every signature that can be used by this module. - assert_eq!( - translation.exported_signatures.len(), - compiled_trampolines.len() - ); - let mut trampolines = Vec::with_capacity(translation.exported_signatures.len()); - for (i, func) in translation - .exported_signatures - .iter() - .zip(&compiled_trampolines) - { - assert!(func.traps.is_empty()); - trampolines.push(builder.trampoline(*i, &func)); - } + builder.finish(); - let symbols = builder.finish()?; - - self.append_dwarf(obj, translation, &funcs, tunables, &symbols)?; if tunables.generate_address_map { addrs.append_to(obj); } traps.append_to(obj); - Ok(( - funcs - .into_iter() - .zip(func_starts) - .map(|((_, mut f), start)| { - f.info.start = start; - f.info - }) - .collect(), - trampolines, - )) + Ok(ret) } fn emit_trampoline_obj( @@ -426,14 +393,21 @@ impl wasmtime_environ::Compiler for Compiler { ty: &WasmFuncType, host_fn: usize, obj: &mut Object<'static>, - ) -> Result<(Trampoline, Trampoline)> { + ) -> Result<(FunctionLoc, FunctionLoc)> { let host_to_wasm = self.host_to_wasm_trampoline(ty)?; let wasm_to_host = self.wasm_to_host_trampoline(ty, host_fn)?; - let module = Module::new(); - let mut builder = ModuleTextBuilder::new(obj, &module, &*self.isa); - let a = builder.trampoline(SignatureIndex::new(0), &host_to_wasm); - let b = builder.trampoline(SignatureIndex::new(1), &wasm_to_host); - builder.finish()?; + let mut builder = ModuleTextBuilder::new(obj, &*self.isa, 2); + let (_, a) = builder.append_func("host_to_wasm", &host_to_wasm, |_| unreachable!()); + let (_, b) = builder.append_func("wasm_to_host", &wasm_to_host, |_| unreachable!()); + let a = FunctionLoc { + start: u32::try_from(a.start).unwrap(), + length: u32::try_from(a.end - a.start).unwrap(), + }; + let b = FunctionLoc { + start: u32::try_from(b.start).unwrap(), + length: u32::try_from(b.end - b.start).unwrap(), + }; + builder.finish(); Ok((a, b)) } @@ -469,6 +443,92 @@ impl wasmtime_environ::Compiler for Compiler { fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler { self } + + fn append_dwarf( + &self, + obj: &mut Object<'_>, + translation: &ModuleTranslation<'_>, + funcs: &PrimaryMap, + ) -> Result<()> { + let ofs = VMOffsets::new( + self.isa + .triple() + .architecture + .pointer_width() + .unwrap() + .bytes(), + &translation.module, + ); + + let memory_offset = if ofs.num_imported_memories > 0 { + ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) + } else if ofs.num_defined_memories > 0 { + // The addition of shared memory makes the following assumption, + // "owned memory index = 0", possibly false. If the first memory + // is a shared memory, the base pointer will not be stored in + // the `owned_memories` array. The following code should + // eventually be fixed to not only handle shared memories but + // also multiple memories. + assert_eq!( + ofs.num_defined_memories, ofs.num_owned_memories, + "the memory base pointer may be incorrect due to sharing memory" + ); + ModuleMemoryOffset::Defined( + ofs.vmctx_vmmemory_definition_base(OwnedMemoryIndex::new(0)), + ) + } else { + ModuleMemoryOffset::None + }; + let compiled_funcs = funcs + .iter() + .map(|(_, (_, func))| func.downcast_ref().unwrap()) + .collect(); + let dwarf_sections = crate::debug::emit_dwarf( + &*self.isa, + &translation.debuginfo, + &compiled_funcs, + &memory_offset, + ) + .with_context(|| "failed to emit DWARF debug information")?; + + let (debug_bodies, debug_relocs): (Vec<_>, Vec<_>) = dwarf_sections + .iter() + .map(|s| ((s.name, &s.body), (s.name, &s.relocs))) + .unzip(); + let mut dwarf_sections_ids = HashMap::new(); + for (name, body) in debug_bodies { + let segment = obj.segment_name(StandardSegment::Debug).to_vec(); + let section_id = obj.add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug); + dwarf_sections_ids.insert(name, section_id); + obj.append_section_data(section_id, &body, 1); + } + + // Write all debug data relocations. + for (name, relocs) in debug_relocs { + let section_id = *dwarf_sections_ids.get(name).unwrap(); + for reloc in relocs { + let target_symbol = match reloc.target { + DwarfSectionRelocTarget::Func(index) => funcs[DefinedFuncIndex::new(index)].0, + DwarfSectionRelocTarget::Section(name) => { + obj.section_symbol(dwarf_sections_ids[name]) + } + }; + obj.add_relocation( + section_id, + object::write::Relocation { + offset: u64::from(reloc.offset), + size: reloc.size << 3, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol: target_symbol, + addend: i64::from(reloc.addend), + }, + )?; + } + } + + Ok(()) + } } #[cfg(feature = "incremental-cache")] @@ -826,6 +886,7 @@ impl Compiler { .into_iter() .map(mach_trap_to_trap) .collect(); + let alignment = compiled_code.alignment; let unwind_info = if isa.flags().unwind_info() { compiled_code @@ -841,96 +902,11 @@ impl Compiler { relocations: Default::default(), sized_stack_slots: Default::default(), value_labels_ranges: Default::default(), - info: Default::default(), address_map: Default::default(), traps, + alignment, }) } - - pub fn append_dwarf( - &self, - obj: &mut Object<'_>, - translation: &ModuleTranslation<'_>, - funcs: &CompiledFunctions, - tunables: &Tunables, - func_symbols: &PrimaryMap, - ) -> Result<()> { - if !tunables.generate_native_debuginfo || funcs.len() == 0 { - return Ok(()); - } - let ofs = VMOffsets::new( - self.isa - .triple() - .architecture - .pointer_width() - .unwrap() - .bytes(), - &translation.module, - ); - - let memory_offset = if ofs.num_imported_memories > 0 { - ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) - } else if ofs.num_defined_memories > 0 { - // The addition of shared memory makes the following assumption, - // "owned memory index = 0", possibly false. If the first memory - // is a shared memory, the base pointer will not be stored in - // the `owned_memories` array. The following code should - // eventually be fixed to not only handle shared memories but - // also multiple memories. - assert_eq!( - ofs.num_defined_memories, ofs.num_owned_memories, - "the memory base pointer may be incorrect due to sharing memory" - ); - ModuleMemoryOffset::Defined( - ofs.vmctx_vmmemory_definition_base(OwnedMemoryIndex::new(0)), - ) - } else { - ModuleMemoryOffset::None - }; - let dwarf_sections = - crate::debug::emit_dwarf(&*self.isa, &translation.debuginfo, &funcs, &memory_offset) - .with_context(|| "failed to emit DWARF debug information")?; - - let (debug_bodies, debug_relocs): (Vec<_>, Vec<_>) = dwarf_sections - .iter() - .map(|s| ((s.name, &s.body), (s.name, &s.relocs))) - .unzip(); - let mut dwarf_sections_ids = HashMap::new(); - for (name, body) in debug_bodies { - let segment = obj.segment_name(StandardSegment::Debug).to_vec(); - let section_id = obj.add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug); - dwarf_sections_ids.insert(name, section_id); - obj.append_section_data(section_id, &body, 1); - } - - // Write all debug data relocations. - for (name, relocs) in debug_relocs { - let section_id = *dwarf_sections_ids.get(name).unwrap(); - for reloc in relocs { - let target_symbol = match reloc.target { - DwarfSectionRelocTarget::Func(index) => { - func_symbols[DefinedFuncIndex::new(index)] - } - DwarfSectionRelocTarget::Section(name) => { - obj.section_symbol(dwarf_sections_ids[name]) - } - }; - obj.add_relocation( - section_id, - object::write::Relocation { - offset: u64::from(reloc.offset), - size: reloc.size << 3, - kind: RelocationKind::Absolute, - encoding: RelocationEncoding::Generic, - symbol: target_symbol, - addend: i64::from(reloc.addend), - }, - )?; - } - } - - Ok(()) - } } // Collects an iterator of `InstructionAddressMap` into a `Vec` for insertion diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index d8543de541..1a4758eeb0 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -1,20 +1,16 @@ //! Compilation support for the component model. use crate::compiler::{Compiler, CompilerContext}; -use crate::obj::ModuleTextBuilder; use crate::CompiledFunction; use anyhow::Result; use cranelift_codegen::ir::{self, InstBuilder, MemFlags}; use cranelift_frontend::FunctionBuilder; -use object::write::Object; use std::any::Any; -use std::ops::Range; use wasmtime_environ::component::{ - AlwaysTrapInfo, CanonicalOptions, Component, ComponentCompiler, ComponentTypes, FixedEncoding, - FunctionInfo, LowerImport, LoweredIndex, RuntimeAlwaysTrapIndex, RuntimeMemoryIndex, - RuntimeTranscoderIndex, Transcode, Transcoder, VMComponentOffsets, + CanonicalOptions, Component, ComponentCompiler, ComponentTypes, FixedEncoding, LowerImport, + RuntimeMemoryIndex, Transcode, Transcoder, VMComponentOffsets, }; -use wasmtime_environ::{PrimaryMap, PtrSize, SignatureIndex, Trampoline, TrapCode, WasmFuncType}; +use wasmtime_environ::{PtrSize, WasmFuncType}; impl ComponentCompiler for Compiler { fn compile_lowered_trampoline( @@ -235,82 +231,6 @@ impl ComponentCompiler for Compiler { }); Ok(Box::new(func)) } - - fn emit_obj( - &self, - lowerings: PrimaryMap>, - always_trap: PrimaryMap>, - transcoders: PrimaryMap>, - trampolines: Vec<(SignatureIndex, Box)>, - obj: &mut Object<'static>, - ) -> Result<( - PrimaryMap, - PrimaryMap, - PrimaryMap, - Vec, - )> { - let module = Default::default(); - let mut text = ModuleTextBuilder::new(obj, &module, &*self.isa); - - let range2info = |range: Range| FunctionInfo { - start: u32::try_from(range.start).unwrap(), - length: u32::try_from(range.end - range.start).unwrap(), - }; - let ret_lowerings = lowerings - .iter() - .map(|(i, lowering)| { - let lowering = lowering.downcast_ref::().unwrap(); - assert!(lowering.traps.is_empty()); - let range = text.named_func( - &format!("_wasm_component_lowering_trampoline{}", i.as_u32()), - &lowering, - ); - range2info(range) - }) - .collect(); - let ret_always_trap = always_trap - .iter() - .map(|(i, func)| { - let func = func.downcast_ref::().unwrap(); - assert_eq!(func.traps.len(), 1); - assert_eq!(func.traps[0].trap_code, TrapCode::AlwaysTrapAdapter); - let name = format!("_wasmtime_always_trap{}", i.as_u32()); - let range = text.named_func(&name, func); - AlwaysTrapInfo { - info: range2info(range), - trap_offset: func.traps[0].code_offset, - } - }) - .collect(); - - let ret_transcoders = transcoders - .iter() - .map(|(i, func)| { - let func = func.downcast_ref::().unwrap(); - let name = format!("_wasmtime_transcoder{}", i.as_u32()); - let range = text.named_func(&name, func); - range2info(range) - }) - .collect(); - - let ret_trampolines = trampolines - .iter() - .map(|(i, func)| { - let func = func.downcast_ref::().unwrap(); - assert!(func.traps.is_empty()); - text.trampoline(*i, func) - }) - .collect(); - - text.finish()?; - - Ok(( - ret_lowerings, - ret_always_trap, - ret_transcoders, - ret_trampolines, - )) - } } impl Compiler { diff --git a/crates/cranelift/src/debug/transform/address_transform.rs b/crates/cranelift/src/debug/transform/address_transform.rs index ead45b2fe0..4c71f714bc 100644 --- a/crates/cranelift/src/debug/transform/address_transform.rs +++ b/crates/cranelift/src/debug/transform/address_transform.rs @@ -605,7 +605,7 @@ impl AddressTransform { #[cfg(test)] mod tests { use super::{build_function_lookup, get_wasm_code_offset, AddressTransform}; - use crate::{CompiledFunction, CompiledFunctions, FunctionAddressMap}; + use crate::{CompiledFunction, FunctionAddressMap}; use cranelift_entity::PrimaryMap; use gimli::write::Address; use std::iter::FromIterator; @@ -650,13 +650,6 @@ mod tests { } } - fn create_simple_module(address_map: FunctionAddressMap) -> CompiledFunctions { - PrimaryMap::from_iter(vec![CompiledFunction { - address_map, - ..Default::default() - }]) - } - #[test] fn test_build_function_lookup_simple() { let input = create_simple_func(11); @@ -735,7 +728,11 @@ mod tests { #[test] fn test_addr_translate() { - let input = create_simple_module(create_simple_func(11)); + let func = CompiledFunction { + address_map: create_simple_func(11), + ..Default::default() + }; + let input = PrimaryMap::from_iter([&func]); let at = AddressTransform::new( &input, &WasmFileInfo { diff --git a/crates/cranelift/src/debug/transform/expression.rs b/crates/cranelift/src/debug/transform/expression.rs index ea6506a4a0..65a1c169b7 100644 --- a/crates/cranelift/src/debug/transform/expression.rs +++ b/crates/cranelift/src/debug/transform/expression.rs @@ -1118,7 +1118,7 @@ mod tests { use wasmtime_environ::WasmFileInfo; let mut module_map = PrimaryMap::new(); let code_section_offset: u32 = 100; - module_map.push(CompiledFunction { + let func = CompiledFunction { address_map: FunctionAddressMap { instructions: vec![ InstructionAddressMap { @@ -1145,7 +1145,8 @@ mod tests { body_len: 30, }, ..Default::default() - }); + }; + module_map.push(&func); let fi = WasmFileInfo { code_section_offset: code_section_offset.into(), funcs: Vec::new(), diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 837c461bbb..ddefe3cfdd 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -10,7 +10,7 @@ use cranelift_entity::PrimaryMap; use cranelift_wasm::{DefinedFuncIndex, FuncIndex, WasmFuncType, WasmType}; use target_lexicon::{Architecture, CallingConvention}; use wasmtime_environ::{ - FilePos, FunctionInfo, InstructionAddressMap, ModuleTranslation, ModuleTypes, TrapInformation, + FilePos, InstructionAddressMap, ModuleTranslation, ModuleTypes, TrapInformation, }; pub use builder::builder; @@ -21,7 +21,7 @@ mod debug; mod func_environ; mod obj; -type CompiledFunctions = PrimaryMap; +type CompiledFunctions<'a> = PrimaryMap; /// Compiled function: machine code body, jump table offsets, and unwind information. #[derive(Default)] @@ -43,9 +43,7 @@ pub struct CompiledFunction { relocations: Vec, value_labels_ranges: cranelift_codegen::ValueLabelsRanges, sized_stack_slots: ir::StackSlots, - - // TODO: Add dynamic_stack_slots? - info: FunctionInfo, + alignment: u32, } /// Function and its instructions addresses mappings. diff --git a/crates/cranelift/src/obj.rs b/crates/cranelift/src/obj.rs index 28d7094980..757b4705db 100644 --- a/crates/cranelift/src/obj.rs +++ b/crates/cranelift/src/obj.rs @@ -26,8 +26,7 @@ use object::write::{Object, SectionId, StandardSegment, Symbol, SymbolId, Symbol use object::{Architecture, SectionKind, SymbolFlags, SymbolKind, SymbolScope}; use std::convert::TryFrom; use std::ops::Range; -use wasmtime_environ::obj; -use wasmtime_environ::{DefinedFuncIndex, Module, PrimaryMap, SignatureIndex, Trampoline}; +use wasmtime_environ::FuncIndex; const TEXT_SECTION_NAME: &[u8] = b".text"; @@ -46,24 +45,23 @@ pub struct ModuleTextBuilder<'a> { obj: &'a mut Object<'static>, /// The WebAssembly module we're generating code for. - module: &'a Module, - text_section: SectionId, unwind_info: UnwindInfoBuilder<'a>, - /// The corresponding symbol for each function, inserted as they're defined. - /// - /// If an index isn't here yet then it hasn't been defined yet. - func_symbols: PrimaryMap, - /// In-progress text section that we're using cranelift's `MachBuffer` to /// build to resolve relocations (calls) between functions. text: Box, } impl<'a> ModuleTextBuilder<'a> { - pub fn new(obj: &'a mut Object<'static>, module: &'a Module, isa: &'a dyn TargetIsa) -> Self { + /// Creates a new builder for the text section of an executable. + /// + /// The `.text` section will be appended to the specified `obj` along with + /// any unwinding or such information as necessary. The `num_funcs` + /// parameter indicates the number of times the `append_func` function will + /// be called. The `finish` function will panic if this contract is not met. + pub fn new(obj: &'a mut Object<'static>, isa: &'a dyn TargetIsa, num_funcs: usize) -> Self { // Entire code (functions and trampolines) will be placed // in the ".text" section. let text_section = obj.add_section( @@ -72,37 +70,40 @@ impl<'a> ModuleTextBuilder<'a> { SectionKind::Text, ); - let num_defined = module.functions.len() - module.num_imported_funcs; Self { isa, obj, - module, text_section, - func_symbols: PrimaryMap::with_capacity(num_defined), unwind_info: Default::default(), - text: isa.text_section_builder(num_defined as u32), + text: isa.text_section_builder(num_funcs), } } /// Appends the `func` specified named `name` to this object. /// + /// The `resolve_reloc_target` closure is used to resolve a relocation + /// target to an adjacent function which has already been added or will be + /// added to this object. The argument is the relocation target specified + /// within `CompiledFunction` and the return value must be an index where + /// the target will be defined by the `n`th call to `append_func`. + /// /// Returns the symbol associated with the function as well as the range /// that the function resides within the text section. pub fn append_func( &mut self, - labeled: bool, - name: Vec, + name: &str, func: &'a CompiledFunction, + resolve_reloc_target: impl Fn(FuncIndex) -> usize, ) -> (SymbolId, Range) { let body_len = func.body.len() as u64; let off = self.text.append( - labeled, + true, &func.body, - self.isa.function_alignment().max(func.info.alignment), + self.isa.function_alignment().max(func.alignment), ); let symbol_id = self.obj.add_symbol(Symbol { - name, + name: name.as_bytes().to_vec(), value: off, size: body_len, kind: SymbolKind::Text, @@ -125,13 +126,11 @@ impl<'a> ModuleTextBuilder<'a> { // file, but if it can't handle it then we pass through the // relocation. RelocationTarget::UserFunc(index) => { - let defined_index = self.module.defined_func_index(index).unwrap(); - if self.text.resolve_reloc( - off + u64::from(r.offset), - r.reloc, - r.addend, - defined_index.as_u32(), - ) { + let target = resolve_reloc_target(index); + if self + .text + .resolve_reloc(off + u64::from(r.offset), r.reloc, r.addend, target) + { continue; } @@ -160,31 +159,6 @@ impl<'a> ModuleTextBuilder<'a> { (symbol_id, off..off + body_len) } - /// Appends a function to this object file. - /// - /// This is expected to be called in-order for ascending `index` values. - pub fn func(&mut self, index: DefinedFuncIndex, func: &'a CompiledFunction) -> Range { - let name = obj::func_symbol_name(self.module.func_index(index)); - let (symbol_id, range) = self.append_func(true, name.into_bytes(), func); - assert_eq!(self.func_symbols.push(symbol_id), index); - range - } - - pub fn trampoline(&mut self, sig: SignatureIndex, func: &'a CompiledFunction) -> Trampoline { - let name = obj::trampoline_symbol_name(sig); - let range = self.named_func(&name, func); - Trampoline { - signature: sig, - start: range.start, - length: u32::try_from(range.end - range.start).unwrap(), - } - } - - pub fn named_func(&mut self, name: &str, func: &'a CompiledFunction) -> Range { - let (_, range) = self.append_func(false, name.as_bytes().to_vec(), func); - range - } - /// Forces "veneers" to be used for inter-function calls in the text /// section which means that in-bounds optimized addresses are never used. /// @@ -210,7 +184,7 @@ impl<'a> ModuleTextBuilder<'a> { /// /// Note that this will also write out the unwind information sections if /// necessary. - pub fn finish(mut self) -> Result> { + pub fn finish(mut self) { // Finish up the text section now that we're done adding functions. let text = self.text.finish(); self.obj @@ -220,8 +194,6 @@ impl<'a> ModuleTextBuilder<'a> { // Append the unwind information for all our functions, if necessary. self.unwind_info .append_section(self.isa, self.obj, self.text_section); - - Ok(self.func_symbols) } } diff --git a/crates/environ/src/address_map.rs b/crates/environ/src/address_map.rs index af7b79278b..7a219543f1 100644 --- a/crates/environ/src/address_map.rs +++ b/crates/environ/src/address_map.rs @@ -1,6 +1,6 @@ //! Data structures to provide transformation of the source -// addresses of a WebAssembly module into the native code. +use crate::obj::ELF_WASMTIME_ADDRMAP; use object::write::{Object, StandardSegment}; use object::{Bytes, LittleEndian, SectionKind, U32Bytes}; use serde::{Deserialize, Serialize}; @@ -65,35 +65,6 @@ pub struct AddressMapSection { last_offset: u32, } -/// A custom Wasmtime-specific section of our compilation image which stores -/// mapping data from offsets in the image to offset in the original wasm -/// binary. -/// -/// This section has a custom binary encoding. Currently its encoding is: -/// -/// * The section starts with a 32-bit little-endian integer. This integer is -/// how many entries are in the following two arrays. -/// * Next is an array with the previous count number of 32-bit little-endian -/// integers. This array is a sorted list of relative offsets within the text -/// section. This is intended to be a lookup array to perform a binary search -/// on an offset within the text section on this array. -/// * Finally there is another array, with the same count as before, also of -/// 32-bit little-endian integers. These integers map 1:1 with the previous -/// array of offsets, and correspond to what the original offset was in the -/// wasm file. -/// -/// Decoding this section is intentionally simple, it only requires loading a -/// 32-bit little-endian integer plus some bounds checks. Reading this section -/// is done with the `lookup_file_pos` function below. Reading involves -/// performing a binary search on the first array using the index found for the -/// native code offset to index into the second array and find the wasm code -/// offset. -/// -/// At this time this section has an alignment of 1, which means all reads of it -/// are unaligned. Additionally at this time the 32-bit encodings chosen here -/// mean that >=4gb text sections are not supported. -pub const ELF_WASMTIME_ADDRMAP: &str = ".wasmtime.addrmap"; - impl AddressMapSection { /// Pushes a new set of instruction mapping information for a function added /// in the exectuable. diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index 52635cbf04..12126990e4 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -1,13 +1,14 @@ //! A `Compilation` contains the compiled function bodies for a WebAssembly //! module. +use crate::obj; use crate::{ - DefinedFuncIndex, FilePos, FunctionBodyData, ModuleTranslation, ModuleTypes, PrimaryMap, - SignatureIndex, StackMap, Tunables, WasmError, WasmFuncType, + DefinedFuncIndex, FilePos, FuncIndex, FunctionBodyData, ModuleTranslation, ModuleTypes, + PrimaryMap, StackMap, Tunables, WasmError, WasmFuncType, }; use anyhow::Result; -use object::write::Object; -use object::{Architecture, BinaryFormat}; +use object::write::{Object, SymbolId}; +use object::{Architecture, BinaryFormat, FileFlags}; use serde::{Deserialize, Serialize}; use std::any::Any; use std::borrow::Cow; @@ -20,30 +21,19 @@ use thiserror::Error; /// and stack maps. #[derive(Serialize, Deserialize, Default)] #[allow(missing_docs)] -pub struct FunctionInfo { +pub struct WasmFunctionInfo { pub start_srcloc: FilePos, - pub stack_maps: Vec, - - /// Offset in the text section of where this function starts. - pub start: u64, - /// The size of the compiled function, in bytes. - pub length: u32, - - /// The alignment requirements of this function, in bytes. - pub alignment: u32, + pub stack_maps: Box<[StackMapInformation]>, } -/// Information about a compiled trampoline which the host can call to enter -/// wasm. -#[derive(Serialize, Deserialize)] -#[allow(missing_docs)] -pub struct Trampoline { - /// The signature this trampoline is for - pub signature: SignatureIndex, - - /// Offset in the text section of where this function starts. - pub start: u64, - /// The size of the compiled function, in bytes. +/// Description of where a function is located in the text section of a +/// compiled image. +#[derive(Copy, Clone, Serialize, Deserialize)] +pub struct FunctionLoc { + /// The byte offset from the start of the text section where this + /// function starts. + pub start: u32, + /// The byte length of this function's function body. pub length: u32, } @@ -155,6 +145,14 @@ pub enum SettingKind { Preset, } +/// Types of objects that can be created by `Compiler::object` +pub enum ObjectKind { + /// A core wasm compilation artifact + Module, + /// A component compilation artifact + Component, +} + /// An implementation of a compiler which can compile WebAssembly functions to /// machine code and perform other miscellaneous tasks needed by the JIT runtime. pub trait Compiler: Send + Sync { @@ -170,7 +168,7 @@ pub trait Compiler: Send + Sync { data: FunctionBodyData<'_>, tunables: &Tunables, types: &ModuleTypes, - ) -> Result, CompileError>; + ) -> Result<(WasmFunctionInfo, Box), CompileError>; /// Creates a function of type `VMTrampoline` which will then call the /// function pointer argument which has the `ty` type provided. @@ -179,34 +177,35 @@ pub trait Compiler: Send + Sync { ty: &WasmFuncType, ) -> Result, CompileError>; - /// Collects the results of compilation into an in-memory object. + /// Appends a list of compiled functions to an in-memory object. /// /// This function will receive the same `Box` produced as part of - /// `compile_function`, as well as the general compilation environment with - /// the translation. THe `trampolines` argument is generated by - /// `compile_host_to_wasm_trampoline` for each of - /// `module.exported_signatures`. This method is expected to populate - /// information in the object file such as: + /// compilation from functions like `compile_function`, + /// compile_host_to_wasm_trampoline`, and other component-related shims. + /// Internally this will take all of these functions and add information to + /// the object such as: /// /// * Compiled code in a `.text` section /// * Unwind information in Wasmtime-specific sections - /// * DWARF debugging information for the host, if `emit_dwarf` is `true` - /// and the compiler supports it. /// * Relocations, if necessary, for the text section /// - /// The final result of compilation will contain more sections inserted by - /// the compiler-agnostic runtime. + /// Each function is accompanied with its desired symbol name and the return + /// value of this function is the symbol for each function as well as where + /// each function was placed within the object. /// - /// This function returns information about the compiled functions (where - /// they are in the text section) along with where trampolines are located. - fn emit_obj( + /// The `resolve_reloc` argument is intended to resolving relocations + /// between function, chiefly resolving intra-module calls within one core + /// wasm module. The closure here takes two arguments: first the index + /// within `funcs` that is being resolved and next the `FuncIndex` which is + /// the relocation target to resolve. The return value is an index within + /// `funcs` that the relocation points to. + fn append_code( &self, - module: &ModuleTranslation, - funcs: PrimaryMap>, - trampolines: Vec>, - tunables: &Tunables, obj: &mut Object<'static>, - ) -> Result<(PrimaryMap, Vec)>; + funcs: &[(String, Box)], + tunables: &Tunables, + resolve_reloc: &dyn Fn(usize, FuncIndex) -> usize, + ) -> Result>; /// Inserts two functions for host-to-wasm and wasm-to-host trampolines into /// the `obj` provided. @@ -220,7 +219,7 @@ pub trait Compiler: Send + Sync { ty: &WasmFuncType, host_fn: usize, obj: &mut Object<'static>, - ) -> Result<(Trampoline, Trampoline)>; + ) -> Result<(FunctionLoc, FunctionLoc)>; /// Creates a new `Object` file which is used to build the results of a /// compilation into. @@ -228,11 +227,11 @@ pub trait Compiler: Send + Sync { /// The returned object file will have an appropriate /// architecture/endianness for `self.triple()`, but at this time it is /// always an ELF file, regardless of target platform. - fn object(&self) -> Result> { + fn object(&self, kind: ObjectKind) -> Result> { use target_lexicon::Architecture::*; let triple = self.triple(); - Ok(Object::new( + let mut obj = Object::new( BinaryFormat::Elf, match triple.architecture { X86_32(_) => Architecture::I386, @@ -249,7 +248,16 @@ pub trait Compiler: Send + Sync { target_lexicon::Endianness::Little => object::Endianness::Little, target_lexicon::Endianness::Big => object::Endianness::Big, }, - )) + ); + obj.flags = FileFlags::Elf { + os_abi: obj::ELFOSABI_WASMTIME, + e_flags: match kind { + ObjectKind::Module => obj::EF_WASMTIME_MODULE, + ObjectKind::Component => obj::EF_WASMTIME_COMPONENT, + }, + abi_version: 0, + }; + Ok(obj) } /// Returns the target triple that this compiler is compiling for. @@ -276,6 +284,15 @@ pub trait Compiler: Send + Sync { /// `Self` in which case this function would simply return `self`. #[cfg(feature = "component-model")] fn component_compiler(&self) -> &dyn crate::component::ComponentCompiler; + + /// Appends generated DWARF sections to the `obj` specified for the compiled + /// functions. + fn append_dwarf( + &self, + obj: &mut Object<'_>, + translation: &ModuleTranslation<'_>, + funcs: &PrimaryMap, + ) -> Result<()>; } /// Value of a configured setting for a [`Compiler`] diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs index 212b6ca3c2..a939a02c5c 100644 --- a/crates/environ/src/component/compiler.rs +++ b/crates/environ/src/component/compiler.rs @@ -1,34 +1,8 @@ -use crate::component::{ - Component, ComponentTypes, LowerImport, LoweredIndex, RuntimeAlwaysTrapIndex, - RuntimeTranscoderIndex, Transcoder, -}; -use crate::{PrimaryMap, SignatureIndex, Trampoline, WasmFuncType}; +use crate::component::{Component, ComponentTypes, LowerImport, Transcoder}; +use crate::WasmFuncType; use anyhow::Result; -use object::write::Object; -use serde::{Deserialize, Serialize}; use std::any::Any; -/// Description of where a trampoline is located in the text section of a -/// compiled image. -#[derive(Serialize, Deserialize)] -pub struct FunctionInfo { - /// The byte offset from the start of the text section where this trampoline - /// starts. - pub start: u32, - /// The byte length of this trampoline's function body. - pub length: u32, -} - -/// Description of an "always trap" function generated by -/// `ComponentCompiler::compile_always_trap`. -#[derive(Serialize, Deserialize)] -pub struct AlwaysTrapInfo { - /// Information about the extent of this generated function. - pub info: FunctionInfo, - /// The offset from `start` of where the trapping instruction is located. - pub trap_offset: u32, -} - /// Compilation support necessary for components. pub trait ComponentCompiler: Send + Sync { /// Creates a trampoline for a `canon.lower`'d host function. @@ -79,26 +53,4 @@ pub trait ComponentCompiler: Send + Sync { transcoder: &Transcoder, types: &ComponentTypes, ) -> Result>; - - /// Emits the `lowerings` and `trampolines` specified into the in-progress - /// ELF object specified by `obj`. - /// - /// Returns a map of trampoline information for where to find them all in - /// the text section. - /// - /// Note that this will also prepare unwinding information for all the - /// trampolines as necessary. - fn emit_obj( - &self, - lowerings: PrimaryMap>, - always_trap: PrimaryMap>, - transcoders: PrimaryMap>, - tramplines: Vec<(SignatureIndex, Box)>, - obj: &mut Object<'static>, - ) -> Result<( - PrimaryMap, - PrimaryMap, - PrimaryMap, - Vec, - )>; } diff --git a/crates/environ/src/obj.rs b/crates/environ/src/obj.rs index 73ea699993..526ef8b7dc 100644 --- a/crates/environ/src/obj.rs +++ b/crates/environ/src/obj.rs @@ -1,33 +1,121 @@ //! Utilities for working with object files that operate as Wasmtime's //! serialization and intermediate format for compiled modules. -use crate::{EntityRef, FuncIndex, SignatureIndex}; +/// Filler for the `os_abi` field of the ELF header. +/// +/// This is just a constant that seems reasonable in the sense it's unlikely to +/// clash with others. +pub const ELFOSABI_WASMTIME: u8 = 200; -const FUNCTION_PREFIX: &str = "_wasm_function_"; -const TRAMPOLINE_PREFIX: &str = "_trampoline_"; +/// Flag for the `e_flags` field in the ELF header indicating a compiled +/// module. +pub const EF_WASMTIME_MODULE: u32 = 1 << 0; -/// Returns the symbol name in an object file for the corresponding wasm -/// function index in a module. -pub fn func_symbol_name(index: FuncIndex) -> String { - format!("{}{}", FUNCTION_PREFIX, index.index()) -} +/// Flag for the `e_flags` field in the ELF header indicating a compiled +/// component. +pub const EF_WASMTIME_COMPONENT: u32 = 1 << 1; -/// Attempts to extract the corresponding function index from a symbol possibly produced by -/// `func_symbol_name`. -pub fn try_parse_func_name(name: &str) -> Option { - let n = name.strip_prefix(FUNCTION_PREFIX)?.parse().ok()?; - Some(FuncIndex::new(n)) -} +/// A custom Wasmtime-specific section of our compilation image which stores +/// mapping data from offsets in the image to offset in the original wasm +/// binary. +/// +/// This section has a custom binary encoding. Currently its encoding is: +/// +/// * The section starts with a 32-bit little-endian integer. This integer is +/// how many entries are in the following two arrays. +/// * Next is an array with the previous count number of 32-bit little-endian +/// integers. This array is a sorted list of relative offsets within the text +/// section. This is intended to be a lookup array to perform a binary search +/// on an offset within the text section on this array. +/// * Finally there is another array, with the same count as before, also of +/// 32-bit little-endian integers. These integers map 1:1 with the previous +/// array of offsets, and correspond to what the original offset was in the +/// wasm file. +/// +/// Decoding this section is intentionally simple, it only requires loading a +/// 32-bit little-endian integer plus some bounds checks. Reading this section +/// is done with the `lookup_file_pos` function below. Reading involves +/// performing a binary search on the first array using the index found for the +/// native code offset to index into the second array and find the wasm code +/// offset. +/// +/// At this time this section has an alignment of 1, which means all reads of it +/// are unaligned. Additionally at this time the 32-bit encodings chosen here +/// mean that >=4gb text sections are not supported. +pub const ELF_WASMTIME_ADDRMAP: &str = ".wasmtime.addrmap"; -/// Returns the symbol name in an object file for the corresponding trampoline -/// for the given signature in a module. -pub fn trampoline_symbol_name(index: SignatureIndex) -> String { - format!("{}{}", TRAMPOLINE_PREFIX, index.index()) -} +/// A custom binary-encoded section of wasmtime compilation artifacts which +/// encodes the ability to map an offset in the text section to the trap code +/// that it corresponds to. +/// +/// This section is used at runtime to determine what flavor fo trap happened to +/// ensure that embedders and debuggers know the reason for the wasm trap. The +/// encoding of this section is custom to Wasmtime and managed with helpers in +/// the `object` crate: +/// +/// * First the section has a 32-bit little endian integer indicating how many +/// trap entries are in the section. +/// * Next is an array, of the same length as read before, of 32-bit +/// little-endian integers. These integers are offsets into the text section +/// of the compilation image. +/// * Finally is the same count number of bytes. Each of these bytes corresponds +/// to a trap code. +/// +/// This section is decoded by `lookup_trap_code` below which will read the +/// section count, slice some bytes to get the various arrays, and then perform +/// a binary search on the offsets array to find the an index corresponding to +/// the pc being looked up. If found the same index in the trap array (the array +/// of bytes) is the trap code for that offset. +/// +/// Note that at this time this section has an alignment of 1. Additionally due +/// to the 32-bit encodings for offsets this doesn't support images >=4gb. +pub const ELF_WASMTIME_TRAPS: &str = ".wasmtime.traps"; -/// Attempts to extract the corresponding signature index from a symbol -/// possibly produced by `trampoline_symbol_name`. -pub fn try_parse_trampoline_name(name: &str) -> Option { - let n = name.strip_prefix(TRAMPOLINE_PREFIX)?.parse().ok()?; - Some(SignatureIndex::new(n)) -} +/// A custom section which consists of just 1 byte which is either 0 or 1 as to +/// whether BTI is enabled. +pub const ELF_WASM_BTI: &str = ".wasmtime.bti"; + +/// A bincode-encoded section containing engine-specific metadata used to +/// double-check that an artifact can be loaded into the current host. +pub const ELF_WASM_ENGINE: &str = ".wasmtime.engine"; + +/// This is the name of the section in the final ELF image which contains +/// concatenated data segments from the original wasm module. +/// +/// This section is simply a list of bytes and ranges into this section are +/// stored within a `Module` for each data segment. Memory initialization and +/// passive segment management all index data directly located in this section. +/// +/// Note that this implementation does not afford any method of leveraging the +/// `data.drop` instruction to actually release the data back to the OS. The +/// data section is simply always present in the ELF image. If we wanted to +/// release the data it's probably best to figure out what the best +/// implementation is for it at the time given a particular set of constraints. +pub const ELF_WASM_DATA: &'static str = ".rodata.wasm"; + +/// This is the name of the section in the final ELF image which contains a +/// `bincode`-encoded `CompiledModuleInfo`. +/// +/// This section is optionally decoded in `CompiledModule::from_artifacts` +/// depending on whether or not a `CompiledModuleInfo` is already available. In +/// cases like `Module::new` where compilation directly leads into consumption, +/// it's available. In cases like `Module::deserialize` this section must be +/// decoded to get all the relevant information. +pub const ELF_WASMTIME_INFO: &'static str = ".wasmtime.info"; + +/// This is the name of the section in the final ELF image which contains a +/// concatenated list of all function names. +/// +/// This section is optionally included in the final artifact depending on +/// whether the wasm module has any name data at all (or in the future if we add +/// an option to not preserve name data). This section is a concatenated list of +/// strings where `CompiledModuleInfo::func_names` stores offsets/lengths into +/// this section. +/// +/// Note that the goal of this section is to avoid having to decode names at +/// module-load time if we can. Names are typically only used for debugging or +/// things like backtraces so there's no need to eagerly load all of them. By +/// storing the data in a separate section the hope is that the data, which is +/// sometimes quite large (3MB seen for spidermonkey-compiled-to-wasm), can be +/// paged in lazily from an mmap and is never paged in if we never reference it. +pub const ELF_NAME_DATA: &'static str = ".name.wasm"; diff --git a/crates/environ/src/trap_encoding.rs b/crates/environ/src/trap_encoding.rs index 1a56bb2618..7fe7cec7e2 100644 --- a/crates/environ/src/trap_encoding.rs +++ b/crates/environ/src/trap_encoding.rs @@ -1,3 +1,4 @@ +use crate::obj::ELF_WASMTIME_TRAPS; use object::write::{Object, StandardSegment}; use object::{Bytes, LittleEndian, SectionKind, U32Bytes}; use std::convert::TryFrom; @@ -16,33 +17,6 @@ pub struct TrapEncodingBuilder { last_offset: u32, } -/// A custom binary-encoded section of wasmtime compilation artifacts which -/// encodes the ability to map an offset in the text section to the trap code -/// that it corresponds to. -/// -/// This section is used at runtime to determine what flavor fo trap happened to -/// ensure that embedders and debuggers know the reason for the wasm trap. The -/// encoding of this section is custom to Wasmtime and managed with helpers in -/// the `object` crate: -/// -/// * First the section has a 32-bit little endian integer indicating how many -/// trap entries are in the section. -/// * Next is an array, of the same length as read before, of 32-bit -/// little-endian integers. These integers are offsets into the text section -/// of the compilation image. -/// * Finally is the same count number of bytes. Each of these bytes corresponds -/// to a trap code. -/// -/// This section is decoded by `lookup_trap_code` below which will read the -/// section count, slice some bytes to get the various arrays, and then perform -/// a binary search on the offsets array to find the an index corresponding to -/// the pc being looked up. If found the same index in the trap array (the array -/// of bytes) is the trap code for that offset. -/// -/// Note that at this time this section has an alignment of 1. Additionally due -/// to the 32-bit encodings for offsets this doesn't support images >=4gb. -pub const ELF_WASMTIME_TRAPS: &str = ".wasmtime.traps"; - /// Information about trap. #[derive(Debug, PartialEq, Eq, Clone)] pub struct TrapInformation { diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 2f024d8e16..30423035a2 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -1,16 +1,16 @@ //! Memory management for executable code. +use crate::subslice_range; use crate::unwind::UnwindRegistration; use anyhow::{anyhow, bail, Context, Result}; use object::read::{File, Object, ObjectSection}; -use std::ffi::c_void; +use std::mem; use std::mem::ManuallyDrop; +use std::ops::Range; +use wasmtime_environ::obj; +use wasmtime_environ::FunctionLoc; use wasmtime_jit_icache_coherence as icache_coherence; -use wasmtime_runtime::MmapVec; - -/// Name of the section in ELF files indicating that branch protection was -/// enabled for the compiled code. -pub const ELF_WASM_BTI: &str = ".wasmtime.bti"; +use wasmtime_runtime::{MmapVec, VMTrampoline}; /// Management of executable memory within a `MmapVec` /// @@ -22,6 +22,20 @@ pub struct CodeMemory { mmap: ManuallyDrop, unwind_registration: ManuallyDrop>, published: bool, + enable_branch_protection: bool, + + // Ranges within `self.mmap` of where the particular sections lie. + text: Range, + unwind: Range, + trap_data: Range, + wasm_data: Range, + address_map_data: Range, + func_name_data: Range, + info_data: Range, + + /// Map of dwarf sections indexed by `gimli::SectionId` which points to the + /// range within `code_memory`'s mmap as to the contents of the section. + dwarf_sections: Vec>, } impl Drop for CodeMemory { @@ -39,32 +53,115 @@ fn _assert() { _assert_send_sync::(); } -/// Result of publishing a `CodeMemory`, containing references to the parsed -/// internals. -pub struct Publish<'a> { - /// The parsed ELF image that resides within the original `MmapVec`. - pub obj: File<'a>, - - /// Reference to the entire `MmapVec` and its contents. - pub mmap: &'a MmapVec, - - /// Reference to just the text section of the object file, a subslice of - /// `mmap`. - pub text: &'a [u8], -} - impl CodeMemory { /// Creates a new `CodeMemory` by taking ownership of the provided /// `MmapVec`. /// /// The returned `CodeMemory` manages the internal `MmapVec` and the /// `publish` method is used to actually make the memory executable. - pub fn new(mmap: MmapVec) -> Self { - Self { + pub fn new(mmap: MmapVec) -> Result { + use gimli::SectionId::*; + + let obj = File::parse(&mmap[..]) + .with_context(|| "failed to parse internal compilation artifact")?; + + let mut text = 0..0; + let mut unwind = 0..0; + let mut enable_branch_protection = None; + let mut trap_data = 0..0; + let mut wasm_data = 0..0; + let mut address_map_data = 0..0; + let mut func_name_data = 0..0; + let mut info_data = 0..0; + let mut dwarf_sections = Vec::new(); + for section in obj.sections() { + let data = section.data()?; + let name = section.name()?; + let range = subslice_range(data, &mmap); + + // Double-check that sections are all aligned properly. + if section.align() != 0 && data.len() != 0 { + if (data.as_ptr() as u64 - mmap.as_ptr() as u64) % section.align() != 0 { + bail!( + "section `{}` isn't aligned to {:#x}", + section.name().unwrap_or("ERROR"), + section.align() + ); + } + } + + let mut gimli = |id: gimli::SectionId| { + let idx = id as usize; + if dwarf_sections.len() <= idx { + dwarf_sections.resize(idx + 1, 0..0); + } + dwarf_sections[idx] = range.clone(); + }; + + match name { + obj::ELF_WASM_BTI => match data.len() { + 1 => enable_branch_protection = Some(data[0] != 0), + _ => bail!("invalid `{name}` section"), + }, + ".text" => { + text = range; + + // Double-check there are no relocations in the text section. At + // this time relocations are not expected at all from loaded code + // since everything should be resolved at compile time. Handling + // must be added here, though, if relocations pop up. + assert!(section.relocations().count() == 0); + } + UnwindRegistration::SECTION_NAME => unwind = range, + obj::ELF_WASM_DATA => wasm_data = range, + obj::ELF_WASMTIME_ADDRMAP => address_map_data = range, + obj::ELF_WASMTIME_TRAPS => trap_data = range, + obj::ELF_NAME_DATA => func_name_data = range, + obj::ELF_WASMTIME_INFO => info_data = range, + + // Register dwarf sections into the `dwarf_sections` + // array which is indexed by `gimli::SectionId` + ".debug_abbrev.wasm" => gimli(DebugAbbrev), + ".debug_addr.wasm" => gimli(DebugAddr), + ".debug_aranges.wasm" => gimli(DebugAranges), + ".debug_frame.wasm" => gimli(DebugFrame), + ".eh_frame.wasm" => gimli(EhFrame), + ".eh_frame_hdr.wasm" => gimli(EhFrameHdr), + ".debug_info.wasm" => gimli(DebugInfo), + ".debug_line.wasm" => gimli(DebugLine), + ".debug_line_str.wasm" => gimli(DebugLineStr), + ".debug_loc.wasm" => gimli(DebugLoc), + ".debug_loc_lists.wasm" => gimli(DebugLocLists), + ".debug_macinfo.wasm" => gimli(DebugMacinfo), + ".debug_macro.wasm" => gimli(DebugMacro), + ".debug_pub_names.wasm" => gimli(DebugPubNames), + ".debug_pub_types.wasm" => gimli(DebugPubTypes), + ".debug_ranges.wasm" => gimli(DebugRanges), + ".debug_rng_lists.wasm" => gimli(DebugRngLists), + ".debug_str.wasm" => gimli(DebugStr), + ".debug_str_offsets.wasm" => gimli(DebugStrOffsets), + ".debug_types.wasm" => gimli(DebugTypes), + ".debug_cu_index.wasm" => gimli(DebugCuIndex), + ".debug_tu_index.wasm" => gimli(DebugTuIndex), + + _ => log::debug!("ignoring section {name}"), + } + } + Ok(Self { mmap: ManuallyDrop::new(mmap), unwind_registration: ManuallyDrop::new(None), published: false, - } + enable_branch_protection: enable_branch_protection + .ok_or_else(|| anyhow!("missing `{}` section", obj::ELF_WASM_BTI))?, + text, + unwind, + trap_data, + address_map_data, + func_name_data, + dwarf_sections, + info_data, + wasm_data, + }) } /// Returns a reference to the underlying `MmapVec` this memory owns. @@ -72,6 +169,67 @@ impl CodeMemory { &self.mmap } + /// Returns the contents of the text section of the ELF executable this + /// represents. + pub fn text(&self) -> &[u8] { + &self.mmap[self.text.clone()] + } + + /// Returns the data in the corresponding dwarf section, or an empty slice + /// if the section wasn't present. + pub fn dwarf_section(&self, section: gimli::SectionId) -> &[u8] { + let range = self + .dwarf_sections + .get(section as usize) + .cloned() + .unwrap_or(0..0); + &self.mmap[range] + } + + /// Returns the data in the `ELF_NAME_DATA` section. + pub fn func_name_data(&self) -> &[u8] { + &self.mmap[self.func_name_data.clone()] + } + + /// Returns the concatenated list of all data associated with this wasm + /// module. + /// + /// This is used for initialization of memories and all data ranges stored + /// in a `Module` are relative to the slice returned here. + pub fn wasm_data(&self) -> &[u8] { + &self.mmap[self.wasm_data.clone()] + } + + /// Returns the encoded address map section used to pass to + /// `wasmtime_environ::lookup_file_pos`. + pub fn address_map_data(&self) -> &[u8] { + &self.mmap[self.address_map_data.clone()] + } + + /// Returns the contents of the `ELF_WASMTIME_INFO` section, or an empty + /// slice if it wasn't found. + pub fn wasmtime_info(&self) -> &[u8] { + &self.mmap[self.info_data.clone()] + } + + /// Returns the contents of the `ELF_WASMTIME_TRAPS` section, or an empty + /// slice if it wasn't found. + pub fn trap_data(&self) -> &[u8] { + &self.mmap[self.trap_data.clone()] + } + + /// Returns a `VMTrampoline` function pointer for the given function in the + /// text section. + /// + /// # Unsafety + /// + /// This function is unsafe as there's no guarantee that the returned + /// function pointer is valid. + pub unsafe fn vmtrampoline(&self, loc: FunctionLoc) -> VMTrampoline { + let ptr = self.text()[loc.start as usize..][..loc.length as usize].as_ptr(); + mem::transmute::<*const u8, VMTrampoline>(ptr) + } + /// Publishes the internal ELF image to be ready for execution. /// /// This method can only be called once and will panic if called twice. This @@ -82,99 +240,37 @@ impl CodeMemory { /// * Register unwinding information with the OS /// /// After this function executes all JIT code should be ready to execute. - /// The various parsed results of the internals of the `MmapVec` are - /// returned through the `Publish` structure. - pub fn publish(&mut self) -> Result> { + pub fn publish(&mut self) -> Result<()> { assert!(!self.published); self.published = true; - let mut ret = Publish { - obj: File::parse(&self.mmap[..]) - .with_context(|| "failed to parse internal compilation artifact")?, - mmap: &self.mmap, - text: &[], - }; - let mmap_ptr = self.mmap.as_ptr() as u64; - - // Sanity-check that all sections are aligned correctly and - // additionally probe for a few sections that we're interested in. - let mut enable_branch_protection = None; - let mut text = None; - for section in ret.obj.sections() { - let data = match section.data() { - Ok(data) => data, - Err(_) => continue, - }; - if section.align() == 0 || data.len() == 0 { - continue; - } - if (data.as_ptr() as u64 - mmap_ptr) % section.align() != 0 { - bail!( - "section `{}` isn't aligned to {:#x}", - section.name().unwrap_or("ERROR"), - section.align() - ); - } - - match section.name().unwrap_or("") { - ELF_WASM_BTI => match data.len() { - 1 => enable_branch_protection = Some(data[0] != 0), - _ => bail!("invalid `{ELF_WASM_BTI}` section"), - }, - ".text" => { - ret.text = data; - text = Some(section); - } - _ => {} - } + if self.text().is_empty() { + return Ok(()); } - let enable_branch_protection = - enable_branch_protection.ok_or_else(|| anyhow!("missing `{ELF_WASM_BTI}` section"))?; - let text = match text { - Some(text) => text, - None => return Ok(ret), - }; // The unsafety here comes from a few things: // - // * First in `apply_reloc` we're walking around the `File` that the - // `object` crate has to get a mutable view into the text section. - // Currently the `object` crate doesn't support easily parsing a file - // and updating small bits and pieces of it, so we work around it for - // now. ELF's file format should guarantee that `text_mut` doesn't - // collide with any memory accessed by `text.relocations()`. + // * We're actually updating some page protections to executable memory. // - // * Second we're actually updating some page protections to executable - // memory. - // - // * Finally we're registering unwinding information which relies on the + // * We're registering unwinding information which relies on the // correctness of the information in the first place. This applies to // both the actual unwinding tables as well as the validity of the // pointers we pass in itself. unsafe { - let text_mut = - std::slice::from_raw_parts_mut(ret.text.as_ptr() as *mut u8, ret.text.len()); - let text_offset = ret.text.as_ptr() as usize - ret.mmap.as_ptr() as usize; - let text_range = text_offset..text_offset + text_mut.len(); - - // Double-check there are no relocations in the text section. At - // this time relocations are not expected at all from loaded code - // since everything should be resolved at compile time. Handling - // must be added here, though, if relocations pop up. - assert!(text.relocations().count() == 0); + let text = self.text(); // Clear the newly allocated code from cache if the processor requires it // // Do this before marking the memory as R+X, technically we should be able to do it after // but there are some CPU's that have had errata about doing this with read only memory. - icache_coherence::clear_cache(ret.text.as_ptr() as *const c_void, ret.text.len()) + icache_coherence::clear_cache(text.as_ptr().cast(), text.len()) .expect("Failed cache clear"); // Switch the executable portion from read/write to // read/execute, notably not using read/write/execute to prevent // modifications. self.mmap - .make_executable(text_range.clone(), enable_branch_protection) + .make_executable(self.text.clone(), self.enable_branch_protection) .expect("unable to make memory executable"); // Flush any in-flight instructions from the pipeline @@ -184,26 +280,22 @@ impl CodeMemory { // `UnwindRegistration` implementation to inform the general // runtime that there's unwinding information available for all // our just-published JIT functions. - *self.unwind_registration = register_unwind_info(&ret.obj, ret.text)?; + self.register_unwind_info()?; } - Ok(ret) + Ok(()) } -} -unsafe fn register_unwind_info(obj: &File, text: &[u8]) -> Result> { - let unwind_info = match obj - .section_by_name(UnwindRegistration::section_name()) - .and_then(|s| s.data().ok()) - { - Some(info) => info, - None => return Ok(None), - }; - if unwind_info.len() == 0 { - return Ok(None); + unsafe fn register_unwind_info(&mut self) -> Result<()> { + if self.unwind.len() == 0 { + return Ok(()); + } + let text = self.text(); + let unwind_info = &self.mmap[self.unwind.clone()]; + let registration = + UnwindRegistration::new(text.as_ptr(), unwind_info.as_ptr(), unwind_info.len()) + .context("failed to create unwind info registration")?; + *self.unwind_registration = Some(registration); + Ok(()) } - Ok(Some( - UnwindRegistration::new(text.as_ptr(), unwind_info.as_ptr(), unwind_info.len()) - .context("failed to create unwind info registration")?, - )) } diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 4498157c15..730daa0596 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -6,66 +6,25 @@ use crate::code_memory::CodeMemory; use crate::debug::create_gdbjit_image; use crate::ProfilingAgent; -use anyhow::{anyhow, bail, Context, Error, Result}; -use object::write::{Object, StandardSegment, WritableBuffer}; -use object::{File, Object as _, ObjectSection, SectionKind}; +use anyhow::{bail, Context, Error, Result}; +use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; +use object::SectionKind; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; use std::ops::Range; use std::str; use std::sync::Arc; use thiserror::Error; +use wasmtime_environ::obj; use wasmtime_environ::{ - CompileError, DefinedFuncIndex, FuncIndex, FunctionInfo, Module, ModuleTranslation, PrimaryMap, - SignatureIndex, StackMapInformation, Trampoline, Tunables, ELF_WASMTIME_ADDRMAP, - ELF_WASMTIME_TRAPS, + CompileError, DefinedFuncIndex, FuncIndex, FunctionLoc, MemoryInitialization, Module, + ModuleTranslation, PrimaryMap, SignatureIndex, StackMapInformation, Tunables, WasmFunctionInfo, }; use wasmtime_runtime::{ CompiledModuleId, CompiledModuleIdAllocator, GdbJitImageRegistration, InstantiationError, MmapVec, VMFunctionBody, VMTrampoline, }; -/// This is the name of the section in the final ELF image which contains -/// concatenated data segments from the original wasm module. -/// -/// This section is simply a list of bytes and ranges into this section are -/// stored within a `Module` for each data segment. Memory initialization and -/// passive segment management all index data directly located in this section. -/// -/// Note that this implementation does not afford any method of leveraging the -/// `data.drop` instruction to actually release the data back to the OS. The -/// data section is simply always present in the ELF image. If we wanted to -/// release the data it's probably best to figure out what the best -/// implementation is for it at the time given a particular set of constraints. -const ELF_WASM_DATA: &'static str = ".rodata.wasm"; - -/// This is the name of the section in the final ELF image which contains a -/// `bincode`-encoded `CompiledModuleInfo`. -/// -/// This section is optionally decoded in `CompiledModule::from_artifacts` -/// depending on whether or not a `CompiledModuleInfo` is already available. In -/// cases like `Module::new` where compilation directly leads into consumption, -/// it's available. In cases like `Module::deserialize` this section must be -/// decoded to get all the relevant information. -const ELF_WASMTIME_INFO: &'static str = ".wasmtime.info"; - -/// This is the name of the section in the final ELF image which contains a -/// concatenated list of all function names. -/// -/// This section is optionally included in the final artifact depending on -/// whether the wasm module has any name data at all (or in the future if we add -/// an option to not preserve name data). This section is a concatenated list of -/// strings where `CompiledModuleInfo::func_names` stores offsets/lengths into -/// this section. -/// -/// Note that the goal of this section is to avoid having to decode names at -/// module-load time if we can. Names are typically only used for debugging or -/// things like backtraces so there's no need to eagerly load all of them. By -/// storing the data in a separate section the hope is that the data, which is -/// sometimes quite large (3MB seen for spidermonkey-compiled-to-wasm), can be -/// paged in lazily from an mmap and is never paged in if we never reference it. -const ELF_NAME_DATA: &'static str = ".name.wasm"; - /// An error condition while setting up a wasm instance, be it validation, /// compilation, or instantiation. #[derive(Error, Debug)] @@ -98,14 +57,14 @@ pub struct CompiledModuleInfo { module: Module, /// Metadata about each compiled function. - funcs: PrimaryMap, + funcs: PrimaryMap, /// Sorted list, by function index, of names we have for this module. func_names: Vec, /// The trampolines compiled into the text section and their start/length /// relative to the start of the text section. - trampolines: Vec, + pub trampolines: Vec<(SignatureIndex, FunctionLoc)>, /// General compilation metadata. meta: Metadata, @@ -138,365 +97,345 @@ struct Metadata { has_wasm_debuginfo: bool, } -/// Finishes compilation of the `translation` specified, producing the final -/// compilation artifact and auxiliary information. +/// Helper structure to create an ELF file as a compilation artifact. /// -/// This function will consume the final results of compiling a wasm module -/// and finish the ELF image in-progress as part of `obj` by appending any -/// compiler-agnostic sections. -/// -/// The auxiliary `CompiledModuleInfo` structure returned here has also been -/// serialized into the object returned, but if the caller will quickly -/// turn-around and invoke `CompiledModule::from_artifacts` after this then the -/// information can be passed to that method to avoid extra deserialization. -/// This is done to avoid a serialize-then-deserialize for API calls like -/// `Module::new` where the compiled module is immediately going to be used. -/// -/// The `MmapVec` returned here contains the compiled image and resides in -/// mmap'd memory for easily switching permissions to executable afterwards. -pub fn finish_compile( - translation: ModuleTranslation<'_>, - mut obj: Object, - funcs: PrimaryMap, - trampolines: Vec, - tunables: &Tunables, -) -> Result<(MmapVec, CompiledModuleInfo)> { - let ModuleTranslation { - mut module, - debuginfo, - has_unparsed_debuginfo, - data, - data_align, - passive_data, - .. - } = translation; +/// This structure exposes the process which Wasmtime will encode a core wasm +/// module into an ELF file, notably managing data sections and all that good +/// business going into the final file. +pub struct ObjectBuilder<'a> { + /// The `object`-crate-defined ELF file write we're using. + obj: Object<'a>, - // Place all data from the wasm module into a section which will the - // source of the data later at runtime. - let data_id = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASM_DATA.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - let mut total_data_len = 0; - for (i, data) in data.iter().enumerate() { - // The first data segment has its alignment specified as the alignment - // for the entire section, but everything afterwards is adjacent so it - // has alignment of 1. - let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; - obj.append_section_data(data_id, data, align); - total_data_len += data.len(); - } - for data in passive_data.iter() { - obj.append_section_data(data_id, data, 1); - } + /// General compilation configuration. + tunables: &'a Tunables, - // If any names are present in the module then the `ELF_NAME_DATA` section - // is create and appended. - let mut func_names = Vec::new(); - if debuginfo.name_section.func_names.len() > 0 { - let name_id = obj.add_section( + /// The section identifier for "rodata" which is where wasm data segments + /// will go. + data: SectionId, + + /// The section identifier for function name information, or otherwise where + /// the `name` custom section of wasm is copied into. + /// + /// This is optional and lazily created on demand. + names: Option, +} + +impl<'a> ObjectBuilder<'a> { + /// Creates a new builder for the `obj` specified. + pub fn new(mut obj: Object<'a>, tunables: &'a Tunables) -> ObjectBuilder<'a> { + let data = obj.add_section( obj.segment_name(StandardSegment::Data).to_vec(), - ELF_NAME_DATA.as_bytes().to_vec(), + obj::ELF_WASM_DATA.as_bytes().to_vec(), SectionKind::ReadOnlyData, ); - let mut sorted_names = debuginfo.name_section.func_names.iter().collect::>(); - sorted_names.sort_by_key(|(idx, _name)| *idx); - for (idx, name) in sorted_names { - let offset = obj.append_section_data(name_id, name.as_bytes(), 1); - let offset = match u32::try_from(offset) { - Ok(offset) => offset, - Err(_) => bail!("name section too large (> 4gb)"), - }; - let len = u32::try_from(name.len()).unwrap(); - func_names.push(FunctionName { - idx: *idx, - offset, - len, - }); + ObjectBuilder { + obj, + tunables, + data, + names: None, } } - // Update passive data offsets since they're all located after the other - // data in the module. - for (_, range) in module.passive_data_map.iter_mut() { - range.start = range.start.checked_add(total_data_len as u32).unwrap(); - range.end = range.end.checked_add(total_data_len as u32).unwrap(); - } - - // Insert the wasm raw wasm-based debuginfo into the output, if - // requested. Note that this is distinct from the native debuginfo - // possibly generated by the native compiler, hence these sections - // getting wasm-specific names. - if tunables.parse_wasm_debuginfo { - push_debug(&mut obj, &debuginfo.dwarf.debug_abbrev); - push_debug(&mut obj, &debuginfo.dwarf.debug_addr); - push_debug(&mut obj, &debuginfo.dwarf.debug_aranges); - push_debug(&mut obj, &debuginfo.dwarf.debug_info); - push_debug(&mut obj, &debuginfo.dwarf.debug_line); - push_debug(&mut obj, &debuginfo.dwarf.debug_line_str); - push_debug(&mut obj, &debuginfo.dwarf.debug_str); - push_debug(&mut obj, &debuginfo.dwarf.debug_str_offsets); - push_debug(&mut obj, &debuginfo.debug_ranges); - push_debug(&mut obj, &debuginfo.debug_rnglists); - } - - // Encode a `CompiledModuleInfo` structure into the `ELF_WASMTIME_INFO` - // section of this image. This is not necessary when the returned module - // is never serialized to disk, which is also why we return a copy of - // the `CompiledModuleInfo` structure to the caller in case they don't - // want to deserialize this value immediately afterwards from the - // section. Otherwise, though, this is necessary to reify a `Module` on - // the other side from disk-serialized artifacts in - // `Module::deserialize` (a Wasmtime API). - let info_id = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASMTIME_INFO.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - let mut bytes = Vec::new(); - let info = CompiledModuleInfo { - module, - funcs, - trampolines, - func_names, - meta: Metadata { - native_debug_info_present: tunables.generate_native_debuginfo, + /// Completes compilation of the `translation` specified, inserting + /// everything necessary into the `Object` being built. + /// + /// This function will consume the final results of compiling a wasm module + /// and finish the ELF image in-progress as part of `self.obj` by appending + /// any compiler-agnostic sections. + /// + /// The auxiliary `CompiledModuleInfo` structure returned here has also been + /// serialized into the object returned, but if the caller will quickly + /// turn-around and invoke `CompiledModule::from_artifacts` after this then + /// the information can be passed to that method to avoid extra + /// deserialization. This is done to avoid a serialize-then-deserialize for + /// API calls like `Module::new` where the compiled module is immediately + /// going to be used. + /// + /// The various arguments here are: + /// + /// * `translation` - the core wasm translation that's being completed. + /// + /// * `funcs` - compilation metadata about functions within the translation + /// as well as where the functions are located in the text section. + /// + /// * `trampolines` - list of all trampolines necessary for this module + /// and where they're located in the text section. + /// + /// Returns the `CompiledModuleInfo` corresopnding to this core wasm module + /// as a result of this append operation. This is then serialized into the + /// final artifact by the caller. + pub fn append( + &mut self, + translation: ModuleTranslation<'_>, + funcs: PrimaryMap, + trampolines: Vec<(SignatureIndex, FunctionLoc)>, + ) -> Result { + let ModuleTranslation { + mut module, + debuginfo, has_unparsed_debuginfo, - code_section_offset: debuginfo.wasm_file.code_section_offset, - has_wasm_debuginfo: tunables.parse_wasm_debuginfo, - }, - }; - bincode::serialize_into(&mut bytes, &info)?; - obj.append_section_data(info_id, &bytes, 1); + data, + data_align, + passive_data, + .. + } = translation; - return Ok((mmap_vec_from_obj(obj)?, info)); + // Place all data from the wasm module into a section which will the + // source of the data later at runtime. This additionally keeps track of + // the offset of + let mut total_data_len = 0; + let data_offset = self + .obj + .append_section_data(self.data, &[], data_align.unwrap_or(1)); + for (i, data) in data.iter().enumerate() { + // The first data segment has its alignment specified as the alignment + // for the entire section, but everything afterwards is adjacent so it + // has alignment of 1. + let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; + self.obj.append_section_data(self.data, data, align); + total_data_len += data.len(); + } + for data in passive_data.iter() { + self.obj.append_section_data(self.data, data, 1); + } - fn push_debug<'a, T>(obj: &mut Object, section: &T) + // If any names are present in the module then the `ELF_NAME_DATA` section + // is create and appended. + let mut func_names = Vec::new(); + if debuginfo.name_section.func_names.len() > 0 { + let name_id = *self.names.get_or_insert_with(|| { + self.obj.add_section( + self.obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_NAME_DATA.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ) + }); + let mut sorted_names = debuginfo.name_section.func_names.iter().collect::>(); + sorted_names.sort_by_key(|(idx, _name)| *idx); + for (idx, name) in sorted_names { + let offset = self.obj.append_section_data(name_id, name.as_bytes(), 1); + let offset = match u32::try_from(offset) { + Ok(offset) => offset, + Err(_) => bail!("name section too large (> 4gb)"), + }; + let len = u32::try_from(name.len()).unwrap(); + func_names.push(FunctionName { + idx: *idx, + offset, + len, + }); + } + } + + // Data offsets in `MemoryInitialization` are offsets within the + // `translation.data` list concatenated which is now present in the data + // segment that's appended to the object. Increase the offsets by + // `self.data_size` to account for any previously added module. + let data_offset = u32::try_from(data_offset).unwrap(); + match &mut module.memory_initialization { + MemoryInitialization::Segmented(list) => { + for segment in list { + segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); + segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); + } + } + MemoryInitialization::Static { map } => { + for (_, segment) in map { + if let Some(segment) = segment { + segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); + segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); + } + } + } + } + + // Data offsets for passive data are relative to the start of + // `translation.passive_data` which was appended to the data segment + // of this object, after active data in `translation.data`. Update the + // offsets to account prior modules added in addition to active data. + let data_offset = data_offset + u32::try_from(total_data_len).unwrap(); + for (_, range) in module.passive_data_map.iter_mut() { + range.start = range.start.checked_add(data_offset).unwrap(); + range.end = range.end.checked_add(data_offset).unwrap(); + } + + // Insert the wasm raw wasm-based debuginfo into the output, if + // requested. Note that this is distinct from the native debuginfo + // possibly generated by the native compiler, hence these sections + // getting wasm-specific names. + if self.tunables.parse_wasm_debuginfo { + self.push_debug(&debuginfo.dwarf.debug_abbrev); + self.push_debug(&debuginfo.dwarf.debug_addr); + self.push_debug(&debuginfo.dwarf.debug_aranges); + self.push_debug(&debuginfo.dwarf.debug_info); + self.push_debug(&debuginfo.dwarf.debug_line); + self.push_debug(&debuginfo.dwarf.debug_line_str); + self.push_debug(&debuginfo.dwarf.debug_str); + self.push_debug(&debuginfo.dwarf.debug_str_offsets); + self.push_debug(&debuginfo.debug_ranges); + self.push_debug(&debuginfo.debug_rnglists); + } + + Ok(CompiledModuleInfo { + module, + funcs, + trampolines, + func_names, + meta: Metadata { + native_debug_info_present: self.tunables.generate_native_debuginfo, + has_unparsed_debuginfo, + code_section_offset: debuginfo.wasm_file.code_section_offset, + has_wasm_debuginfo: self.tunables.parse_wasm_debuginfo, + }, + }) + } + + fn push_debug<'b, T>(&mut self, section: &T) where - T: gimli::Section>, + T: gimli::Section>, { let data = section.reader().slice(); if data.is_empty() { return; } - let section_id = obj.add_section( - obj.segment_name(StandardSegment::Debug).to_vec(), + let section_id = self.obj.add_section( + self.obj.segment_name(StandardSegment::Debug).to_vec(), format!("{}.wasm", T::id().name()).into_bytes(), SectionKind::Debug, ); - obj.append_section_data(section_id, data, 1); + self.obj.append_section_data(section_id, data, 1); } -} -/// Creates a new `MmapVec` from serializing the specified `obj`. -/// -/// The returned `MmapVec` will contain the serialized version of `obj` and -/// is sized appropriately to the exact size of the object serialized. -pub fn mmap_vec_from_obj(obj: Object) -> Result { - let mut result = ObjectMmap::default(); - return match obj.emit(&mut result) { - Ok(()) => { - assert!(result.mmap.is_some(), "no reserve"); - let mmap = result.mmap.expect("reserve not called"); - assert_eq!(mmap.len(), result.len); - Ok(mmap) - } - Err(e) => match result.err.take() { - Some(original) => Err(original.context(e)), - None => Err(e.into()), - }, - }; + /// Creates the `ELF_WASMTIME_INFO` section from the given serializable data + /// structure. + pub fn serialize_info(&mut self, info: &T) + where + T: serde::Serialize, + { + let section = self.obj.add_section( + self.obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_WASMTIME_INFO.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + let data = bincode::serialize(info).unwrap(); + self.obj.set_section_data(section, data, 1); + } - /// Helper struct to implement the `WritableBuffer` trait from the `object` - /// crate. + /// Creates a new `MmapVec` from `self.` /// - /// This enables writing an object directly into an mmap'd memory so it's - /// immediately usable for execution after compilation. This implementation - /// relies on a call to `reserve` happening once up front with all the needed - /// data, and the mmap internally does not attempt to grow afterwards. - #[derive(Default)] - struct ObjectMmap { - mmap: Option, - len: usize, - err: Option, - } - - impl WritableBuffer for ObjectMmap { - fn len(&self) -> usize { - self.len - } - - fn reserve(&mut self, additional: usize) -> Result<(), ()> { - assert!(self.mmap.is_none(), "cannot reserve twice"); - self.mmap = match MmapVec::with_capacity(additional) { - Ok(mmap) => Some(mmap), - Err(e) => { - self.err = Some(e); - return Err(()); - } - }; - Ok(()) - } - - fn resize(&mut self, new_len: usize) { - // Resizing always appends 0 bytes and since new mmaps start out as 0 - // bytes we don't actually need to do anything as part of this other - // than update our own length. - if new_len <= self.len { - return; + /// The returned `MmapVec` will contain the serialized version of `self` + /// and is sized appropriately to the exact size of the object serialized. + pub fn finish(self) -> Result { + let mut result = ObjectMmap::default(); + return match self.obj.emit(&mut result) { + Ok(()) => { + assert!(result.mmap.is_some(), "no reserve"); + let mmap = result.mmap.expect("reserve not called"); + assert_eq!(mmap.len(), result.len); + Ok(mmap) } - self.len = new_len; + Err(e) => match result.err.take() { + Some(original) => Err(original.context(e)), + None => Err(e.into()), + }, + }; + + /// Helper struct to implement the `WritableBuffer` trait from the `object` + /// crate. + /// + /// This enables writing an object directly into an mmap'd memory so it's + /// immediately usable for execution after compilation. This implementation + /// relies on a call to `reserve` happening once up front with all the needed + /// data, and the mmap internally does not attempt to grow afterwards. + #[derive(Default)] + struct ObjectMmap { + mmap: Option, + len: usize, + err: Option, } - fn write_bytes(&mut self, val: &[u8]) { - let mmap = self.mmap.as_mut().expect("write before reserve"); - mmap[self.len..][..val.len()].copy_from_slice(val); - self.len += val.len(); + impl WritableBuffer for ObjectMmap { + fn len(&self) -> usize { + self.len + } + + fn reserve(&mut self, additional: usize) -> Result<(), ()> { + assert!(self.mmap.is_none(), "cannot reserve twice"); + self.mmap = match MmapVec::with_capacity(additional) { + Ok(mmap) => Some(mmap), + Err(e) => { + self.err = Some(e); + return Err(()); + } + }; + Ok(()) + } + + fn resize(&mut self, new_len: usize) { + // Resizing always appends 0 bytes and since new mmaps start out as 0 + // bytes we don't actually need to do anything as part of this other + // than update our own length. + if new_len <= self.len { + return; + } + self.len = new_len; + } + + fn write_bytes(&mut self, val: &[u8]) { + let mmap = self.mmap.as_mut().expect("write before reserve"); + mmap[self.len..][..val.len()].copy_from_slice(val); + self.len += val.len(); + } } } } /// A compiled wasm module, ready to be instantiated. pub struct CompiledModule { - wasm_data: Range, - address_map_data: Range, - trap_data: Range, module: Arc, - funcs: PrimaryMap, - trampolines: Vec, + funcs: PrimaryMap, + trampolines: Vec<(SignatureIndex, FunctionLoc)>, meta: Metadata, - code: Range, - code_memory: CodeMemory, + code_memory: Arc, dbg_jit_registration: Option, /// A unique ID used to register this module with the engine. unique_id: CompiledModuleId, func_names: Vec, - func_name_data: Range, - /// Map of dwarf sections indexed by `gimli::SectionId` which points to the - /// range within `code_memory`'s mmap as to the contents of the section. - dwarf_sections: Vec>, } impl CompiledModule { /// Creates `CompiledModule` directly from a precompiled artifact. /// - /// The `mmap` argument is expecte to be the result of a previous call to - /// `finish_compile` above. This is an ELF image, at this time, which - /// contains all necessary information to create a `CompiledModule` from a - /// compilation. + /// The `code_memory` argument is expected to be the result of a previous + /// call to `ObjectBuilder::finish` above. This is an ELF image, at this + /// time, which contains all necessary information to create a + /// `CompiledModule` from a compilation. /// - /// This method also takes `info`, an optionally-provided deserialization of - /// the artifacts' compilation metadata section. If this information is not - /// provided (e.g. it's set to `None`) then the information will be + /// This method also takes `info`, an optionally-provided deserialization + /// of the artifacts' compilation metadata section. If this information is + /// not provided then the information will be /// deserialized from the image of the compilation artifacts. Otherwise it - /// will be assumed to be what would otherwise happen if the section were to - /// be deserialized. + /// will be assumed to be what would otherwise happen if the section were + /// to be deserialized. /// /// The `profiler` argument here is used to inform JIT profiling runtimes /// about new code that is loaded. pub fn from_artifacts( - mmap: MmapVec, - mut info: Option, + code_memory: Arc, + info: CompiledModuleInfo, profiler: &dyn ProfilingAgent, id_allocator: &CompiledModuleIdAllocator, ) -> Result { - use gimli::SectionId::*; - - // Parse the `code_memory` as an object file and extract information - // about where all of its sections are located, stored into the - // `CompiledModule` created here. - // - // Note that dwarf sections here specifically are those that are carried - // over directly from the original wasm module's dwarf sections, not the - // wasmtime-generated host DWARF sections. - let obj = File::parse(&mmap[..]).context("failed to parse internal elf file")?; - let mut wasm_data = None; - let mut address_map_data = None; - let mut func_name_data = None; - let mut trap_data = None; - let mut code = None; - let mut dwarf_sections = Vec::new(); - for section in obj.sections() { - let name = section.name()?; - let data = section.data()?; - let range = subslice_range(data, &mmap); - let mut gimli = |id: gimli::SectionId| { - let idx = id as usize; - if dwarf_sections.len() <= idx { - dwarf_sections.resize(idx + 1, 0..0); - } - dwarf_sections[idx] = range.clone(); - }; - - match name { - ELF_WASM_DATA => wasm_data = Some(range), - ELF_WASMTIME_ADDRMAP => address_map_data = Some(range), - ELF_WASMTIME_TRAPS => trap_data = Some(range), - ELF_NAME_DATA => func_name_data = Some(range), - ".text" => code = Some(range), - - // Parse the metadata if it's not already available - // in-memory. - ELF_WASMTIME_INFO => { - if info.is_none() { - info = Some( - bincode::deserialize(data) - .context("failed to deserialize wasmtime module info")?, - ); - } - } - - // Register dwarf sections into the `dwarf_sections` - // array which is indexed by `gimli::SectionId` - ".debug_abbrev.wasm" => gimli(DebugAbbrev), - ".debug_addr.wasm" => gimli(DebugAddr), - ".debug_aranges.wasm" => gimli(DebugAranges), - ".debug_frame.wasm" => gimli(DebugFrame), - ".eh_frame.wasm" => gimli(EhFrame), - ".eh_frame_hdr.wasm" => gimli(EhFrameHdr), - ".debug_info.wasm" => gimli(DebugInfo), - ".debug_line.wasm" => gimli(DebugLine), - ".debug_line_str.wasm" => gimli(DebugLineStr), - ".debug_loc.wasm" => gimli(DebugLoc), - ".debug_loc_lists.wasm" => gimli(DebugLocLists), - ".debug_macinfo.wasm" => gimli(DebugMacinfo), - ".debug_macro.wasm" => gimli(DebugMacro), - ".debug_pub_names.wasm" => gimli(DebugPubNames), - ".debug_pub_types.wasm" => gimli(DebugPubTypes), - ".debug_ranges.wasm" => gimli(DebugRanges), - ".debug_rng_lists.wasm" => gimli(DebugRngLists), - ".debug_str.wasm" => gimli(DebugStr), - ".debug_str_offsets.wasm" => gimli(DebugStrOffsets), - ".debug_types.wasm" => gimli(DebugTypes), - ".debug_cu_index.wasm" => gimli(DebugCuIndex), - ".debug_tu_index.wasm" => gimli(DebugTuIndex), - - _ => log::debug!("ignoring section {name}"), - } - } - - let info = info.ok_or_else(|| anyhow!("failed to find wasm info section"))?; - let mut ret = Self { module: Arc::new(info.module), funcs: info.funcs, trampolines: info.trampolines, - wasm_data: wasm_data.ok_or_else(|| anyhow!("missing wasm data section"))?, - address_map_data: address_map_data.unwrap_or(0..0), - func_name_data: func_name_data.unwrap_or(0..0), - trap_data: trap_data.ok_or_else(|| anyhow!("missing trap data section"))?, - code: code.ok_or_else(|| anyhow!("missing code section"))?, dbg_jit_registration: None, - code_memory: CodeMemory::new(mmap), + code_memory, meta: info.meta, unique_id: id_allocator.alloc(), func_names: info.func_names, - dwarf_sections, }; - ret.code_memory - .publish() - .context("failed to publish code memory")?; ret.register_debug_and_profiling(profiler)?; Ok(ret) @@ -505,8 +444,8 @@ impl CompiledModule { fn register_debug_and_profiling(&mut self, profiler: &dyn ProfilingAgent) -> Result<()> { // Register GDB JIT images; initialize profiler and load the wasm module. if self.meta.native_debug_info_present { - let code = self.code(); - let bytes = create_gdbjit_image(self.mmap().to_vec(), (code.as_ptr(), code.len())) + let text = self.text(); + let bytes = create_gdbjit_image(self.mmap().to_vec(), (text.as_ptr(), text.len())) .map_err(SetupError::DebugInfo)?; profiler.module_load(self, Some(&bytes)); let reg = GdbJitImageRegistration::register(bytes); @@ -529,33 +468,16 @@ impl CompiledModule { self.code_memory.mmap() } - /// Returns the concatenated list of all data associated with this wasm - /// module. - /// - /// This is used for initialization of memories and all data ranges stored - /// in a `Module` are relative to the slice returned here. - pub fn wasm_data(&self) -> &[u8] { - &self.mmap()[self.wasm_data.clone()] - } - - /// Returns the encoded address map section used to pass to - /// `wasmtime_environ::lookup_file_pos`. - pub fn address_map_data(&self) -> &[u8] { - &self.mmap()[self.address_map_data.clone()] - } - - /// Returns the encoded trap information for this compiled image. - /// - /// For more information see `wasmtime_environ::trap_encoding`. - pub fn trap_data(&self) -> &[u8] { - &self.mmap()[self.trap_data.clone()] + /// Returns the underlying owned mmap of this compiled image. + pub fn code_memory(&self) -> &Arc { + &self.code_memory } /// Returns the text section of the ELF image for this compiled module. /// /// This memory should have the read/execute permissions. - pub fn code(&self) -> &[u8] { - &self.mmap()[self.code.clone()] + pub fn text(&self) -> &[u8] { + self.code_memory.text() } /// Return a reference-counting pointer to a module. @@ -574,7 +496,7 @@ impl CompiledModule { // `from_utf8_unchecked` if we really wanted since this section is // guaranteed to only have valid utf-8 data. Until it's a problem it's // probably best to double-check this though. - let data = &self.mmap()[self.func_name_data.clone()]; + let data = self.code_memory().func_name_data(); Some(str::from_utf8(&data[name.offset as usize..][..name.len as usize]).unwrap()) } @@ -588,9 +510,9 @@ impl CompiledModule { pub fn finished_functions( &self, ) -> impl ExactSizeIterator + '_ { - let code = self.code(); - self.funcs.iter().map(move |(i, info)| { - let func = &code[info.start as usize..][..info.length as usize]; + let text = self.text(); + self.funcs.iter().map(move |(i, (_, loc))| { + let func = &text[loc.start as usize..][..loc.length as usize]; ( i, std::ptr::slice_from_raw_parts(func.as_ptr().cast::(), func.len()), @@ -600,15 +522,15 @@ impl CompiledModule { /// Returns the per-signature trampolines for this module. pub fn trampolines(&self) -> impl Iterator + '_ { - let code = self.code(); - self.trampolines.iter().map(move |info| { + let text = self.text(); + self.trampolines.iter().map(move |(signature, loc)| { ( - info.signature, + *signature, unsafe { - let ptr = &code[info.start as usize]; + let ptr = &text[loc.start as usize]; std::mem::transmute::<*const u8, VMTrampoline>(ptr) }, - info.length as usize, + loc.length as usize, ) }) } @@ -623,7 +545,7 @@ impl CompiledModule { ) -> impl Iterator { self.finished_functions() .map(|(_, f)| f) - .zip(self.funcs.values().map(|f| f.stack_maps.as_slice())) + .zip(self.funcs.values().map(|f| &f.0.stack_maps[..])) } /// Lookups a defined function by a program counter value. @@ -631,14 +553,14 @@ impl CompiledModule { /// Returns the defined function index and the relative address of /// `text_offset` within the function itself. pub fn func_by_text_offset(&self, text_offset: usize) -> Option<(DefinedFuncIndex, u32)> { - let text_offset = text_offset as u64; + let text_offset = u32::try_from(text_offset).unwrap(); let index = match self .funcs - .binary_search_values_by_key(&text_offset, |info| { - debug_assert!(info.length > 0); + .binary_search_values_by_key(&text_offset, |(_, loc)| { + debug_assert!(loc.length > 0); // Return the inclusive "end" of the function - info.start + u64::from(info.length) - 1 + loc.start + loc.length - 1 }) { Ok(k) => { // Exact match, pc is at the end of this function @@ -652,22 +574,33 @@ impl CompiledModule { } }; - let body = self.funcs.get(index)?; - let start = body.start; - let end = body.start + u64::from(body.length); + let (_, loc) = self.funcs.get(index)?; + let start = loc.start; + let end = loc.start + loc.length; if text_offset < start || end < text_offset { return None; } - Some((index, (text_offset - body.start) as u32)) + Some((index, text_offset - loc.start)) + } + + /// Gets the function location information for a given function index. + pub fn func_loc(&self, index: DefinedFuncIndex) -> &FunctionLoc { + &self + .funcs + .get(index) + .expect("defined function should be present") + .1 } /// Gets the function information for a given function index. - pub fn func_info(&self, index: DefinedFuncIndex) -> &FunctionInfo { - self.funcs + pub fn wasm_func_info(&self, index: DefinedFuncIndex) -> &WasmFunctionInfo { + &self + .funcs .get(index) .expect("defined function should be present") + .0 } /// Creates a new symbolication context which can be used to further @@ -681,12 +614,7 @@ impl CompiledModule { return Ok(None); } let dwarf = gimli::Dwarf::load(|id| -> Result<_> { - let range = self - .dwarf_sections - .get(id as usize) - .cloned() - .unwrap_or(0..0); - let data = &self.mmap()[range]; + let data = self.code_memory().dwarf_section(id); Ok(EndianSlice::new(data, gimli::LittleEndian)) })?; let cx = addr2line::Context::from_dwarf(dwarf) @@ -709,7 +637,7 @@ impl CompiledModule { /// If this function returns `false` then `lookup_file_pos` will always /// return `None`. pub fn has_address_map(&self) -> bool { - !self.address_map_data().is_empty() + !self.code_memory.address_map_data().is_empty() } /// Returns the bounds, in host memory, of where this module's compiled diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 8ed600382b..7cba8ed153 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -27,10 +27,9 @@ mod instantiate; mod profiling; mod unwind; -pub use crate::code_memory::{CodeMemory, ELF_WASM_BTI}; +pub use crate::code_memory::CodeMemory; pub use crate::instantiate::{ - finish_compile, mmap_vec_from_obj, subslice_range, CompiledModule, CompiledModuleInfo, - SetupError, SymbolizeContext, + subslice_range, CompiledModule, CompiledModuleInfo, ObjectBuilder, SetupError, SymbolizeContext, }; pub use demangling::*; pub use profiling::*; diff --git a/crates/jit/src/unwind/systemv.rs b/crates/jit/src/unwind/systemv.rs index 331b0de1e8..66ba66f2c7 100644 --- a/crates/jit/src/unwind/systemv.rs +++ b/crates/jit/src/unwind/systemv.rs @@ -14,6 +14,8 @@ extern "C" { } impl UnwindRegistration { + pub const SECTION_NAME: &str = ".eh_frame"; + /// Registers precompiled unwinding information with the system. /// /// The `_base_address` field is ignored here (only used on other @@ -67,10 +69,6 @@ impl UnwindRegistration { Ok(UnwindRegistration { registrations }) } - - pub fn section_name() -> &'static str { - ".eh_frame" - } } impl Drop for UnwindRegistration { diff --git a/crates/jit/src/unwind/winx64.rs b/crates/jit/src/unwind/winx64.rs index 23ccbe285c..f0f663077d 100644 --- a/crates/jit/src/unwind/winx64.rs +++ b/crates/jit/src/unwind/winx64.rs @@ -10,6 +10,8 @@ pub struct UnwindRegistration { } impl UnwindRegistration { + pub const SECTION_NAME: &str = ".pdata"; + pub unsafe fn new( base_address: *const u8, unwind_info: *const u8, @@ -31,10 +33,6 @@ impl UnwindRegistration { functions: unwind_info as usize, }) } - - pub fn section_name() -> &'static str { - ".pdata" - } } impl Drop for UnwindRegistration { diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 7cd729a889..5675a79cd7 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -493,7 +493,7 @@ impl Instance { let (func_ptr, vmctx) = if let Some(def_index) = self.module().defined_func_index(index) { ( (self.runtime_info.image_base() - + self.runtime_info.function_info(def_index).start as usize) + + self.runtime_info.function_loc(def_index).start as usize) as *mut _, VMOpaqueContext::from_vmcontext(self.vmctx_ptr()), ) diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 3082c3ae84..97e2eea93d 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -1129,7 +1129,7 @@ mod test { use super::*; use crate::{CompiledModuleId, Imports, MemoryImage, StorePtr, VMSharedSignatureIndex}; use std::sync::Arc; - use wasmtime_environ::{DefinedFuncIndex, DefinedMemoryIndex, FunctionInfo, SignatureIndex}; + use wasmtime_environ::{DefinedFuncIndex, DefinedMemoryIndex, FunctionLoc, SignatureIndex}; pub(crate) fn empty_runtime_info( module: Arc, @@ -1143,7 +1143,7 @@ mod test { fn image_base(&self) -> usize { 0 } - fn function_info(&self, _: DefinedFuncIndex) -> &FunctionInfo { + fn function_loc(&self, _: DefinedFuncIndex) -> &FunctionLoc { unimplemented!() } fn signature(&self, _: SignatureIndex) -> VMSharedSignatureIndex { diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 399f209e9e..87e485dfdc 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -25,7 +25,7 @@ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use wasmtime_environ::DefinedFuncIndex; use wasmtime_environ::DefinedMemoryIndex; -use wasmtime_environ::FunctionInfo; +use wasmtime_environ::FunctionLoc; use wasmtime_environ::SignatureIndex; #[macro_use] @@ -183,7 +183,7 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static { /// Descriptors about each compiled function, such as the offset from /// `image_base`. - fn function_info(&self, func_index: DefinedFuncIndex) -> &FunctionInfo; + fn function_loc(&self, func_index: DefinedFuncIndex) -> &FunctionLoc; /// Returns the `MemoryImage` structure used for copy-on-write /// initialization of the memory, if it's applicable. diff --git a/crates/wasmtime/src/code.rs b/crates/wasmtime/src/code.rs new file mode 100644 index 0000000000..a0d1037e70 --- /dev/null +++ b/crates/wasmtime/src/code.rs @@ -0,0 +1,103 @@ +use crate::signatures::SignatureCollection; +use std::sync::Arc; +#[cfg(feature = "component-model")] +use wasmtime_environ::component::ComponentTypes; +use wasmtime_environ::ModuleTypes; +use wasmtime_jit::CodeMemory; + +/// Metadata in Wasmtime about a loaded compiled artifact in memory which is +/// ready to execute. +/// +/// This structure is used in both `Module` and `Component`. For components it's +/// notably shared amongst the core wasm modules within a component and the +/// component itself. For core wasm modules this is uniquely owned within a +/// `Module`. +pub struct CodeObject { + /// Actual underlying mmap which is executable and contains other compiled + /// information. + /// + /// Note the `Arc` here is used to share this with `CompiledModule` and the + /// global module registry of traps. While probably not strictly necessary + /// and could be avoided with some refactorings is a hopefully a relatively + /// minor `Arc` for now. + mmap: Arc, + + /// Registered shared signature for the loaded object. + /// + /// Note that this type has a significant destructor which unregisters + /// signatures within the `Engine` it was originally tied to, and this ends + /// up corresponding to the liftime of a `Component` or `Module`. + signatures: SignatureCollection, + + /// Type information for the loaded object. + /// + /// This is either a `ModuleTypes` or a `ComponentTypes` depending on the + /// top-level creator of this code. + types: Types, +} + +impl CodeObject { + pub fn new(mmap: Arc, signatures: SignatureCollection, types: Types) -> CodeObject { + // The corresopnding unregister for this is below in `Drop for + // CodeObject`. + crate::module::register_code(&mmap); + + CodeObject { + mmap, + signatures, + types, + } + } + + pub fn code_memory(&self) -> &Arc { + &self.mmap + } + + #[cfg(feature = "component-model")] + pub fn types(&self) -> &Types { + &self.types + } + + pub fn module_types(&self) -> &ModuleTypes { + self.types.module_types() + } + + pub fn signatures(&self) -> &SignatureCollection { + &self.signatures + } +} + +impl Drop for CodeObject { + fn drop(&mut self) { + crate::module::unregister_code(&self.mmap); + } +} + +pub enum Types { + Module(ModuleTypes), + #[cfg(feature = "component-model")] + Component(Arc), +} + +impl Types { + fn module_types(&self) -> &ModuleTypes { + match self { + Types::Module(m) => m, + #[cfg(feature = "component-model")] + Types::Component(c) => c.module_types(), + } + } +} + +impl From for Types { + fn from(types: ModuleTypes) -> Types { + Types::Module(types) + } +} + +#[cfg(feature = "component-model")] +impl From> for Types { + fn from(types: Arc) -> Types { + Types::Component(types) + } +} diff --git a/crates/wasmtime/src/component/component.rs b/crates/wasmtime/src/component/component.rs index aea26c6d95..1467636b49 100644 --- a/crates/wasmtime/src/component/component.rs +++ b/crates/wasmtime/src/component/component.rs @@ -1,21 +1,21 @@ +use crate::code::CodeObject; use crate::signatures::SignatureCollection; use crate::{Engine, Module}; use anyhow::{bail, Context, Result}; -use std::any::Any; -use std::collections::HashMap; -use std::collections::HashSet; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap}; use std::fs; -use std::ops::Range; +use std::mem; use std::path::Path; use std::ptr::NonNull; use std::sync::Arc; use wasmtime_environ::component::{ - AlwaysTrapInfo, ComponentTypes, FunctionInfo, GlobalInitializer, LoweredIndex, - RuntimeAlwaysTrapIndex, RuntimeTranscoderIndex, StaticModuleIndex, Translator, + ComponentTypes, GlobalInitializer, LoweredIndex, RuntimeAlwaysTrapIndex, + RuntimeTranscoderIndex, StaticModuleIndex, Translator, }; -use wasmtime_environ::{PrimaryMap, ScopeVec, SignatureIndex, Trampoline, TrapCode}; -use wasmtime_jit::CodeMemory; -use wasmtime_runtime::VMFunctionBody; +use wasmtime_environ::{EntityRef, FunctionLoc, ObjectKind, PrimaryMap, ScopeVec, SignatureIndex}; +use wasmtime_jit::{CodeMemory, CompiledModuleInfo}; +use wasmtime_runtime::{MmapVec, VMFunctionBody, VMTrampoline}; /// A compiled WebAssembly Component. // @@ -26,48 +26,57 @@ pub struct Component { } struct ComponentInner { - /// Type information calculated during translation about this component. - component: wasmtime_environ::component::Component, - /// Core wasm modules that the component defined internally, indexed by the /// compile-time-assigned `ModuleUpvarIndex`. static_modules: PrimaryMap, - /// Registered core wasm signatures of this component, or otherwise the - /// mapping of the component-local `SignatureIndex` to the engine-local - /// `VMSharedSignatureIndex`. - signatures: SignatureCollection, + /// Code-related information such as the compiled artifact, type + /// information, etc. + /// + /// Note that the `Arc` here is used to share this allocation with internal + /// modules. + code: Arc, - /// Type information about this component and all the various types it - /// defines internally. All type indices for `component` will point into - /// this field. - types: Arc, + /// Metadata produced during compilation. + info: CompiledComponentInfo, +} - /// The in-memory ELF image of the compiled functions for this component. - trampoline_obj: CodeMemory, - - /// The index ranges within `trampoline_obj`'s mmap memory for the entire - /// text section. - text: Range, +#[derive(Serialize, Deserialize)] +struct CompiledComponentInfo { + /// Type information calculated during translation about this component. + component: wasmtime_environ::component::Component, /// Where lowered function trampolines are located within the `text` - /// section of `trampoline_obj`. + /// section of `code_memory`. /// /// These trampolines are the function pointer within the /// `VMCallerCheckedAnyfunc` and will delegate indirectly to a host function /// pointer when called. - lowerings: PrimaryMap, + lowerings: PrimaryMap, /// Where the "always trap" functions are located within the `text` section - /// of `trampoline_obj`. + /// of `code_memory`. /// /// These functions are "degenerate functions" here solely to implement - /// functions that are `canon lift`'d then immediately `canon lower`'d. - always_trap: PrimaryMap, + /// functions that are `canon lift`'d then immediately `canon lower`'d. The + /// `u32` value here is the offset of the trap instruction from the start fo + /// the function. + always_trap: PrimaryMap, /// Where all the cranelift-generated transcode functions are located in the /// compiled image of this component. - transcoders: PrimaryMap, + transcoders: PrimaryMap, + + /// Extra trampolines other than those contained in static modules + /// necessary for this component. + trampolines: Vec<(SignatureIndex, FunctionLoc)>, +} + +#[derive(Serialize, Deserialize)] +pub(crate) struct ComponentArtifacts { + info: CompiledComponentInfo, + types: ComponentTypes, + static_modules: PrimaryMap, } impl Component { @@ -121,273 +130,344 @@ impl Component { .check_compatible_with_native_host() .context("compilation settings are not compatible with the native host")?; + let (mmap, artifacts) = Component::build_artifacts(engine, binary)?; + let mut code_memory = CodeMemory::new(mmap)?; + code_memory.publish()?; + Component::from_parts(engine, Arc::new(code_memory), Some(artifacts)) + } + + /// Same as [`Module::deserialize`], but for components. + /// + /// Note that the file referenced here must contain contents previously + /// produced by [`Engine::precompile_component`] or + /// [`Component::serialize`]. + /// + /// For more information see the [`Module::deserialize`] method. + /// + /// [`Module::deserialize`]: crate::Module::deserialize + pub unsafe fn deserialize(engine: &Engine, bytes: impl AsRef<[u8]>) -> Result { + let code = engine.load_code_bytes(bytes.as_ref(), ObjectKind::Component)?; + Component::from_parts(engine, code, None) + } + + /// Same as [`Module::deserialize_file`], but for components. + /// + /// For more information see the [`Component::deserialize`] and + /// [`Module::deserialize_file`] methods. + /// + /// [`Module::deserialize_file`]: crate::Module::deserialize_file + pub unsafe fn deserialize_file(engine: &Engine, path: impl AsRef) -> Result { + let code = engine.load_code_file(path.as_ref(), ObjectKind::Component)?; + Component::from_parts(engine, code, None) + } + + /// Performs the compilation phase for a component, translating and + /// validating the provided wasm binary to machine code. + /// + /// This method will compile all nested core wasm binaries in addition to + /// any necessary extra functions required for operation with components. + /// The output artifact here is the serialized object file contained within + /// an owned mmap along with metadata about the compilation itself. + #[cfg(compiler)] + pub(crate) fn build_artifacts( + engine: &Engine, + binary: &[u8], + ) -> Result<(MmapVec, ComponentArtifacts)> { let tunables = &engine.config().tunables; + let compiler = engine.compiler(); let scope = ScopeVec::new(); let mut validator = wasmparser::Validator::new_with_features(engine.config().features.clone()); let mut types = Default::default(); - let (component, modules) = Translator::new(tunables, &mut validator, &mut types, &scope) - .translate(binary) - .context("failed to parse WebAssembly module")?; - let types = Arc::new(types.finish()); + let (component, mut modules) = + Translator::new(tunables, &mut validator, &mut types, &scope) + .translate(binary) + .context("failed to parse WebAssembly module")?; + let types = types.finish(); - let provided_trampolines = modules + // Compile all core wasm modules, in parallel, which will internally + // compile all their functions in parallel as well. + let module_funcs = engine.run_maybe_parallel(modules.values_mut().collect(), |module| { + Module::compile_functions(engine, module, types.module_types()) + })?; + + // Compile all host-to-wasm trampolines where the required set of + // trampolines is unioned from all core wasm modules plus what the + // component itself needs. + let module_trampolines = modules .iter() .flat_map(|(_, m)| m.exported_signatures.iter().copied()) - .collect::>(); + .collect::>(); + let trampolines = module_trampolines + .iter() + .copied() + .chain( + // All lowered functions will require a trampoline to be available in + // case they're used when entering wasm. For example a lowered function + // could be immediately lifted in which case we'll need a trampoline to + // call that lowered function. + // + // Most of the time trampolines can come from the core wasm modules + // since lifted functions come from core wasm. For these esoteric cases + // though we may have to compile trampolines specifically into the + // component object as well in case core wasm doesn't provide the + // necessary trampoline. + component.initializers.iter().filter_map(|init| match init { + GlobalInitializer::LowerImport(i) => Some(i.canonical_abi), + GlobalInitializer::AlwaysTrap(i) => Some(i.canonical_abi), + _ => None, + }), + ) + .collect::>(); + let compiled_trampolines = engine + .run_maybe_parallel(trampolines.iter().cloned().collect(), |i| { + compiler.compile_host_to_wasm_trampoline(&types[i]) + })?; - let (static_modules, trampolines) = engine.join_maybe_parallel( - // In one (possibly) parallel task all the modules found within this - // component are compiled. Note that this will further parallelize - // function compilation internally too. - || -> Result<_> { - let upvars = modules.into_iter().map(|(_, t)| t).collect::>(); - let modules = engine.run_maybe_parallel(upvars, |module| { - let (mmap, info) = - Module::compile_functions(engine, module, types.module_types())?; - // FIXME: the `SignatureCollection` here is re-registering - // the entire list of wasm types within `types` on each - // invocation. That's ok semantically but is quite slow to - // do so. This should build up a mapping from - // `SignatureIndex` to `VMSharedSignatureIndex` once and - // then reuse that for each module somehow. - Module::from_parts(engine, mmap, Some((info, types.clone().into()))) - })?; + // Compile all transcoders required which adapt from a + // core-wasm-specific ABI (e.g. 32 or 64-bit) into the host transcoder + // ABI through an indirect libcall. + let transcoders = component + .initializers + .iter() + .filter_map(|init| match init { + GlobalInitializer::Transcoder(i) => Some(i), + _ => None, + }) + .collect(); + let transcoders = engine.run_maybe_parallel(transcoders, |info| { + compiler + .component_compiler() + .compile_transcoder(&component, info, &types) + })?; - Ok(modules.into_iter().collect::>()) - }, - // In another (possibly) parallel task we compile lowering - // trampolines necessary found in the component. - || Component::compile_component(engine, &component, &types, &provided_trampolines), - ); - let static_modules = static_modules?; - let (lowerings, always_trap, transcoders, trampolines, trampoline_obj) = trampolines?; - let mut trampoline_obj = CodeMemory::new(trampoline_obj); - let code = trampoline_obj.publish()?; - let text = wasmtime_jit::subslice_range(code.text, code.mmap); + // Compile all "always trap" functions which are small typed shims that + // exits to solely trap immediately for components. + let always_trap = component + .initializers + .iter() + .filter_map(|init| match init { + GlobalInitializer::AlwaysTrap(i) => Some(i), + _ => None, + }) + .collect(); + let always_trap = engine.run_maybe_parallel(always_trap, |info| { + compiler + .component_compiler() + .compile_always_trap(&types[info.canonical_abi]) + })?; - // This map is used to register all known tramplines in the - // `SignatureCollection` created below. This is later consulted during - // `ModuleRegistry::lookup_trampoline` if a trampoline needs to be - // located for a signature index where the original function pointer - // is that of the `trampolines` created above. - // - // This situation arises when a core wasm module imports a lowered - // function and then immediately exports it. Wasmtime will lookup an - // entry trampoline for the exported function which is actually a - // lowered host function, hence an entry in the `trampolines` variable - // above, and the type of that function will be stored in this - // `vmtrampolines` map since the type is guaranteed to have escaped - // from at least one of the modules we compiled prior. - let mut vmtrampolines = HashMap::new(); - for (_, module) in static_modules.iter() { - for (idx, trampoline, _) in module.compiled_module().trampolines() { - vmtrampolines.insert(idx, trampoline); + // Compile all "lowerings" which are adapters that go from core wasm + // into the host which will process the canonical ABI. + let lowerings = component + .initializers + .iter() + .filter_map(|init| match init { + GlobalInitializer::LowerImport(i) => Some(i), + _ => None, + }) + .collect(); + let lowerings = engine.run_maybe_parallel(lowerings, |lowering| { + compiler + .component_compiler() + .compile_lowered_trampoline(&component, lowering, &types) + })?; + + // Collect the results of all of the function-based compilations above + // into one large list of functions to get appended into the text + // section of the final module. + let mut funcs = Vec::new(); + let mut module_func_start_index = Vec::new(); + let mut func_index_to_module_index = Vec::new(); + let mut func_infos = Vec::new(); + for (i, list) in module_funcs.into_iter().enumerate() { + module_func_start_index.push(func_index_to_module_index.len()); + let mut infos = Vec::new(); + for (j, (info, func)) in list.into_iter().enumerate() { + func_index_to_module_index.push(i); + let name = format!("_wasm{i}_function{j}"); + funcs.push((name, func)); + infos.push(info); } + func_infos.push(infos); } - for trampoline in trampolines { - vmtrampolines.insert(trampoline.signature, unsafe { - let ptr = - code.text[trampoline.start as usize..][..trampoline.length as usize].as_ptr(); - std::mem::transmute::<*const u8, wasmtime_runtime::VMTrampoline>(ptr) - }); + for (sig, func) in trampolines.iter().zip(compiled_trampolines) { + let name = format!("_wasm_trampoline{}", sig.as_u32()); + funcs.push((name, func)); + } + let ntranscoders = transcoders.len(); + for (i, func) in transcoders.into_iter().enumerate() { + let name = format!("_wasm_component_transcoder{i}"); + funcs.push((name, func)); + } + let nalways_trap = always_trap.len(); + for (i, func) in always_trap.into_iter().enumerate() { + let name = format!("_wasm_component_always_trap{i}"); + funcs.push((name, func)); + } + let nlowerings = lowerings.len(); + for (i, func) in lowerings.into_iter().enumerate() { + let name = format!("_wasm_component_lowering{i}"); + funcs.push((name, func)); } - // FIXME: for the same reason as above where each module is - // re-registering everything this should only be registered once. This - // is benign for now but could do with refactorings later on. + let mut object = compiler.object(ObjectKind::Component)?; + let locs = compiler.append_code(&mut object, &funcs, tunables, &|i, idx| { + // Map from the `i`th function which is requesting the relocation to + // the index in `modules` that the function belongs to. Using that + // metadata we can resolve `idx: FuncIndex` to a `DefinedFuncIndex` + // to the index of that module's function that's being called. + // + // Note that this will panic if `i` is a function beyond the initial + // set of core wasm module functions. That's intentional, however, + // since trampolines and otherwise should not have relocations to + // resolve. + let module_index = func_index_to_module_index[i]; + let defined_index = modules[StaticModuleIndex::new(module_index)] + .module + .defined_func_index(idx) + .unwrap(); + // Additionally use the module index to determine where that + // module's list of functions started at to factor in as an offset + // as well. + let offset = module_func_start_index[module_index]; + defined_index.index() + offset + })?; + engine.append_compiler_info(&mut object); + engine.append_bti(&mut object); + + // Disassemble the result of the appending to the text section, where + // each function is in the module, into respective maps. + let mut locs = locs.into_iter().map(|(_sym, loc)| loc); + let funcs = func_infos + .into_iter() + .map(|infos| { + infos + .into_iter() + .zip(&mut locs) + .collect::>() + }) + .collect::>(); + let signature_to_trampoline = trampolines + .iter() + .cloned() + .zip(&mut locs) + .collect::>(); + let transcoders = locs + .by_ref() + .take(ntranscoders) + .collect::>(); + let always_trap = locs + .by_ref() + .take(nalways_trap) + .collect::>(); + let lowerings = locs + .by_ref() + .take(nlowerings) + .collect::>(); + assert!(locs.next().is_none()); + + // Convert all `ModuleTranslation` instances into `CompiledModuleInfo` + // through an `ObjectBuilder` here. This is then used to create the + // final `mmap` which is the final compilation artifact. + let mut builder = wasmtime_jit::ObjectBuilder::new(object, tunables); + let mut static_modules = PrimaryMap::new(); + for ((_, module), funcs) in modules.into_iter().zip(funcs) { + // Build the list of trampolines for this module from its set of + // exported signatures, which is the list of expected trampolines, + // from the set of trampolines that were compiled for everything + // within this component. + let trampolines = module + .exported_signatures + .iter() + .map(|sig| (*sig, signature_to_trampoline[sig])) + .collect(); + let info = builder.append(module, funcs, trampolines)?; + static_modules.push(info); + } + + let info = CompiledComponentInfo { + always_trap, + component, + lowerings, + trampolines: trampolines + .difference(&module_trampolines) + .map(|i| (*i, signature_to_trampoline[i])) + .collect(), + transcoders, + }; + let artifacts = ComponentArtifacts { + info, + types, + static_modules, + }; + builder.serialize_info(&artifacts); + + let mmap = builder.finish()?; + Ok((mmap, artifacts)) + } + + /// Final assembly step for a component from its in-memory representation. + /// + /// If the `artifacts` are specified as `None` here then they will be + /// deserialized from `code_memory`. + fn from_parts( + engine: &Engine, + code_memory: Arc, + artifacts: Option, + ) -> Result { + let ComponentArtifacts { + info, + types, + static_modules, + } = match artifacts { + Some(artifacts) => artifacts, + None => bincode::deserialize(code_memory.wasmtime_info())?, + }; + + // Create a signature registration with the `Engine` for all trampolines + // and core wasm types found within this component, both for the + // component and for all included core wasm modules. let signatures = SignatureCollection::new_for_module( engine.signatures(), types.module_types(), - vmtrampolines.into_iter(), + static_modules + .iter() + .flat_map(|(_, m)| m.trampolines.iter().copied()) + .chain(info.trampolines.iter().copied()) + .map(|(sig, loc)| { + let trampoline = code_memory.text()[loc.start as usize..].as_ptr(); + (sig, unsafe { + mem::transmute::<*const u8, VMTrampoline>(trampoline) + }) + }), ); - // Assert that this `always_trap` list is sorted which is relied on in - // `register_component` as well as `Component::lookup_trap_code` below. - assert!(always_trap - .values() - .as_slice() - .windows(2) - .all(|window| { window[0].info.start < window[1].info.start })); + // Assemble the `CodeObject` artifact which is shared by all core wasm + // modules as well as the final component. + let types = Arc::new(types); + let code = Arc::new(CodeObject::new(code_memory, signatures, types.into())); + + // Convert all information about static core wasm modules into actual + // `Module` instances by converting each `CompiledModuleInfo`, the + // `types` type information, and the code memory to a runtime object. + let static_modules = static_modules + .into_iter() + .map(|(_, info)| Module::from_parts_raw(engine, code.clone(), info, false)) + .collect::>()?; - crate::module::register_component(code.text, &always_trap); Ok(Component { inner: Arc::new(ComponentInner { - component, static_modules, - types, - signatures, - trampoline_obj, - text, - lowerings, - always_trap, - transcoders, + code, + info, }), }) } - #[cfg(compiler)] - fn compile_component( - engine: &Engine, - component: &wasmtime_environ::component::Component, - types: &ComponentTypes, - provided_trampolines: &HashSet, - ) -> Result<( - PrimaryMap, - PrimaryMap, - PrimaryMap, - Vec, - wasmtime_runtime::MmapVec, - )> { - let results = engine.join_maybe_parallel( - || compile_lowerings(engine, component, types), - || -> Result<_> { - Ok(engine.join_maybe_parallel( - || compile_always_trap(engine, component, types), - || -> Result<_> { - Ok(engine.join_maybe_parallel( - || compile_transcoders(engine, component, types), - || compile_trampolines(engine, component, types, provided_trampolines), - )) - }, - )) - }, - ); - let (lowerings, other) = results; - let (always_trap, other) = other?; - let (transcoders, trampolines) = other?; - let mut obj = engine.compiler().object()?; - let (lower, traps, transcoders, trampolines) = - engine.compiler().component_compiler().emit_obj( - lowerings?, - always_trap?, - transcoders?, - trampolines?, - &mut obj, - )?; - engine.append_bti(&mut obj); - return Ok(( - lower, - traps, - transcoders, - trampolines, - wasmtime_jit::mmap_vec_from_obj(obj)?, - )); - - fn compile_lowerings( - engine: &Engine, - component: &wasmtime_environ::component::Component, - types: &ComponentTypes, - ) -> Result>> { - let lowerings = component - .initializers - .iter() - .filter_map(|init| match init { - GlobalInitializer::LowerImport(i) => Some(i), - _ => None, - }) - .collect::>(); - Ok(engine - .run_maybe_parallel(lowerings, |lowering| { - engine - .compiler() - .component_compiler() - .compile_lowered_trampoline(&component, lowering, &types) - })? - .into_iter() - .collect()) - } - - fn compile_always_trap( - engine: &Engine, - component: &wasmtime_environ::component::Component, - types: &ComponentTypes, - ) -> Result>> { - let always_trap = component - .initializers - .iter() - .filter_map(|init| match init { - GlobalInitializer::AlwaysTrap(i) => Some(i), - _ => None, - }) - .collect::>(); - Ok(engine - .run_maybe_parallel(always_trap, |info| { - engine - .compiler() - .component_compiler() - .compile_always_trap(&types[info.canonical_abi]) - })? - .into_iter() - .collect()) - } - - fn compile_transcoders( - engine: &Engine, - component: &wasmtime_environ::component::Component, - types: &ComponentTypes, - ) -> Result>> { - let always_trap = component - .initializers - .iter() - .filter_map(|init| match init { - GlobalInitializer::Transcoder(i) => Some(i), - _ => None, - }) - .collect::>(); - Ok(engine - .run_maybe_parallel(always_trap, |info| { - engine - .compiler() - .component_compiler() - .compile_transcoder(component, info, types) - })? - .into_iter() - .collect()) - } - - fn compile_trampolines( - engine: &Engine, - component: &wasmtime_environ::component::Component, - types: &ComponentTypes, - provided_trampolines: &HashSet, - ) -> Result)>> { - // All lowered functions will require a trampoline to be available in - // case they're used when entering wasm. For example a lowered function - // could be immediately lifted in which case we'll need a trampoline to - // call that lowered function. - // - // Most of the time trampolines can come from the core wasm modules - // since lifted functions come from core wasm. For these esoteric cases - // though we may have to compile trampolines specifically into the - // component object as well in case core wasm doesn't provide the - // necessary trampoline. - let required_trampolines = component - .initializers - .iter() - .filter_map(|init| match init { - GlobalInitializer::LowerImport(i) => Some(i.canonical_abi), - GlobalInitializer::AlwaysTrap(i) => Some(i.canonical_abi), - _ => None, - }) - .collect::>(); - let mut trampolines_to_compile = required_trampolines - .difference(&provided_trampolines) - .collect::>(); - // Ensure a deterministically compiled artifact by sorting this list - // which was otherwise created with nondeterministically ordered hash - // tables. - trampolines_to_compile.sort(); - engine.run_maybe_parallel(trampolines_to_compile.clone(), |i| { - let ty = &types[*i]; - Ok((*i, engine.compiler().compile_host_to_wasm_trampoline(ty)?)) - }) - } - } - pub(crate) fn env_component(&self) -> &wasmtime_environ::component::Component { - &self.inner.component + &self.inner.info.component } pub(crate) fn static_module(&self, idx: StaticModuleIndex) -> &Module { @@ -395,65 +475,56 @@ impl Component { } pub(crate) fn types(&self) -> &Arc { - &self.inner.types + match self.inner.code.types() { + crate::code::Types::Component(types) => types, + // The only creator of a `Component` is itself which uses the other + // variant, so this shouldn't be possible. + crate::code::Types::Module(_) => unreachable!(), + } } pub(crate) fn signatures(&self) -> &SignatureCollection { - &self.inner.signatures + self.inner.code.signatures() } pub(crate) fn text(&self) -> &[u8] { - self.inner.text() + self.inner.code.code_memory().text() } pub(crate) fn lowering_ptr(&self, index: LoweredIndex) -> NonNull { - let info = &self.inner.lowerings[index]; + let info = &self.inner.info.lowerings[index]; self.func(info) } pub(crate) fn always_trap_ptr(&self, index: RuntimeAlwaysTrapIndex) -> NonNull { - let info = &self.inner.always_trap[index]; - self.func(&info.info) + let loc = &self.inner.info.always_trap[index]; + self.func(loc) } pub(crate) fn transcoder_ptr(&self, index: RuntimeTranscoderIndex) -> NonNull { - let info = &self.inner.transcoders[index]; + let info = &self.inner.info.transcoders[index]; self.func(info) } - fn func(&self, info: &FunctionInfo) -> NonNull { + fn func(&self, loc: &FunctionLoc) -> NonNull { let text = self.text(); - let trampoline = &text[info.start as usize..][..info.length as usize]; + let trampoline = &text[loc.start as usize..][..loc.length as usize]; NonNull::new(trampoline.as_ptr() as *mut VMFunctionBody).unwrap() } - /// Looks up a trap code for the instruction at `offset` where the offset - /// specified is relative to the start of this component's text section. - pub(crate) fn lookup_trap_code(&self, offset: usize) -> Option { - let offset = u32::try_from(offset).ok()?; - // Currently traps only come from "always trap" adapters so that map is - // the only map that's searched. - match self - .inner - .always_trap - .values() - .as_slice() - .binary_search_by_key(&offset, |info| info.info.start + info.trap_offset) - { - Ok(_) => Some(TrapCode::AlwaysTrapAdapter), - Err(_) => None, - } + pub(crate) fn code_object(&self) -> &Arc { + &self.inner.code } -} -impl ComponentInner { - fn text(&self) -> &[u8] { - &self.trampoline_obj.mmap()[self.text.clone()] - } -} - -impl Drop for ComponentInner { - fn drop(&mut self) { - crate::module::unregister_component(self.text()); + /// Same as [`Module::serialize`], except for a component. + /// + /// Note that the artifact produced here must be passed to + /// [`Component::deserialize`] and is not compatible for use with + /// [`Module`]. + /// + /// [`Module::serialize`]: crate::Module::serialize + /// [`Module`]: crate::Module + pub fn serialize(&self) -> Result> { + Ok(self.code_object().code_memory().mmap().to_vec()) } } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index fff01ebaaf..51cbeedae7 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -11,8 +11,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; #[cfg(feature = "cache")] use wasmtime_cache::CacheConfig; -use wasmtime_environ::FlagValue; -use wasmtime_jit::ProfilingAgent; +use wasmtime_environ::obj; +use wasmtime_environ::{FlagValue, ObjectKind}; +use wasmtime_jit::{CodeMemory, ProfilingAgent}; use wasmtime_runtime::{debug_builtins, CompiledModuleIdAllocator, InstanceAllocator, MmapVec}; mod serialization; @@ -225,6 +226,19 @@ impl Engine { Ok(mmap.to_vec()) } + /// Same as [`Engine::precompile_module`] except for a + /// [`Component`](crate::component::Component) + #[cfg(compiler)] + #[cfg_attr(nightlydoc, doc(cfg(feature = "cranelift")))] // see build.rs + #[cfg(feature = "component-model")] + #[cfg_attr(nightlydoc, doc(cfg(feature = "component-model")))] + pub fn precompile_component(&self, bytes: &[u8]) -> Result> { + #[cfg(feature = "wat")] + let bytes = wat::parse_bytes(&bytes)?; + let (mmap, _) = crate::component::Component::build_artifacts(self, &bytes)?; + Ok(mmap.to_vec()) + } + pub(crate) fn run_maybe_parallel< A: Send, B: Send, @@ -561,7 +575,7 @@ impl Engine { pub(crate) fn append_bti(&self, obj: &mut Object<'_>) { let section = obj.add_section( obj.segment_name(StandardSegment::Data).to_vec(), - wasmtime_jit::ELF_WASM_BTI.as_bytes().to_vec(), + obj::ELF_WASM_BTI.as_bytes().to_vec(), SectionKind::ReadOnlyData, ); let contents = if self.compiler().is_branch_protection_enabled() { @@ -572,21 +586,38 @@ impl Engine { obj.append_section_data(section, &[contents], 1); } - pub(crate) fn load_mmap_bytes(&self, bytes: &[u8]) -> Result { - self.load_mmap(MmapVec::from_slice(bytes)?) + /// Loads a `CodeMemory` from the specified in-memory slice, copying it to a + /// uniquely owned mmap. + /// + /// The `expected` marker here is whether the bytes are expected to be a + /// precompiled module or a component. + pub(crate) fn load_code_bytes( + &self, + bytes: &[u8], + expected: ObjectKind, + ) -> Result> { + self.load_code(MmapVec::from_slice(bytes)?, expected) } - pub(crate) fn load_mmap_file(&self, path: &Path) -> Result { - self.load_mmap( + /// Like `load_code_bytes`, but crates a mmap from a file on disk. + pub(crate) fn load_code_file( + &self, + path: &Path, + expected: ObjectKind, + ) -> Result> { + self.load_code( MmapVec::from_file(path).with_context(|| { format!("failed to create file mapping for: {}", path.display()) })?, + expected, ) } - fn load_mmap(&self, mmap: MmapVec) -> Result { - serialization::check_compatible(self, &mmap)?; - Ok(mmap) + fn load_code(&self, mmap: MmapVec, expected: ObjectKind) -> Result> { + serialization::check_compatible(self, &mmap, expected)?; + let mut code = CodeMemory::new(mmap)?; + code.publish()?; + Ok(Arc::new(code)) } } diff --git a/crates/wasmtime/src/engine/serialization.rs b/crates/wasmtime/src/engine/serialization.rs index 304a9797a8..8021b9ee2c 100644 --- a/crates/wasmtime/src/engine/serialization.rs +++ b/crates/wasmtime/src/engine/serialization.rs @@ -24,15 +24,15 @@ use crate::{Engine, ModuleVersionStrategy}; use anyhow::{anyhow, bail, Context, Result}; use object::write::{Object, StandardSegment}; -use object::{File, Object as _, ObjectSection, SectionKind}; +use object::{File, FileFlags, Object as _, ObjectSection, SectionKind}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::str::FromStr; -use wasmtime_environ::{FlagValue, Tunables}; +use wasmtime_environ::obj; +use wasmtime_environ::{FlagValue, ObjectKind, Tunables}; use wasmtime_runtime::MmapVec; const VERSION: u8 = 0; -const ELF_WASM_ENGINE: &str = ".wasmtime.engine"; /// Produces a blob of bytes by serializing the `engine`'s configuration data to /// be checked, perhaps in a different process, with the `check_compatible` @@ -44,7 +44,7 @@ const ELF_WASM_ENGINE: &str = ".wasmtime.engine"; pub fn append_compiler_info(engine: &Engine, obj: &mut Object<'_>) { let section = obj.add_section( obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASM_ENGINE.as_bytes().to_vec(), + obj::ELF_WASM_ENGINE.as_bytes().to_vec(), SectionKind::ReadOnlyData, ); let mut data = Vec::new(); @@ -73,16 +73,37 @@ pub fn append_compiler_info(engine: &Engine, obj: &mut Object<'_>) { /// provided here, notably compatible wasm features are enabled, compatible /// compiler options, etc. If a mismatch is found and the compilation metadata /// specified is incompatible then an error is returned. -pub fn check_compatible(engine: &Engine, mmap: &MmapVec) -> Result<()> { +pub fn check_compatible(engine: &Engine, mmap: &MmapVec, expected: ObjectKind) -> Result<()> { + // Parse the input `mmap` as an ELF file and see if the header matches the + // Wasmtime-generated header. This includes a Wasmtime-specific `os_abi` and + // the `e_flags` field should indicate whether `expected` matches or not. + // + // Note that errors generated here could mean that a precompiled module was + // loaded as a component, or vice versa, both of which aren't supposed to + // work. + // // Ideally we'd only `File::parse` once and avoid the linear // `section_by_name` search here but the general serialization code isn't // structured well enough to make this easy and additionally it's not really // a perf issue right now so doing that is left for another day's // refactoring. let obj = File::parse(&mmap[..]).context("failed to parse precompiled artifact as an ELF")?; + let expected_e_flags = match expected { + ObjectKind::Module => obj::EF_WASMTIME_MODULE, + ObjectKind::Component => obj::EF_WASMTIME_COMPONENT, + }; + match obj.flags() { + FileFlags::Elf { + os_abi: obj::ELFOSABI_WASMTIME, + abi_version: 0, + e_flags, + } if e_flags == expected_e_flags => {} + _ => bail!("incompatible object file format"), + } + let data = obj - .section_by_name(ELF_WASM_ENGINE) - .ok_or_else(|| anyhow!("failed to find section `{ELF_WASM_ENGINE}`"))? + .section_by_name(obj::ELF_WASM_ENGINE) + .ok_or_else(|| anyhow!("failed to find section `{}`", obj::ELF_WASM_ENGINE))? .data()?; let (first, data) = data .split_first() diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index c86fbeab10..189dc5d878 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -389,6 +389,7 @@ #[macro_use] mod func; +mod code; mod config; mod engine; mod externals; diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index f5123ac4f1..b6e8336dd3 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -1,3 +1,4 @@ +use crate::code::CodeObject; use crate::{ signatures::SignatureCollection, types::{ExportType, ExternType, ImportType}, @@ -5,29 +6,25 @@ use crate::{ }; use anyhow::{bail, Context, Result}; use once_cell::sync::OnceCell; +use std::any::Any; use std::fs; use std::mem; use std::ops::Range; use std::path::Path; use std::sync::Arc; use wasmparser::{Parser, ValidPayload, Validator}; -#[cfg(feature = "component-model")] -use wasmtime_environ::component::ComponentTypes; use wasmtime_environ::{ - DefinedFuncIndex, DefinedMemoryIndex, FunctionInfo, ModuleEnvironment, ModuleTranslation, - ModuleTypes, PrimaryMap, SignatureIndex, + DefinedFuncIndex, DefinedMemoryIndex, FunctionLoc, ModuleEnvironment, ModuleTranslation, + ModuleTypes, ObjectKind, PrimaryMap, SignatureIndex, WasmFunctionInfo, }; -use wasmtime_jit::{CompiledModule, CompiledModuleInfo}; +use wasmtime_jit::{CodeMemory, CompiledModule, CompiledModuleInfo}; use wasmtime_runtime::{ CompiledModuleId, MemoryImage, MmapVec, ModuleMemoryImages, VMSharedSignatureIndex, }; mod registry; -mod serialization; -pub use registry::{is_wasm_trap_pc, ModuleRegistry}; -#[cfg(feature = "component-model")] -pub use registry::{register_component, unregister_component}; +pub use registry::{is_wasm_trap_pc, register_code, unregister_code, ModuleRegistry}; /// A compiled WebAssembly module, ready to be instantiated. /// @@ -106,11 +103,15 @@ struct ModuleInner { engine: Engine, /// The compiled artifacts for this module that will be instantiated and /// executed. - module: Arc, - /// Type information of this module. - types: Types, - /// Registered shared signature for the module. - signatures: SignatureCollection, + module: CompiledModule, + + /// Runtime information such as the underlying mmap, type information, etc. + /// + /// Note that this `Arc` is used to share information between compiled + /// modules within a component. For bare core wasm modules created with + /// `Module::new`, for example, this is a uniquely owned `Arc`. + code: Arc, + /// A set of initialization images for memories, if any. /// /// Note that this is behind a `OnceCell` to lazily create this image. On @@ -119,6 +120,9 @@ struct ModuleInner { /// improves memory usage for modules that are created but may not ever be /// instantiated. memory_images: OnceCell>, + + /// Flag indicating whether this module can be serialized or not. + serializable: bool, } impl Module { @@ -287,7 +291,7 @@ impl Module { cfg_if::cfg_if! { if #[cfg(feature = "cache")] { let state = (HashedEngineCompileEnv(engine), binary); - let (mmap, info_and_types) = wasmtime_cache::ModuleCacheEntry::new( + let (code, info_and_types) = wasmtime_cache::ModuleCacheEntry::new( "wasmtime", engine.cache_config(), ) @@ -295,48 +299,58 @@ impl Module { &state, // Cache miss, compute the actual artifacts - |(engine, wasm)| Module::build_artifacts(engine.0, wasm), + |(engine, wasm)| -> Result<_> { + let (mmap, info) = Module::build_artifacts(engine.0, wasm)?; + let code = publish_mmap(mmap)?; + Ok((code, info)) + }, // Implementation of how to serialize artifacts - |(_engine, _wasm), (mmap, _info_and_types)| { - Some(mmap.to_vec()) + |(_engine, _wasm), (code, _info_and_types)| { + Some(code.mmap().to_vec()) }, // Cache hit, deserialize the provided artifacts |(engine, _wasm), serialized_bytes| { - let mmap = engine.0.load_mmap_bytes(&serialized_bytes).ok()?; - Some((mmap, None)) + let code = engine.0.load_code_bytes(&serialized_bytes, ObjectKind::Module).ok()?; + Some((code, None)) }, )?; } else { let (mmap, info_and_types) = Module::build_artifacts(engine, binary)?; + let code = publish_mmap(mmap)?; } }; let info_and_types = info_and_types.map(|(info, types)| (info, types.into())); - Self::from_parts(engine, mmap, info_and_types) + return Self::from_parts(engine, code, info_and_types); + + fn publish_mmap(mmap: MmapVec) -> Result> { + let mut code = CodeMemory::new(mmap)?; + code.publish()?; + Ok(Arc::new(code)) + } } - /// Converts an input binary-encoded WebAssembly module to compilation - /// artifacts and type information. + /// Compiles a binary-encoded WebAssembly module to an artifact usable by + /// Wasmtime. /// /// This is where compilation actually happens of WebAssembly modules and - /// translation/parsing/validation of the binary input occurs. The actual - /// result here is a combination of: + /// translation/parsing/validation of the binary input occurs. The binary + /// artifact represented in the `MmapVec` returned here is an in-memory ELF + /// file in an owned area of virtual linear memory where permissions (such + /// as the executable bit) can be applied. /// - /// * The compilation artifacts for the module found within `wasm`, as - /// returned by `wasmtime_jit::finish_compile`. - /// * Type information about the module returned. All returned modules have - /// local type information with indices that refer to these returned - /// tables. - /// * A boolean value indicating whether forward-edge CFI has been applied - /// to the compiled module. + /// Additionally compilation returns an `Option` here which is always + /// `Some`, notably compiled metadata about the module in addition to the + /// type information found within. #[cfg(compiler)] pub(crate) fn build_artifacts( engine: &Engine, wasm: &[u8], ) -> Result<(MmapVec, Option<(CompiledModuleInfo, ModuleTypes)>)> { let tunables = &engine.config().tunables; + let compiler = engine.compiler(); // First a `ModuleEnvironment` is created which records type information // about the wasm module. This is where the WebAssembly is parsed and @@ -346,66 +360,127 @@ impl Module { wasmparser::Validator::new_with_features(engine.config().features.clone()); let parser = wasmparser::Parser::new(0); let mut types = Default::default(); - let translation = ModuleEnvironment::new(tunables, &mut validator, &mut types) + let mut translation = ModuleEnvironment::new(tunables, &mut validator, &mut types) .translate(parser, wasm) .context("failed to parse WebAssembly module")?; let types = types.finish(); - let (mmap, info) = Module::compile_functions(engine, translation, &types)?; - Ok((mmap, Some((info, types)))) - } - #[cfg(compiler)] - pub(crate) fn compile_functions( - engine: &Engine, - mut translation: ModuleTranslation<'_>, - types: &ModuleTypes, - ) -> Result<(MmapVec, CompiledModuleInfo)> { - let tunables = &engine.config().tunables; - let functions = mem::take(&mut translation.function_body_inputs); - let functions = functions.into_iter().collect::>(); - let compiler = engine.compiler(); + // Afterwards compile all functions and trampolines required by the + // module. + let signatures = translation.exported_signatures.clone(); let (funcs, trampolines) = engine.join_maybe_parallel( // In one (possibly) parallel task all wasm functions are compiled // in parallel. Note that this is also where the actual validation // of all function bodies happens as well. - || -> Result<_> { - let funcs = engine.run_maybe_parallel(functions, |(index, func)| { - let offset = func.body.range().start; - let result = - compiler.compile_function(&translation, index, func, tunables, types); - result.with_context(|| { - let index = translation.module.func_index(index); - let name = match translation.debuginfo.name_section.func_names.get(&index) { - Some(name) => format!(" (`{}`)", name), - None => String::new(), - }; - let index = index.as_u32(); - format!( - "failed to compile wasm function {index}{name} at offset {offset:#x}" - ) - }) - })?; - - Ok(funcs.into_iter().collect()) - }, + || Self::compile_functions(engine, &mut translation, &types), // In another (possibly) parallel task all trampolines necessary // for untyped host-to-wasm entry are compiled. Note that this // isn't really expected to take all that long, it's moreso "well // if we're using rayon why not use it here too". || -> Result<_> { - engine.run_maybe_parallel(translation.exported_signatures.clone(), |sig| { + engine.run_maybe_parallel(signatures, |sig| { let ty = &types[sig]; Ok(compiler.compile_host_to_wasm_trampoline(ty)?) }) }, ); - // Collect all the function results into a final ELF object. - let mut obj = engine.compiler().object()?; - let (funcs, trampolines) = - engine - .compiler() - .emit_obj(&translation, funcs?, trampolines?, tunables, &mut obj)?; + // Weave the separate list of compiled functions into one list, storing + // the other metadata off to the side for now. + let funcs = funcs?; + let trampolines = trampolines?; + let mut func_infos = PrimaryMap::with_capacity(funcs.len()); + let mut compiled_funcs = Vec::with_capacity(funcs.len() + trampolines.len()); + for (info, func) in funcs { + let idx = func_infos.push(info); + let sym = format!( + "_wasm_function_{}", + translation.module.func_index(idx).as_u32() + ); + compiled_funcs.push((sym, func)); + } + for (sig, func) in translation.exported_signatures.iter().zip(trampolines) { + let sym = format!("_trampoline_{}", sig.as_u32()); + compiled_funcs.push((sym, func)); + } + + // Emplace all compiled functions into the object file with any other + // sections associated with code as well. + let mut obj = engine.compiler().object(ObjectKind::Module)?; + let locs = compiler.append_code(&mut obj, &compiled_funcs, tunables, &|i, idx| { + assert!(i < func_infos.len()); + let defined = translation.module.defined_func_index(idx).unwrap(); + defined.as_u32() as usize + })?; + + // If requested, generate and add dwarf information. + if tunables.generate_native_debuginfo && !func_infos.is_empty() { + let mut locs = locs.iter(); + let mut funcs = compiled_funcs.iter(); + let funcs = (0..func_infos.len()) + .map(|_| (locs.next().unwrap().0, &*funcs.next().unwrap().1)) + .collect(); + compiler.append_dwarf(&mut obj, &translation, &funcs)?; + } + + // Process all the results of compilation into a final state for our + // internal representation. + let mut locs = locs.into_iter(); + let funcs = func_infos + .into_iter() + .map(|(_, info)| (info, locs.next().unwrap().1)) + .collect(); + let trampolines = translation + .exported_signatures + .iter() + .cloned() + .map(|i| (i, locs.next().unwrap().1)) + .collect(); + assert!(locs.next().is_none()); + + // Insert `Engine` and type-level information into the compiled + // artifact so if this module is deserialized later it contains all + // information necessary. + // + // Note that `append_compiler_info` and `append_types` here in theory + // can both be skipped if this module will never get serialized. + // They're only used during deserialization and not during runtime for + // the module itself. Currently there's no need for that, however, so + // it's left as an exercise for later. + engine.append_compiler_info(&mut obj); + engine.append_bti(&mut obj); + + let mut obj = wasmtime_jit::ObjectBuilder::new(obj, tunables); + let info = obj.append(translation, funcs, trampolines)?; + obj.serialize_info(&(&info, &types)); + let mmap = obj.finish()?; + + Ok((mmap, Some((info, types)))) + } + + #[cfg(compiler)] + pub(crate) fn compile_functions( + engine: &Engine, + translation: &mut ModuleTranslation<'_>, + types: &ModuleTypes, + ) -> Result)>> { + let tunables = &engine.config().tunables; + let functions = mem::take(&mut translation.function_body_inputs); + let functions = functions.into_iter().collect::>(); + let compiler = engine.compiler(); + let funcs = engine.run_maybe_parallel(functions, |(index, func)| { + let offset = func.body.range().start; + let result = compiler.compile_function(&translation, index, func, tunables, types); + result.with_context(|| { + let index = translation.module.func_index(index); + let name = match translation.debuginfo.name_section.func_names.get(&index) { + Some(name) => format!(" (`{}`)", name), + None => String::new(), + }; + let index = index.as_u32(); + format!("failed to compile wasm function {index}{name} at offset {offset:#x}") + }) + })?; // If configured attempt to use static memory initialization which // can either at runtime be implemented as a single memcpy to @@ -422,23 +497,7 @@ impl Module { // table lazy init. translation.try_func_table_init(); - // Insert `Engine` and type-level information into the compiled - // artifact so if this module is deserialized later it contains all - // information necessary. - // - // Note that `append_compiler_info` and `append_types` here in theory - // can both be skipped if this module will never get serialized. - // They're only used during deserialization and not during runtime for - // the module itself. Currently there's no need for that, however, so - // it's left as an exercise for later. - engine.append_compiler_info(&mut obj); - engine.append_bti(&mut obj); - serialization::append_types(types, &mut obj); - - let (mmap, info) = - wasmtime_jit::finish_compile(translation, obj, funcs, trampolines, tunables)?; - - Ok((mmap, info)) + Ok(funcs) } /// Deserializes an in-memory compiled module previously created with @@ -484,8 +543,8 @@ impl Module { /// blobs across versions of wasmtime you can be safely guaranteed that /// future versions of wasmtime will reject old cache entries). pub unsafe fn deserialize(engine: &Engine, bytes: impl AsRef<[u8]>) -> Result { - let mmap = engine.load_mmap_bytes(bytes.as_ref())?; - Module::from_parts(engine, mmap, None) + let code = engine.load_code_bytes(bytes.as_ref(), ObjectKind::Module)?; + Module::from_parts(engine, code, None) } /// Same as [`deserialize`], except that the contents of `path` are read to @@ -512,48 +571,75 @@ impl Module { /// reflect the current state of the file, not necessarily the origianl /// state of the file. pub unsafe fn deserialize_file(engine: &Engine, path: impl AsRef) -> Result { - let mmap = engine.load_mmap_file(path.as_ref())?; - Module::from_parts(engine, mmap, None) + let code = engine.load_code_file(path.as_ref(), ObjectKind::Module)?; + Module::from_parts(engine, code, None) } - pub(crate) fn from_parts( + /// Entrypoint for creating a `Module` for all above functions, both + /// of the AOT and jit-compiled cateogries. + /// + /// In all cases the compilation artifact, `code_memory`, is provided here. + /// The `info_and_types` argument is `None` when a module is being + /// deserialized from a precompiled artifact or it's `Some` if it was just + /// compiled and the values are already available. + fn from_parts( engine: &Engine, - mmap: MmapVec, - info_and_types: Option<(CompiledModuleInfo, Types)>, + code_memory: Arc, + info_and_types: Option<(CompiledModuleInfo, ModuleTypes)>, ) -> Result { + // Acquire this module's metadata and type information, deserializing + // it from the provided artifact if it wasn't otherwise provided + // already. let (info, types) = match info_and_types { - Some((info, types)) => (Some(info), types), - None => (None, serialization::deserialize_types(&mmap)?.into()), + Some((info, types)) => (info, types), + None => bincode::deserialize(code_memory.wasmtime_info())?, }; - let module = Arc::new(CompiledModule::from_artifacts( - mmap, + + // Register function type signatures into the engine for the lifetime + // of the `Module` that will be returned. This notably also builds up + // maps for trampolines to be used for this module when inserted into + // stores. + // + // Note that the unsafety here should be ok since the `trampolines` + // field should only point to valid trampoline function pointers + // within the text section. + let signatures = SignatureCollection::new_for_module( + engine.signatures(), + &types, + info.trampolines + .iter() + .map(|(idx, f)| (*idx, unsafe { code_memory.vmtrampoline(*f) })), + ); + + // Package up all our data into a `CodeObject` and delegate to the final + // step of module compilation. + let code = Arc::new(CodeObject::new(code_memory, signatures, types.into())); + Module::from_parts_raw(engine, code, info, true) + } + + pub(crate) fn from_parts_raw( + engine: &Engine, + code: Arc, + info: CompiledModuleInfo, + serializable: bool, + ) -> Result { + let module = CompiledModule::from_artifacts( + code.code_memory().clone(), info, engine.profiler(), engine.unique_id_allocator(), - )?); + )?; // Validate the module can be used with the current allocator engine.allocator().validate(module.module())?; - let signatures = SignatureCollection::new_for_module( - engine.signatures(), - types.module_types(), - module.trampolines().map(|(idx, f, _)| (idx, f)), - ); - - // We're about to create a `Module` for real now so enter this module - // into the global registry of modules so we can resolve traps - // appropriately. Note that the corresponding `unregister` happens below - // in `Drop for ModuleInner`. - registry::register_module(&module); - Ok(Self { inner: Arc::new(ModuleInner { engine: engine.clone(), - types, - signatures, + code, memory_images: OnceCell::new(), module, + serializable, }), }) } @@ -615,6 +701,26 @@ impl Module { #[cfg(compiler)] #[cfg_attr(nightlydoc, doc(cfg(feature = "cranelift")))] // see build.rs pub fn serialize(&self) -> Result> { + // The current representation of compiled modules within a compiled + // component means that it cannot be serialized. The mmap returned here + // is the mmap for the entire component and while it contains all + // necessary data to deserialize this particular module it's all + // embedded within component-specific information. + // + // It's not the hardest thing in the world to support this but it's + // expected that there's not much of a use case at this time. In theory + // all that needs to be done is to edit the `.wasmtime.info` section + // to contains this module's metadata instead of the metadata for the + // whole component. The metadata itself is fairly trivially + // recreateable here it's more that there's no easy one-off API for + // editing the sections of an ELF object to use here. + // + // Overall for now this simply always returns an error in this + // situation. If you're reading this and feel that the situation should + // be different please feel free to open an issue. + if !self.inner.serializable { + bail!("cannot serialize a module exported from a component"); + } Ok(self.compiled_module().mmap().to_vec()) } @@ -622,16 +728,20 @@ impl Module { &self.inner.module } + fn code_object(&self) -> &Arc { + &self.inner.code + } + pub(crate) fn env_module(&self) -> &wasmtime_environ::Module { self.compiled_module().module() } pub(crate) fn types(&self) -> &ModuleTypes { - self.inner.types.module_types() + self.inner.code.module_types() } pub(crate) fn signatures(&self) -> &SignatureCollection { - &self.inner.signatures + self.inner.code.signatures() } /// Returns identifier/name that this [`Module`] has. This name @@ -944,15 +1054,15 @@ impl wasmtime_runtime::ModuleRuntimeInfo for ModuleInner { } fn signature(&self, index: SignatureIndex) -> VMSharedSignatureIndex { - self.signatures.as_module_map()[index] + self.code.signatures().as_module_map()[index] } fn image_base(&self) -> usize { - self.module.code().as_ptr() as usize + self.module.text().as_ptr() as usize } - fn function_info(&self, index: DefinedFuncIndex) -> &FunctionInfo { - self.module.func_info(index) + fn function_loc(&self, index: DefinedFuncIndex) -> &FunctionLoc { + self.module.func_loc(index) } fn memory_image(&self, memory: DefinedMemoryIndex) -> Result>> { @@ -965,19 +1075,19 @@ impl wasmtime_runtime::ModuleRuntimeInfo for ModuleInner { } fn wasm_data(&self) -> &[u8] { - self.module.wasm_data() + self.module.code_memory().wasm_data() } fn signature_ids(&self) -> &[VMSharedSignatureIndex] { - self.signatures.as_module_map().values().as_slice() + self.code.signatures().as_module_map().values().as_slice() } } impl wasmtime_runtime::ModuleInfo for ModuleInner { fn lookup_stack_map(&self, pc: usize) -> Option<&wasmtime_environ::StackMap> { - let text_offset = pc - self.module.code().as_ptr() as usize; + let text_offset = pc - self.module.text().as_ptr() as usize; let (index, func_offset) = self.module.func_by_text_offset(text_offset)?; - let info = self.module.func_info(index); + let info = self.module.wasm_func_info(index); // Do a binary search to find the stack map for the given offset. let index = match info @@ -999,12 +1109,6 @@ impl wasmtime_runtime::ModuleInfo for ModuleInner { } } -impl Drop for ModuleInner { - fn drop(&mut self) { - registry::unregister_module(&self.module); - } -} - /// A barebones implementation of ModuleRuntimeInfo that is useful for /// cases where a purpose-built environ::Module is used and a full /// CompiledModule does not exist (for example, for tests or for the @@ -1013,7 +1117,6 @@ pub(crate) struct BareModuleInfo { module: Arc, image_base: usize, one_signature: Option<(SignatureIndex, VMSharedSignatureIndex)>, - function_info: PrimaryMap, } impl BareModuleInfo { @@ -1022,7 +1125,6 @@ impl BareModuleInfo { module, image_base: 0, one_signature: None, - function_info: PrimaryMap::default(), } } @@ -1034,7 +1136,6 @@ impl BareModuleInfo { module, image_base: 0, one_signature, - function_info: PrimaryMap::default(), } } @@ -1060,8 +1161,8 @@ impl wasmtime_runtime::ModuleRuntimeInfo for BareModuleInfo { self.image_base } - fn function_info(&self, index: DefinedFuncIndex) -> &FunctionInfo { - &self.function_info[index] + fn function_loc(&self, _index: DefinedFuncIndex) -> &FunctionLoc { + unreachable!() } fn memory_image(&self, _memory: DefinedMemoryIndex) -> Result>> { @@ -1084,35 +1185,6 @@ impl wasmtime_runtime::ModuleRuntimeInfo for BareModuleInfo { } } -pub(crate) enum Types { - Module(ModuleTypes), - #[cfg(feature = "component-model")] - Component(Arc), -} - -impl Types { - fn module_types(&self) -> &ModuleTypes { - match self { - Types::Module(m) => m, - #[cfg(feature = "component-model")] - Types::Component(c) => c.module_types(), - } - } -} - -impl From for Types { - fn from(types: ModuleTypes) -> Types { - Types::Module(types) - } -} - -#[cfg(feature = "component-model")] -impl From> for Types { - fn from(types: Arc) -> Types { - Types::Component(types) - } -} - /// Helper method to construct a `ModuleMemoryImages` for an associated /// `CompiledModule`. fn memory_images(engine: &Engine, module: &CompiledModule) -> Result> { @@ -1129,5 +1201,5 @@ fn memory_images(engine: &Engine, module: &CompiledModule) -> Result, + // The value here is the start address and the information about what's + // loaded at that address. + loaded_code: BTreeMap, // Preserved for keeping data segments alive or similar modules_without_code: Vec, } -enum ModuleOrComponent { - Module(Module), - #[cfg(feature = "component-model")] - Component(Component), -} +struct LoadedCode { + /// Representation of loaded code which could be either a component or a + /// module. + code: Arc, -fn start(module: &Module) -> usize { - assert!(!module.compiled_module().code().is_empty()); - module.compiled_module().code().as_ptr() as usize + /// Modules found within `self.code`, keyed by start address here of the + /// address of the first function in the module. + modules: BTreeMap, } impl ModuleRegistry { @@ -54,25 +50,32 @@ impl ModuleRegistry { self.module(pc).map(|(m, _)| m.module_info()) } - fn module(&self, pc: usize) -> Option<(&Module, usize)> { - match self.module_or_component(pc)? { - (ModuleOrComponent::Module(m), offset) => Some((m, offset)), - #[cfg(feature = "component-model")] - (ModuleOrComponent::Component(_), _) => None, - } - } - - fn module_or_component(&self, pc: usize) -> Option<(&ModuleOrComponent, usize)> { - let (end, (start, module)) = self.modules_with_code.range(pc..).next()?; + fn code(&self, pc: usize) -> Option<(&LoadedCode, usize)> { + let (end, (start, code)) = self.loaded_code.range(pc..).next()?; if pc < *start || *end < pc { return None; } - Some((module, pc - *start)) + Some((code, pc - *start)) + } + + fn module(&self, pc: usize) -> Option<(&Module, usize)> { + let (code, offset) = self.code(pc)?; + Some((code.module(pc)?, offset)) } /// Registers a new module with the registry. pub fn register_module(&mut self, module: &Module) { - let compiled_module = module.compiled_module(); + self.register(module.code_object(), Some(module)) + } + + #[cfg(feature = "component-model")] + pub fn register_component(&mut self, component: &Component) { + self.register(component.code_object(), None) + } + + /// Registers a new module with the registry. + fn register(&mut self, code: &Arc, module: Option<&Module>) { + let text = code.code_memory().text(); // If there's not actually any functions in this module then we may // still need to preserve it for its data segments. Instances of this @@ -80,86 +83,58 @@ impl ModuleRegistry { // and for schemes that perform lazy initialization which could use the // module in the future. For that reason we continue to register empty // modules and retain them. - if compiled_module.finished_functions().len() == 0 { - self.modules_without_code.push(module.clone()); - } else { - // The module code range is exclusive for end, so make it inclusive as it - // may be a valid PC value - let start_addr = start(module); - let end_addr = start_addr + compiled_module.code().len() - 1; - self.register( - start_addr, - end_addr, - ModuleOrComponent::Module(module.clone()), - ); - } - } - - #[cfg(feature = "component-model")] - pub fn register_component(&mut self, component: &Component) { - // If there's no text section associated with this component (e.g. no - // lowered functions) then there's nothing to register, otherwise it's - // registered along the same lines as modules above. - // - // Note that empty components don't need retaining here since it doesn't - // have data segments like empty modules. - let text = component.text(); if text.is_empty() { + self.modules_without_code.extend(module.cloned()); return; } - let start = text.as_ptr() as usize; - self.register( - start, - start + text.len() - 1, - ModuleOrComponent::Component(component.clone()), - ); - } - /// Registers a new module with the registry. - fn register(&mut self, start_addr: usize, end_addr: usize, item: ModuleOrComponent) { - // Ensure the module isn't already present in the registry - // This is expected when a module is instantiated multiple times in the - // same store - if let Some((other_start, _)) = self.modules_with_code.get(&end_addr) { + // The module code range is exclusive for end, so make it inclusive as + // it may be a valid PC value + let start_addr = text.as_ptr() as usize; + let end_addr = start_addr + text.len() - 1; + + // If this module is already present in the registry then that means + // it's either an overlapping image, for example for two modules + // found within a component, or it's a second instantiation of the same + // module. Delegate to `push_module` to find out. + if let Some((other_start, prev)) = self.loaded_code.get_mut(&end_addr) { assert_eq!(*other_start, start_addr); + if let Some(module) = module { + prev.push_module(module); + } return; } // Assert that this module's code doesn't collide with any other // registered modules - if let Some((_, (prev_start, _))) = self.modules_with_code.range(start_addr..).next() { + if let Some((_, (prev_start, _))) = self.loaded_code.range(start_addr..).next() { assert!(*prev_start > end_addr); } - if let Some((prev_end, _)) = self.modules_with_code.range(..=start_addr).next_back() { + if let Some((prev_end, _)) = self.loaded_code.range(..=start_addr).next_back() { assert!(*prev_end < start_addr); } - let prev = self.modules_with_code.insert(end_addr, (start_addr, item)); + let mut item = LoadedCode { + code: code.clone(), + modules: Default::default(), + }; + if let Some(module) = module { + item.push_module(module); + } + let prev = self.loaded_code.insert(end_addr, (start_addr, item)); assert!(prev.is_none()); } /// Looks up a trampoline from an anyfunc. pub fn lookup_trampoline(&self, anyfunc: &VMCallerCheckedAnyfunc) -> Option { - let signatures = match self - .module_or_component(anyfunc.func_ptr.as_ptr() as usize)? - .0 - { - ModuleOrComponent::Module(m) => m.signatures(), - #[cfg(feature = "component-model")] - ModuleOrComponent::Component(c) => c.signatures(), - }; - signatures.trampoline(anyfunc.type_index) + let (code, _offset) = self.code(anyfunc.func_ptr.as_ptr() as usize)?; + code.code.signatures().trampoline(anyfunc.type_index) } /// Fetches trap information about a program counter in a backtrace. pub fn lookup_trap_code(&self, pc: usize) -> Option { - match self.module_or_component(pc)? { - (ModuleOrComponent::Module(module), offset) => { - wasmtime_environ::lookup_trap_code(module.compiled_module().trap_data(), offset) - } - #[cfg(feature = "component-model")] - (ModuleOrComponent::Component(component), offset) => component.lookup_trap_code(offset), - } + let (code, offset) = self.code(pc)?; + wasmtime_environ::lookup_trap_code(code.code.code_memory().trap_data(), offset) } /// Fetches frame information about a program counter in a backtrace. @@ -171,26 +146,49 @@ impl ModuleRegistry { /// boolean indicates whether the engine used to compile this module is /// using environment variables to control debuginfo parsing. pub(crate) fn lookup_frame_info(&self, pc: usize) -> Option<(FrameInfo, &Module)> { - match self.module_or_component(pc)? { - (ModuleOrComponent::Module(module), offset) => { - let info = FrameInfo::new(module, offset)?; - Some((info, module)) - } - #[cfg(feature = "component-model")] - (ModuleOrComponent::Component(_), _) => { - // FIXME: should investigate whether it's worth preserving - // frame information on a `Component` to resolve a frame here. - // Note that this can be traced back to either a lowered - // function via a trampoline or an "always trap" function at - // this time which may be useful debugging information to have. - None - } - } + let (module, offset) = self.module(pc)?; + let info = FrameInfo::new(module, offset)?; + Some((info, module)) } } -// This is the global module registry that stores information for all modules -// that are currently in use by any `Store`. +impl LoadedCode { + fn push_module(&mut self, module: &Module) { + let func = match module.compiled_module().finished_functions().next() { + Some((_, func)) => func, + // There are no compiled functions in this module so there's no + // need to push onto `self.modules` which is only used for frame + // information lookup for a trap which only symbolicates defined + // functions. + None => return, + }; + let start = unsafe { (*func).as_ptr() as usize }; + + match self.modules.entry(start) { + // This module is already present, and it should be the same as + // `module`. + Entry::Occupied(m) => { + debug_assert!(Arc::ptr_eq(&module.inner, &m.get().inner)); + } + // This module was not already present, so now it's time to insert. + Entry::Vacant(v) => { + v.insert(module.clone()); + } + } + } + + fn module(&self, pc: usize) -> Option<&Module> { + // The `modules` map is keyed on the start address of the first + // function in the module, so find the first module whose start address + // is less than the `pc`. That may be the wrong module but lookup + // within the module should fail in that case. + let (_start, module) = self.modules.range(..=pc).next_back()?; + Some(module) + } +} + +// This is the global code registry that stores information for all loaded code +// objects that are currently in use by any `Store` in the current process. // // The purpose of this map is to be called from signal handlers to determine // whether a program counter is a wasm trap or not. Specifically macOS has @@ -201,23 +199,16 @@ impl ModuleRegistry { // supports removal. Any time anything is registered with a `ModuleRegistry` // it is also automatically registered with the singleton global module // registry. When a `ModuleRegistry` is destroyed then all of its entries -// are removed from the global module registry. -static GLOBAL_MODULES: Lazy> = Lazy::new(Default::default); +// are removed from the global registry. +static GLOBAL_CODE: Lazy> = Lazy::new(Default::default); -type GlobalModuleRegistry = BTreeMap; - -#[derive(Clone)] -enum TrapInfo { - Module(Arc), - #[cfg(feature = "component-model")] - Component(Arc>), -} +type GlobalRegistry = BTreeMap)>; /// Returns whether the `pc`, according to globally registered information, /// is a wasm trap or not. pub fn is_wasm_trap_pc(pc: usize) -> bool { - let (trap_info, text_offset) = { - let all_modules = GLOBAL_MODULES.read().unwrap(); + let (code, text_offset) = { + let all_modules = GLOBAL_CODE.read().unwrap(); let (end, (start, module)) = match all_modules.range(pc..).next() { Some(info) => info, @@ -229,16 +220,7 @@ pub fn is_wasm_trap_pc(pc: usize) -> bool { (module.clone(), pc - *start) }; - match trap_info { - TrapInfo::Module(module) => { - wasmtime_environ::lookup_trap_code(module.trap_data(), text_offset).is_some() - } - #[cfg(feature = "component-model")] - TrapInfo::Component(traps) => { - let offset = u32::try_from(text_offset).unwrap(); - traps.binary_search(&offset).is_ok() - } - } + wasmtime_environ::lookup_trap_code(code.trap_data(), text_offset).is_some() } /// Registers a new region of code. @@ -247,66 +229,33 @@ pub fn is_wasm_trap_pc(pc: usize) -> bool { /// prevent leaking memory. /// /// This is required to enable traps to work correctly since the signal handler -/// will lookup in the `GLOBAL_MODULES` list to determine which a particular pc +/// will lookup in the `GLOBAL_CODE` list to determine which a particular pc /// is a trap or not. -pub fn register_module(module: &Arc) { - let code = module.code(); - if code.is_empty() { +pub fn register_code(code: &Arc) { + let text = code.text(); + if text.is_empty() { return; } - let start = code.as_ptr() as usize; - let end = start + code.len() - 1; - let prev = GLOBAL_MODULES + let start = text.as_ptr() as usize; + let end = start + text.len() - 1; + let prev = GLOBAL_CODE .write() .unwrap() - .insert(end, (start, TrapInfo::Module(module.clone()))); + .insert(end, (start, code.clone())); assert!(prev.is_none()); } -/// Unregisters a module from the global map. +/// Unregisters a code mmap from the global map. /// /// Must have been previously registered with `register`. -pub fn unregister_module(module: &Arc) { - let code = module.code(); - if code.is_empty() { - return; - } - let end = (code.as_ptr() as usize) + code.len() - 1; - let module = GLOBAL_MODULES.write().unwrap().remove(&end); - assert!(module.is_some()); -} - -/// Same as `register_module`, but for components -#[cfg(feature = "component-model")] -pub fn register_component(text: &[u8], traps: &PrimaryMap) { +pub fn unregister_code(code: &Arc) { + let text = code.text(); if text.is_empty() { return; } - let start = text.as_ptr() as usize; - let end = start + text.len(); - let info = Arc::new( - traps - .iter() - .map(|(_, info)| info.info.start + info.trap_offset) - .collect::>(), - ); - let prev = GLOBAL_MODULES - .write() - .unwrap() - .insert(end, (start, TrapInfo::Component(info))); - assert!(prev.is_none()); -} - -/// Same as `unregister_module`, but for components -#[cfg(feature = "component-model")] -pub fn unregister_component(text: &[u8]) { - if text.is_empty() { - return; - } - let start = text.as_ptr() as usize; - let end = start + text.len(); - let info = GLOBAL_MODULES.write().unwrap().remove(&end); - assert!(info.is_some()); + let end = (text.as_ptr() as usize) + text.len() - 1; + let code = GLOBAL_CODE.write().unwrap().remove(&end); + assert!(code.is_some()); } #[test] diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs deleted file mode 100644 index 21d4efa58f..0000000000 --- a/crates/wasmtime/src/module/serialization.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! Support for serializing type information for a `Module`. -//! -//! Wasmtime AOT compiled artifacts are ELF files where relevant data is stored -//! in relevant sections. This module implements the serialization format for -//! type information, or the `ModuleTypes` structure. -//! -//! This structure lives in a section of the final artifact at this time. It is -//! appended after compilation has otherwise completed and additionally is -//! deserialized from the entirety of the section. -//! -//! Implementation details are "just bincode it all" right now with no further -//! clever tricks about representation. Currently this works out more-or-less -//! ok since the type information is typically relatively small per-module. - -use anyhow::{anyhow, Result}; -use object::write::{Object, StandardSegment}; -use object::{File, Object as _, ObjectSection, SectionKind}; -use wasmtime_environ::ModuleTypes; -use wasmtime_runtime::MmapVec; - -const ELF_WASM_TYPES: &str = ".wasmtime.types"; - -pub fn append_types(types: &ModuleTypes, obj: &mut Object<'_>) { - let section = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASM_TYPES.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - let data = bincode::serialize(types).unwrap(); - obj.set_section_data(section, data, 1); -} - -pub fn deserialize_types(mmap: &MmapVec) -> Result { - // Ideally we'd only `File::parse` once and avoid the linear - // `section_by_name` search here but the general serialization code isn't - // structured well enough to make this easy and additionally it's not really - // a perf issue right now so doing that is left for another day's - // refactoring. - let obj = File::parse(&mmap[..])?; - let data = obj - .section_by_name(ELF_WASM_TYPES) - .ok_or_else(|| anyhow!("failed to find section `{ELF_WASM_TYPES}`"))? - .data()?; - Ok(bincode::deserialize(data)?) -} diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 8406873708..a80babdb83 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -66,11 +66,16 @@ unsafe extern "C" fn stub_fn( } #[cfg(compiler)] -fn register_trampolines(profiler: &dyn ProfilingAgent, image: &object::File<'_>) { - use object::{Object as _, ObjectSection, ObjectSymbol, SectionKind, SymbolKind}; +fn register_trampolines(profiler: &dyn ProfilingAgent, code: &CodeMemory) { + use object::{File, Object as _, ObjectSection, ObjectSymbol, SectionKind, SymbolKind}; let pid = std::process::id(); let tid = pid; + let image = match File::parse(&code.mmap()[..]) { + Ok(image) => image, + Err(_) => return, + }; + let text_base = match image.sections().find(|s| s.kind() == SectionKind::Text) { Some(section) => match section.data() { Ok(data) => data.as_ptr() as usize, @@ -107,7 +112,9 @@ pub fn create_function( where F: Fn(*mut VMContext, &mut [ValRaw]) -> Result<(), Trap> + Send + Sync + 'static, { - let mut obj = engine.compiler().object()?; + let mut obj = engine + .compiler() + .object(wasmtime_environ::ObjectKind::Module)?; let (t1, t2) = engine.compiler().emit_trampoline_obj( ft.as_wasm_func_type(), stub_fn:: as usize, @@ -115,20 +122,21 @@ where )?; engine.append_compiler_info(&mut obj); engine.append_bti(&mut obj); - let obj = wasmtime_jit::mmap_vec_from_obj(obj)?; + let obj = wasmtime_jit::ObjectBuilder::new(obj, &engine.config().tunables).finish()?; // Copy the results of JIT compilation into executable memory, and this will // also take care of unwind table registration. - let mut code_memory = CodeMemory::new(obj); - let code = code_memory.publish()?; + let mut code_memory = CodeMemory::new(obj)?; + code_memory.publish()?; - register_trampolines(engine.profiler(), &code.obj); + register_trampolines(engine.profiler(), &code_memory); // Extract the host/wasm trampolines from the results of compilation since // we know their start/length. - let host_trampoline = code.text[t1.start as usize..][..t1.length as usize].as_ptr(); - let wasm_trampoline = code.text[t2.start as usize..].as_ptr() as *mut _; + let text = code_memory.text(); + let host_trampoline = text[t1.start as usize..][..t1.length as usize].as_ptr(); + let wasm_trampoline = text[t2.start as usize..].as_ptr() as *mut _; let wasm_trampoline = NonNull::new(wasm_trampoline).unwrap(); let sig = engine.signatures().register(ft.as_wasm_func_type()); diff --git a/crates/wasmtime/src/trap.rs b/crates/wasmtime/src/trap.rs index 086fac434a..b9c2ed2421 100644 --- a/crates/wasmtime/src/trap.rs +++ b/crates/wasmtime/src/trap.rs @@ -523,8 +523,9 @@ impl FrameInfo { pub(crate) fn new(module: &Module, text_offset: usize) -> Option { let module = module.compiled_module(); let (index, _func_offset) = module.func_by_text_offset(text_offset)?; - let info = module.func_info(index); - let instr = wasmtime_environ::lookup_file_pos(module.address_map_data(), text_offset); + let info = module.wasm_func_info(index); + let instr = + wasmtime_environ::lookup_file_pos(module.code_memory().address_map_data(), text_offset); // In debug mode for now assert that we found a mapping for `pc` within // the function, because otherwise something is buggy along the way and diff --git a/crates/winch/Cargo.toml b/crates/winch/Cargo.toml index 3096e00e43..7b3ff5be09 100644 --- a/crates/winch/Cargo.toml +++ b/crates/winch/Cargo.toml @@ -12,6 +12,7 @@ winch-codegen = { workspace = true } target-lexicon = { workspace = true } wasmtime-environ = { workspace = true } anyhow = { workspace = true } +object = { workspace = true } [features] default = ["all-arch", "component-model"] diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs index d1a473ff54..f3806c754c 100644 --- a/crates/winch/src/compiler.rs +++ b/crates/winch/src/compiler.rs @@ -1,6 +1,9 @@ use anyhow::Result; +use object::write::{Object, SymbolId}; +use std::any::Any; use wasmtime_environ::{ - CompileError, DefinedFuncIndex, FunctionBodyData, ModuleTranslation, ModuleTypes, Tunables, + CompileError, DefinedFuncIndex, FuncIndex, FunctionBodyData, FunctionLoc, ModuleTranslation, + ModuleTypes, PrimaryMap, Tunables, WasmFunctionInfo, }; use winch_codegen::isa::TargetIsa; @@ -22,28 +25,24 @@ impl wasmtime_environ::Compiler for Compiler { _data: FunctionBodyData<'_>, _tunables: &Tunables, _types: &ModuleTypes, - ) -> Result, CompileError> { + ) -> Result<(WasmFunctionInfo, Box), CompileError> { todo!() } fn compile_host_to_wasm_trampoline( &self, _ty: &wasmtime_environ::WasmFuncType, - ) -> Result, CompileError> { + ) -> Result, CompileError> { todo!() } - fn emit_obj( + fn append_code( &self, - _module: &ModuleTranslation, - _funcs: wasmtime_environ::PrimaryMap>, - _trampolines: Vec>, + _obj: &mut Object<'static>, + _funcs: &[(String, Box)], _tunables: &Tunables, - _obj: &mut wasmtime_environ::object::write::Object<'static>, - ) -> Result<( - wasmtime_environ::PrimaryMap, - Vec, - )> { + _resolve_reloc: &dyn Fn(usize, FuncIndex) -> usize, + ) -> Result> { todo!() } @@ -52,7 +51,7 @@ impl wasmtime_environ::Compiler for Compiler { _ty: &wasmtime_environ::WasmFuncType, _host_fn: usize, _obj: &mut wasmtime_environ::object::write::Object<'static>, - ) -> Result<(wasmtime_environ::Trampoline, wasmtime_environ::Trampoline)> { + ) -> Result<(FunctionLoc, FunctionLoc)> { todo!() } @@ -80,4 +79,13 @@ impl wasmtime_environ::Compiler for Compiler { fn component_compiler(&self) -> &dyn wasmtime_environ::component::ComponentCompiler { todo!() } + + fn append_dwarf( + &self, + _obj: &mut Object<'_>, + _translation: &ModuleTranslation<'_>, + _funcs: &PrimaryMap, + ) -> Result<()> { + todo!() + } } diff --git a/src/commands/compile.rs b/src/commands/compile.rs index 801706aff2..0988e5749d 100644 --- a/src/commands/compile.rs +++ b/src/commands/compile.rs @@ -77,7 +77,7 @@ impl CompileCommand { ); } - let input = fs::read(&self.module).with_context(|| "failed to read input file")?; + let input = wat::parse_file(&self.module).with_context(|| "failed to read input file")?; let output = self.output.take().unwrap_or_else(|| { let mut output: PathBuf = self.module.file_name().unwrap().into(); @@ -85,6 +85,16 @@ impl CompileCommand { output }); + // If the component-model proposal is enabled and the binary we're + // compiling looks like a component, tested by sniffing the first 8 + // bytes with the current component model proposal. + #[cfg(feature = "component-model")] + { + if input.starts_with(b"\0asm\x0a\0\x01\0") { + fs::write(output, engine.precompile_component(&input)?)?; + return Ok(()); + } + } fs::write(output, engine.precompile_module(&input)?)?; Ok(()) diff --git a/tests/all/component_model.rs b/tests/all/component_model.rs index 1aad7658d6..e5a4787333 100644 --- a/tests/all/component_model.rs +++ b/tests/all/component_model.rs @@ -5,6 +5,7 @@ use std::iter; use wasmtime::component::Component; use wasmtime_component_util::REALLOC_AND_FREE; +mod aot; mod r#async; mod dynamic; mod func; diff --git a/tests/all/component_model/aot.rs b/tests/all/component_model/aot.rs new file mode 100644 index 0000000000..49f3937c48 --- /dev/null +++ b/tests/all/component_model/aot.rs @@ -0,0 +1,99 @@ +use anyhow::Result; +use wasmtime::component::{Component, Linker}; +use wasmtime::{Module, Store}; + +#[test] +fn module_component_mismatch() -> Result<()> { + let engine = super::engine(); + let module = Module::new(&engine, "(module)")?.serialize()?; + let component = Component::new(&engine, "(component)")?.serialize()?; + + unsafe { + assert!(Module::deserialize(&engine, &component).is_err()); + assert!(Component::deserialize(&engine, &module).is_err()); + } + + Ok(()) +} + +#[test] +fn bare_bones() -> Result<()> { + let engine = super::engine(); + let component = Component::new(&engine, "(component)")?.serialize()?; + assert_eq!(component, engine.precompile_component(b"(component)")?); + + let component = unsafe { Component::deserialize(&engine, &component)? }; + let mut store = Store::new(&engine, ()); + Linker::new(&engine).instantiate(&mut store, &component)?; + + Ok(()) +} + +#[test] +fn mildly_more_interesting() -> Result<()> { + let engine = super::engine(); + let component = Component::new( + &engine, + r#" + (component + (core module $a + (func (export "a") (result i32) + i32.const 100) + ) + (core instance $a (instantiate $a)) + + (core module $b + (import "a" "a" (func $import (result i32))) + (func (export "a") (result i32) + call $import + i32.const 3 + i32.add) + ) + (core instance $b (instantiate $b (with "a" (instance $a)))) + + (func (export "a") (result u32) + (canon lift (core func $b "a")) + ) + ) + "#, + )? + .serialize()?; + + let component = unsafe { Component::deserialize(&engine, &component)? }; + let mut store = Store::new(&engine, ()); + let instance = Linker::new(&engine).instantiate(&mut store, &component)?; + let func = instance.get_typed_func::<(), (u32,), _>(&mut store, "a")?; + assert_eq!(func.call(&mut store, ())?, (103,)); + + Ok(()) +} + +#[test] +fn deserialize_from_serialized() -> Result<()> { + let engine = super::engine(); + let buffer1 = Component::new(&engine, "(component (core module))")?.serialize()?; + let buffer2 = unsafe { Component::deserialize(&engine, &buffer1)?.serialize()? }; + assert!(buffer1 == buffer2); + Ok(()) +} + +// This specifically tests the current behavior that it's an error, but this can +// be made to work if necessary in the future. Currently the implementation of +// `serialize` is not conducive to easily implementing this feature and +// otherwise it's not seen as too important to implement. +#[test] +fn cannot_serialize_exported_module() -> Result<()> { + let engine = super::engine(); + let component = Component::new( + &engine, + r#"(component + (core module $m) + (export "" (core module $m)) + )"#, + )?; + let mut store = Store::new(&engine, ()); + let instance = Linker::new(&engine).instantiate(&mut store, &component)?; + let module = instance.get_module(&mut store, "").unwrap(); + assert!(module.serialize().is_err()); + Ok(()) +} diff --git a/tests/all/module_serialize.rs b/tests/all/module_serialize.rs index 475b256990..121afc3716 100644 --- a/tests/all/module_serialize.rs +++ b/tests/all/module_serialize.rs @@ -103,3 +103,15 @@ fn test_deserialize_from_file() -> Result<()> { Ok(()) } } + +#[test] +fn deserialize_from_serialized() -> Result<()> { + let engine = Engine::default(); + let buffer1 = serialize( + &engine, + "(module (func (export \"run\") (result i32) i32.const 42))", + )?; + let buffer2 = unsafe { Module::deserialize(&engine, &buffer1)?.serialize()? }; + assert!(buffer1 == buffer2); + Ok(()) +} diff --git a/winch/Cargo.toml b/winch/Cargo.toml index 7c08c07161..6689ce10d2 100644 --- a/winch/Cargo.toml +++ b/winch/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" [dependencies] winch-codegen = { workspace = true } wasmtime-environ = { workspace = true } -target-lexicon = { workspace = true } +target-lexicon = { workspace = true } anyhow = { workspace = true } wasmparser = { workspace = true } clap = { workspace = true }