diff --git a/crates/debug/src/lib.rs b/crates/debug/src/lib.rs index e831570cf0..bf1dd7c92b 100644 --- a/crates/debug/src/lib.rs +++ b/crates/debug/src/lib.rs @@ -3,117 +3,99 @@ #![allow(clippy::cast_ptr_alignment)] use anyhow::Error; -use faerie::{Artifact, Decl}; -use gimli::write::{Address, FrameTable}; +use faerie::{Artifact, Decl, SectionKind}; use more_asserts::assert_gt; use target_lexicon::BinaryFormat; -use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; -use wasmtime_environ::{Compilation, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges}; +use wasmtime_environ::isa::TargetIsa; pub use crate::read_debuginfo::{read_debuginfo, DebugInfoData, WasmFileInfo}; -pub use crate::transform::transform_dwarf; -pub use crate::write_debuginfo::{emit_dwarf, ResolvedSymbol, SymbolResolver}; +pub use crate::write_debuginfo::{emit_dwarf, DwarfSection}; mod gc; mod read_debuginfo; mod transform; mod write_debuginfo; -struct FunctionRelocResolver {} -impl SymbolResolver for FunctionRelocResolver { - fn resolve_symbol(&self, symbol: usize, addend: i64) -> ResolvedSymbol { - let name = format!("_wasm_function_{}", symbol); - ResolvedSymbol::Reloc { name, addend } +pub fn write_debugsections(obj: &mut Artifact, sections: Vec) -> Result<(), Error> { + let (bodies, relocs) = sections + .into_iter() + .map(|s| ((s.name.clone(), s.body), (s.name, s.relocs))) + .unzip::<_, _, Vec<_>, Vec<_>>(); + for (name, body) in bodies { + obj.declare_with(name, Decl::section(SectionKind::Debug), body)?; } -} - -fn create_frame_table<'a>( - isa: &dyn TargetIsa, - infos: impl Iterator>, -) -> Option { - let mut table = FrameTable::default(); - - let cie_id = table.add_cie(isa.create_systemv_cie()?); - - for (i, info) in infos.enumerate() { - if let Some(UnwindInfo::SystemV(info)) = info { - table.add_fde( - cie_id, - info.to_fde(Address::Symbol { - symbol: i, - addend: 0, - }), - ); + for (name, relocs) in relocs { + for reloc in relocs { + obj.link_with( + faerie::Link { + from: &name, + to: &reloc.target, + at: reloc.offset as u64, + }, + faerie::Reloc::Debug { + size: reloc.size, + addend: reloc.addend, + }, + )?; } } - Some(table) -} - -pub fn emit_debugsections( - obj: &mut Artifact, - vmctx_info: &ModuleVmctxInfo, - isa: &dyn TargetIsa, - debuginfo_data: &DebugInfoData, - at: &ModuleAddressMap, - ranges: &ValueLabelsRanges, - compilation: &Compilation, -) -> Result<(), Error> { - let resolver = FunctionRelocResolver {}; - let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?; - let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info)); - - emit_dwarf(obj, dwarf, &resolver, frame_table)?; Ok(()) } -struct ImageRelocResolver<'a> { - func_offsets: &'a Vec, -} - -impl<'a> SymbolResolver for ImageRelocResolver<'a> { - fn resolve_symbol(&self, symbol: usize, addend: i64) -> ResolvedSymbol { - let func_start = self.func_offsets[symbol]; - ResolvedSymbol::PhysicalAddress(func_start + addend as u64) +fn patch_dwarf_sections(sections: &mut [DwarfSection], funcs: &[*const u8]) { + for section in sections { + const FUNC_SYMBOL_PREFIX: &str = "_wasm_function_"; + for reloc in section.relocs.iter() { + if !reloc.target.starts_with(FUNC_SYMBOL_PREFIX) { + // Fixing only "all" section relocs -- all functions are merged + // into one blob. + continue; + } + let func_index = reloc.target[FUNC_SYMBOL_PREFIX.len()..] + .parse::() + .expect("func index"); + let target = (funcs[func_index] as u64).wrapping_add(reloc.addend as i64 as u64); + let entry_ptr = section.body + [reloc.offset as usize..reloc.offset as usize + reloc.size as usize] + .as_mut_ptr(); + unsafe { + match reloc.size { + 4 => std::ptr::write(entry_ptr as *mut u32, target as u32), + 8 => std::ptr::write(entry_ptr as *mut u64, target), + _ => panic!("unexpected reloc entry size"), + } + } + } + section + .relocs + .retain(|r| !r.target.starts_with(FUNC_SYMBOL_PREFIX)); } } -pub fn emit_debugsections_image( +pub fn write_debugsections_image( isa: &dyn TargetIsa, - debuginfo_data: &DebugInfoData, - vmctx_info: &ModuleVmctxInfo, - at: &ModuleAddressMap, - ranges: &ValueLabelsRanges, - funcs: &[(*const u8, usize)], - compilation: &Compilation, + mut sections: Vec, + code_region: (*const u8, usize), + funcs: &[*const u8], ) -> Result, Error> { - let func_offsets = &funcs - .iter() - .map(|(ptr, _)| *ptr as u64) - .collect::>(); let mut obj = Artifact::new(isa.triple().clone(), String::from("module")); - let resolver = ImageRelocResolver { func_offsets }; - let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?; - // Assuming all functions in the same code block, looking min/max of its range. + assert!(!code_region.0.is_null() && code_region.1 > 0); assert_gt!(funcs.len(), 0); - let mut segment_body: (usize, usize) = (!0, 0); - for (body_ptr, body_len) in funcs { - segment_body.0 = std::cmp::min(segment_body.0, *body_ptr as usize); - segment_body.1 = std::cmp::max(segment_body.1, *body_ptr as usize + body_len); - } - let segment_body = (segment_body.0 as *const u8, segment_body.1 - segment_body.0); - let body = unsafe { std::slice::from_raw_parts(segment_body.0, segment_body.1) }; + let body = unsafe { std::slice::from_raw_parts(code_region.0, code_region.1) }; obj.declare_with("all", Decl::function(), body.to_vec())?; - let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info)); - emit_dwarf(&mut obj, dwarf, &resolver, frame_table)?; + // Get DWARF sections and patch relocs + patch_dwarf_sections(&mut sections, funcs); + + write_debugsections(&mut obj, sections)?; // LLDB is too "magical" about mach-o, generating elf let mut bytes = obj.emit_as(BinaryFormat::Elf)?; // elf is still missing details... - convert_faerie_elf_to_loadable_file(&mut bytes, segment_body.0); + convert_faerie_elf_to_loadable_file(&mut bytes, code_region.0); // let mut file = ::std::fs::File::create(::std::path::Path::new("test.o")).expect("file"); // ::std::io::Write::write(&mut file, &bytes).expect("write"); @@ -170,7 +152,7 @@ fn convert_faerie_elf_to_loadable_file(bytes: &mut Vec, code_ptr: *const u8) } assert!(segment.is_none()); - // Functions was added at emit_debugsections_image as .text.all. + // Functions was added at write_debugsections_image as .text.all. // Patch vaddr, and save file location and its size. unsafe { *(bytes.as_ptr().offset(off + 0x10) as *mut u64) = code_ptr as u64; diff --git a/crates/debug/src/write_debuginfo.rs b/crates/debug/src/write_debuginfo.rs index 8fe80f0ceb..016f0d3413 100644 --- a/crates/debug/src/write_debuginfo.rs +++ b/crates/debug/src/write_debuginfo.rs @@ -1,84 +1,63 @@ -use faerie::artifact::{Decl, SectionKind}; -use faerie::*; +pub use crate::read_debuginfo::{read_debuginfo, DebugInfoData, WasmFileInfo}; +pub use crate::transform::transform_dwarf; use gimli::write::{Address, Dwarf, EndianVec, FrameTable, Result, Sections, Writer}; use gimli::{RunTimeEndian, SectionId}; +use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; +use wasmtime_environ::{Compilation, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges}; #[derive(Clone)] -struct DebugReloc { - offset: u32, - size: u8, - name: String, - addend: i64, +pub struct DwarfSectionReloc { + pub target: String, + pub offset: u32, + pub addend: i32, + pub size: u8, } -pub enum ResolvedSymbol { - PhysicalAddress(u64), - Reloc { name: String, addend: i64 }, +pub struct DwarfSection { + pub name: String, + pub body: Vec, + pub relocs: Vec, } -pub trait SymbolResolver { - fn resolve_symbol(&self, symbol: usize, addend: i64) -> ResolvedSymbol; -} - -pub fn emit_dwarf( - artifact: &mut Artifact, +fn emit_dwarf_sections( mut dwarf: Dwarf, - symbol_resolver: &dyn SymbolResolver, frames: Option, -) -> anyhow::Result<()> { - let endian = RunTimeEndian::Little; - - let mut sections = Sections::new(WriterRelocate::new(endian, symbol_resolver)); +) -> anyhow::Result> { + let mut sections = Sections::new(WriterRelocate::default()); dwarf.write(&mut sections)?; if let Some(frames) = frames { frames.write_debug_frame(&mut sections.debug_frame)?; } + + let mut result = Vec::new(); sections.for_each_mut(|id, s| -> anyhow::Result<()> { - artifact.declare_with( - id.name(), - Decl::section(SectionKind::Debug), - s.writer.take(), - )?; - Ok(()) - })?; - sections.for_each_mut(|id, s| -> anyhow::Result<()> { - for reloc in &s.relocs { - artifact.link_with( - faerie::Link { - from: id.name(), - to: &reloc.name, - at: u64::from(reloc.offset), - }, - faerie::Reloc::Debug { - size: reloc.size, - addend: reloc.addend as i32, - }, - )?; - } + let name = id.name().to_string(); + let body = s.writer.take(); + let mut relocs = vec![]; + ::std::mem::swap(&mut relocs, &mut s.relocs); + result.push(DwarfSection { name, body, relocs }); Ok(()) })?; - Ok(()) + Ok(result) } #[derive(Clone)] -pub struct WriterRelocate<'a> { - relocs: Vec, +pub struct WriterRelocate { + relocs: Vec, writer: EndianVec, - symbol_resolver: &'a dyn SymbolResolver, } -impl<'a> WriterRelocate<'a> { - pub fn new(endian: RunTimeEndian, symbol_resolver: &'a dyn SymbolResolver) -> Self { +impl Default for WriterRelocate { + fn default() -> Self { WriterRelocate { relocs: Vec::new(), - writer: EndianVec::new(endian), - symbol_resolver, + writer: EndianVec::new(RunTimeEndian::Little), } } } -impl<'a> Writer for WriterRelocate<'a> { +impl Writer for WriterRelocate { type Endian = RunTimeEndian; fn endian(&self) -> Self::Endian { @@ -101,31 +80,27 @@ impl<'a> Writer for WriterRelocate<'a> { match address { Address::Constant(val) => self.write_udata(val, size), Address::Symbol { symbol, addend } => { - match self.symbol_resolver.resolve_symbol(symbol, addend as i64) { - ResolvedSymbol::PhysicalAddress(addr) => self.write_udata(addr, size), - ResolvedSymbol::Reloc { name, addend } => { - let offset = self.len() as u64; - self.relocs.push(DebugReloc { - offset: offset as u32, - size, - name, - addend, - }); - self.write_udata(addend as u64, size) - } - } + let target = format!("_wasm_function_{}", symbol); + let offset = self.len() as u32; + self.relocs.push(DwarfSectionReloc { + target, + offset, + size, + addend: addend as i32, + }); + self.write_udata(addend as u64, size) } } } fn write_offset(&mut self, val: usize, section: SectionId, size: u8) -> Result<()> { let offset = self.len() as u32; - let name = section.name().to_string(); - self.relocs.push(DebugReloc { + let target = section.name().to_string(); + self.relocs.push(DwarfSectionReloc { + target, offset, size, - name, - addend: val as i64, + addend: val as i32, }); self.write_udata(val as u64, size) } @@ -137,13 +112,50 @@ impl<'a> Writer for WriterRelocate<'a> { section: SectionId, size: u8, ) -> Result<()> { - let name = section.name().to_string(); - self.relocs.push(DebugReloc { + let target = section.name().to_string(); + self.relocs.push(DwarfSectionReloc { + target, offset: offset as u32, size, - name, - addend: val as i64, + addend: val as i32, }); self.write_udata_at(offset, val as u64, size) } } + +fn create_frame_table<'a>( + isa: &dyn TargetIsa, + infos: impl Iterator>, +) -> Option { + let mut table = FrameTable::default(); + + let cie_id = table.add_cie(isa.create_systemv_cie()?); + + for (i, info) in infos.enumerate() { + if let Some(UnwindInfo::SystemV(info)) = info { + table.add_fde( + cie_id, + info.to_fde(Address::Symbol { + symbol: i, + addend: 0, + }), + ); + } + } + + Some(table) +} + +pub fn emit_dwarf( + isa: &dyn TargetIsa, + debuginfo_data: &DebugInfoData, + at: &ModuleAddressMap, + vmctx_info: &ModuleVmctxInfo, + ranges: &ValueLabelsRanges, + compilation: &Compilation, +) -> anyhow::Result> { + let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?; + let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info)); + let sections = emit_dwarf_sections(dwarf, frame_table)?; + Ok(sections) +} diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 134a8dca30..b7e0a3851f 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -6,20 +6,27 @@ use std::mem::ManuallyDrop; use std::{cmp, mem}; use wasmtime_environ::{ isa::{unwind::UnwindInfo, TargetIsa}, - Compilation, CompiledFunction, + Compilation, CompiledFunction, Relocation, Relocations, }; use wasmtime_runtime::{Mmap, VMFunctionBody}; +type CodeMemoryRelocations = Vec<(u32, Vec)>; + struct CodeMemoryEntry { mmap: ManuallyDrop, registry: ManuallyDrop, + relocs: CodeMemoryRelocations, } impl CodeMemoryEntry { fn with_capacity(cap: usize) -> Result { let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?); let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize)); - Ok(Self { mmap, registry }) + Ok(Self { + mmap, + registry, + relocs: vec![], + }) } fn range(&self) -> (usize, usize) { @@ -66,16 +73,19 @@ impl CodeMemory { /// Allocate a continuous memory block for a single compiled function. /// TODO: Reorganize the code that calls this to emit code directly into the /// mmap region rather than into a Vec that we need to copy in. - pub fn allocate_for_function( + pub fn allocate_for_function<'a>( &mut self, - func: &CompiledFunction, + func: &'a CompiledFunction, + relocs: impl Iterator, ) -> Result<&mut [VMFunctionBody], String> { let size = Self::function_allocation_size(func); - let (buf, registry, start) = self.allocate(size)?; + let (buf, registry, start, m_relocs) = self.allocate(size)?; let (_, _, vmfunc) = Self::copy_function(func, start as u32, buf, registry); + Self::copy_relocs(m_relocs, start as u32, relocs); + Ok(vmfunc) } @@ -83,20 +93,23 @@ impl CodeMemory { pub fn allocate_for_compilation( &mut self, compilation: &Compilation, + relocations: &Relocations, ) -> Result, String> { let total_len = compilation .into_iter() .fold(0, |acc, func| acc + Self::function_allocation_size(func)); - let (mut buf, registry, start) = self.allocate(total_len)?; + let (mut buf, registry, start, m_relocs) = self.allocate(total_len)?; let mut result = Vec::with_capacity(compilation.len()); let mut start = start as u32; - for func in compilation.into_iter() { + for (func, relocs) in compilation.into_iter().zip(relocations.values()) { let (next_start, next_buf, vmfunc) = Self::copy_function(func, start, buf, registry); result.push(vmfunc); + Self::copy_relocs(m_relocs, start, relocs.iter()); + start = next_start; buf = next_buf; } @@ -112,6 +125,7 @@ impl CodeMemory { for CodeMemoryEntry { mmap: m, registry: r, + relocs, } in &mut self.entries[self.published..] { // Remove write access to the pages due to the relocation fixups. @@ -124,6 +138,10 @@ impl CodeMemory { } .expect("unable to make memory readonly and executable"); } + + // Relocs data in not needed anymore -- clearing. + // TODO use relocs to serialize the published code. + relocs.clear(); } self.published = self.entries.len(); @@ -141,7 +159,18 @@ impl CodeMemory { /// * The offset within the current mmap that the slice starts at /// /// TODO: Add an alignment flag. - fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize), String> { + fn allocate( + &mut self, + size: usize, + ) -> Result< + ( + &mut [u8], + &mut UnwindRegistry, + usize, + &mut CodeMemoryRelocations, + ), + String, + > { assert!(size > 0); if match &self.current { @@ -160,6 +189,7 @@ impl CodeMemory { &mut e.mmap.as_mut_slice()[old_position..self.position], &mut e.registry, old_position, + &mut e.relocs, )) } @@ -176,6 +206,14 @@ impl CodeMemory { } } + fn copy_relocs<'a>( + entry_relocs: &'_ mut CodeMemoryRelocations, + start: u32, + relocs: impl Iterator, + ) { + entry_relocs.push((start, relocs.cloned().collect())); + } + /// Copies the data of the compiled function to the given buffer. /// /// This will also add the function to the current unwind registry. @@ -249,4 +287,19 @@ impl CodeMemory { .iter() .map(|entry| entry.range()) } + + /// Returns all relocations for the unpublished memory. + pub fn unpublished_relocations<'a>( + &'a self, + ) -> impl Iterator + 'a { + self.entries[self.published..] + .iter() + .chain(self.current.iter()) + .flat_map(|entry| { + entry.relocs.iter().flat_map(move |(start, relocs)| { + let base_ptr = unsafe { entry.mmap.as_ptr().add(*start as usize) }; + relocs.iter().map(move |r| (base_ptr, r)) + }) + }) + } } diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index 7b7408ad66..a53656e99f 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -8,15 +8,14 @@ use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::Context; use cranelift_codegen::{binemit, ir}; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; -use std::collections::HashMap; -use wasmtime_debug::{emit_debugsections_image, DebugInfoData}; +use wasmtime_debug::{emit_dwarf, DebugInfoData, DwarfSection}; use wasmtime_environ::entity::{EntityRef, PrimaryMap}; use wasmtime_environ::isa::{TargetFrontendConfig, TargetIsa}; use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex, SignatureIndex}; use wasmtime_environ::{ - CacheConfig, CompileError, CompiledFunction, Compiler as _C, ModuleAddressMap, + CacheConfig, CompileError, CompiledFunction, Compiler as _C, Module, ModuleAddressMap, ModuleMemoryOffset, ModuleTranslation, ModuleVmctxInfo, Relocation, RelocationTarget, - Relocations, Traps, Tunables, VMOffsets, + Relocations, Traps, Tunables, VMOffsets, ValueLabelsRanges, }; use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline}; @@ -71,15 +70,73 @@ fn _assert_compiler_send_sync() { _assert::(); } +fn transform_dwarf_data( + isa: &dyn TargetIsa, + module: &Module, + debug_data: &DebugInfoData, + address_transform: &ModuleAddressMap, + value_ranges: &ValueLabelsRanges, + stack_slots: PrimaryMap, + compilation: &wasmtime_environ::Compilation, +) -> Result, SetupError> { + let target_config = isa.frontend_config(); + let ofs = VMOffsets::new(target_config.pointer_bytes(), &module.local); + + let module_vmctx_info = { + ModuleVmctxInfo { + memory_offset: if ofs.num_imported_memories > 0 { + ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) + } else if ofs.num_defined_memories > 0 { + ModuleMemoryOffset::Defined( + ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)), + ) + } else { + ModuleMemoryOffset::None + }, + stack_slots, + } + }; + emit_dwarf( + isa, + debug_data, + &address_transform, + &module_vmctx_info, + &value_ranges, + &compilation, + ) + .map_err(SetupError::DebugInfo) +} + +fn get_code_range( + compilation: &wasmtime_environ::Compilation, + finished_functions: &PrimaryMap, +) -> (*const u8, usize) { + if finished_functions.is_empty() { + return (::std::ptr::null(), 0); + } + // Assuming all functions in the same code block, looking min/max of its range. + let (start, end) = finished_functions.iter().fold::<(usize, usize), _>( + (!0, 0), + |(start, end), (i, body_ptr)| { + let body_ptr = (*body_ptr) as *const u8 as usize; + let body_len = compilation.get(i).body.len(); + ( + ::std::cmp::min(start, body_ptr), + ::std::cmp::max(end, body_ptr + body_len), + ) + }, + ); + (start as *const u8, end - start) +} + #[allow(missing_docs)] pub struct Compilation { pub code_memory: CodeMemory, pub finished_functions: PrimaryMap, - pub relocations: Relocations, + pub code_range: (*const u8, usize), pub trampolines: PrimaryMap, - pub trampoline_relocations: HashMap>, pub jt_offsets: PrimaryMap, - pub dbg_image: Option>, + pub dwarf_sections: Vec, pub traps: Traps, pub address_transform: ModuleAddressMap, } @@ -130,15 +187,29 @@ impl Compiler { } .map_err(SetupError::Compile)?; + let dwarf_sections = if debug_data.is_some() && !compilation.is_empty() { + transform_dwarf_data( + &*self.isa, + &translation.module, + debug_data.as_ref().unwrap(), + &address_transform, + &value_ranges, + stack_slots, + &compilation, + )? + } else { + vec![] + }; + // Allocate all of the compiled functions into executable memory, // copying over their contents. - let finished_functions = - allocate_functions(&mut code_memory, &compilation).map_err(|message| { - SetupError::Instantiate(InstantiationError::Resource(format!( - "failed to allocate memory for functions: {}", - message - ))) - })?; + let finished_functions = allocate_functions(&mut code_memory, &compilation, &relocations) + .map_err(|message| { + SetupError::Instantiate(InstantiationError::Resource(format!( + "failed to allocate memory for functions: {}", + message + ))) + })?; // Eagerly generate a entry trampoline for every type signature in the // module. This should be "relatively lightweight" for most modules and @@ -146,9 +217,8 @@ impl Compiler { // tables) have a trampoline when invoked through the wasmtime API. let mut cx = FunctionBuilderContext::new(); let mut trampolines = PrimaryMap::new(); - let mut trampoline_relocations = HashMap::new(); - for (index, (_, native_sig)) in translation.module.local.signatures.iter() { - let (trampoline, relocations) = make_trampoline( + for (_, (_, native_sig)) in translation.module.local.signatures.iter() { + let trampoline = make_trampoline( &*self.isa, &mut code_memory, &mut cx, @@ -156,66 +226,18 @@ impl Compiler { std::mem::size_of::(), )?; trampolines.push(trampoline); - - // Typically trampolines do not have relocations, so if one does - // show up be sure to log it in case anyone's listening and there's - // an accidental bug. - if relocations.len() > 0 { - log::info!("relocations found in trampoline for {:?}", native_sig); - trampoline_relocations.insert(index, relocations); - } } - // Translate debug info (DWARF) only if at least one function is present. - let dbg_image = if debug_data.is_some() && !finished_functions.is_empty() { - let target_config = self.isa.frontend_config(); - let ofs = VMOffsets::new(target_config.pointer_bytes(), &translation.module.local); - - let mut funcs = Vec::new(); - for (i, allocated) in finished_functions.into_iter() { - let ptr = (*allocated) as *const u8; - let body_len = compilation.get(i).body.len(); - funcs.push((ptr, body_len)); - } - let module_vmctx_info = { - ModuleVmctxInfo { - memory_offset: if ofs.num_imported_memories > 0 { - ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0))) - } else if ofs.num_defined_memories > 0 { - ModuleMemoryOffset::Defined( - ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)), - ) - } else { - ModuleMemoryOffset::None - }, - stack_slots, - } - }; - let bytes = emit_debugsections_image( - &*self.isa, - debug_data.as_ref().unwrap(), - &module_vmctx_info, - &address_transform, - &value_ranges, - &funcs, - &compilation, - ) - .map_err(SetupError::DebugInfo)?; - Some(bytes) - } else { - None - }; - let jt_offsets = compilation.get_jt_offsets(); + let code_range = get_code_range(&compilation, &finished_functions); Ok(Compilation { code_memory, finished_functions, - relocations, + code_range, trampolines, - trampoline_relocations, jt_offsets, - dbg_image, + dwarf_sections, traps, address_transform, }) @@ -229,7 +251,7 @@ pub fn make_trampoline( fn_builder_ctx: &mut FunctionBuilderContext, signature: &ir::Signature, value_size: usize, -) -> Result<(VMTrampoline, Vec), SetupError> { +) -> Result { let pointer_type = isa.pointer_type(); let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv); @@ -337,28 +359,29 @@ pub fn make_trampoline( })?; let ptr = code_memory - .allocate_for_function(&CompiledFunction { - body: code_buf, - jt_offsets: context.func.jt_offsets, - unwind_info, - }) + .allocate_for_function( + &CompiledFunction { + body: code_buf, + jt_offsets: context.func.jt_offsets, + unwind_info, + }, + reloc_sink.relocs.iter(), + ) .map_err(|message| SetupError::Instantiate(InstantiationError::Resource(message)))? .as_ptr(); - Ok(( - unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) }, - reloc_sink.relocs, - )) + Ok(unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) }) } fn allocate_functions( code_memory: &mut CodeMemory, compilation: &wasmtime_environ::Compilation, + relocations: &Relocations, ) -> Result, String> { if compilation.is_empty() { return Ok(PrimaryMap::new()); } - let fat_ptrs = code_memory.allocate_for_compilation(compilation)?; + let fat_ptrs = code_memory.allocate_for_compilation(compilation, relocations)?; // Second, create a PrimaryMap from result vector of pointers. let mut result = PrimaryMap::with_capacity(compilation.len()); @@ -374,10 +397,17 @@ fn allocate_functions( /// this `RelocSink` just asserts that it doesn't recieve most of them, but /// handles libcall ones. #[derive(Default)] -struct RelocSink { +pub struct RelocSink { relocs: Vec, } +impl RelocSink { + /// Returns collected relocations. + pub fn relocs(&self) -> &[Relocation] { + &self.relocs + } +} + impl binemit::RelocSink for RelocSink { fn reloc_block( &mut self, diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index c8cdc937a9..12152c7084 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -4,21 +4,21 @@ //! steps. use crate::code_memory::CodeMemory; -use crate::compiler::Compiler; +use crate::compiler::{Compilation, Compiler}; use crate::imports::resolve_imports; use crate::link::link_module; use crate::resolver::Resolver; use std::any::Any; use std::collections::HashMap; -use std::io::Write; use std::sync::Arc; use thiserror::Error; -use wasmtime_debug::read_debuginfo; +use wasmtime_debug::{read_debuginfo, write_debugsections_image, DwarfSection}; use wasmtime_environ::entity::{BoxedSlice, PrimaryMap}; +use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex}; use wasmtime_environ::{ CompileError, DataInitializer, DataInitializerLocation, Module, ModuleAddressMap, - ModuleEnvironment, Traps, + ModuleEnvironment, ModuleTranslation, Traps, }; use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::VMInterrupts; @@ -91,41 +91,53 @@ impl CompiledModule { debug_data = Some(read_debuginfo(&data)?); } - let compilation = compiler.compile(&translation, debug_data)?; + let Compilation { + mut code_memory, + finished_functions, + code_range, + trampolines, + jt_offsets, + dwarf_sections, + traps, + address_transform, + } = compiler.compile(&translation, debug_data)?; - let module = translation.module; + let ModuleTranslation { + module, + data_initializers, + .. + } = translation; - link_module(&module, &compilation); + link_module(&mut code_memory, &module, &finished_functions, &jt_offsets); // Make all code compiled thus far executable. - let mut code_memory = compilation.code_memory; code_memory.publish(compiler.isa()); - let data_initializers = translation - .data_initializers + let data_initializers = data_initializers .into_iter() .map(OwnedDataInitializer::new) .collect::>() .into_boxed_slice(); - // Initialize profiler and load the wasm module - profiler.module_load( - &module, - &compilation.finished_functions, - compilation.dbg_image.as_deref(), - ); + // Register GDB JIT images; initialize profiler and load the wasm module. + let dbg_jit_registration = if !dwarf_sections.is_empty() { + let bytes = create_dbg_image( + dwarf_sections, + compiler.isa(), + code_range, + &finished_functions, + )?; + + profiler.module_load(&module, &finished_functions, Some(&bytes)); - let dbg_jit_registration = if let Some(img) = compilation.dbg_image { - let mut bytes = Vec::new(); - bytes.write_all(&img).expect("all written"); let reg = GdbJitImageRegistration::register(bytes); Some(reg) } else { + profiler.module_load(&module, &finished_functions, None); None }; - let finished_functions = - FinishedFunctions(compilation.finished_functions.into_boxed_slice()); + let finished_functions = FinishedFunctions(finished_functions.into_boxed_slice()); Ok(Self { module: Arc::new(module), @@ -134,10 +146,10 @@ impl CompiledModule { dbg_jit_registration, }), finished_functions, - trampolines: compilation.trampolines, + trampolines, data_initializers, - traps: compilation.traps, - address_transform: compilation.address_transform, + traps, + address_transform, }) } @@ -256,3 +268,17 @@ impl OwnedDataInitializer { } } } + +fn create_dbg_image( + dwarf_sections: Vec, + isa: &dyn TargetIsa, + code_range: (*const u8, usize), + finished_functions: &PrimaryMap, +) -> Result, SetupError> { + let funcs = finished_functions + .values() + .map(|allocated: &*mut [VMFunctionBody]| (*allocated) as *const u8) + .collect::>(); + write_debugsections_image(isa, dwarf_sections, code_range, &funcs) + .map_err(SetupError::DebugInfo) +} diff --git a/crates/jit/src/link.rs b/crates/jit/src/link.rs index cc3044b1a5..f344d4ab69 100644 --- a/crates/jit/src/link.rs +++ b/crates/jit/src/link.rs @@ -1,8 +1,11 @@ //! Linking for JIT-compiled code. -use crate::Compilation; +use crate::CodeMemory; use cranelift_codegen::binemit::Reloc; +use cranelift_codegen::ir::JumpTableOffsets; use std::ptr::{read_unaligned, write_unaligned}; +use wasmtime_environ::entity::PrimaryMap; +use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::{Module, Relocation, RelocationTarget}; use wasmtime_runtime::libcalls; use wasmtime_runtime::VMFunctionBody; @@ -10,27 +13,22 @@ use wasmtime_runtime::VMFunctionBody; /// Links a module that has been compiled with `compiled_module` in `wasmtime-environ`. /// /// Performs all required relocations inside the function code, provided the necessary metadata. -pub fn link_module(module: &Module, compilation: &Compilation) { - for (i, function_relocs) in compilation.relocations.iter() { - for r in function_relocs.iter() { - let fatptr: *const [VMFunctionBody] = compilation.finished_functions[i]; - let body = fatptr as *const VMFunctionBody; - apply_reloc(module, compilation, body, r); - } - } - - for (i, function_relocs) in compilation.trampoline_relocations.iter() { - for r in function_relocs.iter() { - println!("tramopline relocation"); - let body = compilation.trampolines[*i] as *const VMFunctionBody; - apply_reloc(module, compilation, body, r); - } +pub fn link_module( + code_memory: &mut CodeMemory, + module: &Module, + finished_functions: &PrimaryMap, + jt_offsets: &PrimaryMap, +) { + for (fatptr, r) in code_memory.unpublished_relocations() { + let body = fatptr as *const VMFunctionBody; + apply_reloc(module, finished_functions, jt_offsets, body, r); } } fn apply_reloc( module: &Module, - compilation: &Compilation, + finished_functions: &PrimaryMap, + jt_offsets: &PrimaryMap, body: *const VMFunctionBody, r: &Relocation, ) { @@ -38,7 +36,7 @@ fn apply_reloc( let target_func_address: usize = match r.reloc_target { RelocationTarget::UserFunc(index) => match module.local.defined_func_index(index) { Some(f) => { - let fatptr: *const [VMFunctionBody] = compilation.finished_functions[f]; + let fatptr: *const [VMFunctionBody] = finished_functions[f]; fatptr as *const VMFunctionBody as usize } None => panic!("direct call to import"), @@ -67,12 +65,11 @@ fn apply_reloc( RelocationTarget::JumpTable(func_index, jt) => { match module.local.defined_func_index(func_index) { Some(f) => { - let offset = *compilation - .jt_offsets + let offset = *jt_offsets .get(f) .and_then(|ofs| ofs.get(jt)) .expect("func jump table"); - let fatptr: *const [VMFunctionBody] = compilation.finished_functions[f]; + let fatptr: *const [VMFunctionBody] = finished_functions[f]; fatptr as *const VMFunctionBody as usize + offset as usize } None => panic!("func index of jump table"), diff --git a/crates/jit/src/trampoline.rs b/crates/jit/src/trampoline.rs index 7ece3e51dc..26011bd5c5 100644 --- a/crates/jit/src/trampoline.rs +++ b/crates/jit/src/trampoline.rs @@ -10,49 +10,7 @@ pub use cranelift_codegen::Context; pub use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; pub mod binemit { + pub use crate::compiler::RelocSink as TrampolineRelocSink; pub use cranelift_codegen::binemit::NullTrapSink; pub use cranelift_codegen::binemit::{CodeOffset, NullStackmapSink, TrapSink}; - - use cranelift_codegen::{binemit, ir}; - - /// We don't expect trampoline compilation to produce any relocations, so - /// this `RelocSink` just asserts that it doesn't recieve any. - pub struct TrampolineRelocSink {} - - impl binemit::RelocSink for TrampolineRelocSink { - fn reloc_block( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _block_offset: binemit::CodeOffset, - ) { - panic!("trampoline compilation should not produce block relocs"); - } - fn reloc_external( - &mut self, - _offset: binemit::CodeOffset, - _srcloc: ir::SourceLoc, - _reloc: binemit::Reloc, - _name: &ir::ExternalName, - _addend: binemit::Addend, - ) { - panic!("trampoline compilation should not produce external symbol relocs"); - } - fn reloc_constant( - &mut self, - _code_offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _constant_offset: ir::ConstantOffset, - ) { - panic!("trampoline compilation should not produce constant relocs"); - } - fn reloc_jt( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _jt: ir::JumpTable, - ) { - panic!("trampoline compilation should not produce jump table relocs"); - } - } } diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 15a80c6cd8..b4a614357b 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -172,7 +172,7 @@ fn make_trampoline( } let mut code_buf: Vec = Vec::new(); - let mut reloc_sink = binemit::TrampolineRelocSink {}; + let mut reloc_sink = binemit::TrampolineRelocSink::default(); let mut trap_sink = binemit::NullTrapSink {}; let mut stackmap_sink = binemit::NullStackmapSink {}; context @@ -192,11 +192,14 @@ fn make_trampoline( .expect("create unwind information"); code_memory - .allocate_for_function(&CompiledFunction { - body: code_buf, - jt_offsets: context.func.jt_offsets, - unwind_info, - }) + .allocate_for_function( + &CompiledFunction { + body: code_buf, + jt_offsets: context.func.jt_offsets, + unwind_info, + }, + reloc_sink.relocs().iter(), + ) .expect("allocate_for_function") } @@ -239,14 +242,13 @@ pub fn create_handle_with_function( // ... and then we also need a trampoline with the standard "trampoline ABI" // which enters into the ABI specified by `ft`. Note that this is only used // if `Func::call` is called on an object created by `Func::new`. - let (trampoline, relocations) = wasmtime_jit::make_trampoline( + let trampoline = wasmtime_jit::make_trampoline( &*isa, &mut code_memory, &mut fn_builder_ctx, &sig, mem::size_of::(), )?; - assert!(relocations.is_empty()); let sig_id = store.register_signature(ft.to_wasm_func_type(), sig); trampolines.insert(sig_id, trampoline); diff --git a/src/obj.rs b/src/obj.rs index 765a185d53..cfa186aae6 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, bail, Context as _, Result}; use faerie::Artifact; use target_lexicon::Triple; use wasmtime::Strategy; -use wasmtime_debug::{emit_debugsections, read_debuginfo}; +use wasmtime_debug::{emit_dwarf, read_debuginfo, write_debugsections}; #[cfg(feature = "lightbeam")] use wasmtime_environ::Lightbeam; use wasmtime_environ::{ @@ -113,16 +113,16 @@ pub fn compile_to_obj( if debug_info { let debug_data = read_debuginfo(wasm).context("failed to emit DWARF")?; - emit_debugsections( - &mut obj, - &module_vmctx_info, + let sections = emit_dwarf( &*isa, &debug_data, &address_transform, + &module_vmctx_info, &value_ranges, &compilation, ) .context("failed to emit debug sections")?; + write_debugsections(&mut obj, sections).context("failed to emit debug sections")?; } Ok(obj) }