T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor CompiledModule to separate compile and linking stages (#1831)

Browse Source 
* Refactor how relocs are stored and handled

* refactor CompiledModule::instantiate and link_module

* Refactor DWARF creation: split generation and serialization

* Separate DWARF data transform from instantiation

* rm LinkContext
This commit is contained in:
Yury Delendik
2020-06-09 15:09:48 -05:00
committed by GitHub
parent ac87ed12bd
commit 4ebbcb82a9
9 changed files with 400 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
crates/debug/src/lib.rs
View File

@@ -3,117 +3,99 @@
#![allow(clippy::cast_ptr_alignment)]
use anyhow::Error;
use faerie::{Artifact, Decl};
use gimli::write::{Address, FrameTable};
use faerie::{Artifact, Decl, SectionKind};
use more_asserts::assert_gt;
use target_lexicon::BinaryFormat;
use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa};
use wasmtime_environ::{Compilation, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges};
use wasmtime_environ::isa::TargetIsa;
pub use crate::read_debuginfo::{read_debuginfo, DebugInfoData, WasmFileInfo};
pub use crate::transform::transform_dwarf;
pub use crate::write_debuginfo::{emit_dwarf, ResolvedSymbol, SymbolResolver};
pub use crate::write_debuginfo::{emit_dwarf, DwarfSection};
mod gc;
mod read_debuginfo;
mod transform;
mod write_debuginfo;
struct FunctionRelocResolver {}
impl SymbolResolver for FunctionRelocResolver {
fn resolve_symbol(&self, symbol: usize, addend: i64) -> ResolvedSymbol {
let name = format!("_wasm_function_{}", symbol);
ResolvedSymbol::Reloc { name, addend }
pub fn write_debugsections(obj: &mut Artifact, sections: Vec<DwarfSection>) -> Result<(), Error> {
let (bodies, relocs) = sections
.into_iter()
.map(|s| ((s.name.clone(), s.body), (s.name, s.relocs)))
.unzip::<_, _, Vec<_>, Vec<_>>();
for (name, body) in bodies {
obj.declare_with(name, Decl::section(SectionKind::Debug), body)?;
}
for (name, relocs) in relocs {
for reloc in relocs {
obj.link_with(
faerie::Link {
from: &name,
to: &reloc.target,
at: reloc.offset as u64,
},
faerie::Reloc::Debug {
size: reloc.size,
addend: reloc.addend,
},
)?;
}
}
fn create_frame_table<'a>(
isa: &dyn TargetIsa,
infos: impl Iterator<Item = &'a Option<UnwindInfo>>,
) -> Option<FrameTable> {
let mut table = FrameTable::default();
let cie_id = table.add_cie(isa.create_systemv_cie()?);
for (i, info) in infos.enumerate() {
if let Some(UnwindInfo::SystemV(info)) = info {
table.add_fde(
cie_id,
info.to_fde(Address::Symbol {
symbol: i,
addend: 0,
}),
);
}
}
Some(table)
}
pub fn emit_debugsections(
obj: &mut Artifact,
vmctx_info: &ModuleVmctxInfo,
isa: &dyn TargetIsa,
debuginfo_data: &DebugInfoData,
at: &ModuleAddressMap,
ranges: &ValueLabelsRanges,
compilation: &Compilation,
) -> Result<(), Error> {
let resolver = FunctionRelocResolver {};
let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?;
let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info));
emit_dwarf(obj, dwarf, &resolver, frame_table)?;
Ok(())
}
struct ImageRelocResolver<'a> {
func_offsets: &'a Vec<u64>,
fn patch_dwarf_sections(sections: &mut [DwarfSection], funcs: &[*const u8]) {
for section in sections {
const FUNC_SYMBOL_PREFIX: &str = "_wasm_function_";
for reloc in section.relocs.iter() {
if !reloc.target.starts_with(FUNC_SYMBOL_PREFIX) {
// Fixing only "all" section relocs -- all functions are merged
// into one blob.
continue;
}
impl<'a> SymbolResolver for ImageRelocResolver<'a> {
fn resolve_symbol(&self, symbol: usize, addend: i64) -> ResolvedSymbol {
let func_start = self.func_offsets[symbol];
ResolvedSymbol::PhysicalAddress(func_start + addend as u64)
let func_index = reloc.target[FUNC_SYMBOL_PREFIX.len()..]
.parse::<usize>()
.expect("func index");
let target = (funcs[func_index] as u64).wrapping_add(reloc.addend as i64 as u64);
let entry_ptr = section.body
[reloc.offset as usize..reloc.offset as usize + reloc.size as usize]
.as_mut_ptr();
unsafe {
match reloc.size {
4 => std::ptr::write(entry_ptr as *mut u32, target as u32),
8 => std::ptr::write(entry_ptr as *mut u64, target),
_ => panic!("unexpected reloc entry size"),
}
}
}
section
.relocs
.retain(|r| !r.target.starts_with(FUNC_SYMBOL_PREFIX));
}
}
pub fn emit_debugsections_image(
pub fn write_debugsections_image(
isa: &dyn TargetIsa,
debuginfo_data: &DebugInfoData,
vmctx_info: &ModuleVmctxInfo,
at: &ModuleAddressMap,
ranges: &ValueLabelsRanges,
funcs: &[(*const u8, usize)],
compilation: &Compilation,
mut sections: Vec<DwarfSection>,
code_region: (*const u8, usize),
funcs: &[*const u8],
) -> Result<Vec<u8>, Error> {
let func_offsets = &funcs
.iter()
.map(|(ptr, _)| *ptr as u64)
.collect::<Vec<u64>>();
let mut obj = Artifact::new(isa.triple().clone(), String::from("module"));
let resolver = ImageRelocResolver { func_offsets };
let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?;
// Assuming all functions in the same code block, looking min/max of its range.
assert!(!code_region.0.is_null() && code_region.1 > 0);
assert_gt!(funcs.len(), 0);
let mut segment_body: (usize, usize) = (!0, 0);
for (body_ptr, body_len) in funcs {
segment_body.0 = std::cmp::min(segment_body.0, *body_ptr as usize);
segment_body.1 = std::cmp::max(segment_body.1, *body_ptr as usize + body_len);
}
let segment_body = (segment_body.0 as *const u8, segment_body.1 - segment_body.0);
let body = unsafe { std::slice::from_raw_parts(segment_body.0, segment_body.1) };
let body = unsafe { std::slice::from_raw_parts(code_region.0, code_region.1) };
obj.declare_with("all", Decl::function(), body.to_vec())?;
let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info));
emit_dwarf(&mut obj, dwarf, &resolver, frame_table)?;
// Get DWARF sections and patch relocs
patch_dwarf_sections(&mut sections, funcs);
write_debugsections(&mut obj, sections)?;
// LLDB is too "magical" about mach-o, generating elf
let mut bytes = obj.emit_as(BinaryFormat::Elf)?;
// elf is still missing details...
convert_faerie_elf_to_loadable_file(&mut bytes, segment_body.0);
convert_faerie_elf_to_loadable_file(&mut bytes, code_region.0);
// let mut file = ::std::fs::File::create(::std::path::Path::new("test.o")).expect("file");
// ::std::io::Write::write(&mut file, &bytes).expect("write");
@@ -170,7 +152,7 @@ fn convert_faerie_elf_to_loadable_file(bytes: &mut Vec<u8>, code_ptr: *const u8)
}
assert!(segment.is_none());
// Functions was added at emit_debugsections_image as .text.all.
// Functions was added at write_debugsections_image as .text.all.
// Patch vaddr, and save file location and its size.
unsafe {
*(bytes.as_ptr().offset(off + 0x10) as *mut u64) = code_ptr as u64;

150
crates/debug/src/write_debuginfo.rs
View File

@@ -1,84 +1,63 @@
use faerie::artifact::{Decl, SectionKind};
use faerie::*;
pub use crate::read_debuginfo::{read_debuginfo, DebugInfoData, WasmFileInfo};
pub use crate::transform::transform_dwarf;
use gimli::write::{Address, Dwarf, EndianVec, FrameTable, Result, Sections, Writer};
use gimli::{RunTimeEndian, SectionId};
use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa};
use wasmtime_environ::{Compilation, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges};
#[derive(Clone)]
struct DebugReloc {
offset: u32,
size: u8,
name: String,
addend: i64,
pub struct DwarfSectionReloc {
pub target: String,
pub offset: u32,
pub addend: i32,
pub size: u8,
}
pub enum ResolvedSymbol {
PhysicalAddress(u64),
Reloc { name: String, addend: i64 },
pub struct DwarfSection {
pub name: String,
pub body: Vec<u8>,
pub relocs: Vec<DwarfSectionReloc>,
}
pub trait SymbolResolver {
fn resolve_symbol(&self, symbol: usize, addend: i64) -> ResolvedSymbol;
}
pub fn emit_dwarf(
artifact: &mut Artifact,
fn emit_dwarf_sections(
mut dwarf: Dwarf,
symbol_resolver: &dyn SymbolResolver,
frames: Option<FrameTable>,
) -> anyhow::Result<()> {
let endian = RunTimeEndian::Little;
let mut sections = Sections::new(WriterRelocate::new(endian, symbol_resolver));
) -> anyhow::Result<Vec<DwarfSection>> {
let mut sections = Sections::new(WriterRelocate::default());
dwarf.write(&mut sections)?;
if let Some(frames) = frames {
frames.write_debug_frame(&mut sections.debug_frame)?;
}
let mut result = Vec::new();
sections.for_each_mut(|id, s| -> anyhow::Result<()> {
artifact.declare_with(
id.name(),
Decl::section(SectionKind::Debug),
s.writer.take(),
)?;
Ok(())
})?;
sections.for_each_mut(|id, s| -> anyhow::Result<()> {
for reloc in &s.relocs {
artifact.link_with(
faerie::Link {
from: id.name(),
to: &reloc.name,
at: u64::from(reloc.offset),
},
faerie::Reloc::Debug {
size: reloc.size,
addend: reloc.addend as i32,
},
)?;
}
let name = id.name().to_string();
let body = s.writer.take();
let mut relocs = vec![];
::std::mem::swap(&mut relocs, &mut s.relocs);
result.push(DwarfSection { name, body, relocs });
Ok(())
})?;
Ok(())
Ok(result)
}
#[derive(Clone)]
pub struct WriterRelocate<'a> {
relocs: Vec<DebugReloc>,
pub struct WriterRelocate {
relocs: Vec<DwarfSectionReloc>,
writer: EndianVec<RunTimeEndian>,
symbol_resolver: &'a dyn SymbolResolver,
}
impl<'a> WriterRelocate<'a> {
pub fn new(endian: RunTimeEndian, symbol_resolver: &'a dyn SymbolResolver) -> Self {
impl Default for WriterRelocate {
fn default() -> Self {
WriterRelocate {
relocs: Vec::new(),
writer: EndianVec::new(endian),
symbol_resolver,
writer: EndianVec::new(RunTimeEndian::Little),
}
}
}
impl<'a> Writer for WriterRelocate<'a> {
impl Writer for WriterRelocate {
type Endian = RunTimeEndian;
fn endian(&self) -> Self::Endian {
@@ -101,31 +80,27 @@ impl<'a> Writer for WriterRelocate<'a> {
match address {
Address::Constant(val) => self.write_udata(val, size),
Address::Symbol { symbol, addend } => {
match self.symbol_resolver.resolve_symbol(symbol, addend as i64) {
ResolvedSymbol::PhysicalAddress(addr) => self.write_udata(addr, size),
ResolvedSymbol::Reloc { name, addend } => {
let offset = self.len() as u64;
self.relocs.push(DebugReloc {
offset: offset as u32,
let target = format!("_wasm_function_{}", symbol);
let offset = self.len() as u32;
self.relocs.push(DwarfSectionReloc {
target,
offset,
size,
name,
addend,
addend: addend as i32,
});
self.write_udata(addend as u64, size)
}
}
}
}
}
fn write_offset(&mut self, val: usize, section: SectionId, size: u8) -> Result<()> {
let offset = self.len() as u32;
let name = section.name().to_string();
self.relocs.push(DebugReloc {
let target = section.name().to_string();
self.relocs.push(DwarfSectionReloc {
target,
offset,
size,
name,
addend: val as i64,
addend: val as i32,
});
self.write_udata(val as u64, size)
}
@@ -137,13 +112,50 @@ impl<'a> Writer for WriterRelocate<'a> {
section: SectionId,
size: u8,
) -> Result<()> {
let name = section.name().to_string();
self.relocs.push(DebugReloc {
let target = section.name().to_string();
self.relocs.push(DwarfSectionReloc {
target,
offset: offset as u32,
size,
name,
addend: val as i64,
addend: val as i32,
});
self.write_udata_at(offset, val as u64, size)
}
}
fn create_frame_table<'a>(
isa: &dyn TargetIsa,
infos: impl Iterator<Item = &'a Option<UnwindInfo>>,
) -> Option<FrameTable> {
let mut table = FrameTable::default();
let cie_id = table.add_cie(isa.create_systemv_cie()?);
for (i, info) in infos.enumerate() {
if let Some(UnwindInfo::SystemV(info)) = info {
table.add_fde(
cie_id,
info.to_fde(Address::Symbol {
symbol: i,
addend: 0,
}),
);
}
}
Some(table)
}
pub fn emit_dwarf(
isa: &dyn TargetIsa,
debuginfo_data: &DebugInfoData,
at: &ModuleAddressMap,
vmctx_info: &ModuleVmctxInfo,
ranges: &ValueLabelsRanges,
compilation: &Compilation,
) -> anyhow::Result<Vec<DwarfSection>> {
let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?;
let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info));
let sections = emit_dwarf_sections(dwarf, frame_table)?;
Ok(sections)
}

69
crates/jit/src/code_memory.rs
View File

@@ -6,20 +6,27 @@ use std::mem::ManuallyDrop;
use std::{cmp, mem};
use wasmtime_environ::{
isa::{unwind::UnwindInfo, TargetIsa},
Compilation, CompiledFunction,
Compilation, CompiledFunction, Relocation, Relocations,
};
use wasmtime_runtime::{Mmap, VMFunctionBody};
type CodeMemoryRelocations = Vec<(u32, Vec<Relocation>)>;
struct CodeMemoryEntry {
mmap: ManuallyDrop<Mmap>,
registry: ManuallyDrop<UnwindRegistry>,
relocs: CodeMemoryRelocations,
}
impl CodeMemoryEntry {
fn with_capacity(cap: usize) -> Result<Self, String> {
let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?);
let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize));
Ok(Self { mmap, registry })
Ok(Self {
mmap,
registry,
relocs: vec![],
})
}
fn range(&self) -> (usize, usize) {
@@ -66,16 +73,19 @@ impl CodeMemory {
/// Allocate a continuous memory block for a single compiled function.
/// TODO: Reorganize the code that calls this to emit code directly into the
/// mmap region rather than into a Vec that we need to copy in.
pub fn allocate_for_function(
pub fn allocate_for_function<'a>(
&mut self,
func: &CompiledFunction,
func: &'a CompiledFunction,
relocs: impl Iterator<Item = &'a Relocation>,
) -> Result<&mut [VMFunctionBody], String> {
let size = Self::function_allocation_size(func);
let (buf, registry, start) = self.allocate(size)?;
let (buf, registry, start, m_relocs) = self.allocate(size)?;
let (_, _, vmfunc) = Self::copy_function(func, start as u32, buf, registry);
Self::copy_relocs(m_relocs, start as u32, relocs);
Ok(vmfunc)
}
@@ -83,20 +93,23 @@ impl CodeMemory {
pub fn allocate_for_compilation(
&mut self,
compilation: &Compilation,
relocations: &Relocations,
) -> Result<Box<[&mut [VMFunctionBody]]>, String> {
let total_len = compilation
.into_iter()
.fold(0, |acc, func| acc + Self::function_allocation_size(func));
let (mut buf, registry, start) = self.allocate(total_len)?;
let (mut buf, registry, start, m_relocs) = self.allocate(total_len)?;
let mut result = Vec::with_capacity(compilation.len());
let mut start = start as u32;
for func in compilation.into_iter() {
for (func, relocs) in compilation.into_iter().zip(relocations.values()) {
let (next_start, next_buf, vmfunc) = Self::copy_function(func, start, buf, registry);
result.push(vmfunc);
Self::copy_relocs(m_relocs, start, relocs.iter());
start = next_start;
buf = next_buf;
}
@@ -112,6 +125,7 @@ impl CodeMemory {
for CodeMemoryEntry {
mmap: m,
registry: r,
relocs,
} in &mut self.entries[self.published..]
{
// Remove write access to the pages due to the relocation fixups.
@@ -124,6 +138,10 @@ impl CodeMemory {
}
.expect("unable to make memory readonly and executable");
}
// Relocs data in not needed anymore -- clearing.
// TODO use relocs to serialize the published code.
relocs.clear();
}
self.published = self.entries.len();
@@ -141,7 +159,18 @@ impl CodeMemory {
/// * The offset within the current mmap that the slice starts at
///
/// TODO: Add an alignment flag.
fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize), String> {
fn allocate(
&mut self,
size: usize,
) -> Result<
(
&mut [u8],
&mut UnwindRegistry,
usize,
&mut CodeMemoryRelocations,
),
String,
> {
assert!(size > 0);
if match &self.current {
@@ -160,6 +189,7 @@ impl CodeMemory {
&mut e.mmap.as_mut_slice()[old_position..self.position],
&mut e.registry,
old_position,
&mut e.relocs,
))
}
@@ -176,6 +206,14 @@ impl CodeMemory {
}
}
fn copy_relocs<'a>(
entry_relocs: &'_ mut CodeMemoryRelocations,
start: u32,
relocs: impl Iterator<Item = &'a Relocation>,
) {
entry_relocs.push((start, relocs.cloned().collect()));
}
/// Copies the data of the compiled function to the given buffer.
///
/// This will also add the function to the current unwind registry.
@@ -249,4 +287,19 @@ impl CodeMemory {
.iter()
.map(|entry| entry.range())
}
/// Returns all relocations for the unpublished memory.
pub fn unpublished_relocations<'a>(
&'a self,
) -> impl Iterator<Item = (*const u8, &'a Relocation)> + 'a {
self.entries[self.published..]
.iter()
.chain(self.current.iter())
.flat_map(|entry| {
entry.relocs.iter().flat_map(move |(start, relocs)| {
let base_ptr = unsafe { entry.mmap.as_ptr().add(*start as usize) };
relocs.iter().map(move |r| (base_ptr, r))
})
})
}
}

174
crates/jit/src/compiler.rs
View File

@@ -8,15 +8,14 @@ use cranelift_codegen::print_errors::pretty_error;
use cranelift_codegen::Context;
use cranelift_codegen::{binemit, ir};
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use std::collections::HashMap;
use wasmtime_debug::{emit_debugsections_image, DebugInfoData};
use wasmtime_debug::{emit_dwarf, DebugInfoData, DwarfSection};
use wasmtime_environ::entity::{EntityRef, PrimaryMap};
use wasmtime_environ::isa::{TargetFrontendConfig, TargetIsa};
use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex, SignatureIndex};
use wasmtime_environ::{
CacheConfig, CompileError, CompiledFunction, Compiler as _C, ModuleAddressMap,
CacheConfig, CompileError, CompiledFunction, Compiler as _C, Module, ModuleAddressMap,
ModuleMemoryOffset, ModuleTranslation, ModuleVmctxInfo, Relocation, RelocationTarget,
Relocations, Traps, Tunables, VMOffsets,
Relocations, Traps, Tunables, VMOffsets, ValueLabelsRanges,
};
use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline};
@@ -71,15 +70,73 @@ fn _assert_compiler_send_sync() {
_assert::<Compiler>();
}
fn transform_dwarf_data(
isa: &dyn TargetIsa,
module: &Module,
debug_data: &DebugInfoData,
address_transform: &ModuleAddressMap,
value_ranges: &ValueLabelsRanges,
stack_slots: PrimaryMap<DefinedFuncIndex, ir::StackSlots>,
compilation: &wasmtime_environ::Compilation,
) -> Result<Vec<DwarfSection>, SetupError> {
let target_config = isa.frontend_config();
let ofs = VMOffsets::new(target_config.pointer_bytes(), &module.local);
let module_vmctx_info = {
ModuleVmctxInfo {
memory_offset: if ofs.num_imported_memories > 0 {
ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
} else if ofs.num_defined_memories > 0 {
ModuleMemoryOffset::Defined(
ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
)
} else {
ModuleMemoryOffset::None
},
stack_slots,
}
};
emit_dwarf(
isa,
debug_data,
&address_transform,
&module_vmctx_info,
&value_ranges,
&compilation,
)
.map_err(SetupError::DebugInfo)
}
fn get_code_range(
compilation: &wasmtime_environ::Compilation,
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
) -> (*const u8, usize) {
if finished_functions.is_empty() {
return (::std::ptr::null(), 0);
}
// Assuming all functions in the same code block, looking min/max of its range.
let (start, end) = finished_functions.iter().fold::<(usize, usize), _>(
(!0, 0),
|(start, end), (i, body_ptr)| {
let body_ptr = (*body_ptr) as *const u8 as usize;
let body_len = compilation.get(i).body.len();
(
::std::cmp::min(start, body_ptr),
::std::cmp::max(end, body_ptr + body_len),
)
},
);
(start as *const u8, end - start)
}
#[allow(missing_docs)]
pub struct Compilation {
pub code_memory: CodeMemory,
pub finished_functions: PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
pub relocations: Relocations,
pub code_range: (*const u8, usize),
pub trampolines: PrimaryMap<SignatureIndex, VMTrampoline>,
pub trampoline_relocations: HashMap<SignatureIndex, Vec<Relocation>>,
pub jt_offsets: PrimaryMap<DefinedFuncIndex, ir::JumpTableOffsets>,
pub dbg_image: Option<Vec<u8>>,
pub dwarf_sections: Vec<DwarfSection>,
pub traps: Traps,
pub address_transform: ModuleAddressMap,
}
@@ -130,10 +187,24 @@ impl Compiler {
}
.map_err(SetupError::Compile)?;
let dwarf_sections = if debug_data.is_some() && !compilation.is_empty() {
transform_dwarf_data(
&*self.isa,
&translation.module,
debug_data.as_ref().unwrap(),
&address_transform,
&value_ranges,
stack_slots,
&compilation,
)?
} else {
vec![]
};
// Allocate all of the compiled functions into executable memory,
// copying over their contents.
let finished_functions =
allocate_functions(&mut code_memory, &compilation).map_err(|message| {
let finished_functions = allocate_functions(&mut code_memory, &compilation, &relocations)
.map_err(|message| {
SetupError::Instantiate(InstantiationError::Resource(format!(
"failed to allocate memory for functions: {}",
message
@@ -146,9 +217,8 @@ impl Compiler {
// tables) have a trampoline when invoked through the wasmtime API.
let mut cx = FunctionBuilderContext::new();
let mut trampolines = PrimaryMap::new();
let mut trampoline_relocations = HashMap::new();
for (index, (_, native_sig)) in translation.module.local.signatures.iter() {
let (trampoline, relocations) = make_trampoline(
for (_, (_, native_sig)) in translation.module.local.signatures.iter() {
let trampoline = make_trampoline(
&*self.isa,
&mut code_memory,
&mut cx,
@@ -156,66 +226,18 @@ impl Compiler {
std::mem::size_of::<u128>(),
)?;
trampolines.push(trampoline);
// Typically trampolines do not have relocations, so if one does
// show up be sure to log it in case anyone's listening and there's
// an accidental bug.
if relocations.len() > 0 {
log::info!("relocations found in trampoline for {:?}", native_sig);
trampoline_relocations.insert(index, relocations);
}
}
// Translate debug info (DWARF) only if at least one function is present.
let dbg_image = if debug_data.is_some() && !finished_functions.is_empty() {
let target_config = self.isa.frontend_config();
let ofs = VMOffsets::new(target_config.pointer_bytes(), &translation.module.local);
let mut funcs = Vec::new();
for (i, allocated) in finished_functions.into_iter() {
let ptr = (*allocated) as *const u8;
let body_len = compilation.get(i).body.len();
funcs.push((ptr, body_len));
}
let module_vmctx_info = {
ModuleVmctxInfo {
memory_offset: if ofs.num_imported_memories > 0 {
ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
} else if ofs.num_defined_memories > 0 {
ModuleMemoryOffset::Defined(
ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
)
} else {
ModuleMemoryOffset::None
},
stack_slots,
}
};
let bytes = emit_debugsections_image(
&*self.isa,
debug_data.as_ref().unwrap(),
&module_vmctx_info,
&address_transform,
&value_ranges,
&funcs,
&compilation,
)
.map_err(SetupError::DebugInfo)?;
Some(bytes)
} else {
None
};
let jt_offsets = compilation.get_jt_offsets();
let code_range = get_code_range(&compilation, &finished_functions);
Ok(Compilation {
code_memory,
finished_functions,
relocations,
code_range,
trampolines,
trampoline_relocations,
jt_offsets,
dbg_image,
dwarf_sections,
traps,
address_transform,
})
@@ -229,7 +251,7 @@ pub fn make_trampoline(
fn_builder_ctx: &mut FunctionBuilderContext,
signature: &ir::Signature,
value_size: usize,
) -> Result<(VMTrampoline, Vec<Relocation>), SetupError> {
) -> Result<VMTrampoline, SetupError> {
let pointer_type = isa.pointer_type();
let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv);
@@ -337,28 +359,29 @@ pub fn make_trampoline(
})?;
let ptr = code_memory
.allocate_for_function(&CompiledFunction {
.allocate_for_function(
&CompiledFunction {
body: code_buf,
jt_offsets: context.func.jt_offsets,
unwind_info,
})
},
reloc_sink.relocs.iter(),
)
.map_err(|message| SetupError::Instantiate(InstantiationError::Resource(message)))?
.as_ptr();
Ok((
unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) },
reloc_sink.relocs,
))
Ok(unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) })
}
fn allocate_functions(
code_memory: &mut CodeMemory,
compilation: &wasmtime_environ::Compilation,
relocations: &Relocations,
) -> Result<PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>, String> {
if compilation.is_empty() {
return Ok(PrimaryMap::new());
}
let fat_ptrs = code_memory.allocate_for_compilation(compilation)?;
let fat_ptrs = code_memory.allocate_for_compilation(compilation, relocations)?;
// Second, create a PrimaryMap from result vector of pointers.
let mut result = PrimaryMap::with_capacity(compilation.len());
@@ -374,10 +397,17 @@ fn allocate_functions(
/// this `RelocSink` just asserts that it doesn't recieve most of them, but
/// handles libcall ones.
#[derive(Default)]
struct RelocSink {
pub struct RelocSink {
relocs: Vec<Relocation>,
}
impl RelocSink {
/// Returns collected relocations.
pub fn relocs(&self) -> &[Relocation] {
&self.relocs
}
}
impl binemit::RelocSink for RelocSink {
fn reloc_block(
&mut self,

74
crates/jit/src/instantiate.rs
View File

@@ -4,21 +4,21 @@
//! steps.
use crate::code_memory::CodeMemory;
use crate::compiler::Compiler;
use crate::compiler::{Compilation, Compiler};
use crate::imports::resolve_imports;
use crate::link::link_module;
use crate::resolver::Resolver;
use std::any::Any;
use std::collections::HashMap;
use std::io::Write;
use std::sync::Arc;
use thiserror::Error;
use wasmtime_debug::read_debuginfo;
use wasmtime_debug::{read_debuginfo, write_debugsections_image, DwarfSection};
use wasmtime_environ::entity::{BoxedSlice, PrimaryMap};
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex};
use wasmtime_environ::{
CompileError, DataInitializer, DataInitializerLocation, Module, ModuleAddressMap,
ModuleEnvironment, Traps,
ModuleEnvironment, ModuleTranslation, Traps,
};
use wasmtime_profiling::ProfilingAgent;
use wasmtime_runtime::VMInterrupts;
@@ -91,41 +91,53 @@ impl CompiledModule {
debug_data = Some(read_debuginfo(&data)?);
}
let compilation = compiler.compile(&translation, debug_data)?;
let Compilation {
mut code_memory,
finished_functions,
code_range,
trampolines,
jt_offsets,
dwarf_sections,
traps,
address_transform,
} = compiler.compile(&translation, debug_data)?;
let module = translation.module;
let ModuleTranslation {
module,
data_initializers,
..
} = translation;
link_module(&module, &compilation);
link_module(&mut code_memory, &module, &finished_functions, &jt_offsets);
// Make all code compiled thus far executable.
let mut code_memory = compilation.code_memory;
code_memory.publish(compiler.isa());
let data_initializers = translation
.data_initializers
let data_initializers = data_initializers
.into_iter()
.map(OwnedDataInitializer::new)
.collect::<Vec<_>>()
.into_boxed_slice();
// Initialize profiler and load the wasm module
profiler.module_load(
&module,
&compilation.finished_functions,
compilation.dbg_image.as_deref(),
);
// Register GDB JIT images; initialize profiler and load the wasm module.
let dbg_jit_registration = if !dwarf_sections.is_empty() {
let bytes = create_dbg_image(
dwarf_sections,
compiler.isa(),
code_range,
&finished_functions,
)?;
profiler.module_load(&module, &finished_functions, Some(&bytes));
let dbg_jit_registration = if let Some(img) = compilation.dbg_image {
let mut bytes = Vec::new();
bytes.write_all(&img).expect("all written");
let reg = GdbJitImageRegistration::register(bytes);
Some(reg)
} else {
profiler.module_load(&module, &finished_functions, None);
None
};
let finished_functions =
FinishedFunctions(compilation.finished_functions.into_boxed_slice());
let finished_functions = FinishedFunctions(finished_functions.into_boxed_slice());
Ok(Self {
module: Arc::new(module),
@@ -134,10 +146,10 @@ impl CompiledModule {
dbg_jit_registration,
}),
finished_functions,
trampolines: compilation.trampolines,
trampolines,
data_initializers,
traps: compilation.traps,
address_transform: compilation.address_transform,
traps,
address_transform,
})
}
@@ -256,3 +268,17 @@ impl OwnedDataInitializer {
}
}
}
fn create_dbg_image(
dwarf_sections: Vec<DwarfSection>,
isa: &dyn TargetIsa,
code_range: (*const u8, usize),
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
) -> Result<Vec<u8>, SetupError> {
let funcs = finished_functions
.values()
.map(|allocated: &*mut [VMFunctionBody]| (*allocated) as *const u8)
.collect::<Vec<_>>();
write_debugsections_image(isa, dwarf_sections, code_range, &funcs)
.map_err(SetupError::DebugInfo)
}

37
crates/jit/src/link.rs
View File

@@ -1,8 +1,11 @@
//! Linking for JIT-compiled code.
use crate::Compilation;
use crate::CodeMemory;
use cranelift_codegen::binemit::Reloc;
use cranelift_codegen::ir::JumpTableOffsets;
use std::ptr::{read_unaligned, write_unaligned};
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::wasm::DefinedFuncIndex;
use wasmtime_environ::{Module, Relocation, RelocationTarget};
use wasmtime_runtime::libcalls;
use wasmtime_runtime::VMFunctionBody;
@@ -10,27 +13,22 @@ use wasmtime_runtime::VMFunctionBody;
/// Links a module that has been compiled with `compiled_module` in `wasmtime-environ`.
///
/// Performs all required relocations inside the function code, provided the necessary metadata.
pub fn link_module(module: &Module, compilation: &Compilation) {
for (i, function_relocs) in compilation.relocations.iter() {
for r in function_relocs.iter() {
let fatptr: *const [VMFunctionBody] = compilation.finished_functions[i];
pub fn link_module(
code_memory: &mut CodeMemory,
module: &Module,
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
jt_offsets: &PrimaryMap<DefinedFuncIndex, JumpTableOffsets>,
) {
for (fatptr, r) in code_memory.unpublished_relocations() {
let body = fatptr as *const VMFunctionBody;
apply_reloc(module, compilation, body, r);
}
}
for (i, function_relocs) in compilation.trampoline_relocations.iter() {
for r in function_relocs.iter() {
println!("tramopline relocation");
let body = compilation.trampolines[*i] as *const VMFunctionBody;
apply_reloc(module, compilation, body, r);
}
apply_reloc(module, finished_functions, jt_offsets, body, r);
}
}
fn apply_reloc(
module: &Module,
compilation: &Compilation,
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
jt_offsets: &PrimaryMap<DefinedFuncIndex, JumpTableOffsets>,
body: *const VMFunctionBody,
r: &Relocation,
) {
@@ -38,7 +36,7 @@ fn apply_reloc(
let target_func_address: usize = match r.reloc_target {
RelocationTarget::UserFunc(index) => match module.local.defined_func_index(index) {
Some(f) => {
let fatptr: *const [VMFunctionBody] = compilation.finished_functions[f];
let fatptr: *const [VMFunctionBody] = finished_functions[f];
fatptr as *const VMFunctionBody as usize
}
None => panic!("direct call to import"),
@@ -67,12 +65,11 @@ fn apply_reloc(
RelocationTarget::JumpTable(func_index, jt) => {
match module.local.defined_func_index(func_index) {
Some(f) => {
let offset = *compilation
.jt_offsets
let offset = *jt_offsets
.get(f)
.and_then(|ofs| ofs.get(jt))
.expect("func jump table");
let fatptr: *const [VMFunctionBody] = compilation.finished_functions[f];
let fatptr: *const [VMFunctionBody] = finished_functions[f];
fatptr as *const VMFunctionBody as usize + offset as usize
}
None => panic!("func index of jump table"),

44
crates/jit/src/trampoline.rs
View File

@@ -10,49 +10,7 @@ pub use cranelift_codegen::Context;
pub use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
pub mod binemit {
pub use crate::compiler::RelocSink as TrampolineRelocSink;
pub use cranelift_codegen::binemit::NullTrapSink;
pub use cranelift_codegen::binemit::{CodeOffset, NullStackmapSink, TrapSink};
use cranelift_codegen::{binemit, ir};
/// We don't expect trampoline compilation to produce any relocations, so
/// this `RelocSink` just asserts that it doesn't recieve any.
pub struct TrampolineRelocSink {}
impl binemit::RelocSink for TrampolineRelocSink {
fn reloc_block(
&mut self,
_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_block_offset: binemit::CodeOffset,
) {
panic!("trampoline compilation should not produce block relocs");
}
fn reloc_external(
&mut self,
_offset: binemit::CodeOffset,
_srcloc: ir::SourceLoc,
_reloc: binemit::Reloc,
_name: &ir::ExternalName,
_addend: binemit::Addend,
) {
panic!("trampoline compilation should not produce external symbol relocs");
}
fn reloc_constant(
&mut self,
_code_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_constant_offset: ir::ConstantOffset,
) {
panic!("trampoline compilation should not produce constant relocs");
}
fn reloc_jt(
&mut self,
_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_jt: ir::JumpTable,
) {
panic!("trampoline compilation should not produce jump table relocs");
}
}
}

12
crates/wasmtime/src/trampoline/func.rs
View File

@@ -172,7 +172,7 @@ fn make_trampoline(
}
let mut code_buf: Vec<u8> = Vec::new();
let mut reloc_sink = binemit::TrampolineRelocSink {};
let mut reloc_sink = binemit::TrampolineRelocSink::default();
let mut trap_sink = binemit::NullTrapSink {};
let mut stackmap_sink = binemit::NullStackmapSink {};
context
@@ -192,11 +192,14 @@ fn make_trampoline(
.expect("create unwind information");
code_memory
.allocate_for_function(&CompiledFunction {
.allocate_for_function(
&CompiledFunction {
body: code_buf,
jt_offsets: context.func.jt_offsets,
unwind_info,
})
},
reloc_sink.relocs().iter(),
)
.expect("allocate_for_function")
}
@@ -239,14 +242,13 @@ pub fn create_handle_with_function(
// ... and then we also need a trampoline with the standard "trampoline ABI"
// which enters into the ABI specified by `ft`. Note that this is only used
// if `Func::call` is called on an object created by `Func::new`.
let (trampoline, relocations) = wasmtime_jit::make_trampoline(
let trampoline = wasmtime_jit::make_trampoline(
&*isa,
&mut code_memory,
&mut fn_builder_ctx,
&sig,
mem::size_of::<u128>(),
)?;
assert!(relocations.is_empty());
let sig_id = store.register_signature(ft.to_wasm_func_type(), sig);
trampolines.insert(sig_id, trampoline);

8
src/obj.rs
View File

@@ -2,7 +2,7 @@ use anyhow::{anyhow, bail, Context as _, Result};
use faerie::Artifact;
use target_lexicon::Triple;
use wasmtime::Strategy;
use wasmtime_debug::{emit_debugsections, read_debuginfo};
use wasmtime_debug::{emit_dwarf, read_debuginfo, write_debugsections};
#[cfg(feature = "lightbeam")]
use wasmtime_environ::Lightbeam;
use wasmtime_environ::{
@@ -113,16 +113,16 @@ pub fn compile_to_obj(
if debug_info {
let debug_data = read_debuginfo(wasm).context("failed to emit DWARF")?;
emit_debugsections(
&mut obj,
&module_vmctx_info,
let sections = emit_dwarf(
&*isa,
&debug_data,
&address_transform,
&module_vmctx_info,
&value_ranges,
&compilation,
)
.context("failed to emit debug sections")?;
write_debugsections(&mut obj, sections).context("failed to emit debug sections")?;
}
Ok(obj)
}