T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor CompiledModule to separate compile and linking stages (#1831)

Browse Source 
* Refactor how relocs are stored and handled

* refactor CompiledModule::instantiate and link_module

* Refactor DWARF creation: split generation and serialization

* Separate DWARF data transform from instantiation

* rm LinkContext
This commit is contained in:
Yury Delendik
2020-06-09 15:09:48 -05:00
committed by GitHub
parent ac87ed12bd
commit 4ebbcb82a9
9 changed files with 400 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
crates/jit/src/code_memory.rs
View File

@@ -6,20 +6,27 @@ use std::mem::ManuallyDrop;
use std::{cmp, mem};
use wasmtime_environ::{
isa::{unwind::UnwindInfo, TargetIsa},
Compilation, CompiledFunction,
Compilation, CompiledFunction, Relocation, Relocations,
};
use wasmtime_runtime::{Mmap, VMFunctionBody};
type CodeMemoryRelocations = Vec<(u32, Vec<Relocation>)>;
struct CodeMemoryEntry {
mmap: ManuallyDrop<Mmap>,
registry: ManuallyDrop<UnwindRegistry>,
relocs: CodeMemoryRelocations,
}
impl CodeMemoryEntry {
fn with_capacity(cap: usize) -> Result<Self, String> {
let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?);
let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize));
Ok(Self { mmap, registry })
Ok(Self {
mmap,
registry,
relocs: vec![],
})
}
fn range(&self) -> (usize, usize) {
@@ -66,16 +73,19 @@ impl CodeMemory {
/// Allocate a continuous memory block for a single compiled function.
/// TODO: Reorganize the code that calls this to emit code directly into the
/// mmap region rather than into a Vec that we need to copy in.
pub fn allocate_for_function(
pub fn allocate_for_function<'a>(
&mut self,
func: &CompiledFunction,
func: &'a CompiledFunction,
relocs: impl Iterator<Item = &'a Relocation>,
) -> Result<&mut [VMFunctionBody], String> {
let size = Self::function_allocation_size(func);
let (buf, registry, start) = self.allocate(size)?;
let (buf, registry, start, m_relocs) = self.allocate(size)?;
let (_, _, vmfunc) = Self::copy_function(func, start as u32, buf, registry);
Self::copy_relocs(m_relocs, start as u32, relocs);
Ok(vmfunc)
}
@@ -83,20 +93,23 @@ impl CodeMemory {
pub fn allocate_for_compilation(
&mut self,
compilation: &Compilation,
relocations: &Relocations,
) -> Result<Box<[&mut [VMFunctionBody]]>, String> {
let total_len = compilation
.into_iter()
.fold(0, |acc, func| acc + Self::function_allocation_size(func));
let (mut buf, registry, start) = self.allocate(total_len)?;
let (mut buf, registry, start, m_relocs) = self.allocate(total_len)?;
let mut result = Vec::with_capacity(compilation.len());
let mut start = start as u32;
for func in compilation.into_iter() {
for (func, relocs) in compilation.into_iter().zip(relocations.values()) {
let (next_start, next_buf, vmfunc) = Self::copy_function(func, start, buf, registry);
result.push(vmfunc);
Self::copy_relocs(m_relocs, start, relocs.iter());
start = next_start;
buf = next_buf;
}
@@ -112,6 +125,7 @@ impl CodeMemory {
for CodeMemoryEntry {
mmap: m,
registry: r,
relocs,
} in &mut self.entries[self.published..]
{
// Remove write access to the pages due to the relocation fixups.
@@ -124,6 +138,10 @@ impl CodeMemory {
}
.expect("unable to make memory readonly and executable");
}
// Relocs data in not needed anymore -- clearing.
// TODO use relocs to serialize the published code.
relocs.clear();
}
self.published = self.entries.len();
@@ -141,7 +159,18 @@ impl CodeMemory {
/// * The offset within the current mmap that the slice starts at
///
/// TODO: Add an alignment flag.
fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize), String> {
fn allocate(
&mut self,
size: usize,
) -> Result<
(
&mut [u8],
&mut UnwindRegistry,
usize,
&mut CodeMemoryRelocations,
),
String,
> {
assert!(size > 0);
if match &self.current {
@@ -160,6 +189,7 @@ impl CodeMemory {
&mut e.mmap.as_mut_slice()[old_position..self.position],
&mut e.registry,
old_position,
&mut e.relocs,
))
}
@@ -176,6 +206,14 @@ impl CodeMemory {
}
}
fn copy_relocs<'a>(
entry_relocs: &'_ mut CodeMemoryRelocations,
start: u32,
relocs: impl Iterator<Item = &'a Relocation>,
) {
entry_relocs.push((start, relocs.cloned().collect()));
}
/// Copies the data of the compiled function to the given buffer.
///
/// This will also add the function to the current unwind registry.
@@ -249,4 +287,19 @@ impl CodeMemory {
.iter()
.map(|entry| entry.range())
}
/// Returns all relocations for the unpublished memory.
pub fn unpublished_relocations<'a>(
&'a self,
) -> impl Iterator<Item = (*const u8, &'a Relocation)> + 'a {
self.entries[self.published..]
.iter()
.chain(self.current.iter())
.flat_map(|entry| {
entry.relocs.iter().flat_map(move |(start, relocs)| {
let base_ptr = unsafe { entry.mmap.as_ptr().add(*start as usize) };
relocs.iter().map(move |r| (base_ptr, r))
})
})
}
}

190
crates/jit/src/compiler.rs
View File

@@ -8,15 +8,14 @@ use cranelift_codegen::print_errors::pretty_error;
use cranelift_codegen::Context;
use cranelift_codegen::{binemit, ir};
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use std::collections::HashMap;
use wasmtime_debug::{emit_debugsections_image, DebugInfoData};
use wasmtime_debug::{emit_dwarf, DebugInfoData, DwarfSection};
use wasmtime_environ::entity::{EntityRef, PrimaryMap};
use wasmtime_environ::isa::{TargetFrontendConfig, TargetIsa};
use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex, SignatureIndex};
use wasmtime_environ::{
CacheConfig, CompileError, CompiledFunction, Compiler as _C, ModuleAddressMap,
CacheConfig, CompileError, CompiledFunction, Compiler as _C, Module, ModuleAddressMap,
ModuleMemoryOffset, ModuleTranslation, ModuleVmctxInfo, Relocation, RelocationTarget,
Relocations, Traps, Tunables, VMOffsets,
Relocations, Traps, Tunables, VMOffsets, ValueLabelsRanges,
};
use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline};
@@ -71,15 +70,73 @@ fn _assert_compiler_send_sync() {
_assert::<Compiler>();
}
fn transform_dwarf_data(
isa: &dyn TargetIsa,
module: &Module,
debug_data: &DebugInfoData,
address_transform: &ModuleAddressMap,
value_ranges: &ValueLabelsRanges,
stack_slots: PrimaryMap<DefinedFuncIndex, ir::StackSlots>,
compilation: &wasmtime_environ::Compilation,
) -> Result<Vec<DwarfSection>, SetupError> {
let target_config = isa.frontend_config();
let ofs = VMOffsets::new(target_config.pointer_bytes(), &module.local);
let module_vmctx_info = {
ModuleVmctxInfo {
memory_offset: if ofs.num_imported_memories > 0 {
ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
} else if ofs.num_defined_memories > 0 {
ModuleMemoryOffset::Defined(
ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
)
} else {
ModuleMemoryOffset::None
},
stack_slots,
}
};
emit_dwarf(
isa,
debug_data,
&address_transform,
&module_vmctx_info,
&value_ranges,
&compilation,
)
.map_err(SetupError::DebugInfo)
}
fn get_code_range(
compilation: &wasmtime_environ::Compilation,
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
) -> (*const u8, usize) {
if finished_functions.is_empty() {
return (::std::ptr::null(), 0);
}
// Assuming all functions in the same code block, looking min/max of its range.
let (start, end) = finished_functions.iter().fold::<(usize, usize), _>(
(!0, 0),
|(start, end), (i, body_ptr)| {
let body_ptr = (*body_ptr) as *const u8 as usize;
let body_len = compilation.get(i).body.len();
(
::std::cmp::min(start, body_ptr),
::std::cmp::max(end, body_ptr + body_len),
)
},
);
(start as *const u8, end - start)
}
#[allow(missing_docs)]
pub struct Compilation {
pub code_memory: CodeMemory,
pub finished_functions: PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
pub relocations: Relocations,
pub code_range: (*const u8, usize),
pub trampolines: PrimaryMap<SignatureIndex, VMTrampoline>,
pub trampoline_relocations: HashMap<SignatureIndex, Vec<Relocation>>,
pub jt_offsets: PrimaryMap<DefinedFuncIndex, ir::JumpTableOffsets>,
pub dbg_image: Option<Vec<u8>>,
pub dwarf_sections: Vec<DwarfSection>,
pub traps: Traps,
pub address_transform: ModuleAddressMap,
}
@@ -130,15 +187,29 @@ impl Compiler {
}
.map_err(SetupError::Compile)?;
let dwarf_sections = if debug_data.is_some() && !compilation.is_empty() {
transform_dwarf_data(
&*self.isa,
&translation.module,
debug_data.as_ref().unwrap(),
&address_transform,
&value_ranges,
stack_slots,
&compilation,
)?
} else {
vec![]
};
// Allocate all of the compiled functions into executable memory,
// copying over their contents.
let finished_functions =
allocate_functions(&mut code_memory, &compilation).map_err(|message| {
SetupError::Instantiate(InstantiationError::Resource(format!(
"failed to allocate memory for functions: {}",
message
)))
})?;
let finished_functions = allocate_functions(&mut code_memory, &compilation, &relocations)
.map_err(|message| {
SetupError::Instantiate(InstantiationError::Resource(format!(
"failed to allocate memory for functions: {}",
message
)))
})?;
// Eagerly generate a entry trampoline for every type signature in the
// module. This should be "relatively lightweight" for most modules and
@@ -146,9 +217,8 @@ impl Compiler {
// tables) have a trampoline when invoked through the wasmtime API.
let mut cx = FunctionBuilderContext::new();
let mut trampolines = PrimaryMap::new();
let mut trampoline_relocations = HashMap::new();
for (index, (_, native_sig)) in translation.module.local.signatures.iter() {
let (trampoline, relocations) = make_trampoline(
for (_, (_, native_sig)) in translation.module.local.signatures.iter() {
let trampoline = make_trampoline(
&*self.isa,
&mut code_memory,
&mut cx,
@@ -156,66 +226,18 @@ impl Compiler {
std::mem::size_of::<u128>(),
)?;
trampolines.push(trampoline);
// Typically trampolines do not have relocations, so if one does
// show up be sure to log it in case anyone's listening and there's
// an accidental bug.
if relocations.len() > 0 {
log::info!("relocations found in trampoline for {:?}", native_sig);
trampoline_relocations.insert(index, relocations);
}
}
// Translate debug info (DWARF) only if at least one function is present.
let dbg_image = if debug_data.is_some() && !finished_functions.is_empty() {
let target_config = self.isa.frontend_config();
let ofs = VMOffsets::new(target_config.pointer_bytes(), &translation.module.local);
let mut funcs = Vec::new();
for (i, allocated) in finished_functions.into_iter() {
let ptr = (*allocated) as *const u8;
let body_len = compilation.get(i).body.len();
funcs.push((ptr, body_len));
}
let module_vmctx_info = {
ModuleVmctxInfo {
memory_offset: if ofs.num_imported_memories > 0 {
ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
} else if ofs.num_defined_memories > 0 {
ModuleMemoryOffset::Defined(
ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
)
} else {
ModuleMemoryOffset::None
},
stack_slots,
}
};
let bytes = emit_debugsections_image(
&*self.isa,
debug_data.as_ref().unwrap(),
&module_vmctx_info,
&address_transform,
&value_ranges,
&funcs,
&compilation,
)
.map_err(SetupError::DebugInfo)?;
Some(bytes)
} else {
None
};
let jt_offsets = compilation.get_jt_offsets();
let code_range = get_code_range(&compilation, &finished_functions);
Ok(Compilation {
code_memory,
finished_functions,
relocations,
code_range,
trampolines,
trampoline_relocations,
jt_offsets,
dbg_image,
dwarf_sections,
traps,
address_transform,
})
@@ -229,7 +251,7 @@ pub fn make_trampoline(
fn_builder_ctx: &mut FunctionBuilderContext,
signature: &ir::Signature,
value_size: usize,
) -> Result<(VMTrampoline, Vec<Relocation>), SetupError> {
) -> Result<VMTrampoline, SetupError> {
let pointer_type = isa.pointer_type();
let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv);
@@ -337,28 +359,29 @@ pub fn make_trampoline(
})?;
let ptr = code_memory
.allocate_for_function(&CompiledFunction {
body: code_buf,
jt_offsets: context.func.jt_offsets,
unwind_info,
})
.allocate_for_function(
&CompiledFunction {
body: code_buf,
jt_offsets: context.func.jt_offsets,
unwind_info,
},
reloc_sink.relocs.iter(),
)
.map_err(|message| SetupError::Instantiate(InstantiationError::Resource(message)))?
.as_ptr();
Ok((
unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) },
reloc_sink.relocs,
))
Ok(unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) })
}
fn allocate_functions(
code_memory: &mut CodeMemory,
compilation: &wasmtime_environ::Compilation,
relocations: &Relocations,
) -> Result<PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>, String> {
if compilation.is_empty() {
return Ok(PrimaryMap::new());
}
let fat_ptrs = code_memory.allocate_for_compilation(compilation)?;
let fat_ptrs = code_memory.allocate_for_compilation(compilation, relocations)?;
// Second, create a PrimaryMap from result vector of pointers.
let mut result = PrimaryMap::with_capacity(compilation.len());
@@ -374,10 +397,17 @@ fn allocate_functions(
/// this `RelocSink` just asserts that it doesn't recieve most of them, but
/// handles libcall ones.
#[derive(Default)]
struct RelocSink {
pub struct RelocSink {
relocs: Vec<Relocation>,
}
impl RelocSink {
/// Returns collected relocations.
pub fn relocs(&self) -> &[Relocation] {
&self.relocs
}
}
impl binemit::RelocSink for RelocSink {
fn reloc_block(
&mut self,

74
crates/jit/src/instantiate.rs
View File

@@ -4,21 +4,21 @@
//! steps.
use crate::code_memory::CodeMemory;
use crate::compiler::Compiler;
use crate::compiler::{Compilation, Compiler};
use crate::imports::resolve_imports;
use crate::link::link_module;
use crate::resolver::Resolver;
use std::any::Any;
use std::collections::HashMap;
use std::io::Write;
use std::sync::Arc;
use thiserror::Error;
use wasmtime_debug::read_debuginfo;
use wasmtime_debug::{read_debuginfo, write_debugsections_image, DwarfSection};
use wasmtime_environ::entity::{BoxedSlice, PrimaryMap};
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex};
use wasmtime_environ::{
CompileError, DataInitializer, DataInitializerLocation, Module, ModuleAddressMap,
ModuleEnvironment, Traps,
ModuleEnvironment, ModuleTranslation, Traps,
};
use wasmtime_profiling::ProfilingAgent;
use wasmtime_runtime::VMInterrupts;
@@ -91,41 +91,53 @@ impl CompiledModule {
debug_data = Some(read_debuginfo(&data)?);
}
let compilation = compiler.compile(&translation, debug_data)?;
let Compilation {
mut code_memory,
finished_functions,
code_range,
trampolines,
jt_offsets,
dwarf_sections,
traps,
address_transform,
} = compiler.compile(&translation, debug_data)?;
let module = translation.module;
let ModuleTranslation {
module,
data_initializers,
..
} = translation;
link_module(&module, &compilation);
link_module(&mut code_memory, &module, &finished_functions, &jt_offsets);
// Make all code compiled thus far executable.
let mut code_memory = compilation.code_memory;
code_memory.publish(compiler.isa());
let data_initializers = translation
.data_initializers
let data_initializers = data_initializers
.into_iter()
.map(OwnedDataInitializer::new)
.collect::<Vec<_>>()
.into_boxed_slice();
// Initialize profiler and load the wasm module
profiler.module_load(
&module,
&compilation.finished_functions,
compilation.dbg_image.as_deref(),
);
// Register GDB JIT images; initialize profiler and load the wasm module.
let dbg_jit_registration = if !dwarf_sections.is_empty() {
let bytes = create_dbg_image(
dwarf_sections,
compiler.isa(),
code_range,
&finished_functions,
)?;
profiler.module_load(&module, &finished_functions, Some(&bytes));
let dbg_jit_registration = if let Some(img) = compilation.dbg_image {
let mut bytes = Vec::new();
bytes.write_all(&img).expect("all written");
let reg = GdbJitImageRegistration::register(bytes);
Some(reg)
} else {
profiler.module_load(&module, &finished_functions, None);
None
};
let finished_functions =
FinishedFunctions(compilation.finished_functions.into_boxed_slice());
let finished_functions = FinishedFunctions(finished_functions.into_boxed_slice());
Ok(Self {
module: Arc::new(module),
@@ -134,10 +146,10 @@ impl CompiledModule {
dbg_jit_registration,
}),
finished_functions,
trampolines: compilation.trampolines,
trampolines,
data_initializers,
traps: compilation.traps,
address_transform: compilation.address_transform,
traps,
address_transform,
})
}
@@ -256,3 +268,17 @@ impl OwnedDataInitializer {
}
}
}
fn create_dbg_image(
dwarf_sections: Vec<DwarfSection>,
isa: &dyn TargetIsa,
code_range: (*const u8, usize),
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
) -> Result<Vec<u8>, SetupError> {
let funcs = finished_functions
.values()
.map(|allocated: &*mut [VMFunctionBody]| (*allocated) as *const u8)
.collect::<Vec<_>>();
write_debugsections_image(isa, dwarf_sections, code_range, &funcs)
.map_err(SetupError::DebugInfo)
}

39
crates/jit/src/link.rs
View File

@@ -1,8 +1,11 @@
//! Linking for JIT-compiled code.
use crate::Compilation;
use crate::CodeMemory;
use cranelift_codegen::binemit::Reloc;
use cranelift_codegen::ir::JumpTableOffsets;
use std::ptr::{read_unaligned, write_unaligned};
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::wasm::DefinedFuncIndex;
use wasmtime_environ::{Module, Relocation, RelocationTarget};
use wasmtime_runtime::libcalls;
use wasmtime_runtime::VMFunctionBody;
@@ -10,27 +13,22 @@ use wasmtime_runtime::VMFunctionBody;
/// Links a module that has been compiled with `compiled_module` in `wasmtime-environ`.
///
/// Performs all required relocations inside the function code, provided the necessary metadata.
pub fn link_module(module: &Module, compilation: &Compilation) {
for (i, function_relocs) in compilation.relocations.iter() {
for r in function_relocs.iter() {
let fatptr: *const [VMFunctionBody] = compilation.finished_functions[i];
let body = fatptr as *const VMFunctionBody;
apply_reloc(module, compilation, body, r);
}
}
for (i, function_relocs) in compilation.trampoline_relocations.iter() {
for r in function_relocs.iter() {
println!("tramopline relocation");
let body = compilation.trampolines[*i] as *const VMFunctionBody;
apply_reloc(module, compilation, body, r);
}
pub fn link_module(
code_memory: &mut CodeMemory,
module: &Module,
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
jt_offsets: &PrimaryMap<DefinedFuncIndex, JumpTableOffsets>,
) {
for (fatptr, r) in code_memory.unpublished_relocations() {
let body = fatptr as *const VMFunctionBody;
apply_reloc(module, finished_functions, jt_offsets, body, r);
}
}
fn apply_reloc(
module: &Module,
compilation: &Compilation,
finished_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
jt_offsets: &PrimaryMap<DefinedFuncIndex, JumpTableOffsets>,
body: *const VMFunctionBody,
r: &Relocation,
) {
@@ -38,7 +36,7 @@ fn apply_reloc(
let target_func_address: usize = match r.reloc_target {
RelocationTarget::UserFunc(index) => match module.local.defined_func_index(index) {
Some(f) => {
let fatptr: *const [VMFunctionBody] = compilation.finished_functions[f];
let fatptr: *const [VMFunctionBody] = finished_functions[f];
fatptr as *const VMFunctionBody as usize
}
None => panic!("direct call to import"),
@@ -67,12 +65,11 @@ fn apply_reloc(
RelocationTarget::JumpTable(func_index, jt) => {
match module.local.defined_func_index(func_index) {
Some(f) => {
let offset = *compilation
.jt_offsets
let offset = *jt_offsets
.get(f)
.and_then(|ofs| ofs.get(jt))
.expect("func jump table");
let fatptr: *const [VMFunctionBody] = compilation.finished_functions[f];
let fatptr: *const [VMFunctionBody] = finished_functions[f];
fatptr as *const VMFunctionBody as usize + offset as usize
}
None => panic!("func index of jump table"),

44
crates/jit/src/trampoline.rs
View File

@@ -10,49 +10,7 @@ pub use cranelift_codegen::Context;
pub use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
pub mod binemit {
pub use crate::compiler::RelocSink as TrampolineRelocSink;
pub use cranelift_codegen::binemit::NullTrapSink;
pub use cranelift_codegen::binemit::{CodeOffset, NullStackmapSink, TrapSink};
use cranelift_codegen::{binemit, ir};
/// We don't expect trampoline compilation to produce any relocations, so
/// this `RelocSink` just asserts that it doesn't recieve any.
pub struct TrampolineRelocSink {}
impl binemit::RelocSink for TrampolineRelocSink {
fn reloc_block(
&mut self,
_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_block_offset: binemit::CodeOffset,
) {
panic!("trampoline compilation should not produce block relocs");
}
fn reloc_external(
&mut self,
_offset: binemit::CodeOffset,
_srcloc: ir::SourceLoc,
_reloc: binemit::Reloc,
_name: &ir::ExternalName,
_addend: binemit::Addend,
) {
panic!("trampoline compilation should not produce external symbol relocs");
}
fn reloc_constant(
&mut self,
_code_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_constant_offset: ir::ConstantOffset,
) {
panic!("trampoline compilation should not produce constant relocs");
}
fn reloc_jt(
&mut self,
_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_jt: ir::JumpTable,
) {
panic!("trampoline compilation should not produce jump table relocs");
}
}
}