T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor where results of compilation are stored (#2086)

Browse Source 
* Refactor where results of compilation are stored

This commit refactors the internals of compilation in Wasmtime to change
where results of individual function compilation are stored. Previously
compilation resulted in many maps being returned, and compilation
results generally held all these maps together. This commit instead
switches this to have all metadata stored in a `CompiledFunction`
instead of having a separate map for each item that can be stored.

The motivation for this is primarily to help out with future
module-linking-related PRs. What exactly "module level" is depends on
how we interpret modules and how many modules are in play, so it's a bit
easier for operations in wasmtime to work at the function level where
possible. This means that we don't have to pass around multiple
different maps and a function index, but instead just one map or just
one entry representing a compiled function.

Additionally this change updates where the parallelism of compilation
happens, pushing it into `wasmtime-jit` instead of `wasmtime-environ`.
This is another goal where `wasmtime-jit` will have more knowledge about
module-level pieces with module linking in play. User-facing-wise this
should be the same in terms of parallel compilation, though.

The ultimate goal of this refactoring is to make it easier for the
results of compilation to actually be a set of wasm modules. This means
we won't be able to have a map-per-metadata where the primary key is the
function index, because there will be many modules within one "object
file".

* Don't clear out fields, just don't store them

Persist a smaller set of fields in `CompilationArtifacts` instead of
trying to clear fields out and dynamically not accessing them.
This commit is contained in:
Alex Crichton
2020-08-03 12:20:51 -05:00
committed by GitHub
parent 026fb8d388
commit 65eaca35dd
31 changed files with 466 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
crates/debug/src/transform/address_transform.rs
View File

@@ -5,8 +5,7 @@ use std::iter::FromIterator;
use wasmtime_environ::entity::{EntityRef, PrimaryMap};
use wasmtime_environ::ir::SourceLoc;
use wasmtime_environ::wasm::DefinedFuncIndex;
use wasmtime_environ::WasmFileInfo;
use wasmtime_environ::{FunctionAddressMap, ModuleAddressMap};
use wasmtime_environ::{CompiledFunctions, FunctionAddressMap, WasmFileInfo};
pub type GeneratedAddress = usize;
pub type WasmAddress = u64;
@@ -187,11 +186,12 @@ fn build_function_lookup(
}
fn build_function_addr_map(
at: &ModuleAddressMap,
funcs: &CompiledFunctions,
code_section_offset: u64,
) -> PrimaryMap<DefinedFuncIndex, FunctionMap> {
let mut map = PrimaryMap::new();
for (_, ft) in at {
for (_, f) in funcs {
let ft = &f.address_map;
let mut fn_map = Vec::new();
for t in &ft.instructions {
if t.srcloc.is_default() {
@@ -447,11 +447,12 @@ impl<'a> Iterator for TransformRangeIter<'a> {
}
impl AddressTransform {
pub fn new(at: &ModuleAddressMap, wasm_file: &WasmFileInfo) -> Self {
pub fn new(funcs: &CompiledFunctions, wasm_file: &WasmFileInfo) -> Self {
let code_section_offset = wasm_file.code_section_offset;
let mut func = BTreeMap::new();
for (i, ft) in at {
for (i, f) in funcs {
let ft = &f.address_map;
let (fn_start, fn_end, lookup) = build_function_lookup(ft, code_section_offset);
func.insert(
@@ -465,7 +466,7 @@ impl AddressTransform {
);
}
let map = build_function_addr_map(at, code_section_offset);
let map = build_function_addr_map(funcs, code_section_offset);
let func = Vec::from_iter(func.into_iter());
AddressTransform { map, func }
}
@@ -606,8 +607,8 @@ mod tests {
use std::iter::FromIterator;
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::ir::SourceLoc;
use wasmtime_environ::WasmFileInfo;
use wasmtime_environ::{FunctionAddressMap, InstructionAddressMap, ModuleAddressMap};
use wasmtime_environ::{CompiledFunction, WasmFileInfo};
use wasmtime_environ::{CompiledFunctions, FunctionAddressMap, InstructionAddressMap};
#[test]
fn test_get_wasm_code_offset() {
@@ -640,8 +641,11 @@ mod tests {
}
}
fn create_simple_module(func: FunctionAddressMap) -> ModuleAddressMap {
PrimaryMap::from_iter(vec![func])
fn create_simple_module(address_map: FunctionAddressMap) -> CompiledFunctions {
PrimaryMap::from_iter(vec![CompiledFunction {
address_map,
..Default::default()
}])
}
#[test]

38
crates/debug/src/transform/expression.rs
View File

@@ -601,6 +601,7 @@ mod tests {
use super::compile_expression;
use super::{AddressTransform, FunctionFrameInfo, ValueLabel, ValueLabelsRanges};
use gimli::{self, Encoding, EndianSlice, Expression, RunTimeEndian};
use wasmtime_environ::CompiledFunction;
macro_rules! expression {
($($i:literal),*) => {
@@ -689,23 +690,26 @@ mod tests {
use wasmtime_environ::{FunctionAddressMap, InstructionAddressMap};
let mut module_map = PrimaryMap::new();
let code_section_offset: u32 = 100;
module_map.push(FunctionAddressMap {
instructions: vec![
InstructionAddressMap {
srcloc: SourceLoc::new(code_section_offset + 12),
code_offset: 5,
code_len: 3,
},
InstructionAddressMap {
srcloc: SourceLoc::new(code_section_offset + 17),
code_offset: 15,
code_len: 8,
},
],
start_srcloc: SourceLoc::new(code_section_offset + 10),
end_srcloc: SourceLoc::new(code_section_offset + 20),
body_offset: 0,
body_len: 30,
module_map.push(CompiledFunction {
address_map: FunctionAddressMap {
instructions: vec![
InstructionAddressMap {
srcloc: SourceLoc::new(code_section_offset + 12),
code_offset: 5,
code_len: 3,
},
InstructionAddressMap {
srcloc: SourceLoc::new(code_section_offset + 17),
code_offset: 15,
code_len: 8,
},
],
start_srcloc: SourceLoc::new(code_section_offset + 10),
end_srcloc: SourceLoc::new(code_section_offset + 20),
body_offset: 0,
body_len: 30,
},
..Default::default()
});
let fi = WasmFileInfo {
code_section_offset: code_section_offset.into(),

18
crates/debug/src/transform/mod.rs
View File

@@ -10,8 +10,7 @@ use gimli::{
use std::collections::HashSet;
use thiserror::Error;
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::DebugInfoData;
use wasmtime_environ::{ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges};
use wasmtime_environ::{CompiledFunctions, DebugInfoData, ModuleMemoryOffset};
pub use address_transform::AddressTransform;
@@ -50,11 +49,10 @@ where
pub fn transform_dwarf(
isa: &dyn TargetIsa,
di: &DebugInfoData,
at: &ModuleAddressMap,
vmctx_info: &ModuleVmctxInfo,
ranges: &ValueLabelsRanges,
funcs: &CompiledFunctions,
memory_offset: &ModuleMemoryOffset,
) -> Result<write::Dwarf, Error> {
let addr_tr = AddressTransform::new(at, &di.wasm_file);
let addr_tr = AddressTransform::new(funcs, &di.wasm_file);
let reachable = build_dependencies(&di.dwarf, &addr_tr)?.get_reachable();
let context = DebugInputContext {
@@ -90,9 +88,9 @@ pub fn transform_dwarf(
unit,
&context,
&addr_tr,
&ranges,
funcs,
memory_offset,
out_encoding,
&vmctx_info,
&mut out_units,
&mut out_strings,
&mut translated,
@@ -107,8 +105,8 @@ pub fn transform_dwarf(
generate_simulated_dwarf(
&addr_tr,
di,
&vmctx_info,
&ranges,
memory_offset,
funcs,
&translated,
out_encoding,
&mut out_units,

18
crates/debug/src/transform/simulate.rs
View File

@@ -11,9 +11,9 @@ use wasmparser::Type as WasmType;
use wasmtime_environ::entity::EntityRef;
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::wasm::{get_vmctx_value_label, DefinedFuncIndex};
use wasmtime_environ::WasmFileInfo;
use wasmtime_environ::{DebugInfoData, FunctionMetadata};
use wasmtime_environ::{ModuleVmctxInfo, ValueLabelsRanges};
use wasmtime_environ::{
CompiledFunctions, DebugInfoData, FunctionMetadata, ModuleMemoryOffset, WasmFileInfo,
};
const PRODUCER_NAME: &str = "wasmtime";
@@ -120,9 +120,9 @@ fn add_wasm_types(
unit: &mut write::Unit,
root_id: write::UnitEntryId,
out_strings: &mut write::StringTable,
vmctx_info: &ModuleVmctxInfo,
memory_offset: &ModuleMemoryOffset,
) -> WasmTypesDieRefs {
let (_wp_die_id, vmctx_die_id) = add_internal_types(unit, root_id, out_strings, vmctx_info);
let (_wp_die_id, vmctx_die_id) = add_internal_types(unit, root_id, out_strings, memory_offset);
macro_rules! def_type {
($id:literal, $size:literal, $enc:path) => {{
@@ -280,8 +280,8 @@ fn check_invalid_chars_in_path(path: PathBuf) -> Option<PathBuf> {
pub fn generate_simulated_dwarf(
addr_tr: &AddressTransform,
di: &DebugInfoData,
vmctx_info: &ModuleVmctxInfo,
ranges: &ValueLabelsRanges,
memory_offset: &ModuleMemoryOffset,
funcs: &CompiledFunctions,
translated: &HashSet<DefinedFuncIndex>,
out_encoding: gimli::Encoding,
out_units: &mut write::UnitTable,
@@ -342,7 +342,7 @@ pub fn generate_simulated_dwarf(
(unit, root_id, name_id)
};
let wasm_types = add_wasm_types(unit, root_id, out_strings, vmctx_info);
let wasm_types = add_wasm_types(unit, root_id, out_strings, memory_offset);
for (i, map) in addr_tr.map().iter() {
let index = i.index();
@@ -389,7 +389,7 @@ pub fn generate_simulated_dwarf(
write::AttributeValue::Udata(wasm_offset),
);
if let Some(frame_info) = get_function_frame_info(vmctx_info, i, ranges) {
if let Some(frame_info) = get_function_frame_info(memory_offset, funcs, i) {
let source_range = addr_tr.func_source_range(i);
generate_vars(
unit,

11
crates/debug/src/transform/unit.rs
View File

@@ -12,7 +12,7 @@ use gimli::{AttributeValue, DebuggingInformationEntry, Unit};
use std::collections::HashSet;
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::wasm::DefinedFuncIndex;
use wasmtime_environ::{ModuleVmctxInfo, ValueLabelsRanges};
use wasmtime_environ::{CompiledFunctions, ModuleMemoryOffset};
struct InheritedAttr<T> {
stack: Vec<(usize, T)>,
@@ -246,9 +246,9 @@ pub(crate) fn clone_unit<'a, R>(
unit: Unit<R, R::Offset>,
context: &DebugInputContext<R>,
addr_tr: &'a AddressTransform,
value_ranges: &'a ValueLabelsRanges,
funcs: &'a CompiledFunctions,
memory_offset: &ModuleMemoryOffset,
out_encoding: gimli::Encoding,
module_info: &ModuleVmctxInfo,
out_units: &mut write::UnitTable,
out_strings: &mut write::StringTable,
translated: &mut HashSet<DefinedFuncIndex>,
@@ -319,7 +319,7 @@ where
)?;
let (wp_die_id, vmctx_die_id) =
add_internal_types(comp_unit, root_id, out_strings, module_info);
add_internal_types(comp_unit, root_id, out_strings, memory_offset);
stack.push(root_id);
(
@@ -371,8 +371,7 @@ where
let range_builder =
RangeInfoBuilder::from_subprogram_die(&unit, entry, context, addr_tr, cu_low_pc)?;
if let RangeInfoBuilder::Function(func_index) = range_builder {
if let Some(frame_info) =
get_function_frame_info(module_info, func_index, value_ranges)
if let Some(frame_info) = get_function_frame_info(memory_offset, funcs, func_index)
{
current_value_range.push(new_stack_len, frame_info);
}

18
crates/debug/src/transform/utils.rs
View File

@@ -4,7 +4,7 @@ use anyhow::Error;
use gimli::write;
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::wasm::DefinedFuncIndex;
use wasmtime_environ::{ModuleMemoryOffset, ModuleVmctxInfo, ValueLabelsRanges};
use wasmtime_environ::{CompiledFunctions, ModuleMemoryOffset};
/// Adds internal Wasm utility types DIEs such as WebAssemblyPtr and
/// WasmtimeVMContext.
@@ -17,7 +17,7 @@ pub(crate) fn add_internal_types(
comp_unit: &mut write::Unit,
root_id: write::UnitEntryId,
out_strings: &mut write::StringTable,
module_info: &ModuleVmctxInfo,
memory_offset: &ModuleMemoryOffset,
) -> (write::UnitEntryId, write::UnitEntryId) {
const WASM_PTR_LEN: u8 = 4;
@@ -70,7 +70,7 @@ pub(crate) fn add_internal_types(
);
// TODO multiple memories
match module_info.memory_offset {
match *memory_offset {
ModuleMemoryOffset::Defined(memory_offset) => {
// The context has defined memory: extend the WasmtimeVMContext size
// past the "memory" field.
@@ -165,19 +165,19 @@ pub(crate) fn append_vmctx_info(
}
pub(crate) fn get_function_frame_info<'a, 'b, 'c>(
module_info: &'b ModuleVmctxInfo,
memory_offset: &ModuleMemoryOffset,
funcs: &'b CompiledFunctions,
func_index: DefinedFuncIndex,
value_ranges: &'c ValueLabelsRanges,
) -> Option<FunctionFrameInfo<'a>>
where
'b: 'a,
'c: 'a,
{
if let Some(value_ranges) = value_ranges.get(func_index) {
if let Some(func) = funcs.get(func_index) {
let frame_info = FunctionFrameInfo {
value_ranges,
memory_offset: module_info.memory_offset.clone(),
stack_slots: &module_info.stack_slots[func_index],
value_ranges: &func.value_labels_ranges,
memory_offset: memory_offset.clone(),
stack_slots: &func.stack_slots,
};
Some(frame_info)
} else {