diff --git a/Cargo.lock b/Cargo.lock index e88dcf86f7..4c84df1ef3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3564,6 +3564,7 @@ dependencies = [ "bincode", "cfg-if 1.0.0", "cpp_demangle", + "cranelift-native", "indexmap", "lazy_static", "libc", @@ -3657,6 +3658,7 @@ name = "wasmtime-cli" version = "0.29.0" dependencies = [ "anyhow", + "cranelift-native", "criterion", "env_logger 0.8.3", "file-per-thread-logger", @@ -3793,11 +3795,6 @@ dependencies = [ "addr2line", "anyhow", "cfg-if 1.0.0", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", "gimli", "log", "more-asserts", diff --git a/Cargo.toml b/Cargo.toml index 260bc4ed87..38783c7800 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ path = "src/bin/wasmtime.rs" doc = false [dependencies] -# Enable all supported architectures by default. wasmtime = { path = "crates/wasmtime", version = "0.29.0", default-features = false, features = ['cache'] } wasmtime-cache = { path = "crates/cache", version = "0.29.0" } wasmtime-debug = { path = "crates/debug", version = "0.29.0" } @@ -45,6 +44,7 @@ rayon = "1.5.0" humantime = "2.0.0" wasmparser = "0.80.0" lazy_static = "1.4.0" +cranelift-native = { path = 'cranelift/native', version = '0.76.0' } [dev-dependencies] env_logger = "0.8.1" diff --git a/cranelift/wasm/src/func_translator.rs b/cranelift/wasm/src/func_translator.rs index d455e70697..bf40466239 100644 --- a/cranelift/wasm/src/func_translator.rs +++ b/cranelift/wasm/src/func_translator.rs @@ -35,6 +35,12 @@ impl FuncTranslator { } } + /// Returns the underlying `FunctionBuilderContext` that this translator + /// uses. + pub fn context(&mut self) -> &mut FunctionBuilderContext { + &mut self.func_ctx + } + /// Translate a binary WebAssembly function. /// /// The `code` slice contains the binary WebAssembly *function code* as it appears in the code diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index 4eabe5ec4b..5df4ea5f38 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -18,3 +18,7 @@ cranelift-frontend = { path = "../../cranelift/frontend", version = "0.76.0" } cranelift-entity = { path = "../../cranelift/entity", version = "0.76.0" } wasmparser = "0.80.0" target-lexicon = "0.12" + +[features] +all-arch = ["cranelift-codegen/all-arch"] +old-x86-backend = ["cranelift-codegen/old-x86-backend"] diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 068f736132..5fad01a24d 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1277,7 +1277,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m index: TypeIndex, ) -> WasmResult { let index = self.module.types[index].unwrap_function(); - let sig = crate::indirect_signature(self.isa, self.types, index); + let sig = crate::indirect_signature(self.isa, &self.types.wasm_signatures[index]); Ok(func.import_signature(sig)) } diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 4261b8c886..601cd4a496 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -89,13 +89,16 @@ // assume no valid stack pointer will ever be `usize::max_value() - 32k`. use crate::func_environ::{get_func_name, FuncEnvironment}; -use cranelift_codegen::ir::{self, ExternalName}; +use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags}; use cranelift_codegen::isa::{CallConv, TargetIsa}; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::MachSrcLoc; use cranelift_codegen::{binemit, isa, Context}; -use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator, SignatureIndex, WasmType}; +use cranelift_frontend::FunctionBuilder; +use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator, WasmFuncType, WasmType}; +use std::cmp; use std::convert::TryFrom; +use std::mem; use std::sync::Mutex; use target_lexicon::CallingConvention; use wasmtime_environ::{ @@ -397,16 +400,13 @@ impl Compiler for Cranelift { }); context.func.stack_limit = Some(stack_limit); let mut func_translator = self.take_translator(); - let result = func_translator.translate_body( + func_translator.translate_body( &mut input.validator, input.body.clone(), &mut context.func, &mut func_env, - ); - if result.is_ok() { - self.save_translator(func_translator); - } - result?; + )?; + self.save_translator(func_translator); let mut code_buf: Vec = Vec::new(); let mut reloc_sink = RelocSink::new(func_index); @@ -452,6 +452,206 @@ impl Compiler for Cranelift { stack_maps: stack_map_sink.finish(), }) } + + fn host_to_wasm_trampoline( + &self, + isa: &dyn isa::TargetIsa, + ty: &WasmFuncType, + ) -> Result { + let value_size = mem::size_of::(); + let pointer_type = isa.pointer_type(); + + // The wasm signature we're calling in this trampoline has the actual + // ABI of the function signature described by `ty` + let wasm_signature = indirect_signature(isa, ty); + + // The host signature has the `VMTrampoline` signature where the ABI is + // fixed. + let mut host_signature = blank_sig(isa, wasmtime_call_conv(isa)); + host_signature.params.push(ir::AbiParam::new(pointer_type)); + host_signature.params.push(ir::AbiParam::new(pointer_type)); + + let mut func_translator = self.take_translator(); + let mut context = Context::new(); + context.func = ir::Function::with_name_signature(ExternalName::user(0, 0), host_signature); + + // This trampoline will load all the parameters from the `values_vec` + // that is passed in and then call the real function (also passed + // indirectly) with the specified ABI. + // + // All the results are then stored into the same `values_vec`. + let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context()); + let block0 = builder.create_block(); + + builder.append_block_params_for_function_params(block0); + builder.switch_to_block(block0); + builder.seal_block(block0); + + let (vmctx_ptr_val, caller_vmctx_ptr_val, callee_value, values_vec_ptr_val) = { + let params = builder.func.dfg.block_params(block0); + (params[0], params[1], params[2], params[3]) + }; + + // Load the argument values out of `values_vec`. + let mflags = ir::MemFlags::trusted(); + let callee_args = wasm_signature + .params + .iter() + .enumerate() + .map(|(i, r)| { + match i { + 0 => vmctx_ptr_val, + 1 => caller_vmctx_ptr_val, + _ => + // i - 2 because vmctx and caller vmctx aren't passed through `values_vec`. + { + builder.ins().load( + r.value_type, + mflags, + values_vec_ptr_val, + ((i - 2) * value_size) as i32, + ) + } + } + }) + .collect::>(); + + // Call the indirect function pointer we were given + let new_sig = builder.import_signature(wasm_signature); + let call = builder + .ins() + .call_indirect(new_sig, callee_value, &callee_args); + let results = builder.func.dfg.inst_results(call).to_vec(); + + // Store the return values into `values_vec`. + let mflags = ir::MemFlags::trusted(); + for (i, r) in results.iter().enumerate() { + builder + .ins() + .store(mflags, *r, values_vec_ptr_val, (i * value_size) as i32); + } + builder.ins().return_(&[]); + builder.finalize(); + + let func = self.finish_trampoline(context, isa)?; + self.save_translator(func_translator); + Ok(func) + } + + fn wasm_to_host_trampoline( + &self, + isa: &dyn isa::TargetIsa, + ty: &WasmFuncType, + host_fn: usize, + ) -> Result { + let pointer_type = isa.pointer_type(); + let wasm_signature = indirect_signature(isa, ty); + // The host signature has an added parameter for the `values_vec` input + // and output. + let mut host_signature = blank_sig(isa, wasmtime_call_conv(isa)); + host_signature.params.push(ir::AbiParam::new(pointer_type)); + + // Compute the size of the values vector. The vmctx and caller vmctx are passed separately. + let value_size = mem::size_of::(); + let values_vec_len = (value_size * cmp::max(ty.params.len(), ty.returns.len())) as u32; + + let mut context = Context::new(); + context.func = + ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wasm_signature); + + let ss = context.func.create_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + values_vec_len, + )); + + let mut func_translator = self.take_translator(); + let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context()); + let block0 = builder.create_block(); + + builder.append_block_params_for_function_params(block0); + builder.switch_to_block(block0); + builder.seal_block(block0); + + let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0); + let mflags = MemFlags::trusted(); + for i in 0..ty.params.len() { + let val = builder.func.dfg.block_params(block0)[i + 2]; + builder + .ins() + .store(mflags, val, values_vec_ptr_val, (i * value_size) as i32); + } + + let block_params = builder.func.dfg.block_params(block0); + let vmctx_ptr_val = block_params[0]; + let caller_vmctx_ptr_val = block_params[1]; + + let callee_args = vec![vmctx_ptr_val, caller_vmctx_ptr_val, values_vec_ptr_val]; + + let new_sig = builder.import_signature(host_signature); + + let callee_value = builder.ins().iconst(pointer_type, host_fn as i64); + builder + .ins() + .call_indirect(new_sig, callee_value, &callee_args); + + let mflags = MemFlags::trusted(); + let mut results = Vec::new(); + for (i, r) in ty.returns.iter().enumerate() { + let load = builder.ins().load( + value_type(isa, *r), + mflags, + values_vec_ptr_val, + (i * value_size) as i32, + ); + results.push(load); + } + builder.ins().return_(&results); + builder.finalize(); + + let func = self.finish_trampoline(context, isa)?; + self.save_translator(func_translator); + Ok(func) + } +} + +impl Cranelift { + fn finish_trampoline( + &self, + mut context: Context, + isa: &dyn TargetIsa, + ) -> Result { + let mut code_buf = Vec::new(); + let mut reloc_sink = TrampolineRelocSink::default(); + let mut trap_sink = binemit::NullTrapSink {}; + let mut stack_map_sink = binemit::NullStackMapSink {}; + context + .compile_and_emit( + isa, + &mut code_buf, + &mut reloc_sink, + &mut trap_sink, + &mut stack_map_sink, + ) + .map_err(|error| { + CompileError::Codegen(pretty_error(&context.func, Some(isa), error)) + })?; + + let unwind_info = context.create_unwind_info(isa).map_err(|error| { + CompileError::Codegen(pretty_error(&context.func, Some(isa), error)) + })?; + + Ok(CompiledFunction { + body: code_buf, + jt_offsets: context.func.jt_offsets, + unwind_info, + relocations: reloc_sink.relocs, + stack_maps: Default::default(), + stack_slots: Default::default(), + traps: Default::default(), + value_labels_ranges: Default::default(), + address_map: Default::default(), + }) + } } pub fn blank_sig(isa: &dyn TargetIsa, call_conv: CallConv) -> ir::Signature { @@ -475,38 +675,29 @@ pub fn wasmtime_call_conv(isa: &dyn TargetIsa) -> CallConv { } } -pub fn push_types( - isa: &dyn TargetIsa, - sig: &mut ir::Signature, - types: &TypeTables, - index: SignatureIndex, -) { - let wasm = &types.wasm_signatures[index]; - - let cvt = |ty: &WasmType| { - ir::AbiParam::new(match ty { - WasmType::I32 => ir::types::I32, - WasmType::I64 => ir::types::I64, - WasmType::F32 => ir::types::F32, - WasmType::F64 => ir::types::F64, - WasmType::V128 => ir::types::I8X16, - WasmType::FuncRef | WasmType::ExternRef => { - wasmtime_environ::reference_type(*ty, isa.pointer_type()) - } - WasmType::ExnRef => unimplemented!(), - }) - }; +pub fn push_types(isa: &dyn TargetIsa, sig: &mut ir::Signature, wasm: &WasmFuncType) { + let cvt = |ty: &WasmType| ir::AbiParam::new(value_type(isa, *ty)); sig.params.extend(wasm.params.iter().map(&cvt)); sig.returns.extend(wasm.returns.iter().map(&cvt)); } -pub fn indirect_signature( - isa: &dyn TargetIsa, - types: &TypeTables, - index: SignatureIndex, -) -> ir::Signature { +fn value_type(isa: &dyn TargetIsa, ty: WasmType) -> ir::types::Type { + match ty { + WasmType::I32 => ir::types::I32, + WasmType::I64 => ir::types::I64, + WasmType::F32 => ir::types::F32, + WasmType::F64 => ir::types::F64, + WasmType::V128 => ir::types::I8X16, + WasmType::FuncRef | WasmType::ExternRef => { + wasmtime_environ::reference_type(ty, isa.pointer_type()) + } + WasmType::ExnRef => unimplemented!(), + } +} + +pub fn indirect_signature(isa: &dyn TargetIsa, wasm: &WasmFuncType) -> ir::Signature { let mut sig = blank_sig(isa, wasmtime_call_conv(isa)); - push_types(isa, &mut sig, types, index); + push_types(isa, &mut sig, wasm); return sig; } @@ -529,6 +720,57 @@ pub fn func_signature( _ => wasmtime_call_conv(isa), }; let mut sig = blank_sig(isa, call_conv); - push_types(isa, &mut sig, types, module.functions[index]); + push_types( + isa, + &mut sig, + &types.wasm_signatures[module.functions[index]], + ); return sig; } + +/// We don't expect trampoline compilation to produce many relocations, so +/// this `RelocSink` just asserts that it doesn't recieve most of them, but +/// handles libcall ones. +#[derive(Default)] +struct TrampolineRelocSink { + relocs: Vec, +} + +impl binemit::RelocSink for TrampolineRelocSink { + fn reloc_external( + &mut self, + offset: binemit::CodeOffset, + _srcloc: ir::SourceLoc, + reloc: binemit::Reloc, + name: &ir::ExternalName, + addend: binemit::Addend, + ) { + let reloc_target = if let ir::ExternalName::LibCall(libcall) = *name { + RelocationTarget::LibCall(libcall) + } else { + panic!("unrecognized external name") + }; + self.relocs.push(Relocation { + reloc, + reloc_target, + offset, + addend, + }); + } + fn reloc_constant( + &mut self, + _code_offset: binemit::CodeOffset, + _reloc: binemit::Reloc, + _constant_offset: ir::ConstantOffset, + ) { + panic!("trampoline compilation should not produce constant relocs"); + } + fn reloc_jt( + &mut self, + _offset: binemit::CodeOffset, + _reloc: binemit::Reloc, + _jt: ir::JumpTable, + ) { + panic!("trampoline compilation should not produce jump table relocs"); + } +} diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index cd2f8b31a9..5490e55cd9 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -4,7 +4,7 @@ use crate::{FunctionAddressMap, FunctionBodyData, ModuleTranslation, Tunables, TypeTables}; use cranelift_codegen::{binemit, ir, isa, isa::unwind::UnwindInfo}; use cranelift_entity::PrimaryMap; -use cranelift_wasm::{DefinedFuncIndex, FuncIndex, WasmError}; +use cranelift_wasm::{DefinedFuncIndex, FuncIndex, WasmError, WasmFuncType}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -106,4 +106,24 @@ pub trait Compiler: Send + Sync { tunables: &Tunables, types: &TypeTables, ) -> Result; + + /// Creates a trampoline which the host can use to enter wasm. The + /// trampoline has type `VMTrampoline` and will call a function of type `ty` + /// specified. + fn host_to_wasm_trampoline( + &self, + isa: &dyn isa::TargetIsa, + ty: &WasmFuncType, + ) -> Result; + + /// Creates a trampoline suitable for a wasm module to import. + /// + /// The trampoline has the type specified by `ty` and will call the function + /// `host_fn` which has type `VMTrampoline`. + fn wasm_to_host_trampoline( + &self, + isa: &dyn isa::TargetIsa, + ty: &WasmFuncType, + host_fn: usize, + ) -> Result; } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 12b321d779..29e87cb8d1 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -17,7 +17,7 @@ pub mod settings { pub mod isa { pub use cranelift_codegen::isa::{ - unwind, Builder, CallConv, RegUnit, TargetFrontendConfig, TargetIsa, + lookup, unwind, Builder, CallConv, RegUnit, TargetFrontendConfig, TargetIsa, }; } diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index d6bb5f613f..5c96a98c23 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -11,11 +11,6 @@ repository = "https://github.com/bytecodealliance/wasmtime" edition = "2018" [dependencies] -cranelift-codegen = { path = "../../cranelift/codegen", version = "0.76.0", features = ["enable-serde"] } -cranelift-entity = { path = "../../cranelift/entity", version = "0.76.0", features = ["enable-serde"] } -cranelift-wasm = { path = "../../cranelift/wasm", version = "0.76.0", features = ["enable-serde"] } -cranelift-native = { path = "../../cranelift/native", version = "0.76.0" } -cranelift-frontend = { path = "../../cranelift/frontend", version = "0.76.0" } wasmtime-environ = { path = "../environ", version = "0.29.0" } wasmtime-runtime = { path = "../runtime", version = "0.29.0" } wasmtime-cranelift = { path = "../cranelift", version = "0.29.0" } @@ -45,10 +40,10 @@ lightbeam = ["wasmtime-lightbeam"] jitdump = ["wasmtime-profiling/jitdump"] vtune = ["wasmtime-profiling/vtune"] parallel-compilation = ["rayon"] -all-arch = ["cranelift-codegen/all-arch"] +all-arch = ["wasmtime-cranelift/all-arch"] # Use the old x86 backend. -old-x86-backend = ["cranelift-codegen/old-x86-backend"] +old-x86-backend = ["wasmtime-cranelift/old-x86-backend"] [badges] maintenance = { status = "actively-developed" } diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 3362ed8b57..89f790abc0 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -5,6 +5,7 @@ use crate::object::{ ObjectUnwindInfo, }; use crate::unwind::UnwindRegistry; +use anyhow::{Context, Result}; use object::read::{File as ObjectFile, Object, ObjectSection, ObjectSymbol}; use region; use std::collections::BTreeMap; @@ -24,8 +25,8 @@ struct CodeMemoryEntry { } impl CodeMemoryEntry { - fn with_capacity(cap: usize) -> Result { - let mmap = ManuallyDrop::new(Mmap::with_at_least(cap).map_err(|e| e.to_string())?); + fn with_capacity(cap: usize) -> Result { + let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?); let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize)); Ok(Self { mmap, @@ -119,7 +120,7 @@ impl CodeMemory { pub fn allocate_for_function<'a>( &mut self, func: &'a CompiledFunction, - ) -> Result<&mut [VMFunctionBody], String> { + ) -> Result<&mut [VMFunctionBody]> { let size = Self::function_allocation_size(func); let (buf, registry, start) = self.allocate(size)?; @@ -167,7 +168,7 @@ impl CodeMemory { /// * The offset within the current mmap that the slice starts at /// /// TODO: Add an alignment flag. - fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize), String> { + fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize)> { assert!(size > 0); if match &self.current { @@ -249,7 +250,7 @@ impl CodeMemory { } /// Pushes the current entry and allocates a new one with the given size. - fn push_current(&mut self, new_size: usize) -> Result<(), String> { + fn push_current(&mut self, new_size: usize) -> Result<()> { let previous = mem::replace( &mut self.current, if new_size == 0 { @@ -279,7 +280,7 @@ impl CodeMemory { &'a mut self, obj: &ObjectFile, unwind_info: &[ObjectUnwindInfo], - ) -> Result, String> { + ) -> Result> { let text_section = obj.section_by_name(".text").unwrap(); if text_section.size() == 0 { @@ -296,7 +297,7 @@ impl CodeMemory { buf.copy_from_slice( text_section .data() - .map_err(|_| "cannot read section data".to_string())?, + .with_context(|| "cannot read section data")?, ); // Track locations of all defined functions and trampolines. diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index cd14dddd30..02c7842b08 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -132,6 +132,11 @@ impl Compiler { &self.features } + /// Return the underlying compiler in use + pub fn compiler(&self) -> &dyn EnvCompiler { + &*self.compiler + } + /// Compile the given function bodies. pub fn compile<'data>( &self, @@ -166,8 +171,7 @@ impl Compiler { vec![] }; - let (obj, unwind_info) = - build_object(&*self.isa, &translation, types, &funcs, dwarf_sections)?; + let (obj, unwind_info) = build_object(self, &translation, types, &funcs, dwarf_sections)?; Ok(Compilation { obj, diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index ca90d0ac29..bc88ca638a 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -7,6 +7,7 @@ use crate::code_memory::CodeMemory; use crate::compiler::{Compilation, Compiler}; use crate::link::link_module; use crate::object::ObjectUnwindInfo; +use anyhow::{Context, Result}; use object::File as ObjectFile; use serde::{Deserialize, Serialize}; use std::ops::Range; @@ -483,16 +484,13 @@ fn build_code_memory( obj: &[u8], module: &Module, unwind_info: &[ObjectUnwindInfo], -) -> Result< - ( - CodeMemory, - (*const u8, usize), - PrimaryMap, - Vec<(SignatureIndex, VMTrampoline)>, - ), - String, -> { - let obj = ObjectFile::parse(obj).map_err(|_| "Unable to read obj".to_string())?; +) -> Result<( + CodeMemory, + (*const u8, usize), + PrimaryMap, + Vec<(SignatureIndex, VMTrampoline)>, +)> { + let obj = ObjectFile::parse(obj).with_context(|| "Unable to read obj")?; let mut code_memory = CodeMemory::new(); diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 9a21a61f7f..aa75e68236 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -27,16 +27,12 @@ mod link; mod object; mod unwind; -pub mod native; -pub mod trampoline; - pub use crate::code_memory::CodeMemory; pub use crate::compiler::{Compilation, CompilationStrategy, Compiler}; pub use crate::instantiate::{ CompilationArtifacts, CompiledModule, ModuleCode, SetupError, SymbolizeContext, TypeTables, }; pub use crate::link::link_module; -pub use wasmtime_cranelift::{blank_sig, wasmtime_call_conv}; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/jit/src/native.rs b/crates/jit/src/native.rs deleted file mode 100644 index 706cbc04b4..0000000000 --- a/crates/jit/src/native.rs +++ /dev/null @@ -1,14 +0,0 @@ -#![allow(missing_docs)] - -use cranelift_codegen; - -pub fn builder() -> cranelift_codegen::isa::Builder { - cranelift_native::builder().expect("host machine is not a supported target") -} - -pub fn builder_without_flags() -> cranelift_codegen::isa::Builder { - cranelift_native::builder_with_options(cranelift_codegen::isa::BackendVariant::Any, false) - .expect("host machine is not a supported target") -} - -pub use cranelift_codegen::isa::lookup; diff --git a/crates/jit/src/object.rs b/crates/jit/src/object.rs index e73fa8cf81..b20dc8c7ee 100644 --- a/crates/jit/src/object.rs +++ b/crates/jit/src/object.rs @@ -1,12 +1,11 @@ //! Object file generation. -use super::trampoline::build_trampoline; -use cranelift_frontend::FunctionBuilderContext; +use crate::Compiler; use object::write::Object; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; use wasmtime_debug::DwarfSection; -use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; +use wasmtime_environ::isa::unwind::UnwindInfo; use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; use wasmtime_environ::{CompiledFunctions, ModuleTranslation, TypeTables}; use wasmtime_obj::{ObjectBuilder, ObjectBuilderTarget}; @@ -22,7 +21,7 @@ pub enum ObjectUnwindInfo { // Builds ELF image from the module `Compilation`. pub(crate) fn build_object( - isa: &dyn TargetIsa, + compiler: &Compiler, translation: &ModuleTranslation, types: &TypeTables, funcs: &CompiledFunctions, @@ -50,10 +49,10 @@ pub(crate) fn build_object( }) .collect::>(); let mut trampolines = Vec::with_capacity(signatures.len()); - let mut cx = FunctionBuilderContext::new(); for i in signatures { - let native_sig = wasmtime_cranelift::indirect_signature(isa, &types, i); - let func = build_trampoline(isa, &mut cx, &native_sig, std::mem::size_of::())?; + let func = compiler + .compiler() + .host_to_wasm_trampoline(compiler.isa(), &types.wasm_signatures[i])?; // Preserve trampoline function unwind info. if let Some(info) = &func.unwind_info { unwind_info.push(ObjectUnwindInfo::Trampoline(i, info.clone())) @@ -61,7 +60,7 @@ pub(crate) fn build_object( trampolines.push((i, func)); } - let target = ObjectBuilderTarget::new(isa.triple().architecture)?; + let target = ObjectBuilderTarget::new(compiler.isa().triple().architecture)?; let mut builder = ObjectBuilder::new(target, &translation.module, funcs); builder .set_code_alignment(CODE_SECTION_ALIGNMENT) diff --git a/crates/jit/src/trampoline.rs b/crates/jit/src/trampoline.rs deleted file mode 100644 index b8262fd93f..0000000000 --- a/crates/jit/src/trampoline.rs +++ /dev/null @@ -1,217 +0,0 @@ -#![allow(missing_docs)] - -use crate::code_memory::CodeMemory; -use crate::instantiate::SetupError; -use cranelift_codegen::ir::InstBuilder; -use cranelift_codegen::isa::TargetIsa; -use wasmtime_environ::{CompileError, CompiledFunction, Relocation, RelocationTarget}; -use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline}; - -pub mod ir { - pub(super) use cranelift_codegen::ir::{ - AbiParam, ConstantOffset, JumpTable, Signature, SourceLoc, - }; - pub use cranelift_codegen::ir::{ - ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, - }; -} -pub use cranelift_codegen::print_errors::pretty_error; -pub use cranelift_codegen::Context; -pub use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; - -pub mod binemit { - pub use cranelift_codegen::binemit::NullTrapSink; - pub(super) use cranelift_codegen::binemit::{Addend, Reloc, RelocSink}; - pub use cranelift_codegen::binemit::{CodeOffset, NullStackMapSink, TrapSink}; -} - -/// Create a trampoline for invoking a function. -pub fn make_trampoline( - isa: &dyn TargetIsa, - code_memory: &mut CodeMemory, - fn_builder_ctx: &mut FunctionBuilderContext, - signature: &ir::Signature, - value_size: usize, -) -> Result { - let compiled_function = build_trampoline(isa, fn_builder_ctx, signature, value_size)?; - - assert!(compiled_function.relocations.is_empty()); - let ptr = code_memory - .allocate_for_function(&compiled_function) - .map_err(|message| { - SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!(message))) - })? - .as_ptr(); - Ok(unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) }) -} - -pub(crate) fn build_trampoline( - isa: &dyn TargetIsa, - fn_builder_ctx: &mut FunctionBuilderContext, - signature: &ir::Signature, - value_size: usize, -) -> Result { - let pointer_type = isa.pointer_type(); - let mut wrapper_sig = - wasmtime_cranelift::blank_sig(isa, wasmtime_cranelift::wasmtime_call_conv(isa)); - - // Add the `callee_address` parameter. - wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); - - // Add the `values_vec` parameter. - wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); - - let mut context = Context::new(); - context.func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wrapper_sig); - - { - let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); - let block0 = builder.create_block(); - - builder.append_block_params_for_function_params(block0); - builder.switch_to_block(block0); - builder.seal_block(block0); - - let (vmctx_ptr_val, caller_vmctx_ptr_val, callee_value, values_vec_ptr_val) = { - let params = builder.func.dfg.block_params(block0); - (params[0], params[1], params[2], params[3]) - }; - - // Load the argument values out of `values_vec`. - let mflags = ir::MemFlags::trusted(); - let callee_args = signature - .params - .iter() - .enumerate() - .map(|(i, r)| { - match i { - 0 => vmctx_ptr_val, - 1 => caller_vmctx_ptr_val, - _ => - // i - 2 because vmctx and caller vmctx aren't passed through `values_vec`. - { - builder.ins().load( - r.value_type, - mflags, - values_vec_ptr_val, - ((i - 2) * value_size) as i32, - ) - } - } - }) - .collect::>(); - - let new_sig = builder.import_signature(signature.clone()); - - let call = builder - .ins() - .call_indirect(new_sig, callee_value, &callee_args); - - let results = builder.func.dfg.inst_results(call).to_vec(); - - // Store the return values into `values_vec`. - let mflags = ir::MemFlags::trusted(); - for (i, r) in results.iter().enumerate() { - builder - .ins() - .store(mflags, *r, values_vec_ptr_val, (i * value_size) as i32); - } - - builder.ins().return_(&[]); - builder.finalize() - } - - let mut code_buf = Vec::new(); - let mut reloc_sink = TrampolineRelocSink::default(); - let mut trap_sink = binemit::NullTrapSink {}; - let mut stack_map_sink = binemit::NullStackMapSink {}; - context - .compile_and_emit( - isa, - &mut code_buf, - &mut reloc_sink, - &mut trap_sink, - &mut stack_map_sink, - ) - .map_err(|error| { - SetupError::Compile(CompileError::Codegen(pretty_error( - &context.func, - Some(isa), - error, - ))) - })?; - - let unwind_info = context.create_unwind_info(isa).map_err(|error| { - SetupError::Compile(CompileError::Codegen(pretty_error( - &context.func, - Some(isa), - error, - ))) - })?; - - Ok(CompiledFunction { - body: code_buf, - jt_offsets: context.func.jt_offsets, - unwind_info, - relocations: reloc_sink.relocs, - stack_maps: Default::default(), - stack_slots: Default::default(), - traps: Default::default(), - value_labels_ranges: Default::default(), - address_map: Default::default(), - }) -} - -/// We don't expect trampoline compilation to produce many relocations, so -/// this `RelocSink` just asserts that it doesn't recieve most of them, but -/// handles libcall ones. -#[derive(Default)] -pub struct TrampolineRelocSink { - relocs: Vec, -} - -impl TrampolineRelocSink { - /// Returns collected relocations. - pub fn relocs(&self) -> &[Relocation] { - &self.relocs - } -} - -impl binemit::RelocSink for TrampolineRelocSink { - fn reloc_external( - &mut self, - offset: binemit::CodeOffset, - _srcloc: ir::SourceLoc, - reloc: binemit::Reloc, - name: &ir::ExternalName, - addend: binemit::Addend, - ) { - let reloc_target = if let ir::ExternalName::LibCall(libcall) = *name { - RelocationTarget::LibCall(libcall) - } else { - panic!("unrecognized external name") - }; - self.relocs.push(Relocation { - reloc, - reloc_target, - offset, - addend, - }); - } - fn reloc_constant( - &mut self, - _code_offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _constant_offset: ir::ConstantOffset, - ) { - panic!("trampoline compilation should not produce constant relocs"); - } - fn reloc_jt( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _jt: ir::JumpTable, - ) { - panic!("trampoline compilation should not produce jump table relocs"); - } -} diff --git a/crates/jit/src/unwind/systemv.rs b/crates/jit/src/unwind/systemv.rs index 79f54ee386..d29819d30f 100644 --- a/crates/jit/src/unwind/systemv.rs +++ b/crates/jit/src/unwind/systemv.rs @@ -1,11 +1,11 @@ //! Module for System V ABI unwind registry. use anyhow::{bail, Result}; -use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; use gimli::{ write::{Address, EhFrame, EndianVec, FrameTable, Writer}, RunTimeEndian, }; +use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; /// Represents a registry of function unwind information for System V ABI. pub struct UnwindRegistry { diff --git a/crates/jit/src/unwind/winx64.rs b/crates/jit/src/unwind/winx64.rs index c2f37067ab..f46d2edfd3 100644 --- a/crates/jit/src/unwind/winx64.rs +++ b/crates/jit/src/unwind/winx64.rs @@ -1,7 +1,7 @@ //! Module for Windows x64 ABI unwind registry. use anyhow::{bail, Result}; -use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; +use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use winapi::um::winnt; /// Represents a registry of function unwind information for Windows x64 ABI. diff --git a/crates/lightbeam/wasmtime/src/lib.rs b/crates/lightbeam/wasmtime/src/lib.rs index 10c9a0dceb..338ccab118 100644 --- a/crates/lightbeam/wasmtime/src/lib.rs +++ b/crates/lightbeam/wasmtime/src/lib.rs @@ -9,7 +9,7 @@ use cranelift_codegen::isa; use lightbeam::{CodeGenSession, NullOffsetSink, Sinks}; use wasmtime_environ::wasm::{ DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, - GlobalIndex, MemoryIndex, TableIndex, TypeIndex, + GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmFuncType, }; use wasmtime_environ::{ BuiltinFunctionIndex, CompileError, CompiledFunction, Compiler, FunctionBodyData, Module, @@ -75,6 +75,23 @@ impl Compiler for Lightbeam { jt_offsets: Default::default(), }) } + + fn host_to_wasm_trampoline( + &self, + _isa: &dyn isa::TargetIsa, + _ty: &WasmFuncType, + ) -> Result { + unimplemented!() + } + + fn wasm_to_host_trampoline( + &self, + _isa: &dyn isa::TargetIsa, + _ty: &WasmFuncType, + _host_fn: usize, + ) -> Result { + unimplemented!() + } } /// Implementation of a relocation sink that just saves all the information for later diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 4c0ee28516..8bf29e6a2f 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -13,6 +13,7 @@ edition = "2018" rustdoc-args = ["--cfg", "nightlydoc"] [dependencies] +cranelift-native = { path = '../../cranelift/native', version = '0.76.0' } wasmtime-runtime = { path = "../runtime", version = "0.29.0" } wasmtime-environ = { path = "../environ", version = "0.29.0" } wasmtime-jit = { path = "../jit", version = "0.29.0" } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 431515c655..5eb4b46c93 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -12,7 +12,7 @@ use wasmparser::WasmFeatures; use wasmtime_cache::CacheConfig; use wasmtime_environ::settings::{self, Configurable, SetError}; use wasmtime_environ::{isa, isa::TargetIsa, Tunables}; -use wasmtime_jit::{native, CompilationStrategy, Compiler}; +use wasmtime_jit::{CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; use wasmtime_runtime::{ InstanceAllocator, OnDemandInstanceAllocator, PoolingInstanceAllocator, RuntimeMemoryCreator, @@ -378,7 +378,7 @@ impl Config { let mut ret = Self { tunables: Tunables::default(), flags, - isa_flags: native::builder(), + isa_flags: cranelift_native::builder().expect("host machine is not a supported target"), strategy: CompilationStrategy::Auto, #[cfg(feature = "cache")] cache_config: CacheConfig::new_cache_disabled(), @@ -417,7 +417,7 @@ impl Config { /// This method will error if the given target triple is not supported. pub fn target(&mut self, target: &str) -> Result<&mut Self> { use std::str::FromStr; - self.isa_flags = native::lookup( + self.isa_flags = wasmtime_environ::isa::lookup( target_lexicon::Triple::from_str(target).map_err(|e| anyhow::anyhow!(e))?, )?; diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index a4107558ba..dbe3299b07 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -3,22 +3,12 @@ use crate::{Engine, FuncType, Trap}; use anyhow::Result; use std::any::Any; -use std::cmp; -use std::mem; use std::panic::{self, AssertUnwindSafe}; use std::sync::Arc; use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::SignatureIndex; -use wasmtime_environ::{ir, wasm, CompiledFunction, Module, ModuleType}; -use wasmtime_jit::trampoline::ir::{ - ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, -}; -use wasmtime_jit::trampoline::{ - self, binemit, pretty_error, Context, FunctionBuilder, FunctionBuilderContext, -}; +use wasmtime_environ::{wasm, Module, ModuleType}; use wasmtime_jit::CodeMemory; -use wasmtime_jit::{blank_sig, wasmtime_call_conv}; use wasmtime_runtime::{ Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, VMContext, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, @@ -82,121 +72,6 @@ unsafe extern "C" fn stub_fn( } } -/// Create a trampoline for invoking a function. -fn make_trampoline( - isa: &dyn TargetIsa, - code_memory: &mut CodeMemory, - fn_builder_ctx: &mut FunctionBuilderContext, - signature: &ir::Signature, -) -> *mut [VMFunctionBody] { - // Mostly reverse copy of the similar method from wasmtime's - // wasmtime-jit/src/compiler.rs. - let pointer_type = isa.pointer_type(); - let mut stub_sig = blank_sig(isa, wasmtime_call_conv(isa)); - - // Add the `values_vec` parameter. - stub_sig.params.push(ir::AbiParam::new(pointer_type)); - - // Compute the size of the values vector. The vmctx and caller vmctx are passed separately. - let value_size = mem::size_of::(); - let values_vec_len = ((value_size as usize) - * cmp::max(signature.params.len() - 2, signature.returns.len())) - as u32; - - let mut context = Context::new(); - context.func = Function::with_name_signature(ExternalName::user(0, 0), signature.clone()); - - let ss = context.func.create_stack_slot(StackSlotData::new( - StackSlotKind::ExplicitSlot, - values_vec_len, - )); - - { - let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); - let block0 = builder.create_block(); - - builder.append_block_params_for_function_params(block0); - builder.switch_to_block(block0); - builder.seal_block(block0); - - let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0); - let mflags = MemFlags::trusted(); - for i in 2..signature.params.len() { - let val = builder.func.dfg.block_params(block0)[i]; - builder.ins().store( - mflags, - val, - values_vec_ptr_val, - ((i - 2) * value_size) as i32, - ); - } - - let block_params = builder.func.dfg.block_params(block0); - let vmctx_ptr_val = block_params[0]; - let caller_vmctx_ptr_val = block_params[1]; - - let callee_args = vec![vmctx_ptr_val, caller_vmctx_ptr_val, values_vec_ptr_val]; - - let new_sig = builder.import_signature(stub_sig); - - let callee_value = builder - .ins() - .iconst(pointer_type, stub_fn as *const VMFunctionBody as i64); - builder - .ins() - .call_indirect(new_sig, callee_value, &callee_args); - - let mflags = MemFlags::trusted(); - let mut results = Vec::new(); - for (i, r) in signature.returns.iter().enumerate() { - let load = builder.ins().load( - r.value_type, - mflags, - values_vec_ptr_val, - (i * value_size) as i32, - ); - results.push(load); - } - builder.ins().return_(&results); - builder.finalize() - } - - let mut code_buf: Vec = Vec::new(); - let mut reloc_sink = trampoline::TrampolineRelocSink::default(); - let mut trap_sink = binemit::NullTrapSink {}; - let mut stack_map_sink = binemit::NullStackMapSink {}; - context - .compile_and_emit( - isa, - &mut code_buf, - &mut reloc_sink, - &mut trap_sink, - &mut stack_map_sink, - ) - .map_err(|error| pretty_error(&context.func, Some(isa), error)) - .expect("compile_and_emit"); - - let unwind_info = context - .create_unwind_info(isa) - .map_err(|error| pretty_error(&context.func, Some(isa), error)) - .expect("create unwind information"); - - assert!(reloc_sink.relocs().is_empty()); - code_memory - .allocate_for_function(&CompiledFunction { - body: code_buf, - jt_offsets: context.func.jt_offsets, - unwind_info, - relocations: Default::default(), - address_map: Default::default(), - stack_maps: Default::default(), - stack_slots: Default::default(), - traps: Default::default(), - value_labels_ranges: Default::default(), - }) - .expect("allocate_for_function") -} - pub fn create_function( ft: &FuncType, func: Box Result<(), Trap> + Send + Sync>, @@ -205,36 +80,25 @@ pub fn create_function( // Note that we specifically enable reference types here in our ISA because // `Func::new` is intended to be infallible, but our signature may use // reference types which requires safepoints. - let isa = engine.config().target_isa_with_reference_types(); - - let mut sig = blank_sig(&*isa, wasmtime_call_conv(&*isa)); - sig.params.extend( - ft.params() - .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), - ); - sig.returns.extend( - ft.results() - .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), - ); - - let mut fn_builder_ctx = FunctionBuilderContext::new(); - let mut code_memory = CodeMemory::new(); - - let wasm_trampoline = - make_trampoline(isa.as_ref(), &mut code_memory, &mut fn_builder_ctx, &sig); - - // ... and then we also need a trampoline with the standard "trampoline ABI" - // which enters into the ABI specified by `ft`. Note that this is only used - // if `Func::call` is called on an object created by `Func::new`. - let host_trampoline = trampoline::make_trampoline( - &*isa, - &mut code_memory, - &mut fn_builder_ctx, - &sig, - mem::size_of::(), + let isa = &*engine.config().target_isa_with_reference_types(); + let wasm_trampoline = engine.compiler().compiler().wasm_to_host_trampoline( + isa, + ft.as_wasm_func_type(), + stub_fn as usize, )?; + let host_trampoline = engine + .compiler() + .compiler() + .host_to_wasm_trampoline(isa, ft.as_wasm_func_type())?; - code_memory.publish(isa.as_ref()); + let mut code_memory = CodeMemory::new(); + let host_trampoline = code_memory + .allocate_for_function(&host_trampoline)? + .as_ptr(); + let wasm_trampoline = + code_memory.allocate_for_function(&wasm_trampoline)? as *mut [VMFunctionBody]; + + code_memory.publish(isa); let sig = engine.signatures().register(ft.as_wasm_func_type()); @@ -244,6 +108,8 @@ pub fn create_function( sig, Box::new(TrampolineState { func, code_memory }), )?; + let host_trampoline = + std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(host_trampoline); Ok((instance, host_trampoline)) } } diff --git a/src/commands/settings.rs b/src/commands/settings.rs index f1949e8c27..28d53ac6f8 100644 --- a/src/commands/settings.rs +++ b/src/commands/settings.rs @@ -4,7 +4,6 @@ use anyhow::{anyhow, Result}; use std::str::FromStr; use structopt::StructOpt; use wasmtime_environ::settings::{self, Setting, SettingKind}; -use wasmtime_jit::native; /// Displays available Cranelift settings for a target. #[derive(StructOpt)] @@ -19,10 +18,10 @@ impl SettingsCommand { /// Executes the command. pub fn execute(self) -> Result<()> { let settings = match &self.target { - Some(target) => { - native::lookup(target_lexicon::Triple::from_str(target).map_err(|e| anyhow!(e))?)? - } - None => native::builder(), + Some(target) => wasmtime_environ::isa::lookup( + target_lexicon::Triple::from_str(target).map_err(|e| anyhow!(e))?, + )?, + None => cranelift_native::builder().unwrap(), }; let mut enums = (Vec::new(), 0, "Enum settings:"); diff --git a/src/obj.rs b/src/obj.rs index d691842613..0166c6c580 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -4,7 +4,7 @@ use target_lexicon::Triple; use wasmparser::WasmFeatures; use wasmtime::Strategy; use wasmtime_environ::{settings, settings::Configurable, ModuleEnvironment, Tunables}; -use wasmtime_jit::{native, Compiler}; +use wasmtime_jit::Compiler; /// Creates object file from binary wasm data. pub fn compile_to_obj( @@ -16,8 +16,8 @@ pub fn compile_to_obj( debug_info: bool, ) -> Result { let isa_builder = match target { - Some(target) => native::lookup(target.clone())?, - None => native::builder(), + Some(target) => wasmtime_environ::isa::lookup(target.clone())?, + None => cranelift_native::builder().unwrap(), }; let mut flag_builder = settings::builder(); let mut features = WasmFeatures::default();