From dd5974654c419466c9b5e7d32c7a35ddc7e691ae Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 4 Nov 2020 12:00:54 -0800 Subject: [PATCH 01/63] peepmatic: Make the test-we-can-get-and-rebuild peephole optimizers test work on arm64 This change means that running cargo test --features "enable-peepmatic rebuild-peephole-optimizers" inside `cranelift/codegen` will rebuild peephole optimizers on not only x86_64 but also aarch64. --- cranelift/codegen/src/peepmatic.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/cranelift/codegen/src/peepmatic.rs b/cranelift/codegen/src/peepmatic.rs index bf0f440865..d676dbad93 100644 --- a/cranelift/codegen/src/peepmatic.rs +++ b/cranelift/codegen/src/peepmatic.rs @@ -1312,19 +1312,31 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { } #[cfg(test)] -#[cfg(feature = "x86")] +#[cfg(any(feature = "x64", feature = "x86", feature = "arm64"))] mod tests { use super::*; - use crate::isa::lookup; + use crate::isa::{lookup, TargetIsa}; use crate::settings::{builder, Flags}; use std::str::FromStr; use target_lexicon::triple; + fn isa() -> Box { + // We need a triple to instantiate and run the peephole optimizer, but we + // don't care which one when we're just trying to trigger a rebuild of the + // peephole optimizer (it doesn't affect the serialized bytes at all). + let triple = if cfg!(any(feature = "x64", feature = "x86")) { + triple!("x86_64") + } else if cfg!(feature = "arm64") { + triple!("aarch64") + } else { + panic!("unknown arch") + }; + lookup(triple).unwrap().finish(Flags::new(builder())) + } + #[test] fn get_peepmatic_preopt() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); + let isa = isa(); let _ = preopt(&*isa); } } From 11a3bdfc6ad8e0987630519e401ffcefe5a62109 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 14:13:06 +0100 Subject: [PATCH 02/63] Catch overflows when performing relocations --- cranelift/simplejit/src/compiled_blob.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/cranelift/simplejit/src/compiled_blob.rs b/cranelift/simplejit/src/compiled_blob.rs index dc8563dabd..e00fbeecf0 100644 --- a/cranelift/simplejit/src/compiled_blob.rs +++ b/cranelift/simplejit/src/compiled_blob.rs @@ -21,27 +21,24 @@ impl CompiledBlob { } in &self.relocs { debug_assert!((offset as usize) < self.size); - let at = unsafe { self.ptr.offset(offset as isize) }; + let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) }; let base = get_definition(name); - // TODO: Handle overflow. - let what = unsafe { base.offset(addend as isize) }; + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; match reloc { Reloc::Abs4 => { - // TODO: Handle overflow. #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] unsafe { - write_unaligned(at as *mut u32, what as u32) + write_unaligned(at as *mut u32, u32::try_from(what as usize).unwrap()) }; } Reloc::Abs8 => { #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] unsafe { - write_unaligned(at as *mut u64, what as u64) + write_unaligned(at as *mut u64, u64::try_from(what as usize).unwrap()) }; } Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => { - // TODO: Handle overflow. - let pcrel = ((what as isize) - (at as isize)) as i32; + let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap(); #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] unsafe { write_unaligned(at as *mut i32, pcrel) From eaa2c5b3c25accd1a46fb9bb5bf10ee9a7ae02ac Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 15:06:52 +0100 Subject: [PATCH 03/63] Implement GOT relocations in SimpleJIT --- cranelift/codegen/src/ir/libcall.rs | 28 ++++ cranelift/simplejit/src/backend.rs | 167 +++++++++++++++++++++-- cranelift/simplejit/src/compiled_blob.rs | 26 +++- 3 files changed, 207 insertions(+), 14 deletions(-) diff --git a/cranelift/codegen/src/ir/libcall.rs b/cranelift/codegen/src/ir/libcall.rs index 9dc134e480..632b04a4c3 100644 --- a/cranelift/codegen/src/ir/libcall.rs +++ b/cranelift/codegen/src/ir/libcall.rs @@ -63,6 +63,7 @@ pub enum LibCall { /// Elf __tls_get_addr ElfTlsGetAddr, + // When adding a new variant make sure to add it to `all_libcalls` too. } impl fmt::Display for LibCall { @@ -136,6 +137,33 @@ impl LibCall { _ => return None, }) } + + /// Get a list of all known `LibCall`'s. + pub fn all_libcalls() -> &'static [LibCall] { + use LibCall::*; + &[ + Probestack, + UdivI64, + SdivI64, + UremI64, + SremI64, + IshlI64, + UshrI64, + SshrI64, + CeilF32, + CeilF64, + FloorF32, + FloorF64, + TruncF32, + TruncF64, + NearestF32, + NearestF64, + Memcpy, + Memset, + Memmove, + ElfTlsGetAddr, + ] + } } /// Get a function reference for `libcall` in `func`, following the signature diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index d211eb5feb..5ed2e3fce3 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -22,6 +22,7 @@ use std::convert::TryInto; use std::ffi::CString; use std::io::Write; use std::ptr; +use std::ptr::NonNull; use target_lexicon::PointerWidth; #[cfg(windows)] use winapi; @@ -73,7 +74,7 @@ impl SimpleJITBuilder { isa: Box, libcall_names: Box String + Send + Sync>, ) -> Self { - debug_assert!(!isa.flags().is_pic(), "SimpleJIT requires non-PIC code"); + assert!(isa.flags().is_pic(), "SimpleJIT requires PIC code"); let symbols = HashMap::new(); Self { isa, @@ -129,6 +130,9 @@ pub struct SimpleJITModule { libcall_names: Box String>, memory: MemoryHandle, declarations: ModuleDeclarations, + function_got_entries: SecondaryMap>>, + data_object_got_entries: SecondaryMap>>, + libcall_got_entries: HashMap>, compiled_functions: SecondaryMap>, compiled_data_objects: SecondaryMap>, functions_to_finalize: Vec, @@ -164,7 +168,7 @@ impl SimpleJITModule { .or_else(|| lookup_with_dlsym(name)) } - fn get_definition(&self, name: &ir::ExternalName) -> *const u8 { + fn get_address(&self, name: &ir::ExternalName) -> *const u8 { match *name { ir::ExternalName::User { .. } => { let (name, linkage) = if ModuleDeclarations::is_function(name) { @@ -203,10 +207,37 @@ impl SimpleJITModule { } } + fn get_got_address(&self, name: &ir::ExternalName) -> *const u8 { + match *name { + ir::ExternalName::User { .. } => { + if ModuleDeclarations::is_function(name) { + let func_id = FuncId::from_name(name); + self.function_got_entries[func_id] + .unwrap() + .as_ptr() + .cast::() + } else { + let data_id = DataId::from_name(name); + self.data_object_got_entries[data_id] + .unwrap() + .as_ptr() + .cast::() + } + } + ir::ExternalName::LibCall(ref libcall) => self + .libcall_got_entries + .get(libcall) + .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) + .as_ptr() + .cast::(), + _ => panic!("invalid ExternalName {}", name), + } + } + /// Returns the address of a finalized function. pub fn get_finalized_function(&self, func_id: FuncId) -> *const u8 { let info = &self.compiled_functions[func_id]; - debug_assert!( + assert!( !self.functions_to_finalize.iter().any(|x| *x == func_id), "function not yet finalized" ); @@ -218,7 +249,7 @@ impl SimpleJITModule { /// Returns the address and size of a finalized data object. pub fn get_finalized_data(&self, data_id: DataId) -> (*const u8, usize) { let info = &self.compiled_data_objects[data_id]; - debug_assert!( + assert!( !self.data_objects_to_finalize.iter().any(|x| *x == data_id), "data object not yet finalized" ); @@ -255,19 +286,25 @@ impl SimpleJITModule { pub fn finalize_definitions(&mut self) { for func in std::mem::take(&mut self.functions_to_finalize) { let decl = self.declarations.get_function_decl(func); - debug_assert!(decl.linkage.is_definable()); - let func = self.compiled_functions[func] + assert!(decl.linkage.is_definable()); + let func_blob = self.compiled_functions[func] .as_ref() .expect("function must be compiled before it can be finalized"); - func.perform_relocations(|name| self.get_definition(name)); + func_blob.perform_relocations( + |name| unreachable!("non-GOT/PLT relocation in function {} to {}", func, name), + |name| self.get_got_address(name), + ); } for data in std::mem::take(&mut self.data_objects_to_finalize) { let decl = self.declarations.get_data_decl(data); - debug_assert!(decl.linkage.is_definable()); + assert!(decl.linkage.is_definable()); let data = self.compiled_data_objects[data] .as_ref() .expect("data object must be compiled before it can be finalized"); - data.perform_relocations(|name| self.get_definition(name)); + data.perform_relocations( + |name| self.get_address(name), + |name| self.get_got_address(name), + ); } // Now that we're done patching, prepare the memory for execution! @@ -277,18 +314,47 @@ impl SimpleJITModule { /// Create a new `SimpleJITModule`. pub fn new(builder: SimpleJITBuilder) -> Self { - let memory = MemoryHandle { + let mut memory = MemoryHandle { code: Memory::new(), readonly: Memory::new(), writable: Memory::new(), }; + let mut libcall_got_entries = HashMap::new(); + + // Pre-create a GOT entry for each libcall. + for &libcall in ir::LibCall::all_libcalls() { + let got_entry = memory + .writable + .allocate( + std::mem::size_of::<*const u8>(), + std::mem::align_of::<*const u8>().try_into().unwrap(), + ) + .unwrap() + .cast::<*const u8>(); + libcall_got_entries.insert(libcall, NonNull::new(got_entry).unwrap()); + let sym = (builder.libcall_names)(libcall); + if let Some(addr) = builder + .symbols + .get(&sym) + .copied() + .or_else(|| lookup_with_dlsym(&sym)) + { + unsafe { + std::ptr::write(got_entry, addr); + } + } + } + Self { isa: builder.isa, symbols: builder.symbols, libcall_names: builder.libcall_names, memory, declarations: ModuleDeclarations::default(), + function_got_entries: SecondaryMap::new(), + data_object_got_entries: SecondaryMap::new(), + libcall_got_entries, compiled_functions: SecondaryMap::new(), compiled_data_objects: SecondaryMap::new(), functions_to_finalize: Vec::new(), @@ -315,6 +381,23 @@ impl<'simple_jit_backend> Module for SimpleJITModule { let (id, _decl) = self .declarations .declare_function(name, linkage, signature)?; + if self.function_got_entries[id].is_none() { + let got_entry = self + .memory + .writable + .allocate( + std::mem::size_of::<*const u8>(), + std::mem::align_of::<*const u8>().try_into().unwrap(), + ) + .unwrap() + .cast::<*const u8>(); + self.function_got_entries[id] = Some(NonNull::new(got_entry).unwrap()); + // FIXME populate got entries with a null pointer when defined + let val = self.lookup_symbol(name).unwrap_or(std::ptr::null()); + unsafe { + std::ptr::write(got_entry, val); + } + } Ok(id) } @@ -329,9 +412,63 @@ impl<'simple_jit_backend> Module for SimpleJITModule { let (id, _decl) = self .declarations .declare_data(name, linkage, writable, tls)?; + if self.data_object_got_entries[id].is_none() { + let got_entry = self + .memory + .writable + .allocate( + std::mem::size_of::<*const u8>(), + std::mem::align_of::<*const u8>().try_into().unwrap(), + ) + .unwrap() + .cast::<*const u8>(); + self.data_object_got_entries[id] = Some(NonNull::new(got_entry).unwrap()); + // FIXME populate got entries with a null pointer when defined + let val = self.lookup_symbol(name).unwrap_or(std::ptr::null()); + unsafe { + std::ptr::write(got_entry, val); + } + } Ok(id) } + /// Use this when you're building the IR of a function to reference a function. + /// + /// TODO: Coalesce redundant decls and signatures. + /// TODO: Look into ways to reduce the risk of using a FuncRef in the wrong function. + fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { + let decl = self.declarations.get_function_decl(func); + let signature = in_func.import_signature(decl.signature.clone()); + in_func.import_function(ir::ExtFuncData { + name: ir::ExternalName::user(0, func.as_u32()), + signature, + colocated: false, + }) + } + + /// Use this when you're building the IR of a function to reference a data object. + /// + /// TODO: Same as above. + fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue { + let decl = self.declarations.get_data_decl(data); + func.create_global_value(ir::GlobalValueData::Symbol { + name: ir::ExternalName::user(1, data.as_u32()), + offset: ir::immediates::Imm64::new(0), + colocated: false, + tls: decl.tls, + }) + } + + /// TODO: Same as above. + fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { + ctx.import_function(ir::ExternalName::user(0, func.as_u32())) + } + + /// TODO: Same as above. + fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { + ctx.import_global_value(ir::ExternalName::user(1, data.as_u32())) + } + fn define_function( &mut self, id: FuncId, @@ -381,7 +518,11 @@ impl<'simple_jit_backend> Module for SimpleJITModule { size, relocs: reloc_sink.relocs, }); + // FIXME immediately perform relocations self.functions_to_finalize.push(id); + unsafe { + std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); + } Ok(ModuleCompiledFunction { size: code_size }) } @@ -425,6 +566,9 @@ impl<'simple_jit_backend> Module for SimpleJITModule { relocs: relocs.to_vec(), }); self.functions_to_finalize.push(id); + unsafe { + std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); + } Ok(ModuleCompiledFunction { size: total_size }) } @@ -489,6 +633,9 @@ impl<'simple_jit_backend> Module for SimpleJITModule { self.compiled_data_objects[id] = Some(CompiledBlob { ptr, size, relocs }); self.data_objects_to_finalize.push(id); + unsafe { + std::ptr::write(self.data_object_got_entries[id].unwrap().as_ptr(), ptr); + } Ok(()) } diff --git a/cranelift/simplejit/src/compiled_blob.rs b/cranelift/simplejit/src/compiled_blob.rs index e00fbeecf0..5c9b268203 100644 --- a/cranelift/simplejit/src/compiled_blob.rs +++ b/cranelift/simplejit/src/compiled_blob.rs @@ -1,6 +1,7 @@ use cranelift_codegen::binemit::Reloc; use cranelift_codegen::ir::ExternalName; use cranelift_module::RelocRecord; +use std::convert::TryFrom; #[derive(Clone)] pub(crate) struct CompiledBlob { @@ -10,7 +11,11 @@ pub(crate) struct CompiledBlob { } impl CompiledBlob { - pub(crate) fn perform_relocations(&self, get_definition: impl Fn(&ExternalName) -> *const u8) { + pub(crate) fn perform_relocations( + &self, + get_address: impl Fn(&ExternalName) -> *const u8, + get_got_entry: impl Fn(&ExternalName) -> *const u8, + ) { use std::ptr::write_unaligned; for &RelocRecord { @@ -22,29 +27,42 @@ impl CompiledBlob { { debug_assert!((offset as usize) < self.size); let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) }; - let base = get_definition(name); - let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; match reloc { Reloc::Abs4 => { + let base = get_address(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] unsafe { write_unaligned(at as *mut u32, u32::try_from(what as usize).unwrap()) }; } Reloc::Abs8 => { + let base = get_address(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] unsafe { write_unaligned(at as *mut u64, u64::try_from(what as usize).unwrap()) }; } Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => { + let base = get_address(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap(); #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] unsafe { write_unaligned(at as *mut i32, pcrel) }; } - Reloc::X86GOTPCRel4 | Reloc::X86CallPLTRel4 => panic!("unexpected PIC relocation"), + Reloc::X86GOTPCRel4 => { + let base = get_got_entry(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; + let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap(); + #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] + unsafe { + write_unaligned(at as *mut i32, pcrel) + }; + } + Reloc::X86CallPLTRel4 => todo!("PLT relocation"), _ => unimplemented!(), } } From 5458473765bd044da013e8ad08a2018ec4733b9b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 16:19:16 +0100 Subject: [PATCH 04/63] Implement PLT relocations for SimpleJIT --- cranelift/simplejit/src/backend.rs | 74 ++++++++++++++++++++++-- cranelift/simplejit/src/compiled_blob.rs | 11 +++- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index 5ed2e3fce3..e7a5d72e97 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -18,7 +18,7 @@ use cranelift_native; use libc; use log::info; use std::collections::HashMap; -use std::convert::TryInto; +use std::convert::{TryFrom, TryInto}; use std::ffi::CString; use std::io::Write; use std::ptr; @@ -131,8 +131,10 @@ pub struct SimpleJITModule { memory: MemoryHandle, declarations: ModuleDeclarations, function_got_entries: SecondaryMap>>, + function_plt_entries: SecondaryMap>>, data_object_got_entries: SecondaryMap>>, libcall_got_entries: HashMap>, + libcall_plt_entries: HashMap>, compiled_functions: SecondaryMap>, compiled_data_objects: SecondaryMap>, functions_to_finalize: Vec, @@ -168,6 +170,18 @@ impl SimpleJITModule { .or_else(|| lookup_with_dlsym(name)) } + unsafe fn write_plt_entry_bytes(plt_ptr: *mut [u8; 16], got_ptr: *mut *const u8) { + assert!(cfg!(target_arch = "x86_64"), "PLT is currently only supported on x86_64"); + // jmp *got_ptr; ud2; ud2; ud2; ud2; ud2 + let mut plt_val = [ + 0xff, 0x25, 0, 0, 0, 0, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, + ]; + let what = got_ptr as isize - 4; + let at = plt_ptr as isize + 2; + plt_val[2..6].copy_from_slice(&i32::to_ne_bytes(i32::try_from(what - at).unwrap())); + std::ptr::write(plt_ptr, plt_val); + } + fn get_address(&self, name: &ir::ExternalName) -> *const u8 { match *name { ir::ExternalName::User { .. } => { @@ -234,6 +248,29 @@ impl SimpleJITModule { } } + fn get_plt_address(&self, name: &ir::ExternalName) -> *const u8 { + match *name { + ir::ExternalName::User { .. } => { + if ModuleDeclarations::is_function(name) { + let func_id = FuncId::from_name(name); + self.function_plt_entries[func_id] + .unwrap() + .as_ptr() + .cast::() + } else { + unreachable!("PLT relocations can only have functions as target"); + } + } + ir::ExternalName::LibCall(ref libcall) => self + .libcall_plt_entries + .get(libcall) + .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) + .as_ptr() + .cast::(), + _ => panic!("invalid ExternalName {}", name), + } + } + /// Returns the address of a finalized function. pub fn get_finalized_function(&self, func_id: FuncId) -> *const u8 { let info = &self.compiled_functions[func_id]; @@ -293,6 +330,7 @@ impl SimpleJITModule { func_blob.perform_relocations( |name| unreachable!("non-GOT/PLT relocation in function {} to {}", func, name), |name| self.get_got_address(name), + |name| self.get_plt_address(name), ); } for data in std::mem::take(&mut self.data_objects_to_finalize) { @@ -304,6 +342,7 @@ impl SimpleJITModule { data.perform_relocations( |name| self.get_address(name), |name| self.get_got_address(name), + |name| self.get_plt_address(name), ); } @@ -321,6 +360,7 @@ impl SimpleJITModule { }; let mut libcall_got_entries = HashMap::new(); + let mut libcall_plt_entries = HashMap::new(); // Pre-create a GOT entry for each libcall. for &libcall in ir::LibCall::all_libcalls() { @@ -334,15 +374,27 @@ impl SimpleJITModule { .cast::<*const u8>(); libcall_got_entries.insert(libcall, NonNull::new(got_entry).unwrap()); let sym = (builder.libcall_names)(libcall); - if let Some(addr) = builder + let addr = if let Some(addr) = builder .symbols .get(&sym) .copied() .or_else(|| lookup_with_dlsym(&sym)) { - unsafe { - std::ptr::write(got_entry, addr); - } + addr + } else { + continue; + }; + unsafe { + std::ptr::write(got_entry, addr); + } + let plt_entry = memory + .code + .allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT) + .unwrap() + .cast::<[u8; 16]>(); + libcall_plt_entries.insert(libcall, NonNull::new(plt_entry).unwrap()); + unsafe { + Self::write_plt_entry_bytes(plt_entry, got_entry); } } @@ -353,8 +405,10 @@ impl SimpleJITModule { memory, declarations: ModuleDeclarations::default(), function_got_entries: SecondaryMap::new(), + function_plt_entries: SecondaryMap::new(), data_object_got_entries: SecondaryMap::new(), libcall_got_entries, + libcall_plt_entries, compiled_functions: SecondaryMap::new(), compiled_data_objects: SecondaryMap::new(), functions_to_finalize: Vec::new(), @@ -397,6 +451,16 @@ impl<'simple_jit_backend> Module for SimpleJITModule { unsafe { std::ptr::write(got_entry, val); } + let plt_entry = self + .memory + .code + .allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT) + .unwrap() + .cast::<[u8; 16]>(); + self.function_plt_entries[id] = Some(NonNull::new(plt_entry).unwrap()); + unsafe { + Self::write_plt_entry_bytes(plt_entry, got_entry); + } } Ok(id) } diff --git a/cranelift/simplejit/src/compiled_blob.rs b/cranelift/simplejit/src/compiled_blob.rs index 5c9b268203..d44497ae9e 100644 --- a/cranelift/simplejit/src/compiled_blob.rs +++ b/cranelift/simplejit/src/compiled_blob.rs @@ -15,6 +15,7 @@ impl CompiledBlob { &self, get_address: impl Fn(&ExternalName) -> *const u8, get_got_entry: impl Fn(&ExternalName) -> *const u8, + get_plt_entry: impl Fn(&ExternalName) -> *const u8, ) { use std::ptr::write_unaligned; @@ -62,7 +63,15 @@ impl CompiledBlob { write_unaligned(at as *mut i32, pcrel) }; } - Reloc::X86CallPLTRel4 => todo!("PLT relocation"), + Reloc::X86CallPLTRel4 => { + let base = get_plt_entry(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; + let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap(); + #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] + unsafe { + write_unaligned(at as *mut i32, pcrel) + }; + } _ => unimplemented!(), } } From 8a4749af51273dde8b38ee60c583774260edbda2 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 16:33:04 +0100 Subject: [PATCH 05/63] Immediately perform relocations when defining a function --- cranelift/simplejit/src/backend.rs | 36 ++++++++++++------------------ 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index e7a5d72e97..e5f90e9db8 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -137,7 +137,6 @@ pub struct SimpleJITModule { libcall_plt_entries: HashMap>, compiled_functions: SecondaryMap>, compiled_data_objects: SecondaryMap>, - functions_to_finalize: Vec, data_objects_to_finalize: Vec, } @@ -171,7 +170,10 @@ impl SimpleJITModule { } unsafe fn write_plt_entry_bytes(plt_ptr: *mut [u8; 16], got_ptr: *mut *const u8) { - assert!(cfg!(target_arch = "x86_64"), "PLT is currently only supported on x86_64"); + assert!( + cfg!(target_arch = "x86_64"), + "PLT is currently only supported on x86_64" + ); // jmp *got_ptr; ud2; ud2; ud2; ud2; ud2 let mut plt_val = [ 0xff, 0x25, 0, 0, 0, 0, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, @@ -274,10 +276,6 @@ impl SimpleJITModule { /// Returns the address of a finalized function. pub fn get_finalized_function(&self, func_id: FuncId) -> *const u8 { let info = &self.compiled_functions[func_id]; - assert!( - !self.functions_to_finalize.iter().any(|x| *x == func_id), - "function not yet finalized" - ); info.as_ref() .expect("function must be compiled before it can be finalized") .ptr @@ -321,18 +319,6 @@ impl SimpleJITModule { /// Use `get_finalized_function` and `get_finalized_data` to obtain the final /// artifacts. pub fn finalize_definitions(&mut self) { - for func in std::mem::take(&mut self.functions_to_finalize) { - let decl = self.declarations.get_function_decl(func); - assert!(decl.linkage.is_definable()); - let func_blob = self.compiled_functions[func] - .as_ref() - .expect("function must be compiled before it can be finalized"); - func_blob.perform_relocations( - |name| unreachable!("non-GOT/PLT relocation in function {} to {}", func, name), - |name| self.get_got_address(name), - |name| self.get_plt_address(name), - ); - } for data in std::mem::take(&mut self.data_objects_to_finalize) { let decl = self.declarations.get_data_decl(data); assert!(decl.linkage.is_definable()); @@ -411,7 +397,6 @@ impl SimpleJITModule { libcall_plt_entries, compiled_functions: SecondaryMap::new(), compiled_data_objects: SecondaryMap::new(), - functions_to_finalize: Vec::new(), data_objects_to_finalize: Vec::new(), } } @@ -582,11 +567,14 @@ impl<'simple_jit_backend> Module for SimpleJITModule { size, relocs: reloc_sink.relocs, }); - // FIXME immediately perform relocations - self.functions_to_finalize.push(id); unsafe { std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); } + self.compiled_functions[id].as_ref().unwrap().perform_relocations( + |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), + |name| self.get_got_address(name), + |name| self.get_plt_address(name), + ); Ok(ModuleCompiledFunction { size: code_size }) } @@ -629,10 +617,14 @@ impl<'simple_jit_backend> Module for SimpleJITModule { size, relocs: relocs.to_vec(), }); - self.functions_to_finalize.push(id); unsafe { std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); } + self.compiled_functions[id].as_ref().unwrap().perform_relocations( + |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), + |name| self.get_got_address(name), + |name| self.get_plt_address(name), + ); Ok(ModuleCompiledFunction { size: total_size }) } From bf9e5d94480c085bb7c8d6165ade1ccd1a4444f0 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 16:41:23 +0100 Subject: [PATCH 06/63] Use a PLT reference for function relocations in data objects This ensures that all functions can be replaced without having to perform relocations again. --- cranelift/simplejit/src/backend.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index e5f90e9db8..97abd744f6 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -188,14 +188,7 @@ impl SimpleJITModule { match *name { ir::ExternalName::User { .. } => { let (name, linkage) = if ModuleDeclarations::is_function(name) { - let func_id = FuncId::from_name(name); - match &self.compiled_functions[func_id] { - Some(compiled) => return compiled.ptr, - None => { - let decl = self.declarations.get_function_decl(func_id); - (&decl.name, decl.linkage) - } - } + return self.get_plt_address(name); } else { let data_id = DataId::from_name(name); match &self.compiled_data_objects[data_id] { From cdbbcf7e13dd3a29993158847a9204a96d389ff2 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 18:58:28 +0100 Subject: [PATCH 07/63] Add plt entries to perf jit map --- cranelift/simplejit/src/backend.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index 97abd744f6..38b61c04d1 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -435,6 +435,11 @@ impl<'simple_jit_backend> Module for SimpleJITModule { .allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT) .unwrap() .cast::<[u8; 16]>(); + self.record_function_for_perf( + plt_entry as *mut _, + std::mem::size_of::<[u8; 16]>(), + &format!("{}@plt", name), + ); self.function_plt_entries[id] = Some(NonNull::new(plt_entry).unwrap()); unsafe { Self::write_plt_entry_bytes(plt_entry, got_entry); From 03c0e7e67811095f78927dbd1f3b070916ee528c Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 18:58:40 +0100 Subject: [PATCH 08/63] Rustfmt --- cranelift/simplejit/src/backend.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index 38b61c04d1..bb7ddef6e1 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -568,11 +568,14 @@ impl<'simple_jit_backend> Module for SimpleJITModule { unsafe { std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); } - self.compiled_functions[id].as_ref().unwrap().perform_relocations( - |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), - |name| self.get_got_address(name), - |name| self.get_plt_address(name), - ); + self.compiled_functions[id] + .as_ref() + .unwrap() + .perform_relocations( + |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), + |name| self.get_got_address(name), + |name| self.get_plt_address(name), + ); Ok(ModuleCompiledFunction { size: code_size }) } @@ -618,11 +621,14 @@ impl<'simple_jit_backend> Module for SimpleJITModule { unsafe { std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); } - self.compiled_functions[id].as_ref().unwrap().perform_relocations( - |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), - |name| self.get_got_address(name), - |name| self.get_plt_address(name), - ); + self.compiled_functions[id] + .as_ref() + .unwrap() + .perform_relocations( + |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), + |name| self.get_got_address(name), + |name| self.get_plt_address(name), + ); Ok(ModuleCompiledFunction { size: total_size }) } From 86d3dc9510bd98de6bd9cc6793d2fc8ee5d17fa1 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 19:29:05 +0100 Subject: [PATCH 09/63] Add prepare_for_function_redefine --- Cargo.lock | 1 + cranelift/module/src/module.rs | 94 ++++++++++++++++++++++++++++++ cranelift/simplejit/Cargo.toml | 1 + cranelift/simplejit/src/backend.rs | 22 +++++++ 4 files changed, 118 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 7e22a7d85a..36dc8c1618 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -477,6 +477,7 @@ dependencies = [ name = "cranelift-simplejit" version = "0.68.0" dependencies = [ + "anyhow", "cranelift", "cranelift-codegen", "cranelift-entity", diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index eb4e668b98..2af5000568 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -490,3 +490,97 @@ pub trait Module { /// Define a data object, producing the data contents from the given `DataContext`. fn define_data(&mut self, data: DataId, data_ctx: &DataContext) -> ModuleResult<()>; } + +impl Module for &mut M { + fn isa(&self) -> &dyn isa::TargetIsa { + (**self).isa() + } + + fn declarations(&self) -> &ModuleDeclarations { + (**self).declarations() + } + + fn get_name(&self, name: &str) -> Option { + (**self).get_name(name) + } + + fn target_config(&self) -> isa::TargetFrontendConfig { + (**self).target_config() + } + + fn make_context(&self) -> Context { + (**self).make_context() + } + + fn clear_context(&self, ctx: &mut Context) { + (**self).clear_context(ctx) + } + + fn make_signature(&self) -> ir::Signature { + (**self).make_signature() + } + + fn clear_signature(&self, sig: &mut ir::Signature) { + (**self).clear_signature(sig) + } + + fn declare_function( + &mut self, + name: &str, + linkage: Linkage, + signature: &ir::Signature, + ) -> ModuleResult { + (**self).declare_function(name, linkage, signature) + } + + fn declare_data( + &mut self, + name: &str, + linkage: Linkage, + writable: bool, + tls: bool, + ) -> ModuleResult { + (**self).declare_data(name, linkage, writable, tls) + } + + fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { + (**self).declare_func_in_func(func, in_func) + } + + fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue { + (**self).declare_data_in_func(data, func) + } + + fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { + (**self).declare_func_in_data(func, ctx) + } + + fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { + (**self).declare_data_in_data(data, ctx) + } + + fn define_function( + &mut self, + func: FuncId, + ctx: &mut Context, + trap_sink: &mut TS, + ) -> ModuleResult + where + TS: binemit::TrapSink, + { + (**self).define_function(func, ctx, trap_sink) + } + + fn define_function_bytes( + &mut self, + func: FuncId, + bytes: &[u8], + relocs: &[RelocRecord], + ) -> ModuleResult { + (**self).define_function_bytes(func, bytes, relocs) + } + + fn define_data(&mut self, data: DataId, data_ctx: &DataContext) -> ModuleResult<()> { + (**self).define_data(data, data_ctx) + } +} diff --git a/cranelift/simplejit/Cargo.toml b/cranelift/simplejit/Cargo.toml index 9c7bea2c3d..ea8165035a 100644 --- a/cranelift/simplejit/Cargo.toml +++ b/cranelift/simplejit/Cargo.toml @@ -14,6 +14,7 @@ cranelift-module = { path = "../module", version = "0.68.0" } cranelift-native = { path = "../native", version = "0.68.0" } cranelift-codegen = { path = "../codegen", version = "0.68.0", default-features = false, features = ["std"] } cranelift-entity = { path = "../entity", version = "0.68.0" } +anyhow = "1.0" region = "2.2.0" libc = { version = "0.2.42" } errno = "0.2.4" diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index bb7ddef6e1..2c90157875 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -393,6 +393,28 @@ impl SimpleJITModule { data_objects_to_finalize: Vec::new(), } } + + /// Allow a single future `define_function` on a previously defined function. This allows for + /// hot code swapping and lazy compilation of functions. + pub fn prepare_for_function_redefine(&mut self, func_id: FuncId) -> ModuleResult<()> { + let decl = self.declarations.get_function_decl(func_id); + if !decl.linkage.is_definable() { + return Err(ModuleError::InvalidImportDefinition(decl.name.clone())); + } + + if self.compiled_functions[func_id].is_none() { + return Err(ModuleError::Backend(anyhow::anyhow!( + "Tried to redefine not yet defined function {}", + decl.name + ))); + } + + self.compiled_functions[func_id] = None; + + // FIXME return some kind of handle that allows for deallocating the function + + Ok(()) + } } impl<'simple_jit_backend> Module for SimpleJITModule { From 8a35cbaf0d5c99dd408b0b8b51af26a800a509c4 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 12 Nov 2020 19:49:42 +0100 Subject: [PATCH 10/63] Enable PIC in SimpleJITBuilder::new --- cranelift/simplejit/src/backend.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index 2c90157875..71dd5c07de 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -51,6 +51,7 @@ impl SimpleJITBuilder { // which might not reach all definitions; we can't handle that here, so // we require long-range relocation types. flag_builder.set("use_colocated_libcalls", "false").unwrap(); + flag_builder.set("is_pic", "true").unwrap(); let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { panic!("host machine is not supported: {}", msg); }); From d777ec675ca77463f6a04320e76d5026e2bd6bc4 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 13 Nov 2020 09:28:51 +0100 Subject: [PATCH 11/63] Transparently change non-PLT libcall relocations to PLT relocations --- cranelift/simplejit/src/backend.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index 71dd5c07de..eb6610907c 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -595,7 +595,18 @@ impl<'simple_jit_backend> Module for SimpleJITModule { .as_ref() .unwrap() .perform_relocations( - |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), + |name| match *name { + ir::ExternalName::User { .. } => { + unreachable!("non GOT or PLT relocation in function {} to {}", id, name) + } + ir::ExternalName::LibCall(ref libcall) => self + .libcall_plt_entries + .get(libcall) + .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) + .as_ptr() + .cast::(), + _ => panic!("invalid ExternalName {}", name), + }, |name| self.get_got_address(name), |name| self.get_plt_address(name), ); From b6d783a1202fd45e3f6dbcc22635ce9d67b85f58 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sun, 22 Nov 2020 12:00:54 -0800 Subject: [PATCH 12/63] Adds support for i32x4.trunc_sat_f32x4_u --- cranelift/codegen/src/isa/x64/lower.rs | 41 +++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index bbe886c24b..2abff1a033 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2776,7 +2776,46 @@ fn lower_insn_to_regs>( dst, )); } else if op == Opcode::FcvtToUintSat { - unimplemented!("f32x4.convert_i32x4_u"); + let tmp1 = ctx.alloc_tmp(RegClass::V128, types::I32X4); + let tmp2 = ctx.alloc_tmp(RegClass::V128, types::I32X4); + + // Converting to unsigned int so if float src is negative or NaN + // will first set to zero. + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp2)); + ctx.emit(Inst::gen_move(dst, src, input_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Maxps, RegMem::from(tmp2), dst)); + + // Set tmp2 to the maximum signed floating point value. + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pcmpeqd, RegMem::from(tmp2), tmp2)); + ctx.emit(Inst::xmm_rmi_reg(SseOpcode::Psrld, RegMemImm::imm(1), tmp2)); + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Cvtdq2ps, + RegMem::from(tmp2), + tmp2, + )); + + ctx.emit(Inst::xmm_mov(SseOpcode::Movaps, RegMem::from(dst), tmp1)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Subps, RegMem::from(tmp2), tmp1)); + let cond = FcmpImm::from(FloatCC::LessThanOrEqual); + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Cmpps, + RegMem::from(tmp1), + tmp2, + cond.encode(), + false, + )); + + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Cvttps2dq, + RegMem::from(tmp1), + tmp1, + )); + + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp1)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp2)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmaxsd, RegMem::from(tmp2), tmp1)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Cvttps2dq, RegMem::from(dst), dst)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Paddd, RegMem::from(tmp1), dst)); } else { // Since this branch is also guarded by a check for vector types // neither Opcode::FcvtToUint nor Opcode::FcvtToSint can reach here From 615a575da11da798f0f58645584863bcb4c1a731 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sun, 22 Nov 2020 20:23:00 -0800 Subject: [PATCH 13/63] Add support for x86_64 packed move lowering for the vcode backend --- cranelift/codegen/src/isa/x64/inst/args.rs | 36 +++++++++ cranelift/codegen/src/isa/x64/inst/emit.rs | 12 +++ .../codegen/src/isa/x64/inst/emit_tests.rs | 75 +++++++++++++++++++ 3 files changed, 123 insertions(+) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index e992288560..2958dd56d0 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -507,6 +507,18 @@ pub enum SseOpcode { Pminuw, Pminud, Pmovmskb, + Pmovsxbd, + Pmovsxbw, + Pmovsxbq, + Pmovsxwd, + Pmovsxwq, + Pmovsxdq, + Pmovzxbd, + Pmovzxbw, + Pmovzxbq, + Pmovzxwd, + Pmovzxwq, + Pmovzxdq, Pmulld, Pmullw, Pmuludq, @@ -692,6 +704,18 @@ impl SseOpcode { | SseOpcode::Pminsd | SseOpcode::Pminuw | SseOpcode::Pminud + | SseOpcode::Pmovsxbd + | SseOpcode::Pmovsxbw + | SseOpcode::Pmovsxbq + | SseOpcode::Pmovsxwd + | SseOpcode::Pmovsxwq + | SseOpcode::Pmovsxdq + | SseOpcode::Pmovzxbd + | SseOpcode::Pmovzxbw + | SseOpcode::Pmovzxbq + | SseOpcode::Pmovzxwd + | SseOpcode::Pmovzxwq + | SseOpcode::Pmovzxdq | SseOpcode::Pmulld | SseOpcode::Ptest | SseOpcode::Roundss @@ -812,6 +836,18 @@ impl fmt::Debug for SseOpcode { SseOpcode::Pminuw => "pminuw", SseOpcode::Pminud => "pminud", SseOpcode::Pmovmskb => "pmovmskb", + SseOpcode::Pmovsxbd => "pmovsxbd", + SseOpcode::Pmovsxbw => "pmovsxbw", + SseOpcode::Pmovsxbq => "pmovsxbq", + SseOpcode::Pmovsxwd => "pmovsxwd", + SseOpcode::Pmovsxwq => "pmovsxwq", + SseOpcode::Pmovsxdq => "pmovsxdq", + SseOpcode::Pmovzxbd => "pmovzxbd", + SseOpcode::Pmovzxbw => "pmovzxbw", + SseOpcode::Pmovzxbq => "pmovzxbq", + SseOpcode::Pmovzxwd => "pmovzxwd", + SseOpcode::Pmovzxwq => "pmovzxwq", + SseOpcode::Pmovzxdq => "pmovzxdq", SseOpcode::Pmulld => "pmulld", SseOpcode::Pmullw => "pmullw", SseOpcode::Pmuludq => "pmuludq", diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 7d15063ad4..7d04e0e800 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1802,6 +1802,18 @@ pub(crate) fn emit( SseOpcode::Pcmpgtw => (LegacyPrefixes::_66, 0x0F65, 2), SseOpcode::Pcmpgtd => (LegacyPrefixes::_66, 0x0F66, 2), SseOpcode::Pcmpgtq => (LegacyPrefixes::_66, 0x0F3837, 3), + SseOpcode::Pmovsxbd => (LegacyPrefixes::_66, 0x0F3821, 3), + SseOpcode::Pmovsxbw => (LegacyPrefixes::_66, 0x0F3820, 3), + SseOpcode::Pmovsxbq => (LegacyPrefixes::_66, 0x0F3822, 3), + SseOpcode::Pmovsxwd => (LegacyPrefixes::_66, 0x0F3823, 3), + SseOpcode::Pmovsxwq => (LegacyPrefixes::_66, 0x0F3824, 3), + SseOpcode::Pmovsxdq => (LegacyPrefixes::_66, 0x0F3825, 3), + SseOpcode::Pmovzxbd => (LegacyPrefixes::_66, 0x0F3831, 3), + SseOpcode::Pmovzxbw => (LegacyPrefixes::_66, 0x0F3830, 3), + SseOpcode::Pmovzxbq => (LegacyPrefixes::_66, 0x0F3832, 3), + SseOpcode::Pmovzxwd => (LegacyPrefixes::_66, 0x0F3833, 3), + SseOpcode::Pmovzxwq => (LegacyPrefixes::_66, 0x0F3834, 3), + SseOpcode::Pmovzxdq => (LegacyPrefixes::_66, 0x0F3835, 3), SseOpcode::Pmaxsb => (LegacyPrefixes::_66, 0x0F383C, 3), SseOpcode::Pmaxsw => (LegacyPrefixes::_66, 0x0FEE, 2), SseOpcode::Pmaxsd => (LegacyPrefixes::_66, 0x0F383D, 3), diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 6c2fe6f2d4..bfcd7a401f 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3183,6 +3183,81 @@ fn test_x64_emit() { "cvttps2dq %xmm9, %xmm8", )); + // ======================================================== + // XMM_RM_R: Packed Move + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovsxbd, RegMem::reg(xmm6), w_xmm8), + "66440F3821C6", + "pmovsxbd %xmm6, %xmm8", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovsxbw, RegMem::reg(xmm9), w_xmm10), + "66450F3820D1", + "pmovsxbw %xmm9, %xmm10", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovsxbq, RegMem::reg(xmm1), w_xmm1), + "660F3822C9", + "pmovsxbq %xmm1, %xmm1", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovsxwd, RegMem::reg(xmm13), w_xmm10), + "66450F3823D5", + "pmovsxwd %xmm13, %xmm10", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovsxwq, RegMem::reg(xmm12), w_xmm12), + "66450F3824E4", + "pmovsxwq %xmm12, %xmm12", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovsxdq, RegMem::reg(xmm10), w_xmm8), + "66450F3825C2", + "pmovsxdq %xmm10, %xmm8", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovzxbd, RegMem::reg(xmm5), w_xmm6), + "660F3831F5", + "pmovzxbd %xmm5, %xmm6", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovzxbw, RegMem::reg(xmm5), w_xmm13), + "66440F3830ED", + "pmovzxbw %xmm5, %xmm13", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovzxbq, RegMem::reg(xmm10), w_xmm11), + "66450F3832DA", + "pmovzxbq %xmm10, %xmm11", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovzxwd, RegMem::reg(xmm2), w_xmm10), + "66440F3833D2", + "pmovzxwd %xmm2, %xmm10", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovzxwq, RegMem::reg(xmm7), w_xmm4), + "660F3834E7", + "pmovzxwq %xmm7, %xmm4", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Pmovzxdq, RegMem::reg(xmm3), w_xmm4), + "660F3835E3", + "pmovzxdq %xmm3, %xmm4", + )); + // XMM_Mov_R_M: float stores insns.push(( Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12)), From f9937575d693a40cfacbebf1d1f3942b17eef96b Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sun, 22 Nov 2020 20:58:13 -0800 Subject: [PATCH 14/63] Add support for SwidenLow and UwidenLow for the X86_64 vcode backend Adds support using lowerings compatible with SSE4.1 --- cranelift/codegen/src/isa/x64/lower.rs | 57 +++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 2abff1a033..8e3636acb1 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2825,7 +2825,62 @@ fn lower_insn_to_regs>( } } } - + Opcode::UwidenHigh | Opcode::UwidenLow | Opcode::SwidenHigh | Opcode::SwidenLow => { + let input_ty = ctx.input_ty(insn, 0); + let output_ty = ctx.output_ty(insn, 0); + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]); + if output_ty.is_vector() { + match op { + Opcode::SwidenLow => match (input_ty, output_ty) { + (types::I8X16, types::I16X8) => { + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxbw, RegMem::from(dst), dst)); + } + (types::I16X8, types::I32X4) => { + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxwd, RegMem::from(dst), dst)); + } + _ => unreachable!(), + }, + Opcode::SwidenHigh => match (input_ty, output_ty) { + (types::I8X16, types::I16X8) => { + unimplemented!("No lowering for {:?}", op); + } + (types::I16X8, types::I32X4) => { + unimplemented!("No lowering for {:?}", op); + } + _ => unreachable!(), + }, + Opcode::UwidenLow => match (input_ty, output_ty) { + (types::I8X16, types::I16X8) => { + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxbw, RegMem::from(dst), dst)); + } + (types::I16X8, types::I32X4) => { + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxwd, RegMem::from(dst), dst)); + } + _ => unreachable!(), + }, + Opcode::UwidenHigh => match (input_ty, output_ty) { + (types::I8X16, types::I16X8) => { + unimplemented!("No lowering for {:?}", op); + } + (types::I16X8, types::I32X4) => { + unimplemented!("No lowering for {:?}", op); + } + _ => unreachable!(), + }, + _ => unreachable!(), + } + } else { + panic!("Unsupported non-vector type for widen instruction {:?}", ty); + } + } + Opcode::Snarrow | Opcode::Unarrow => { + unimplemented!("No lowering for {:?}", op); + } Opcode::Bitcast => { let input_ty = ctx.input_ty(insn, 0); let output_ty = ctx.output_ty(insn, 0); From 124096735bba3072b02e319d92d66fc683d38b83 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sun, 22 Nov 2020 21:55:37 -0800 Subject: [PATCH 15/63] Add support for palignr for X86_64 vcode backend --- cranelift/codegen/src/isa/x64/inst/args.rs | 8 +++++++- cranelift/codegen/src/isa/x64/inst/emit.rs | 1 + cranelift/codegen/src/isa/x64/inst/emit_tests.rs | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 2958dd56d0..feede2e063 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -476,6 +476,7 @@ pub enum SseOpcode { Paddsw, Paddusb, Paddusw, + Palignr, Pand, Pandn, Pavgb, @@ -688,7 +689,11 @@ impl SseOpcode { | SseOpcode::Ucomisd | SseOpcode::Xorpd => SSE2, - SseOpcode::Pabsb | SseOpcode::Pabsw | SseOpcode::Pabsd | SseOpcode::Pshufb => SSSE3, + SseOpcode::Pabsb + | SseOpcode::Pabsw + | SseOpcode::Pabsd + | SseOpcode::Palignr + | SseOpcode::Pshufb => SSSE3, SseOpcode::Insertps | SseOpcode::Pcmpeqq @@ -805,6 +810,7 @@ impl fmt::Debug for SseOpcode { SseOpcode::Paddsw => "paddsw", SseOpcode::Paddusb => "paddusb", SseOpcode::Paddusw => "paddusw", + SseOpcode::Palignr => "palignr", SseOpcode::Pand => "pand", SseOpcode::Pandn => "pandn", SseOpcode::Pavgb => "pavgb", diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 7d04e0e800..ec46d27744 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1970,6 +1970,7 @@ pub(crate) fn emit( SseOpcode::Cmpss => (LegacyPrefixes::_F3, 0x0FC2, 2), SseOpcode::Cmpsd => (LegacyPrefixes::_F2, 0x0FC2, 2), SseOpcode::Insertps => (LegacyPrefixes::_66, 0x0F3A21, 3), + SseOpcode::Palignr => (LegacyPrefixes::_66, 0x0F3A0F, 3), SseOpcode::Pinsrb => (LegacyPrefixes::_66, 0x0F3A20, 3), SseOpcode::Pinsrw => (LegacyPrefixes::_66, 0x0FC4, 2), SseOpcode::Pinsrd => (LegacyPrefixes::_66, 0x0F3A22, 3), diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index bfcd7a401f..59560cd4ef 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3481,6 +3481,11 @@ fn test_x64_emit() { "410FC2FF00", "cmpps $0, %xmm15, %xmm7", )); + insns.push(( + Inst::xmm_rm_r_imm(SseOpcode::Palignr, RegMem::reg(xmm1), w_xmm9, 3, false), + "66440F3A0FC903", + "palignr $3, %xmm1, %xmm9", + )); // ======================================================== // Pertaining to atomics. From 258013cff1bd7e844a903b6543536db37ef9522c Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sun, 22 Nov 2020 22:14:19 -0800 Subject: [PATCH 16/63] Add support for SWidenHigh and UWidenHigh X86_64 for vcode backend Support is based on SSE4.1 --- cranelift/codegen/src/isa/x64/lower.rs | 40 +++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 8e3636acb1..488408b47b 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2845,10 +2845,26 @@ fn lower_insn_to_regs>( }, Opcode::SwidenHigh => match (input_ty, output_ty) { (types::I8X16, types::I16X8) => { - unimplemented!("No lowering for {:?}", op); + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Palignr, + RegMem::reg(src), + dst, + 8, + false, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxbw, RegMem::from(dst), dst)); } (types::I16X8, types::I32X4) => { - unimplemented!("No lowering for {:?}", op); + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Palignr, + RegMem::reg(src), + dst, + 8, + false, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxwd, RegMem::from(dst), dst)); } _ => unreachable!(), }, @@ -2865,10 +2881,26 @@ fn lower_insn_to_regs>( }, Opcode::UwidenHigh => match (input_ty, output_ty) { (types::I8X16, types::I16X8) => { - unimplemented!("No lowering for {:?}", op); + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Palignr, + RegMem::reg(src), + dst, + 8, + false, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxbw, RegMem::from(dst), dst)); } (types::I16X8, types::I32X4) => { - unimplemented!("No lowering for {:?}", op); + ctx.emit(Inst::gen_move(dst, src, output_ty)); + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Palignr, + RegMem::reg(src), + dst, + 8, + false, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxwd, RegMem::from(dst), dst)); } _ => unreachable!(), }, From 2cc501427eb7cca9c5cef4be4ec7b02c461b0278 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sun, 22 Nov 2020 23:14:29 -0800 Subject: [PATCH 17/63] Add remaining X86_64 support for pack w/ signed/unsigned saturation Adds lowering for packssdw, packusdw, packuswb --- cranelift/codegen/src/isa/x64/inst/args.rs | 9 +++++++++ cranelift/codegen/src/isa/x64/inst/emit.rs | 3 +++ .../codegen/src/isa/x64/inst/emit_tests.rs | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index feede2e063..4542f3386d 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -467,7 +467,10 @@ pub enum SseOpcode { Pabsb, Pabsw, Pabsd, + Packssdw, Packsswb, + Packusdw, + Packuswb, Paddb, Paddd, Paddq, @@ -633,7 +636,9 @@ impl SseOpcode { | SseOpcode::Mulpd | SseOpcode::Mulsd | SseOpcode::Orpd + | SseOpcode::Packssdw | SseOpcode::Packsswb + | SseOpcode::Packuswb | SseOpcode::Paddb | SseOpcode::Paddd | SseOpcode::Paddq @@ -696,6 +701,7 @@ impl SseOpcode { | SseOpcode::Pshufb => SSSE3, SseOpcode::Insertps + | SseOpcode::Packusdw | SseOpcode::Pcmpeqq | SseOpcode::Pextrb | SseOpcode::Pextrd @@ -801,7 +807,10 @@ impl fmt::Debug for SseOpcode { SseOpcode::Pabsb => "pabsb", SseOpcode::Pabsw => "pabsw", SseOpcode::Pabsd => "pabsd", + SseOpcode::Packssdw => "packssdw", SseOpcode::Packsswb => "packsswb", + SseOpcode::Packusdw => "packusdw", + SseOpcode::Packuswb => "packuswb", SseOpcode::Paddb => "paddb", SseOpcode::Paddd => "paddd", SseOpcode::Paddq => "paddq", diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index ec46d27744..56ecc0e843 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1781,7 +1781,10 @@ pub(crate) fn emit( SseOpcode::Mulsd => (LegacyPrefixes::_F2, 0x0F59, 2), SseOpcode::Orpd => (LegacyPrefixes::_66, 0x0F56, 2), SseOpcode::Orps => (LegacyPrefixes::None, 0x0F56, 2), + SseOpcode::Packssdw => (LegacyPrefixes::_66, 0x0F6B, 2), SseOpcode::Packsswb => (LegacyPrefixes::_66, 0x0F63, 2), + SseOpcode::Packusdw => (LegacyPrefixes::_66, 0x0F382B, 3), + SseOpcode::Packuswb => (LegacyPrefixes::_66, 0x0F67, 2), SseOpcode::Paddb => (LegacyPrefixes::_66, 0x0FFC, 2), SseOpcode::Paddd => (LegacyPrefixes::_66, 0x0FFE, 2), SseOpcode::Paddq => (LegacyPrefixes::_66, 0x0FD4, 2), diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 59560cd4ef..fb9f0c1c07 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3151,12 +3151,30 @@ fn test_x64_emit() { "pshufb %xmm11, %xmm2", )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Packssdw, RegMem::reg(xmm11), w_xmm12), + "66450F6BE3", + "packssdw %xmm11, %xmm12", + )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Packsswb, RegMem::reg(xmm11), w_xmm2), "66410F63D3", "packsswb %xmm11, %xmm2", )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Packusdw, RegMem::reg(xmm13), w_xmm6), + "66410F382BF5", + "packusdw %xmm13, %xmm6", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Packuswb, RegMem::reg(xmm9), w_xmm4), + "66410F67E1", + "packuswb %xmm9, %xmm4", + )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Punpckhbw, RegMem::reg(xmm3), w_xmm2), "660F68D3", From ade9f12c725e97a360df8b653272d5adae22de6b Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Mon, 23 Nov 2020 09:58:39 -0800 Subject: [PATCH 18/63] Add support for X86_64 SIMD narrow instructions for vcode backend Adds lowering support for: i8x16.narrow_i16x8_s i8x16.narrow_i16x8_u i16x8.narrow_i32x4_s i16x8.narrow_i32x4_u --- cranelift/codegen/src/isa/x64/lower.rs | 35 +++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 488408b47b..826ed9b5be 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2911,7 +2911,40 @@ fn lower_insn_to_regs>( } } Opcode::Snarrow | Opcode::Unarrow => { - unimplemented!("No lowering for {:?}", op); + let input_ty = ctx.input_ty(insn, 0); + let output_ty = ctx.output_ty(insn, 0); + let src1 = put_input_in_reg(ctx, inputs[0]); + let src2 = put_input_in_reg(ctx, inputs[1]); + let dst = get_output_reg(ctx, outputs[0]); + if output_ty.is_vector() { + match op { + Opcode::Snarrow => match (input_ty, output_ty) { + (types::I16X8, types::I8X16) => { + ctx.emit(Inst::gen_move(dst, src1, input_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Packsswb, RegMem::reg(src2), dst)); + } + (types::I32X4, types::I16X8) => { + ctx.emit(Inst::gen_move(dst, src1, input_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Packssdw, RegMem::reg(src2), dst)); + } + _ => unreachable!(), + }, + Opcode::Unarrow => match (input_ty, output_ty) { + (types::I16X8, types::I8X16) => { + ctx.emit(Inst::gen_move(dst, src1, input_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Packuswb, RegMem::reg(src2), dst)); + } + (types::I32X4, types::I16X8) => { + ctx.emit(Inst::gen_move(dst, src1, input_ty)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Packusdw, RegMem::reg(src2), dst)); + } + _ => unreachable!(), + }, + _ => unreachable!(), + } + } else { + panic!("Unsupported non-vector type for widen instruction {:?}", ty); + } } Opcode::Bitcast => { let input_ty = ctx.input_ty(insn, 0); From ff8a4e4f9bd1d96fabf2cbcf4810eb4658156485 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Mon, 23 Nov 2020 13:00:13 -0800 Subject: [PATCH 19/63] Enable simd_conversions spec test --- build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/build.rs b/build.rs index da9822e711..c4fcc1a961 100644 --- a/build.rs +++ b/build.rs @@ -210,6 +210,7 @@ fn experimental_x64_should_panic(testsuite: &str, testname: &str, strategy: &str ("simd", "simd_load_splat") => return false, ("simd", "simd_splat") => return false, ("simd", "simd_store") => return false, + ("simd", "simd_conversions") => return false, ("simd", _) => return true, _ => {} } From 09f3d4e3316114a4af1cddf90f6710801ebac590 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Mon, 23 Nov 2020 18:27:23 -0800 Subject: [PATCH 20/63] Refactor convert from float to unsigned int and add comments --- cranelift/codegen/src/isa/x64/lower.rs | 77 +++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 826ed9b5be..09ded3c948 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2722,6 +2722,7 @@ fn lower_insn_to_regs>( } else { if op == Opcode::FcvtToSintSat { // Sets destination to zero if float is NaN + assert_eq!(types::F32X4, ctx.input_ty(insn, 0)); let tmp = ctx.alloc_tmp(RegClass::V128, types::I32X4); ctx.emit(Inst::xmm_unary_rm_r( SseOpcode::Movapd, @@ -2776,6 +2777,57 @@ fn lower_insn_to_regs>( dst, )); } else if op == Opcode::FcvtToUintSat { + // The algorithm for converting floats to unsigned ints is a little tricky. The + // complication arises because we are converting from a signed 64-bit int with a positive + // integer range from 1..INT_MAX (0x1..0x7FFFFFFF) to an unsigned integer with an extended + // range from (INT_MAX+1)..UINT_MAX. It's this range from (INT_MAX+1)..UINT_MAX + // (0x80000000..0xFFFFFFFF) that needs to be accounted for as a special case since our + // conversion instruction (cvttps2dq) only converts as high as INT_MAX (0x7FFFFFFF), but + // which conveniently setting underflows and overflows (smaller than MIN_INT or larger than + // MAX_INT) to be INT_MAX+1 (0x80000000). Nothing that the range (INT_MAX+1)..UINT_MAX includes + // precisely INT_MAX values we can correctly account for and convert every value in this range + // if we simply subtract INT_MAX+1 before doing the cvttps2dq conversion. After the subtraction + // every value originally (INT_MAX+1)..UINT_MAX is now the range (0..INT_MAX). + // After the conversion we add INT_MAX+1 back to this converted value, noting again that + // values we are trying to account for were already set to INT_MAX+1 during the original conversion. + // We simply have to create a mask and make sure we are adding together only the lanes that need + // to be accounted for. Digesting it all the steps then are: + // + // Step 1 - Account for NaN and negative floats by setting these src values to zero. + // Step 2 - Make a copy (tmp1) of the src value since we need to convert twice for + // reasons described above. + // Step 3 - Convert the original src values. This will convert properly all floats up to INT_MAX + // Step 4 - Subtract INT_MAX from the copy set (tmp1). Note, all zero and negative values are those + // values that were originally in the range (0..INT_MAX). This will come in handy during + // step 7 when we zero negative lanes. + // Step 5 - Create a bit mask for tmp1 that will correspond to all lanes originally less than + // UINT_MAX that are now less than INT_MAX thanks to the subtraction. + // Step 6 - Convert the second set of values (tmp1) + // Step 7 - Prep the converted second set by zeroing out negative lanes (these have already been + // converted correctly with the first set) and by setting overflow lanes to 0x7FFFFFFF + // as this will allow us to properly saturate overflow lanes when adding to 0x80000000 + // Step 8 - Add the orginal converted src and the converted tmp1 where float values originally less + // than and equal to INT_MAX will be unchanged, float values originally between INT_MAX+1 and + // UINT_MAX will add together (INT_MAX) + (SRC - INT_MAX), and float values originally + // greater than UINT_MAX will be saturated to UINT_MAX (0xFFFFFFFF) after adding (0x8000000 + 0x7FFFFFFF). + // + // + // The table below illustrates the result after each step where it matters for the converted set. + // Note the original value range (original src set) is the final dst in Step 8: + // + // Original src set: + // | Original Value Range | Step 1 | Step 3 | Step 8 | + // | -FLT_MIN..FLT_MAX | 0.0..FLT_MAX | 0..INT_MAX(w/overflow) | 0..UINT_MAX(w/saturation) | + // + // Copied src set (tmp1): + // | Step 2 | Step 4 | + // | 0.0..FLT_MAX | (0.0-(INT_MAX+1))..(FLT_MAX-(INT_MAX+1)) | + // + // | Step 6 | Step 7 | + // | (0-(INT_MAX+1))..(UINT_MAX-(INT_MAX+1))(w/overflow) | ((INT_MAX+1)-(INT_MAX+1))..(INT_MAX+1) | + + // Create temporaries + assert_eq!(types::F32X4, ctx.input_ty(insn, 0)); let tmp1 = ctx.alloc_tmp(RegClass::V128, types::I32X4); let tmp2 = ctx.alloc_tmp(RegClass::V128, types::I32X4); @@ -2785,7 +2837,13 @@ fn lower_insn_to_regs>( ctx.emit(Inst::gen_move(dst, src, input_ty)); ctx.emit(Inst::xmm_rm_r(SseOpcode::Maxps, RegMem::from(tmp2), dst)); - // Set tmp2 to the maximum signed floating point value. + // Set tmp2 to INT_MAX+1. It is important to note here that after it looks + // like we are only converting INT_MAX (0x7FFFFFFF) but in fact because + // single precision IEEE-754 floats can only accurately represent contingous + // integers up to 2^23 and outside of this range it rounds to the closest + // integer that it can represent. In the case of INT_MAX, this value gets + // represented as 0x4f000000 which is the integer value (INT_MAX+1). + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pcmpeqd, RegMem::from(tmp2), tmp2)); ctx.emit(Inst::xmm_rmi_reg(SseOpcode::Psrld, RegMemImm::imm(1), tmp2)); ctx.emit(Inst::xmm_rm_r( @@ -2794,8 +2852,17 @@ fn lower_insn_to_regs>( tmp2, )); + // Make a copy of these lanes and then do the first conversion. + // Overflow lanes greater than the maximum allowed signed value will + // set to 0x80000000. Negative and NaN lanes will be 0x0 ctx.emit(Inst::xmm_mov(SseOpcode::Movaps, RegMem::from(dst), tmp1)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Cvttps2dq, RegMem::from(dst), dst)); + + // Set lanes to src - max_signed_int ctx.emit(Inst::xmm_rm_r(SseOpcode::Subps, RegMem::from(tmp2), tmp1)); + + // Create mask for all positive lanes to saturate (i.e. greater than + // or equal to the maxmimum allowable unsigned int). let cond = FcmpImm::from(FloatCC::LessThanOrEqual); ctx.emit(Inst::xmm_rm_r_imm( SseOpcode::Cmpps, @@ -2805,16 +2872,22 @@ fn lower_insn_to_regs>( false, )); + // Convert those set of lanes that have the max_signed_int factored out. ctx.emit(Inst::xmm_rm_r( SseOpcode::Cvttps2dq, RegMem::from(tmp1), tmp1, )); + // Prepare converted lanes by zeroing negative lanes and prepping lanes + // that have positive overflow (based on the mask) by setting these lanes + // to 0x7FFFFFFF ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp1)); ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp2)); ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmaxsd, RegMem::from(tmp2), tmp1)); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Cvttps2dq, RegMem::from(dst), dst)); + + // Add this second set of converted lanes to the original to properly handle + // values greater than max signed int. ctx.emit(Inst::xmm_rm_r(SseOpcode::Paddd, RegMem::from(tmp1), dst)); } else { // Since this branch is also guarded by a check for vector types From 26509cb080b2ec8b77cac816c385f6794e9fcff3 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Sun, 29 Nov 2020 20:38:02 -0800 Subject: [PATCH 21/63] Optimize access to interpreter frame slots Previously, getting or setting a value in a frame of the Cranelift interpreter involved a hash table lookup. Since the interpreter statically knows the number of slots necessary for each called frame, we can use a vector instead and save time on the hash lookup. This also has the advantage that we have a more stable ABI for switching between interpreted and code. --- cranelift/interpreter/src/frame.rs | 169 +++++++++++++++++++++++------ 1 file changed, 138 insertions(+), 31 deletions(-) diff --git a/cranelift/interpreter/src/frame.rs b/cranelift/interpreter/src/frame.rs index d6f3065c47..2a437a34f0 100644 --- a/cranelift/interpreter/src/frame.rs +++ b/cranelift/interpreter/src/frame.rs @@ -2,41 +2,45 @@ use cranelift_codegen::data_value::DataValue; use cranelift_codegen::ir::{Function, Value as ValueRef}; +use cranelift_entity::EntityRef; use log::trace; -use std::collections::HashMap; -/// Holds the mutable elements of an interpretation. At some point I thought about using -/// Cell/RefCell to do field-level mutability, thinking that otherwise I would have to -/// pass around a mutable object (for inst and registers) and an immutable one (for function, -/// could be self)--in the end I decided to do exactly that but perhaps one day that will become -/// untenable. +/// The type used for ensuring [Frame](crate::frame::Frame) entries conform to the expected memory layout. +pub(crate) type Entries = Vec>; + +/// Holds the mutable elements of an interpreted function call. #[derive(Debug)] pub struct Frame<'a> { /// The currently executing function. - pub function: &'a Function, - /// The current mapping of SSA value-references to their actual values. - registers: HashMap, + pub(crate) function: &'a Function, + /// The current mapping of SSA value-references to their actual values. For efficiency, each SSA value is used as an + /// index into the Vec, meaning some slots may be unused. + registers: Entries, } impl<'a> Frame<'a> { - /// Construct a new [Frame] for a function. This allocates a slot in the hash map for each SSA - /// `Value` (renamed to `ValueRef` here) which should mean that no additional allocations are - /// needed while interpreting the frame. + /// Construct a new [Frame] for a function. This allocates a slot in the hash map for each SSA `Value` (renamed to + /// `ValueRef` here) which should mean that no additional allocations are needed while interpreting the frame. pub fn new(function: &'a Function) -> Self { + let num_slots = function.dfg.num_values(); trace!("Create new frame for function: {}", function.signature); Self { function, - registers: HashMap::with_capacity(function.dfg.num_values()), + registers: vec![None; num_slots], } } /// Retrieve the actual value associated with an SSA reference. #[inline] pub fn get(&self, name: ValueRef) -> &DataValue { + assert!(name.index() < self.registers.len()); trace!("Get {}", name); - self.registers - .get(&name) + &self + .registers + .get(name.index()) .unwrap_or_else(|| panic!("unknown value: {}", name)) + .as_ref() + .unwrap_or_else(|| panic!("empty slot: {}", name)) } /// Retrieve multiple SSA references; see `get`. @@ -47,8 +51,9 @@ impl<'a> Frame<'a> { /// Assign `value` to the SSA reference `name`. #[inline] pub fn set(&mut self, name: ValueRef, value: DataValue) -> Option { + assert!(name.index() < self.registers.len()); trace!("Set {} -> {}", name, value); - self.registers.insert(name, value) + std::mem::replace(&mut self.registers[name.index()], Some(value)) } /// Assign to multiple SSA references; see `set`. @@ -66,12 +71,18 @@ impl<'a> Frame<'a> { pub fn rename(&mut self, old_names: &[ValueRef], new_names: &[ValueRef]) { trace!("Renaming {:?} -> {:?}", old_names, new_names); assert_eq!(old_names.len(), new_names.len()); - let mut registers = HashMap::with_capacity(self.registers.len()); - for (on, nn) in old_names.iter().zip(new_names) { - let v = self.registers.get(on).unwrap().clone(); - registers.insert(*nn, v); + let new_registers = vec![None; self.registers.len()]; + let mut old_registers = std::mem::replace(&mut self.registers, new_registers); + self.registers = vec![None; self.registers.len()]; + for (&on, &nn) in old_names.iter().zip(new_names) { + let value = std::mem::replace(&mut old_registers[on.index()], None); + self.registers[nn.index()] = value; } - self.registers = registers; + } + + /// Accessor for the current entries in the frame. + pub fn entries_mut(&mut self) -> &mut [Option] { + &mut self.registers } } @@ -79,8 +90,15 @@ impl<'a> Frame<'a> { mod tests { use super::*; use cranelift_codegen::data_value::DataValue; + use cranelift_codegen::ir::immediates::{Ieee32, Ieee64}; use cranelift_codegen::ir::InstBuilder; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; + use cranelift_reader::parse_functions; + + /// Helper to create a function from CLIF IR. + fn function(code: &str) -> Function { + parse_functions(code).unwrap().into_iter().next().unwrap() + } /// Build an empty function with a single return. fn empty_function() -> Function { @@ -101,23 +119,112 @@ mod tests { } #[test] - fn assignment() { - let func = empty_function(); + fn assignment_and_retrieval() { + let func = function("function %test(i32) -> i32 { block0(v0:i32): return v0 }"); let mut frame = Frame::new(&func); - - let a = ValueRef::with_number(1).unwrap(); + let ssa_value_ref = ValueRef::from_u32(0); let fortytwo = DataValue::I32(42); - frame.set(a, fortytwo.clone()); - assert_eq!(frame.get(a), &fortytwo); + + // Verify that setting a valid SSA ref will make the value retrievable. + frame.set(ssa_value_ref, fortytwo.clone()); + assert_eq!(frame.get(ssa_value_ref), &fortytwo); } #[test] - #[should_panic] - fn no_existing_value() { + fn assignment_to_extra_slots() { + let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }"); + let mut frame = Frame::new(&func); + let ssa_value_ref = ValueRef::from_u32(5); + let fortytwo = DataValue::I32(42); + + // Due to how Cranelift organizes its SSA values, the use of v10 defines 11 slots for values + // to fit in--the following should work. + frame.set(ssa_value_ref, fortytwo.clone()); + assert_eq!(frame.get(ssa_value_ref), &fortytwo); + } + + #[test] + #[should_panic(expected = "assertion failed: name.index() < self.registers.len()")] + fn invalid_assignment() { + let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }"); + let mut frame = Frame::new(&func); + let fortytwo = DataValue::I32(42); + + // Since the SSA value ref points to 42 and the function only has 11 slots, this should + // fail. TODO currently this is a panic under the assumption we will not set indexes outside + // of the valid SSA value range but it might be better as a result. + frame.set(ValueRef::from_u32(11), fortytwo.clone()); + } + + #[test] + #[should_panic(expected = "assertion failed: name.index() < self.registers.len()")] + fn retrieve_nonexistent_value() { let func = empty_function(); let frame = Frame::new(&func); + let ssa_value_ref = ValueRef::from_u32(1); - let a = ValueRef::with_number(1).unwrap(); - frame.get(a); + // Retrieving a non-existent value should return an error. + frame.get(ssa_value_ref); + } + + #[test] + #[should_panic(expected = "empty slot: v5")] + fn retrieve_and_assign_multiple_values() { + let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }"); + let mut frame = Frame::new(&func); + let ssa_value_refs = [ + ValueRef::from_u32(2), + ValueRef::from_u32(4), + ValueRef::from_u32(6), + ]; + let values = vec![ + DataValue::B(true), + DataValue::I8(42), + DataValue::F32(Ieee32::from(0.42)), + ]; + + // We can assign and retrieve multiple (cloned) values. + frame.set_all(&ssa_value_refs, values.clone()); + let retrieved_values = frame.get_all(&ssa_value_refs); + assert_eq!(values, retrieved_values); + + // But if we attempt to retrieve an invalid value we should get an error: + frame.get_all(&[ValueRef::from_u32(2), ValueRef::from_u32(5)]); + } + + #[test] + #[should_panic(expected = "empty slot: v10")] + fn rename() { + let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }"); + let mut frame = Frame::new(&func); + let old_ssa_value_refs = [ValueRef::from_u32(9), ValueRef::from_u32(10)]; + let values = vec![DataValue::B(true), DataValue::F64(Ieee64::from(0.0))]; + frame.set_all(&old_ssa_value_refs, values.clone()); + + // Rename the old SSA values to the new values. + let new_ssa_value_refs = [ValueRef::from_u32(4), ValueRef::from_u32(2)]; + frame.rename(&old_ssa_value_refs, &new_ssa_value_refs); + + // Now we should be able to retrieve new values and the old ones should fail. + assert_eq!(frame.get_all(&new_ssa_value_refs), values); + frame.get(ValueRef::from_u32(10)); + } + + #[test] + #[should_panic(expected = "empty slot: v2")] + fn rename_duplicates_causes_inconsistency() { + let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }"); + let mut frame = Frame::new(&func); + let old_ssa_value_refs = [ValueRef::from_u32(1), ValueRef::from_u32(9)]; + let values = vec![DataValue::B(true), DataValue::F64(Ieee64::from(f64::NAN))]; + frame.set_all(&old_ssa_value_refs, values.clone()); + + // Rename the old SSA values to the new values. + let old_duplicated_ssa_value_refs = [ValueRef::from_u32(1), ValueRef::from_u32(1)]; + let new_ssa_value_refs = [ValueRef::from_u32(4), ValueRef::from_u32(2)]; + frame.rename(&old_duplicated_ssa_value_refs, &new_ssa_value_refs); + + // If we use duplicates then subsequent renamings (v1 -> v2) will be empty. + frame.get(ValueRef::from_u32(2)); } } From 87b1a85cc645be0709d9cfcbf09e36dd162c8d41 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 30 Nov 2020 17:28:30 -0800 Subject: [PATCH 22/63] Fix confusion caused by overloading of FuncRef Prior to this change, the interpreter would use an incorrect `FuncRef` for accessing functions from the function store. This is now clarified and fixed by a new type--`FuncIndex`. --- cranelift/interpreter/src/environment.rs | 74 +++++++++++++++--------- cranelift/interpreter/src/interpreter.rs | 58 +++++++++++++++---- 2 files changed, 95 insertions(+), 37 deletions(-) diff --git a/cranelift/interpreter/src/environment.rs b/cranelift/interpreter/src/environment.rs index b3b9dade60..6c7a9a0b26 100644 --- a/cranelift/interpreter/src/environment.rs +++ b/cranelift/interpreter/src/environment.rs @@ -1,52 +1,74 @@ //! Implements the function environment (e.g. a name-to-function mapping) for interpretation. - use cranelift_codegen::ir::{FuncRef, Function}; +use cranelift_entity::{entity_impl, PrimaryMap}; use std::collections::HashMap; +/// A function store contains all of the functions that are accessible to an interpreter. #[derive(Default, Clone)] pub struct FunctionStore<'a> { - functions: HashMap, - function_name_to_func_ref: HashMap, + functions: PrimaryMap, + function_names: HashMap, } +/// An opaque reference to a [`Function`](Function) stored in the [FunctionStore]. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct FuncIndex(u32); +entity_impl!(FuncIndex, "fn"); + +/// This is a helpful conversion for instantiating a store from a single [Function]. impl<'a> From<&'a Function> for FunctionStore<'a> { - fn from(f: &'a Function) -> Self { - let func_ref = FuncRef::from_u32(0); - let mut function_name_to_func_ref = HashMap::new(); - function_name_to_func_ref.insert(f.name.to_string(), func_ref); - let mut functions = HashMap::new(); - functions.insert(func_ref, f); - Self { - functions, - function_name_to_func_ref, - } + fn from(function: &'a Function) -> Self { + let mut store = FunctionStore::default(); + store.add(function.name.to_string(), function); + store } } impl<'a> FunctionStore<'a> { /// Add a function by name. pub fn add(&mut self, name: String, function: &'a Function) { - let func_ref = FuncRef::with_number(self.function_name_to_func_ref.len() as u32) - .expect("a valid function reference"); - self.function_name_to_func_ref.insert(name, func_ref); - self.functions.insert(func_ref, function); + assert!(!self.function_names.contains_key(&name)); + let index = self.functions.push(function); + self.function_names.insert(name, index); } - /// Retrieve a reference to a function in the environment by its name. - pub fn index_of(&self, name: &str) -> Option { - self.function_name_to_func_ref.get(name).cloned() + /// Retrieve the index of a function in the function store by its `name`. + pub fn index_of(&self, name: &str) -> Option { + self.function_names.get(name).cloned() } - /// Retrieve a function by its function reference. - pub fn get_by_func_ref(&self, func_ref: FuncRef) -> Option<&'a Function> { - self.functions.get(&func_ref).cloned() + /// Retrieve a function by its index in the function store. + pub fn get_by_index(&self, index: FuncIndex) -> Option<&'a Function> { + self.functions.get(index).cloned() } /// Retrieve a function by its name. pub fn get_by_name(&self, name: &str) -> Option<&'a Function> { - let func_ref = self.index_of(name)?; - self.get_by_func_ref(func_ref) + let index = self.index_of(name)?; + self.get_by_index(index) } + + /// Retrieve a function from a [FuncRef] within a [Function]. TODO this should be optimized, if possible, as + /// currently it retrieves the function name as a string and performs string matching. + pub fn get_from_func_ref( + &self, + func_ref: FuncRef, + function: &Function, + ) -> Option<&'a Function> { + self.get_by_name(&get_function_name(func_ref, function)) + } +} + +/// Retrieve a function name from a [FuncRef] within a [Function]. TODO this should be optimized, if possible, as +/// currently it retrieves the function name as a string and performs string matching. +fn get_function_name(func_ref: FuncRef, function: &Function) -> String { + function + .dfg + .ext_funcs + .get(func_ref) + .expect("function to exist") + .name + .to_string() } #[cfg(test)] @@ -77,6 +99,6 @@ mod tests { let signature = Signature::new(CallConv::Fast); let func = &Function::with_name_signature(name, signature); let env: FunctionStore = func.into(); - assert_eq!(env.index_of("%test"), FuncRef::with_number(0)); + assert_eq!(env.index_of("%test"), Some(FuncIndex::from_u32(0))); } } diff --git a/cranelift/interpreter/src/interpreter.rs b/cranelift/interpreter/src/interpreter.rs index f6032b7cdb..89d2ae32a0 100644 --- a/cranelift/interpreter/src/interpreter.rs +++ b/cranelift/interpreter/src/interpreter.rs @@ -2,7 +2,7 @@ //! //! This module partially contains the logic for interpreting Cranelift IR. -use crate::environment::FunctionStore; +use crate::environment::{FuncIndex, FunctionStore}; use crate::frame::Frame; use crate::instruction::DfgInstructionContext; use crate::state::{MemoryError, State}; @@ -34,22 +34,22 @@ impl<'a> Interpreter<'a> { func_name: &str, arguments: &[DataValue], ) -> Result, InterpreterError> { - let func_ref = self + let index = self .state .functions .index_of(func_name) .ok_or_else(|| InterpreterError::UnknownFunctionName(func_name.to_string()))?; - self.call_by_index(func_ref, arguments) + self.call_by_index(index, arguments) } /// Call a function by its index in the [FunctionStore]; this is a proxy for [Interpreter::call]. pub fn call_by_index( &mut self, - func_ref: FuncRef, + index: FuncIndex, arguments: &[DataValue], ) -> Result, InterpreterError> { - match self.state.get_function(func_ref) { - None => Err(InterpreterError::UnknownFunctionReference(func_ref)), + match self.state.functions.get_by_index(index) { + None => Err(InterpreterError::UnknownFunctionIndex(index)), Some(func) => self.call(func, arguments), } } @@ -98,8 +98,9 @@ impl<'a> Interpreter<'a> { .set_all(function.dfg.block_params(block), block_arguments.to_vec()); maybe_inst = layout.first_inst(block) } - ControlFlow::Call(function, arguments) => { - let returned_arguments = self.call(function, &arguments)?.unwrap_return(); + ControlFlow::Call(called_function, arguments) => { + let returned_arguments = + self.call(called_function, &arguments)?.unwrap_return(); self.state .current_frame_mut() .set_all(function.dfg.inst_results(inst), returned_arguments); @@ -123,8 +124,8 @@ pub enum InterpreterError { StepError(#[from] StepError), #[error("reached an unreachable statement")] Unreachable, - #[error("unknown function reference (has it been added to the function store?): {0}")] - UnknownFunctionReference(FuncRef), + #[error("unknown function index (has it been added to the function store?): {0}")] + UnknownFunctionIndex(FuncIndex), #[error("unknown function with name (has it been added to the function store?): {0}")] UnknownFunctionName(String), #[error("value error")] @@ -176,7 +177,8 @@ impl<'a> InterpreterState<'a> { impl<'a> State<'a, DataValue> for InterpreterState<'a> { fn get_function(&self, func_ref: FuncRef) -> Option<&'a Function> { - self.functions.get_by_func_ref(func_ref) + self.functions + .get_from_func_ref(func_ref, self.frame_stack.last().unwrap().function) } fn push_frame(&mut self, function: &'a Function) { self.frame_stack.push(Frame::new(function)); @@ -273,6 +275,40 @@ mod tests { assert_eq!(result, vec![DataValue::B(true)]) } + // This test verifies that functions can refer to each other using the function store. A double indirection is + // required, which is tricky to get right: a referenced function is a FuncRef when called but a FuncIndex inside the + // function store. This test would preferably be a CLIF filetest but the filetest infrastructure only looks at a + // single function at a time--we need more than one function in the store for this test. + #[test] + fn function_references() { + let code = " + function %child(i32) -> i32 { + block0(v0: i32): + v1 = iadd_imm v0, -1 + return v1 + } + + function %parent(i32) -> i32 { + fn42 = %child(i32) -> i32 + block0(v0: i32): + v1 = iadd_imm v0, 1 + v2 = call fn42(v1) + return v2 + }"; + + let mut env = FunctionStore::default(); + let funcs = parse_functions(code).unwrap().to_vec(); + funcs.iter().for_each(|f| env.add(f.name.to_string(), f)); + + let state = InterpreterState::default().with_function_store(env); + let result = Interpreter::new(state) + .call_by_name("%parent", &[DataValue::I32(0)]) + .unwrap() + .unwrap_return(); + + assert_eq!(result, vec![DataValue::I32(0)]) + } + #[test] fn state_heap_roundtrip() -> Result<(), MemoryError> { let mut state = InterpreterState::default(); From 88a8a8993a40a883622f7deb93d8b289d1e51f5a Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 1 Dec 2020 14:01:31 -0600 Subject: [PATCH 23/63] Instantiate nested modules for module linking (#2447) This commit implements the interpretation necessary of the instance section of the module linking proposal. Instantiating a module which itself has nested instantiated instances will now instantiate the nested instances properly. This isn't all that useful without the ability to alias exports off the result, but we can at least observe the side effects of instantiation through the `start` function. cc #2094 --- build.rs | 1 + cranelift/wasm/src/environ/spec.rs | 17 +- cranelift/wasm/src/module_translator.rs | 7 +- cranelift/wasm/src/sections_translator.rs | 40 ++- crates/environ/src/module_environ.rs | 20 +- crates/jit/src/instantiate.rs | 14 +- crates/wasmtime/src/externals.rs | 9 - crates/wasmtime/src/instance.rs | 264 ++++++++++++------ crates/wasmtime/src/module.rs | 2 +- tests/all/wast.rs | 2 + .../module-linking/instantiate.wast | 162 +++++++++++ 11 files changed, 433 insertions(+), 105 deletions(-) create mode 100644 tests/misc_testsuite/module-linking/instantiate.wast diff --git a/build.rs b/build.rs index c4fcc1a961..4937f6bf12 100644 --- a/build.rs +++ b/build.rs @@ -31,6 +31,7 @@ fn main() -> anyhow::Result<()> { test_directory_module(out, "tests/misc_testsuite/bulk-memory-operations", strategy)?; test_directory_module(out, "tests/misc_testsuite/reference-types", strategy)?; test_directory_module(out, "tests/misc_testsuite/multi-memory", strategy)?; + test_directory_module(out, "tests/misc_testsuite/module-linking", strategy)?; Ok(()) })?; diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 6fc513f6be..6d448f8d6d 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -8,8 +8,8 @@ use crate::state::FuncTranslationState; use crate::translation_utils::{ - DataIndex, ElemIndex, EntityType, Event, EventIndex, FuncIndex, Global, GlobalIndex, Memory, - MemoryIndex, Table, TableIndex, TypeIndex, + DataIndex, ElemIndex, EntityIndex, EntityType, Event, EventIndex, FuncIndex, Global, + GlobalIndex, Memory, MemoryIndex, ModuleIndex, Table, TableIndex, TypeIndex, }; use core::convert::From; use core::convert::TryFrom; @@ -22,6 +22,7 @@ use cranelift_frontend::FunctionBuilder; use serde::{Deserialize, Serialize}; use std::boxed::Box; use std::string::ToString; +use std::vec::Vec; use thiserror::Error; use wasmparser::ValidatorResources; use wasmparser::{BinaryReaderError, FuncValidator, FunctionBody, Operator, WasmFeatures}; @@ -969,4 +970,16 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { fn module_end(&mut self, index: usize) { drop(index); } + + /// Indicates that this module will have `amount` instances. + fn reserve_instances(&mut self, amount: u32) { + drop(amount); + } + + /// Declares a new instance which this module will instantiate before it's + /// instantiated. + fn declare_instance(&mut self, module: ModuleIndex, args: Vec) -> WasmResult<()> { + drop((module, args)); + Err(WasmError::Unsupported("wasm instance".to_string())) + } } diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs index 631dedeaf3..5fc60ccbdd 100644 --- a/cranelift/wasm/src/module_translator.rs +++ b/cranelift/wasm/src/module_translator.rs @@ -3,8 +3,9 @@ use crate::environ::{ModuleEnvironment, WasmResult}; use crate::sections_translator::{ parse_data_section, parse_element_section, parse_event_section, parse_export_section, - parse_function_section, parse_global_section, parse_import_section, parse_memory_section, - parse_name_section, parse_start_section, parse_table_section, parse_type_section, + parse_function_section, parse_global_section, parse_import_section, parse_instance_section, + parse_memory_section, parse_name_section, parse_start_section, parse_table_section, + parse_type_section, }; use crate::state::ModuleTranslationState; use cranelift_codegen::timing; @@ -116,7 +117,7 @@ pub fn translate_module<'data>( } Payload::InstanceSection(s) => { validator.instance_section(&s)?; - unimplemented!("module linking not implemented yet") + parse_instance_section(s, environ)?; } Payload::AliasSection(s) => { validator.alias_section(&s)?; diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index 2e45d4bbf3..e9fc525542 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -10,9 +10,9 @@ use crate::environ::{ModuleEnvironment, WasmError, WasmResult}; use crate::state::ModuleTranslationState; use crate::translation_utils::{ - tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityType, Event, EventIndex, - FuncIndex, Global, GlobalIndex, GlobalInit, Memory, MemoryIndex, Table, TableElementType, - TableIndex, TypeIndex, + tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityIndex, EntityType, Event, + EventIndex, FuncIndex, Global, GlobalIndex, GlobalInit, InstanceIndex, Memory, MemoryIndex, + ModuleIndex, Table, TableElementType, TableIndex, TypeIndex, }; use crate::wasm_unsupported; use core::convert::TryFrom; @@ -475,3 +475,37 @@ pub fn parse_name_section<'data>( } Ok(()) } + +/// Parses the Instance section of the wasm module. +pub fn parse_instance_section<'data>( + section: wasmparser::InstanceSectionReader<'data>, + environ: &mut dyn ModuleEnvironment<'data>, +) -> WasmResult<()> { + environ.reserve_types(section.get_count())?; + + for instance in section { + let instance = instance?; + let module = ModuleIndex::from_u32(instance.module()); + let args = instance + .args()? + .into_iter() + .map(|result| { + let (kind, idx) = result?; + Ok(match kind { + ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(idx)), + ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(idx)), + ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(idx)), + ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(idx)), + ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(idx)), + ExternalKind::Instance => EntityIndex::Instance(InstanceIndex::from_u32(idx)), + ExternalKind::Event => unimplemented!(), + + // this won't pass validation + ExternalKind::Type => unreachable!(), + }) + }) + .collect::>>()?; + environ.declare_instance(module, args)?; + } + Ok(()) +} diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 842853a3cc..e085751d77 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -1,4 +1,4 @@ -use crate::module::{MemoryPlan, Module, ModuleType, TableElements, TablePlan}; +use crate::module::{Instance, MemoryPlan, Module, ModuleType, TableElements, TablePlan}; use crate::tunables::Tunables; use cranelift_codegen::ir; use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; @@ -6,8 +6,8 @@ use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_entity::PrimaryMap; use cranelift_wasm::{ self, translate_module, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, - FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, SignatureIndex, Table, TableIndex, - TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, + FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, ModuleIndex, SignatureIndex, Table, + TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, }; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -60,7 +60,7 @@ pub struct ModuleTranslation<'data> { /// Indexes into the returned list of translations that are submodules of /// this module. - pub submodules: Vec, + pub submodules: PrimaryMap, code_index: u32, } @@ -649,6 +649,18 @@ and for re-adding support for interface types you can see this issue: self.result.submodules.push(self.results.len()); self.results.push(finished); } + + fn reserve_instances(&mut self, amt: u32) { + self.result.module.instances.reserve(amt as usize); + } + + fn declare_instance(&mut self, module: ModuleIndex, args: Vec) -> WasmResult<()> { + self.result + .module + .instances + .push(Instance::Instantiate { module, args }); + Ok(()) + } } /// Add environment-specific function parameters. diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 0d27918f51..3ec6f04bf1 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -17,7 +17,7 @@ use thiserror::Error; use wasmtime_debug::create_gdbjit_image; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex}; +use wasmtime_environ::wasm::{DefinedFuncIndex, ModuleIndex, SignatureIndex}; use wasmtime_environ::{ CompileError, DataInitializer, DataInitializerLocation, FunctionAddressMap, Module, ModuleEnvironment, ModuleTranslation, StackMapInformation, TrapInformation, @@ -71,6 +71,10 @@ pub struct CompilationArtifacts { /// Debug info presence flags. debug_info: bool, + + /// Where to find this module's submodule code in the top-level list of + /// modules. + submodules: PrimaryMap, } impl CompilationArtifacts { @@ -98,6 +102,7 @@ impl CompilationArtifacts { let ModuleTranslation { module, data_initializers, + submodules, .. } = translation; @@ -118,6 +123,7 @@ impl CompilationArtifacts { obj: obj.into_boxed_slice(), unwind_info: unwind_info.into_boxed_slice(), data_initializers, + submodules, funcs: funcs .into_iter() .map(|(_, func)| FunctionInfo { @@ -336,6 +342,12 @@ impl CompiledModule { pub fn code(&self) -> &Arc { &self.code } + + /// Returns where the specified submodule lives in this module's + /// array-of-modules (store at the top-level) + pub fn submodule_idx(&self, idx: ModuleIndex) -> usize { + self.artifacts.submodules[idx] + } } /// Similar to `DataInitializer`, but owns its own copy of the data rather diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index 63f2e2678f..bc8d2daffc 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -115,15 +115,6 @@ impl Extern { }; Store::same(my_store, store) } - - pub(crate) fn desc(&self) -> &'static str { - match self { - Extern::Func(_) => "function", - Extern::Table(_) => "table", - Extern::Memory(_) => "memory", - Extern::Global(_) => "global", - } - } } impl From for Extern { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 5c612b1db9..b375b48848 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -1,21 +1,64 @@ use crate::trampoline::StoreInstanceHandle; use crate::{Engine, Export, Extern, Func, Global, Memory, Module, Store, Table, Trap}; -use anyhow::{anyhow, bail, Context, Error, Result}; -use std::any::Any; +use anyhow::{bail, Error, Result}; use std::mem; -use wasmtime_environ::wasm::EntityIndex; +use wasmtime_environ::entity::PrimaryMap; +use wasmtime_environ::wasm::{EntityIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex}; use wasmtime_jit::CompiledModule; use wasmtime_runtime::{ Imports, InstantiationError, StackMapRegistry, VMContext, VMExternRefActivationsTable, - VMFunctionBody, + VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, }; fn instantiate( store: &Store, compiled_module: &CompiledModule, - imports: Imports<'_>, - host: Box, + all_modules: &[CompiledModule], + imports: &mut ImportsBuilder<'_>, ) -> Result { + let env_module = compiled_module.module(); + + // The first part of instantiating any module is to first follow any + // `instantiate` instructions it has as part of the module linking + // proposal. Here we iterate overall those instructions and create the + // instances as necessary. + for instance in env_module.instances.values() { + let (module_idx, args) = match instance { + wasmtime_environ::Instance::Instantiate { module, args } => (*module, args), + wasmtime_environ::Instance::Import(_) => continue, + }; + // Translate the `module_idx` to a top-level module `usize` and then + // use that to extract the child `&CompiledModule` itself. Then we can + // iterate over each of the arguments provided to satisfy its imports. + // + // Note that we directly reach into `imports` below based on indexes + // and push raw value into how to instantiate our submodule. This should + // be safe due to wasm validation ensuring that all our indices are + // in-bounds and all the expected types and such line up. + let module_idx = compiled_module.submodule_idx(module_idx); + let compiled_module = &all_modules[module_idx]; + let mut builder = ImportsBuilder::new(compiled_module.module(), store); + for arg in args { + match *arg { + EntityIndex::Global(i) => { + builder.globals.push(imports.globals[i]); + } + EntityIndex::Table(i) => { + builder.tables.push(imports.tables[i]); + } + EntityIndex::Function(i) => { + builder.functions.push(imports.functions[i]); + } + EntityIndex::Memory(i) => { + builder.memories.push(imports.memories[i]); + } + EntityIndex::Module(_) => unimplemented!(), + EntityIndex::Instance(_) => unimplemented!(), + } + } + instantiate(store, compiled_module, all_modules, &mut builder)?; + } + // Register the module just before instantiation to ensure we have a // trampoline registered for every signature and to preserve the module's // compiled JIT code within the `Store`. @@ -24,11 +67,11 @@ fn instantiate( let config = store.engine().config(); let instance = unsafe { let instance = compiled_module.instantiate( - imports, + imports.imports(), &store.lookup_shared_signature(compiled_module.module()), config.memory_creator.as_ref().map(|a| a as _), store.interrupts(), - host, + Box::new(()), store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, store.stack_map_registry() as *const StackMapRegistry as *mut _, )?; @@ -165,9 +208,27 @@ impl Instance { bail!("cross-`Engine` instantiation is not currently supported"); } - let handle = with_imports(store, module.compiled_module(), imports, |imports| { - instantiate(store, module.compiled_module(), imports, Box::new(())) - })?; + let mut builder = ImportsBuilder::new(module.compiled_module().module(), store); + for import in imports { + // For now we have a restriction that the `Store` that we're working + // with is the same for everything involved here. + if !import.comes_from_same_store(store) { + bail!("cross-`Store` instantiation is not currently supported"); + } + match import { + Extern::Global(e) => builder.global(e)?, + Extern::Func(e) => builder.func(e)?, + Extern::Table(e) => builder.table(e)?, + Extern::Memory(e) => builder.memory(e)?, + } + } + builder.validate_all_imports_provided()?; + let handle = instantiate( + store, + module.compiled_module(), + &module.compiled, + &mut builder, + )?; Ok(Instance { handle, @@ -238,87 +299,126 @@ impl Instance { } } -fn with_imports( - store: &Store, - module: &CompiledModule, - externs: &[Extern], - f: impl FnOnce(Imports<'_>) -> Result, -) -> Result { - let m = module.module(); - if externs.len() != m.imports.len() { - bail!( - "wrong number of imports provided, {} != {}", - externs.len(), - m.imports.len() - ); +struct ImportsBuilder<'a> { + functions: PrimaryMap, + tables: PrimaryMap, + memories: PrimaryMap, + globals: PrimaryMap, + module: &'a wasmtime_environ::Module, + imports: std::slice::Iter<'a, (String, Option, EntityIndex)>, + store: &'a Store, +} + +impl<'a> ImportsBuilder<'a> { + fn new(module: &'a wasmtime_environ::Module, store: &'a Store) -> ImportsBuilder<'a> { + ImportsBuilder { + imports: module.imports.iter(), + module, + store, + functions: PrimaryMap::with_capacity(module.num_imported_funcs), + tables: PrimaryMap::with_capacity(module.num_imported_tables), + memories: PrimaryMap::with_capacity(module.num_imported_memories), + globals: PrimaryMap::with_capacity(module.num_imported_globals), + } } - let mut tables = Vec::new(); - let mut functions = Vec::new(); - let mut globals = Vec::new(); - let mut memories = Vec::new(); - - let mut process = |expected: &EntityIndex, actual: &Extern| { - // For now we have a restriction that the `Store` that we're working - // with is the same for everything involved here. - if !actual.comes_from_same_store(store) { - bail!("cross-`Store` instantiation is not currently supported"); - } - - match *expected { - EntityIndex::Table(i) => tables.push(match actual { - Extern::Table(e) if e.matches_expected(&m.table_plans[i]) => e.vmimport(), - Extern::Table(_) => bail!("table types incompatible"), - _ => bail!("expected table, but found {}", actual.desc()), - }), - EntityIndex::Memory(i) => memories.push(match actual { - Extern::Memory(e) if e.matches_expected(&m.memory_plans[i]) => e.vmimport(), - Extern::Memory(_) => bail!("memory types incompatible"), - _ => bail!("expected memory, but found {}", actual.desc()), - }), - EntityIndex::Global(i) => globals.push(match actual { - Extern::Global(e) if e.matches_expected(&m.globals[i]) => e.vmimport(), - Extern::Global(_) => bail!("global types incompatible"), - _ => bail!("expected global, but found {}", actual.desc()), - }), - EntityIndex::Function(i) => { - let func = match actual { - Extern::Func(e) => e, - _ => bail!("expected function, but found {}", actual.desc()), + fn next_import( + &mut self, + found: &str, + get: impl FnOnce(&wasmtime_environ::Module, &EntityIndex) -> Option, + ) -> Result<()> { + match self.imports.next() { + Some((module, field, idx)) => { + let error = match get(self.module, idx) { + Some(true) => return Ok(()), + Some(false) => { + anyhow::anyhow!("{} types incompatible", found) + } + None => { + let desc = match idx { + EntityIndex::Table(_) => "table", + EntityIndex::Function(_) => "func", + EntityIndex::Memory(_) => "memory", + EntityIndex::Global(_) => "global", + EntityIndex::Instance(_) => "instance", + EntityIndex::Module(_) => "module", + }; + anyhow::anyhow!("expected {}, but found {}", desc, found) + } }; + let import_name = match field { + Some(name) => format!("{}/{}", module, name), + None => module.to_string(), + }; + Err(error.context(format!("incompatible import type for {}", import_name))) + } + None => bail!("too many imports provided"), + } + } + + fn global(&mut self, global: &Global) -> Result<()> { + self.next_import("global", |m, e| match e { + EntityIndex::Global(i) => Some(global.matches_expected(&m.globals[*i])), + _ => None, + })?; + self.globals.push(global.vmimport()); + Ok(()) + } + + fn memory(&mut self, mem: &Memory) -> Result<()> { + self.next_import("memory", |m, e| match e { + EntityIndex::Memory(i) => Some(mem.matches_expected(&m.memory_plans[*i])), + _ => None, + })?; + self.memories.push(mem.vmimport()); + Ok(()) + } + + fn table(&mut self, table: &Table) -> Result<()> { + self.next_import("table", |m, e| match e { + EntityIndex::Table(i) => Some(table.matches_expected(&m.table_plans[*i])), + _ => None, + })?; + self.tables.push(table.vmimport()); + Ok(()) + } + + fn func(&mut self, func: &Func) -> Result<()> { + let store = self.store; + self.next_import("func", |m, e| match e { + EntityIndex::Function(i) => Some( // Look up the `i`th function's type from the module in our // signature registry. If it's not present then we have no // functions registered with that type, so `func` is guaranteed // to not match. - let ty = store + match store .signatures() .borrow() - .lookup(&m.signatures[m.functions[i]]) - .ok_or_else(|| anyhow!("function types incompatible"))?; - if !func.matches_expected(ty) { - bail!("function types incompatible"); - } - functions.push(func.vmimport()); - } - - // FIXME(#2094) - EntityIndex::Module(_i) => unimplemented!(), - EntityIndex::Instance(_i) => unimplemented!(), - } - Ok(()) - }; - - for (expected, actual) in m.imports.iter().zip(externs) { - process(&expected.2, actual).with_context(|| match &expected.1 { - Some(name) => format!("incompatible import type for {}/{}", expected.0, name), - None => format!("incompatible import type for {}", expected.0), + .lookup(&m.signatures[m.functions[*i]]) + { + Some(ty) => func.matches_expected(ty), + None => false, + }, + ), + _ => None, })?; + self.functions.push(func.vmimport()); + Ok(()) } - return f(Imports { - tables: &tables, - functions: &functions, - globals: &globals, - memories: &memories, - }); + fn validate_all_imports_provided(&mut self) -> Result<()> { + if self.imports.next().is_some() { + bail!("not enough imports provided"); + } + Ok(()) + } + + fn imports(&self) -> Imports<'_> { + Imports { + tables: self.tables.values().as_slice(), + globals: self.globals.values().as_slice(), + memories: self.memories.values().as_slice(), + functions: self.functions.values().as_slice(), + } + } } diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index acc1b75bed..875a1896d3 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -81,7 +81,7 @@ use wasmtime_jit::{CompilationArtifacts, CompiledModule}; #[derive(Clone)] pub struct Module { engine: Engine, - compiled: Arc<[CompiledModule]>, + pub(crate) compiled: Arc<[CompiledModule]>, index: usize, } diff --git a/tests/all/wast.rs b/tests/all/wast.rs index b34db6fb9b..9b069c2ebe 100644 --- a/tests/all/wast.rs +++ b/tests/all/wast.rs @@ -13,6 +13,7 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { let simd = wast.iter().any(|s| s == "simd"); let multi_memory = wast.iter().any(|s| s == "multi-memory"); + let module_linking = wast.iter().any(|s| s == "module-linking"); let bulk_mem = multi_memory || wast.iter().any(|s| s == "bulk-memory-operations"); // Some simd tests assume support for multiple tables, which are introduced @@ -24,6 +25,7 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { .wasm_bulk_memory(bulk_mem) .wasm_reference_types(reftypes) .wasm_multi_memory(multi_memory) + .wasm_module_linking(module_linking) .strategy(strategy)? .cranelift_debug_verifier(true); diff --git a/tests/misc_testsuite/module-linking/instantiate.wast b/tests/misc_testsuite/module-linking/instantiate.wast new file mode 100644 index 0000000000..9afe725f52 --- /dev/null +++ b/tests/misc_testsuite/module-linking/instantiate.wast @@ -0,0 +1,162 @@ +(module + (module) + (instance $a (instantiate 0)) +) + +(module $a + (global (export "global") (mut i32) (i32.const 0)) + + (func (export "reset") + i32.const 0 + global.set 0) + + (func $set (export "inc") + i32.const 1 + global.get 0 + i32.add + global.set 0) + + (func (export "get") (result i32) + global.get 0) + + (func (export "load") (result i32) + i32.const 0 + i32.load) + + (memory (export "memory") 1) + (table (export "table") 1 funcref) + (elem (i32.const 0) $set) +) + +;; Imported functions work +(module + (import "a" "inc" (func $set)) + (module + (import "" (func)) + (start 0)) + (instance $a (instantiate 0 (func $set))) +) + +(assert_return (invoke $a "get") (i32.const 1)) + +;; Imported globals work +(module + (import "a" "global" (global $g (mut i32))) + (module + (import "" (global (mut i32))) + (func + i32.const 2 + global.set 0) + (start 0)) + + (instance $a (instantiate 0 (global $g))) +) +(assert_return (invoke $a "get") (i32.const 2)) + +;; Imported tables work +(module + (import "a" "table" (table $t 1 funcref)) + (module + (import "" (table 1 funcref)) + (func + i32.const 0 + call_indirect) + (start 0)) + + (instance $a (instantiate 0 (table $t))) +) +(assert_return (invoke $a "get") (i32.const 3)) + +;; Imported memories work +(module + (import "a" "memory" (memory $m 1)) + (module + (import "" (memory 1)) + (func + i32.const 0 + i32.const 4 + i32.store) + (start 0)) + + (instance $a (instantiate 0 (memory $m))) +) +(assert_return (invoke $a "load") (i32.const 4)) + +;; all at once +(module + (import "a" "inc" (func $f)) + (import "a" "global" (global $g (mut i32))) + (import "a" "table" (table $t 1 funcref)) + (import "a" "memory" (memory $m 1)) + + (module + (import "" (memory 1)) + (import "" (global (mut i32))) + (import "" (table 1 funcref)) + (import "" (func)) + (func $start + call 0 + + i32.const 0 + i32.const 4 + i32.store + + i32.const 0 + call_indirect + + global.get 0 + global.set 0) + (start $start)) + + (instance $a + (instantiate 0 + (memory $m) + (global $g) + (table $t) + (func $f) + ) + ) +) + +;; instantiate lots +(module + (import "a" "inc" (func $f)) + (import "a" "global" (global $g (mut i32))) + (import "a" "table" (table $t 1 funcref)) + (import "a" "memory" (memory $m 1)) + + (module $mm (import "" (memory 1))) + (module $mf (import "" (func))) + (module $mt (import "" (table 1 funcref))) + (module $mg (import "" (global (mut i32)))) + + (instance (instantiate $mm (memory $m))) + (instance (instantiate $mf (func $f))) + (instance (instantiate $mt (table $t))) + (instance (instantiate $mg (global $g))) +) + +;; instantiate nested +(assert_return (invoke $a "reset")) +(assert_return (invoke $a "get") (i32.const 0)) +(module + (import "a" "inc" (func)) + (module + (import "" (func)) + (module + (import "" (func)) + (module + (import "" (func)) + (module + (import "" (func)) + (start 0) + ) + (instance (instantiate 0 (func 0))) + ) + (instance (instantiate 0 (func 0))) + ) + (instance (instantiate 0 (func 0))) + ) + (instance (instantiate 0 (func 0))) +) +(assert_return (invoke $a "get") (i32.const 1)) From 51c1d4bbd6dc2985e8f2ad00731012df071d928b Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 1 Dec 2020 16:56:23 -0600 Subject: [PATCH 24/63] Provide filename/line number information in `Trap` (#2452) * Provide filename/line number information in `Trap` This commit extends the `Trap` type and `Store` to retain DWARF debug information found in a wasm file unconditionally, if it's present. This then enables us to print filenames and line numbers which point back to actual source code when a trap backtrace is printed. Additionally the `FrameInfo` type has been souped up to return filename/line number information as well. The implementation here is pretty simplistic currently. The meat of all the work happens in `gimli` and `addr2line`, and otherwise wasmtime is just schlepping around bytes of dwarf debuginfo here and there! The general goal here is to assist with debugging when using wasmtime because filenames and line numbers are generally orders of magnitude better even when you already have a stack trace. Another nicety here is that backtraces will display inlined frames (learned through debug information), improving the experience in release mode as well. An example of this is that with this file: ```rust fn main() { panic!("hello"); } ``` we get this stack trace: ``` $ rustc foo.rs --target wasm32-wasi -g $ cargo run foo.wasm Finished dev [unoptimized + debuginfo] target(s) in 0.16s Running `target/debug/wasmtime foo.wasm` thread 'main' panicked at 'hello', foo.rs:2:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace Error: failed to run main module `foo.wasm` Caused by: 0: failed to invoke command default 1: wasm trap: unreachable wasm backtrace: 0: 0x6c1c - panic_abort::__rust_start_panic::abort::h2d60298621b1ccbf at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:77:17 - __rust_start_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/panic_abort/src/lib.rs:32:5 1: 0x68c7 - rust_panic at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:626:9 2: 0x65a1 - std::panicking::rust_panic_with_hook::h2345fb0909b53e12 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:596:5 3: 0x1436 - std::panicking::begin_panic::{{closure}}::h106f151a6db8c8fb at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:506:9 4: 0xda8 - std::sys_common::backtrace::__rust_end_short_backtrace::he55aa13f22782798 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:153:18 5: 0x1324 - std::panicking::begin_panic::h1727e7d1d719c76f at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:505:12 6: 0xfde - foo::main::h2db1313a64510850 at /Users/acrichton/code/wasmtime/foo.rs:2:5 7: 0x11d5 - core::ops::function::FnOnce::call_once::h20ee1cc04aeff1fc at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:227:5 8: 0xddf - std::sys_common::backtrace::__rust_begin_short_backtrace::h054493e41e27e69c at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/sys_common/backtrace.rs:137:18 9: 0x1d5a - std::rt::lang_start::{{closure}}::hd83784448d3fcb42 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:66:18 10: 0x69d8 - core::ops::function::impls:: for &F>::call_once::h564d3dad35014917 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:259:13 - std::panicking::try::do_call::hdca4832ace5a8603 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:381:40 - std::panicking::try::ha8624a1a6854b456 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:345:19 - std::panic::catch_unwind::h71421f57cf2bc688 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panic.rs:382:14 - std::rt::lang_start_internal::h260050c92cd470af at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:51:25 11: 0x1d0c - std::rt::lang_start::h0b4bcf3c5e498224 at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/rt.rs:65:5 12: 0xffc - !__original_main 13: 0x393 - __muloti4 at /cargo/registry/src/github.com-1ecc6299db9ec823/compiler_builtins-0.1.35/src/macros.rs:269 ``` This is relatively noisy by default but there's filenames and line numbers! Additionally frame 10 can be seen to have lots of frames inlined into it. All information is always available to the embedder but we could try to handle the `__rust_begin_short_backtrace` and `__rust_end_short_backtrace` markers to trim the backtrace by default as well. The only gotcha here is that it looks like `__muloti4` is out of place. That's because the libc that Rust ships with doesn't have dwarf information, although I'm not sure why we land in that function for symbolizing it... * Add a configuration switch for debuginfo * Control debuginfo by default with `WASM_BACKTRACE_DETAILS` * Try cpp_demangle on demangling as well * Rename to WASMTIME_BACKTRACE_DETAILS --- Cargo.lock | 12 ++ crates/cranelift/src/lib.rs | 4 +- crates/environ/src/module_environ.rs | 20 ++-- crates/environ/src/tunables.rs | 10 +- crates/jit/Cargo.toml | 1 + crates/jit/src/compiler.rs | 2 +- crates/jit/src/instantiate.rs | 159 +++++++++++++++++++++++++-- crates/jit/src/lib.rs | 4 +- crates/lightbeam/wasmtime/src/lib.rs | 2 +- crates/wasmtime/Cargo.toml | 1 + crates/wasmtime/src/config.rs | 55 ++++++++- crates/wasmtime/src/frame_info.rs | 127 +++++++++++++++++++-- crates/wasmtime/src/lib.rs | 2 +- crates/wasmtime/src/trap.rs | 71 ++++++++++-- src/obj.rs | 3 +- tests/all/traps.rs | 157 +++++++++++++++++++++++--- 16 files changed, 568 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aabea2ee2b..40c4034ef9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -331,6 +331,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "cpp_demangle" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44919ecaf6f99e8e737bc239408931c9a01e9a6c74814fee8242dd2506b65390" +dependencies = [ + "cfg-if 1.0.0", + "glob", +] + [[package]] name = "cpu-time" version = "1.0.0" @@ -2342,6 +2352,7 @@ dependencies = [ "backtrace", "bincode", "cfg-if 1.0.0", + "cpp_demangle", "libc", "log", "region", @@ -2522,6 +2533,7 @@ dependencies = [ name = "wasmtime-jit" version = "0.21.0" dependencies = [ + "addr2line", "anyhow", "cfg-if 1.0.0", "cranelift-codegen", diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index a0e241572e..1837ba02d1 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -355,7 +355,7 @@ impl Compiler for Cranelift { context.func.name = get_func_name(func_index); let sig_index = module.functions[func_index]; context.func.signature = translation.native_signatures[sig_index].clone(); - if tunables.debug_info { + if tunables.generate_native_debuginfo { context.func.collect_debug_info(); } @@ -434,7 +434,7 @@ impl Compiler for Cranelift { let address_transform = get_function_address_map(&context, &input, code_buf.len() as u32, isa); - let ranges = if tunables.debug_info { + let ranges = if tunables.generate_native_debuginfo { let ranges = context.build_value_labels_ranges(isa).map_err(|error| { CompileError::Codegen(pretty_error(&context.func, Some(isa), error)) })?; diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index e085751d77..128cc71b31 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -62,6 +62,10 @@ pub struct ModuleTranslation<'data> { /// this module. pub submodules: PrimaryMap, + /// Set if debuginfo was found but it was not parsed due to `Tunables` + /// configuration. + pub has_unparsed_debuginfo: bool, + code_index: u32, } @@ -81,8 +85,8 @@ pub struct DebugInfoData<'a> { pub wasm_file: WasmFileInfo, debug_loc: gimli::DebugLoc>, debug_loclists: gimli::DebugLocLists>, - debug_ranges: gimli::DebugRanges>, - debug_rnglists: gimli::DebugRngLists>, + pub debug_ranges: gimli::DebugRanges>, + pub debug_rnglists: gimli::DebugRngLists>, } #[allow(missing_docs)] @@ -152,9 +156,11 @@ impl<'data> ModuleEnvironment<'data> { } fn register_dwarf_section(&mut self, name: &str, data: &'data [u8]) { - if !self.tunables.debug_info { + if !self.tunables.generate_native_debuginfo && !self.tunables.parse_wasm_debuginfo { + self.result.has_unparsed_debuginfo = true; return; } + if !name.starts_with(".debug_") { return; } @@ -493,7 +499,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data validator: FuncValidator, body: FunctionBody<'data>, ) -> WasmResult<()> { - if self.tunables.debug_info { + if self.tunables.generate_native_debuginfo { let func_index = self.result.code_index + self.result.module.num_imported_funcs as u32; let func_index = FuncIndex::from_u32(func_index); let sig_index = self.result.module.functions[func_index]; @@ -563,7 +569,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn declare_module_name(&mut self, name: &'data str) { self.result.module.name = Some(name.to_string()); - if self.tunables.debug_info { + if self.tunables.generate_native_debuginfo { self.result.debuginfo.name_section.module_name = Some(name); } } @@ -573,7 +579,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data .module .func_names .insert(func_index, name.to_string()); - if self.tunables.debug_info { + if self.tunables.generate_native_debuginfo { self.result .debuginfo .name_section @@ -583,7 +589,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_local_name(&mut self, func_index: FuncIndex, local: u32, name: &'data str) { - if self.tunables.debug_info { + if self.tunables.generate_native_debuginfo { self.result .debuginfo .name_section diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs index 3e1907611d..939861504d 100644 --- a/crates/environ/src/tunables.rs +++ b/crates/environ/src/tunables.rs @@ -10,8 +10,11 @@ pub struct Tunables { /// The size in bytes of the offset guard for dynamic heaps. pub dynamic_memory_offset_guard_size: u64, - /// Whether or not to generate DWARF debug information. - pub debug_info: bool, + /// Whether or not to generate native DWARF debug information. + pub generate_native_debuginfo: bool, + + /// Whether or not to retain DWARF sections in compiled modules. + pub parse_wasm_debuginfo: bool, /// Whether or not to enable the ability to interrupt wasm code dynamically. /// @@ -51,7 +54,8 @@ impl Default for Tunables { /// wasting too much memory. dynamic_memory_offset_guard_size: 0x1_0000, - debug_info: false, + generate_native_debuginfo: false, + parse_wasm_debuginfo: true, interruptable: false, } } diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 5af2425c2d..4f4ceba8e9 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -36,6 +36,7 @@ log = "0.4" gimli = { version = "0.23.0", default-features = false, features = ["write"] } object = { version = "0.22.0", default-features = false, features = ["write"] } serde = { version = "1.0.94", features = ["derive"] } +addr2line = { version = "0.14", default-features = false } [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3.8", features = ["winnt", "impl-default"] } diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index 48fcb2c688..a9a2587c84 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -139,7 +139,7 @@ impl Compiler { .into_iter() .collect::(); - let dwarf_sections = if self.tunables.debug_info && !funcs.is_empty() { + let dwarf_sections = if self.tunables.generate_native_debuginfo && !funcs.is_empty() { transform_dwarf_data( &*self.isa, &translation.module, diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 3ec6f04bf1..4bff020519 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -12,6 +12,7 @@ use object::File as ObjectFile; use rayon::prelude::*; use serde::{Deserialize, Serialize}; use std::any::Any; +use std::ops::Range; use std::sync::Arc; use thiserror::Error; use wasmtime_debug::create_gdbjit_image; @@ -19,8 +20,8 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::{DefinedFuncIndex, ModuleIndex, SignatureIndex}; use wasmtime_environ::{ - CompileError, DataInitializer, DataInitializerLocation, FunctionAddressMap, Module, - ModuleEnvironment, ModuleTranslation, StackMapInformation, TrapInformation, + CompileError, DataInitializer, DataInitializerLocation, DebugInfoData, FunctionAddressMap, + Module, ModuleEnvironment, ModuleTranslation, StackMapInformation, TrapInformation, }; use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::{ @@ -69,12 +70,35 @@ pub struct CompilationArtifacts { /// Descriptions of compiled functions funcs: PrimaryMap, - /// Debug info presence flags. - debug_info: bool, - /// Where to find this module's submodule code in the top-level list of /// modules. submodules: PrimaryMap, + + /// Whether or not native debug information is available in `obj` + native_debug_info_present: bool, + + /// Whether or not the original wasm module contained debug information that + /// we skipped and did not parse. + has_unparsed_debuginfo: bool, + + /// Debug information found in the wasm file, used for symbolicating + /// backtraces. + debug_info: Option, +} + +#[derive(Serialize, Deserialize)] +struct DebugInfo { + data: Box<[u8]>, + code_section_offset: u64, + debug_abbrev: Range, + debug_addr: Range, + debug_info: Range, + debug_line: Range, + debug_line_str: Range, + debug_ranges: Range, + debug_rnglists: Range, + debug_str: Range, + debug_str_offsets: Range, } impl CompilationArtifacts { @@ -103,6 +127,8 @@ impl CompilationArtifacts { module, data_initializers, submodules, + debuginfo, + has_unparsed_debuginfo, .. } = translation; @@ -132,7 +158,13 @@ impl CompilationArtifacts { address_map: func.address_map, }) .collect(), - debug_info: compiler.tunables().debug_info, + native_debug_info_present: compiler.tunables().generate_native_debuginfo, + debug_info: if compiler.tunables().parse_wasm_debuginfo { + Some(debuginfo.into()) + } else { + None + }, + has_unparsed_debuginfo, }) }) .collect::, SetupError>>() @@ -202,7 +234,7 @@ impl CompiledModule { })?; // Register GDB JIT images; initialize profiler and load the wasm module. - let dbg_jit_registration = if artifacts.debug_info { + let dbg_jit_registration = if artifacts.native_debug_info_present { let bytes = create_dbg_image( artifacts.obj.to_vec(), code_range, @@ -348,6 +380,85 @@ impl CompiledModule { pub fn submodule_idx(&self, idx: ModuleIndex) -> usize { self.artifacts.submodules[idx] } + + /// Creates a new symbolication context which can be used to further + /// symbolicate stack traces. + /// + /// Basically this makes a thing which parses debuginfo and can tell you + /// what filename and line number a wasm pc comes from. + pub fn symbolize_context(&self) -> Result, gimli::Error> { + use gimli::EndianSlice; + let info = match &self.artifacts.debug_info { + Some(info) => info, + None => return Ok(None), + }; + // For now we clone the data into the `SymbolizeContext`, but if this + // becomes prohibitive we could always `Arc` it with our own allocation + // here. + let data = info.data.clone(); + let endian = gimli::LittleEndian; + let cx = addr2line::Context::from_sections( + EndianSlice::new(&data[info.debug_abbrev.clone()], endian).into(), + EndianSlice::new(&data[info.debug_addr.clone()], endian).into(), + EndianSlice::new(&data[info.debug_info.clone()], endian).into(), + EndianSlice::new(&data[info.debug_line.clone()], endian).into(), + EndianSlice::new(&data[info.debug_line_str.clone()], endian).into(), + EndianSlice::new(&data[info.debug_ranges.clone()], endian).into(), + EndianSlice::new(&data[info.debug_rnglists.clone()], endian).into(), + EndianSlice::new(&data[info.debug_str.clone()], endian).into(), + EndianSlice::new(&data[info.debug_str_offsets.clone()], endian).into(), + EndianSlice::new(&[], endian), + )?; + Ok(Some(SymbolizeContext { + // See comments on `SymbolizeContext` for why we do this static + // lifetime promotion. + inner: unsafe { + std::mem::transmute::, Addr2LineContext<'static>>(cx) + }, + code_section_offset: info.code_section_offset, + _data: data, + })) + } + + /// Returns whether the original wasm module had unparsed debug information + /// based on the tunables configuration. + pub fn has_unparsed_debuginfo(&self) -> bool { + self.artifacts.has_unparsed_debuginfo + } +} + +type Addr2LineContext<'a> = addr2line::Context>; + +/// A context which contains dwarf debug information to translate program +/// counters back to filenames and line numbers. +pub struct SymbolizeContext { + // Note the `'static` lifetime on `inner`. That's actually a bunch of slices + // which point back into the `_data` field. We currently unsafely manage + // this by saying that when inside the struct it's `'static` (since we own + // the referenced data just next to it) and we only loan out borrowed + // references. + _data: Box<[u8]>, + inner: Addr2LineContext<'static>, + code_section_offset: u64, +} + +impl SymbolizeContext { + /// Returns access to the [`addr2line::Context`] which can be used to query + /// frame information with. + pub fn addr2line(&self) -> &Addr2LineContext<'_> { + // Here we demote our synthetic `'static` lifetime which doesn't + // actually exist back to a lifetime that's tied to `&self`, which + // should be safe. + unsafe { + std::mem::transmute::<&Addr2LineContext<'static>, &Addr2LineContext<'_>>(&self.inner) + } + } + + /// Returns the offset of the code section in the original wasm file, used + /// to calculate lookup values into the DWARF. + pub fn code_section_offset(&self) -> u64 { + self.code_section_offset + } } /// Similar to `DataInitializer`, but owns its own copy of the data rather @@ -432,3 +543,37 @@ fn build_code_memory( Ok((code_memory, code_range, finished_functions, trampolines)) } + +impl From> for DebugInfo { + fn from(raw: DebugInfoData<'_>) -> DebugInfo { + use gimli::Section; + + let mut data = Vec::new(); + let mut push = |section: &[u8]| { + data.extend_from_slice(section); + data.len() - section.len()..data.len() + }; + let debug_abbrev = push(raw.dwarf.debug_abbrev.reader().slice()); + let debug_addr = push(raw.dwarf.debug_addr.reader().slice()); + let debug_info = push(raw.dwarf.debug_info.reader().slice()); + let debug_line = push(raw.dwarf.debug_line.reader().slice()); + let debug_line_str = push(raw.dwarf.debug_line_str.reader().slice()); + let debug_ranges = push(raw.debug_ranges.reader().slice()); + let debug_rnglists = push(raw.debug_rnglists.reader().slice()); + let debug_str = push(raw.dwarf.debug_str.reader().slice()); + let debug_str_offsets = push(raw.dwarf.debug_str_offsets.reader().slice()); + DebugInfo { + data: data.into(), + debug_abbrev, + debug_addr, + debug_info, + debug_line, + debug_line_str, + debug_ranges, + debug_rnglists, + debug_str, + debug_str_offsets, + code_section_offset: raw.wasm_file.code_section_offset, + } + } +} diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 94f268de82..1421ed0544 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -46,7 +46,9 @@ pub mod trampoline; pub use crate::code_memory::CodeMemory; pub use crate::compiler::{Compilation, CompilationStrategy, Compiler}; -pub use crate::instantiate::{CompilationArtifacts, CompiledModule, ModuleCode, SetupError}; +pub use crate::instantiate::{ + CompilationArtifacts, CompiledModule, ModuleCode, SetupError, SymbolizeContext, +}; pub use crate::link::link_module; /// Version number of this crate. diff --git a/crates/lightbeam/wasmtime/src/lib.rs b/crates/lightbeam/wasmtime/src/lib.rs index ed09e04550..e38e48f20d 100644 --- a/crates/lightbeam/wasmtime/src/lib.rs +++ b/crates/lightbeam/wasmtime/src/lib.rs @@ -29,7 +29,7 @@ impl Compiler for Lightbeam { isa: &dyn isa::TargetIsa, tunables: &Tunables, ) -> Result { - if tunables.debug_info { + if tunables.generate_native_debuginfo { return Err(CompileError::DebugInfoNotSupported); } let func_index = translation.module.func_index(i); diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index e0c0d3302c..28de26001e 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -23,6 +23,7 @@ libc = "0.2" cfg-if = "1.0" backtrace = "0.3.42" rustc-demangle = "0.1.16" +cpp_demangle = "0.3.2" log = "0.4.8" wat = { version = "1.0.18", optional = true } smallvec = "1.4.0" diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 0cb776ffdc..5a0686de91 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -32,6 +32,7 @@ pub struct Config { pub(crate) memory_creator: Option, pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, + pub(crate) wasm_backtrace_details_env_used: bool, } impl Config { @@ -61,7 +62,7 @@ impl Config { .set("enable_probestack", "false") .expect("should be valid flag"); - Config { + let mut ret = Config { tunables: Tunables::default(), flags, isa_flags: native::builder(), @@ -71,13 +72,16 @@ impl Config { profiler: Arc::new(NullProfilerAgent), memory_creator: None, max_wasm_stack: 1 << 20, + wasm_backtrace_details_env_used: false, features: WasmFeatures { reference_types: true, bulk_memory: true, multi_value: true, ..WasmFeatures::default() }, - } + }; + ret.wasm_backtrace_details(WasmBacktraceDetails::Environment); + return ret; } /// Configures whether DWARF debug information will be emitted during @@ -85,7 +89,33 @@ impl Config { /// /// By default this option is `false`. pub fn debug_info(&mut self, enable: bool) -> &mut Self { - self.tunables.debug_info = enable; + self.tunables.generate_native_debuginfo = enable; + self + } + + /// Configures backtraces in `Trap` will parse debuginfo in the wasm file to + /// have filename/line number information. + /// + /// When enabled this will causes modules to retain debugging information + /// found in wasm binaries. This debug information will be used when a trap + /// happens to symbolicate each stack frame and attempt to print a + /// filename/line number for each wasm frame in the stack trace. + /// + /// By default this option is `WasmBacktraceDetails::Environment`, meaning + /// that wasm will read `WASMTIME_BACKTRACE_DETAILS` to indicate whether details + /// should be parsed. + pub fn wasm_backtrace_details(&mut self, enable: WasmBacktraceDetails) -> &mut Self { + self.wasm_backtrace_details_env_used = false; + self.tunables.parse_wasm_debuginfo = match enable { + WasmBacktraceDetails::Enable => true, + WasmBacktraceDetails::Disable => false, + WasmBacktraceDetails::Environment => { + self.wasm_backtrace_details_env_used = true; + std::env::var("WASMTIME_BACKTRACE_DETAILS") + .map(|s| s == "1") + .unwrap_or(false) + } + }; self } @@ -640,7 +670,8 @@ impl Default for Config { impl fmt::Debug for Config { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("Config") - .field("debug_info", &self.tunables.debug_info) + .field("debug_info", &self.tunables.generate_native_debuginfo) + .field("parse_wasm_debuginfo", &self.tunables.parse_wasm_debuginfo) .field("strategy", &self.strategy) .field("wasm_threads", &self.features.threads) .field("wasm_reference_types", &self.features.reference_types) @@ -712,3 +743,19 @@ pub enum ProfilingStrategy { /// Collect profiling info using the "ittapi", used with `VTune` on Linux. VTune, } + +/// Select how wasm backtrace detailed information is handled. +#[derive(Debug, Clone, Copy)] +pub enum WasmBacktraceDetails { + /// Support is unconditionally enabled and wasmtime will parse and read + /// debug information. + Enable, + + /// Support is disabled, and wasmtime will not parse debug information for + /// backtrace details. + Disable, + + /// Support for backtrace details is conditional on the + /// `WASMTIME_BACKTRACE_DETAILS` environment variable. + Environment, +} diff --git a/crates/wasmtime/src/frame_info.rs b/crates/wasmtime/src/frame_info.rs index 6a64ec78ec..fbea477d28 100644 --- a/crates/wasmtime/src/frame_info.rs +++ b/crates/wasmtime/src/frame_info.rs @@ -5,7 +5,7 @@ use wasmtime_environ::entity::EntityRef; use wasmtime_environ::ir; use wasmtime_environ::wasm::FuncIndex; use wasmtime_environ::{FunctionAddressMap, Module, TrapInformation}; -use wasmtime_jit::CompiledModule; +use wasmtime_jit::{CompiledModule, SymbolizeContext}; #[derive(Default)] pub struct StoreFrameInfo { @@ -25,6 +25,8 @@ struct ModuleFrameInfo { start: usize, functions: BTreeMap, module: Arc, + symbolize: Option, + has_unparsed_debuginfo: bool, } struct FunctionInfo { @@ -38,8 +40,10 @@ impl StoreFrameInfo { /// Fetches frame information about a program counter in a backtrace. /// /// Returns an object if this `pc` is known to some previously registered - /// module, or returns `None` if no information can be found. - pub fn lookup_frame_info(&self, pc: usize) -> Option { + /// module, or returns `None` if no information can be found. The boolean + /// returned indicates whether the original module has unparsed debug + /// information due to the compiler's configuration. + pub fn lookup_frame_info(&self, pc: usize) -> Option<(FrameInfo, bool)> { let (module, func) = self.func(pc)?; // Use our relative position from the start of the function to find the @@ -72,13 +76,49 @@ impl StoreFrameInfo { Some(pos) => func.instr_map.instructions[pos].srcloc, None => func.instr_map.start_srcloc, }; - Some(FrameInfo { - module_name: module.module.name.clone(), - func_index: func.index.index() as u32, - func_name: module.module.func_names.get(&func.index).cloned(), - instr, - func_start: func.instr_map.start_srcloc, - }) + + // Use our wasm-relative pc to symbolize this frame. If there's a + // symbolication context (dwarf debug info) available then we can try to + // look this up there. + // + // Note that dwarf pcs are code-section-relative, hence the subtraction + // from the location of `instr`. Also note that all errors are ignored + // here for now since technically wasm modules can always have any + // custom section contents. + let mut symbols = Vec::new(); + if let Some(s) = &module.symbolize { + let to_lookup = (instr.bits() as u64) - s.code_section_offset(); + if let Ok(mut frames) = s.addr2line().find_frames(to_lookup) { + while let Ok(Some(frame)) = frames.next() { + symbols.push(FrameSymbol { + name: frame + .function + .as_ref() + .and_then(|l| l.raw_name().ok()) + .map(|s| s.to_string()), + file: frame + .location + .as_ref() + .and_then(|l| l.file) + .map(|s| s.to_string()), + line: frame.location.as_ref().and_then(|l| l.line), + column: frame.location.as_ref().and_then(|l| l.column), + }); + } + } + } + + Some(( + FrameInfo { + module_name: module.module.name.clone(), + func_index: func.index.index() as u32, + func_name: module.module.func_names.get(&func.index).cloned(), + instr, + func_start: func.instr_map.start_srcloc, + symbols, + }, + module.has_unparsed_debuginfo, + )) } /// Returns whether the `pc` specified is contaained within some module's @@ -160,6 +200,8 @@ impl StoreFrameInfo { start: min, functions, module: module.module().clone(), + symbolize: module.symbolize_context().ok().and_then(|c| c), + has_unparsed_debuginfo: module.has_unparsed_debuginfo(), }, ); assert!(prev.is_none()); @@ -180,6 +222,20 @@ pub struct FrameInfo { func_name: Option, func_start: ir::SourceLoc, instr: ir::SourceLoc, + symbols: Vec, +} + +/// Debug information for a symbol that is attached to a [`FrameInfo`]. +/// +/// When DWARF debug information is present in a wasm file then this structure +/// can be found on a [`FrameInfo`] and can be used to learn about filenames, +/// line numbers, etc, which are the origin of a function in a stack trace. +#[derive(Debug)] +pub struct FrameSymbol { + name: Option, + file: Option, + line: Option, + column: Option, } impl FrameInfo { @@ -240,6 +296,55 @@ impl FrameInfo { pub fn func_offset(&self) -> usize { (self.instr.bits() - self.func_start.bits()) as usize } + + /// Returns the debug symbols found, if any, for this function frame. + /// + /// When a wasm program is compiled with DWARF debug information then this + /// function may be populated to return symbols which contain extra debug + /// information about a frame including the filename and line number. If no + /// debug information was found or if it was malformed then this will return + /// an empty array. + pub fn symbols(&self) -> &[FrameSymbol] { + &self.symbols + } +} + +impl FrameSymbol { + /// Returns the function name associated with this symbol. + /// + /// Note that this may not be present with malformed debug information, or + /// the debug information may not include it. Also note that the symbol is + /// frequently mangled, so you might need to run some form of demangling + /// over it. + pub fn name(&self) -> Option<&str> { + self.name.as_deref() + } + + /// Returns the source code filename this symbol was defined in. + /// + /// Note that this may not be present with malformed debug information, or + /// the debug information may not include it. + pub fn file(&self) -> Option<&str> { + self.file.as_deref() + } + + /// Returns the 1-indexed source code line number this symbol was defined + /// on. + /// + /// Note that this may not be present with malformed debug information, or + /// the debug information may not include it. + pub fn line(&self) -> Option { + self.line + } + + /// Returns the 1-indexed source code column number this symbol was defined + /// on. + /// + /// Note that this may not be present with malformed debug information, or + /// the debug information may not include it. + pub fn column(&self) -> Option { + self.column + } } #[test] @@ -270,7 +375,7 @@ fn test_frame_info() -> Result<(), anyhow::Error> { (ptr as usize, ptr as usize + len) }; for pc in start..end { - let frame = info.lookup_frame_info(pc).unwrap(); + let (frame, _) = info.lookup_frame_info(pc).unwrap(); assert!(frame.func_index() == i.as_u32()); } } diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index cf2d69e01b..2248eec5a7 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -253,7 +253,7 @@ mod values; pub use crate::config::*; pub use crate::engine::*; pub use crate::externals::*; -pub use crate::frame_info::FrameInfo; +pub use crate::frame_info::{FrameInfo, FrameSymbol}; pub use crate::func::*; pub use crate::instance::Instance; pub use crate::linker::*; diff --git a/crates/wasmtime/src/trap.rs b/crates/wasmtime/src/trap.rs index 4e5f2b4de4..acbca71dbe 100644 --- a/crates/wasmtime/src/trap.rs +++ b/crates/wasmtime/src/trap.rs @@ -122,6 +122,7 @@ struct TrapInner { reason: TrapReason, wasm_trace: Vec, native_trace: Backtrace, + hint_wasm_backtrace_details_env: bool, } fn _assert_trap_is_sync_and_send(t: &Trap) -> (&dyn Sync, &dyn Send) { @@ -214,6 +215,7 @@ impl Trap { native_trace: Backtrace, ) -> Self { let mut wasm_trace = Vec::new(); + let mut hint_wasm_backtrace_details_env = false; wasmtime_runtime::with_last_info(|last| { // If the `store` passed in is `None` then we look at the `last` // store configured to call wasm, and if that's a `Store` we use @@ -236,9 +238,22 @@ impl Trap { // want to lookup information for the previous instruction // (the call instruction) so we subtract one as the lookup. let pc_to_lookup = if Some(pc) == trap_pc { pc } else { pc - 1 }; - if let Some(info) = store.frame_info().borrow().lookup_frame_info(pc_to_lookup) + if let Some((info, has_unparsed_debuginfo)) = + store.frame_info().borrow().lookup_frame_info(pc_to_lookup) { wasm_trace.push(info); + + // If this frame has unparsed debug information and the + // store's configuration indicates that we were + // respecting the environment variable of whether to + // do this then we will print out a helpful note in + // `Display` to indicate that more detailed information + // in a trap may be available. + if has_unparsed_debuginfo + && store.engine().config().wasm_backtrace_details_env_used + { + hint_wasm_backtrace_details_env = true; + } } } } @@ -248,6 +263,7 @@ impl Trap { reason, wasm_trace, native_trace, + hint_wasm_backtrace_details_env, }), } } @@ -297,15 +313,52 @@ impl fmt::Display for Trap { writeln!(f, "\nwasm backtrace:")?; for (i, frame) in self.trace().iter().enumerate() { let name = frame.module_name().unwrap_or(""); - write!(f, " {}: {:#6x} - {}!", i, frame.module_offset(), name)?; - match frame.func_name() { - Some(name) => match rustc_demangle::try_demangle(name) { - Ok(name) => write!(f, "{}", name)?, - Err(_) => write!(f, "{}", name)?, - }, - None => write!(f, "", frame.func_index())?, + write!(f, " {:>3}: {:#6x} - ", i, frame.module_offset())?; + + let demangle = + |f: &mut fmt::Formatter<'_>, name: &str| match rustc_demangle::try_demangle(name) { + Ok(name) => write!(f, "{}", name), + Err(_) => match cpp_demangle::Symbol::new(name) { + Ok(name) => write!(f, "{}", name), + Err(_) => write!(f, "{}", name), + }, + }; + let write_raw_func_name = |f: &mut fmt::Formatter<'_>| match frame.func_name() { + Some(name) => demangle(f, name), + None => write!(f, "", frame.func_index()), + }; + if frame.symbols().is_empty() { + write!(f, "{}!", name)?; + write_raw_func_name(f)?; + writeln!(f, "")?; + } else { + for (i, symbol) in frame.symbols().iter().enumerate() { + if i > 0 { + write!(f, " - ")?; + } else { + // ... + } + match symbol.name() { + Some(name) => demangle(f, name)?, + None if i == 0 => write_raw_func_name(f)?, + None => write!(f, "")?, + } + writeln!(f, "")?; + if let Some(file) = symbol.file() { + write!(f, " at {}", file)?; + if let Some(line) = symbol.line() { + write!(f, ":{}", line)?; + if let Some(col) = symbol.column() { + write!(f, ":{}", col)?; + } + } + } + writeln!(f, "")?; + } } - writeln!(f, "")?; + } + if self.inner.hint_wasm_backtrace_details_env { + writeln!(f, "note: run with `WASMTIME_BACKTRACE_DETAILS=1` environment variable to display more information")?; } Ok(()) } diff --git a/src/obj.rs b/src/obj.rs index 8e3ebc6b61..a14f79a8f8 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -46,7 +46,8 @@ pub fn compile_to_obj( // TODO: Expose the tunables as command-line flags. let mut tunables = Tunables::default(); - tunables.debug_info = debug_info; + tunables.generate_native_debuginfo = debug_info; + tunables.parse_wasm_debuginfo = debug_info; let compiler = Compiler::new( isa, diff --git a/tests/all/traps.rs b/tests/all/traps.rs index cd9b4b38d7..3f91229e6b 100644 --- a/tests/all/traps.rs +++ b/tests/all/traps.rs @@ -1,5 +1,6 @@ use anyhow::Result; use std::panic::{self, AssertUnwindSafe}; +use std::process::Command; use wasmtime::*; #[test] @@ -167,10 +168,10 @@ fn trap_display_pretty() -> Result<()> { "\ wasm trap: unreachable wasm backtrace: - 0: 0x23 - m!die - 1: 0x27 - m! - 2: 0x2c - m!foo - 3: 0x31 - m! + 0: 0x23 - m!die + 1: 0x27 - m! + 2: 0x2c - m!foo + 3: 0x31 - m! " ); Ok(()) @@ -211,12 +212,12 @@ fn trap_display_multi_module() -> Result<()> { "\ wasm trap: unreachable wasm backtrace: - 0: 0x23 - a!die - 1: 0x27 - a! - 2: 0x2c - a!foo - 3: 0x31 - a! - 4: 0x29 - b!middle - 5: 0x2e - b! + 0: 0x23 - a!die + 1: 0x27 - a! + 2: 0x2c - a!foo + 3: 0x31 - a! + 4: 0x29 - b!middle + 5: 0x2e - b! " ); Ok(()) @@ -422,10 +423,10 @@ fn start_trap_pretty() -> Result<()> { "\ wasm trap: unreachable wasm backtrace: - 0: 0x1d - m!die - 1: 0x21 - m! - 2: 0x26 - m!foo - 3: 0x2b - m!start + 0: 0x1d - m!die + 1: 0x21 - m! + 2: 0x26 - m!foo + 3: 0x2b - m!start " ); Ok(()) @@ -489,3 +490,131 @@ fn heap_out_of_bounds_trap() { TrapCode::MemoryOutOfBounds, ); } + +fn rustc(src: &str) -> Vec { + let td = tempfile::TempDir::new().unwrap(); + let output = td.path().join("foo.wasm"); + let input = td.path().join("input.rs"); + std::fs::write(&input, src).unwrap(); + let result = Command::new("rustc") + .arg(&input) + .arg("-o") + .arg(&output) + .arg("--target") + .arg("wasm32-wasi") + .arg("-g") + .output() + .unwrap(); + if result.status.success() { + return std::fs::read(&output).unwrap(); + } + panic!( + "rustc failed: {}\n{}", + result.status, + String::from_utf8_lossy(&result.stderr) + ); +} + +#[test] +fn parse_dwarf_info() -> Result<()> { + let wasm = rustc( + " + fn main() { + panic!(); + } + ", + ); + let mut config = Config::new(); + config.wasm_backtrace_details(WasmBacktraceDetails::Enable); + let engine = Engine::new(&config); + let store = Store::new(&engine); + let module = Module::new(&engine, &wasm)?; + let mut linker = Linker::new(&store); + wasmtime_wasi::Wasi::new(&store, wasmtime_wasi::WasiCtxBuilder::new().build()?) + .add_to_linker(&mut linker)?; + linker.module("", &module)?; + let run = linker.get_default("")?; + let trap = run.call(&[]).unwrap_err().downcast::()?; + + let mut found = false; + for frame in trap.trace() { + for symbol in frame.symbols() { + if let Some(file) = symbol.file() { + if file.ends_with("input.rs") { + found = true; + assert!(symbol.name().unwrap().contains("main")); + assert_eq!(symbol.line(), Some(3)); + } + } + } + } + assert!(found); + Ok(()) +} + +#[test] +fn no_hint_even_with_dwarf_info() -> Result<()> { + let mut config = Config::new(); + config.wasm_backtrace_details(WasmBacktraceDetails::Disable); + let engine = Engine::new(&config); + let store = Store::new(&engine); + let module = Module::new( + &engine, + r#" + (module + (@custom ".debug_info" (after last) "") + (func $start + unreachable) + (start $start) + ) + "#, + )?; + let trap = Instance::new(&store, &module, &[]) + .err() + .unwrap() + .downcast::()?; + assert_eq!( + trap.to_string(), + "\ +wasm trap: unreachable +wasm backtrace: + 0: 0x1a - !start +" + ); + Ok(()) +} + +#[test] +fn hint_with_dwarf_info() -> Result<()> { + // Skip this test if the env var is already configure, but in CI we're sure + // to run tests without this env var configured. + if std::env::var("WASMTIME_BACKTRACE_DETAILS").is_ok() { + return Ok(()); + } + let store = Store::default(); + let module = Module::new( + store.engine(), + r#" + (module + (@custom ".debug_info" (after last) "") + (func $start + unreachable) + (start $start) + ) + "#, + )?; + let trap = Instance::new(&store, &module, &[]) + .err() + .unwrap() + .downcast::()?; + assert_eq!( + trap.to_string(), + "\ +wasm trap: unreachable +wasm backtrace: + 0: 0x1a - !start +note: run with `WASMTIME_BACKTRACE_DETAILS=1` environment variable to display more information +" + ); + Ok(()) +} From 04e3730ba646c835df59941a31e1dc10144bdd1b Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Wed, 2 Dec 2020 09:58:12 -0800 Subject: [PATCH 25/63] Update to a CentOS 7 docker container for binary compatible builds. CentOS 6 just went EOL at the end of November 2020; as of today, the repository seems to have disappeared, so our CI builds are failing. This PR updates us to CentOS 7, which should be usable until June 30, 2024. --- .github/actions/binary-compatible-builds/main.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/binary-compatible-builds/main.js b/.github/actions/binary-compatible-builds/main.js index 4af9bc7ac2..d2d774f4eb 100755 --- a/.github/actions/binary-compatible-builds/main.js +++ b/.github/actions/binary-compatible-builds/main.js @@ -41,7 +41,6 @@ if (process.env.CENTOS !== undefined) { let path = process.env.PATH; path = `${path}:/rust/bin`; path = `/opt/rh/devtoolset-8/root/usr/bin:${path}`; -path = `/opt/rh/rh-python36/root/usr/bin:${path}`; // Spawn a container daemonized in the background which we'll connect to via // `docker exec`. This'll have access to the current directory. @@ -52,7 +51,7 @@ child_process.execFileSync('docker', [ '-v', `${process.cwd()}:${process.cwd()}`, '-v', `${child_process.execSync('rustc --print sysroot').toString().trim()}:/rust:ro`, '--env', `PATH=${path}`, - 'centos:6', + 'centos:7', ], stdio); // Use ourselves to run future commands @@ -63,7 +62,7 @@ const exec = s => { child_process.execSync(`docker exec centos ${s}`, stdio); }; exec('yum install -y centos-release-scl cmake xz epel-release'); -exec('yum install -y rh-python36 patchelf unzip'); +exec('yum install -y python3 patchelf unzip'); exec('yum install -y devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++'); exec('yum install -y git'); From c9a81f008d5a5e3c63fa3c6162c6f1fbb5bc6f99 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 1 Dec 2020 23:32:44 -0800 Subject: [PATCH 26/63] x64 backend: fix condition-code used for part of explicit heap check. A dynamic heap address computation may create up to two conditional branches: the usual bounds-check, but also (in some cases) an offset-addition overflow check. The x64 backend had reversed the condition code for this check, resulting in an always-trapping execution for a valid offset. I'm somewhat surprised this has existed so long, but I suppose the particular conditions (large offset, small offset guard, dynamic heap) have been somewhat rare in our testing so far. Found via fuzzing in #2453. --- cranelift/codegen/src/isa/x64/mod.rs | 12 +++++----- .../filetests/filetests/isa/x64/heap.clif | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/heap.clif diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index fd4444498d..73183f79e8 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -92,15 +92,15 @@ impl MachBackend for X64Backend { } fn unsigned_add_overflow_condition(&self) -> IntCC { - // Unsigned `>=`; this corresponds to the carry flag set on x86, which happens on - // overflow of an add. - IntCC::UnsignedGreaterThanOrEqual + // Unsigned `<`; this corresponds to the carry flag set on x86, which + // indicates an add has overflowed. + IntCC::UnsignedLessThan } fn unsigned_sub_overflow_condition(&self) -> IntCC { - // unsigned `>=`; this corresponds to the carry flag set on x86, which happens on - // underflow of a subtract (carry is borrow for subtract). - IntCC::UnsignedGreaterThanOrEqual + // unsigned `<`; this corresponds to the carry flag set on x86, which + // indicates a sub has underflowed (carry is borrow for subtract). + IntCC::UnsignedLessThan } #[cfg(feature = "unwind")] diff --git a/cranelift/filetests/filetests/isa/x64/heap.clif b/cranelift/filetests/filetests/isa/x64/heap.clif new file mode 100644 index 0000000000..d9efb083c6 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/heap.clif @@ -0,0 +1,23 @@ +test compile +target x86_64 +feature "experimental_x64" + +function %f(i32, i64 vmctx) -> i64 { + gv0 = vmctx + gv1 = load.i64 notrap aligned gv0+0 + gv2 = load.i32 notrap aligned gv0+8 + heap0 = dynamic gv1, bound gv2, offset_guard 0x1000, index_type i32 + +block0(v0: i32, v1: i64): + + v2 = heap_addr.i64 heap0, v0, 0x8000 + ; check: movl 8(%rsi), %r12d + ; nextln: movq %rdi, %r13 + ; nextln: addl $$32768, %r13d + ; nextln: jnb ; ud2 heap_oob ; + ; nextln: cmpl %r12d, %r13d + ; nextln: jbe label1; j label2 + ; check: Block 1: + + return v2 +} From 60d7f7de0aed98618e14cb686fd6f5843d216100 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 1 Dec 2020 17:01:38 -0800 Subject: [PATCH 27/63] Debug info: two fixes in x64 backend. - Sort by generated-code offset to maintain invariant and avoid gimli panic. - Fix srcloc interaction with branch peephole optimization in MachBuffer: if a srcloc range overlaps with a branch that is truncated, remove that srcloc range. These issues were found while fuzzing the new backend (#2453); I suspect that they arise with the new backend because we can sink instructions (e.g. loads or extends) in more interesting ways than before, but I'm not entirely sure. Test coverage will be via the fuzz corpus once #2453 lands. --- cranelift/codegen/src/machinst/buffer.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index b2187a9b68..7d75dd46d7 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -674,6 +674,12 @@ impl MachBuffer { // (end of buffer) self.data.truncate(b.start as usize); self.fixup_records.truncate(b.fixup); + while let Some(last_srcloc) = self.srclocs.last() { + if last_srcloc.end <= b.start { + break; + } + self.srclocs.pop(); + } // State: // [PRE CODE] // cur_off, Offset b.start, b.labels_at_this_branch: @@ -1184,12 +1190,15 @@ impl MachBuffer { // incorrect. assert!(self.fixup_records.is_empty()); + let mut srclocs = self.srclocs; + srclocs.sort_by_key(|entry| entry.start); + MachBufferFinalized { data: self.data, relocs: self.relocs, traps: self.traps, call_sites: self.call_sites, - srclocs: self.srclocs, + srclocs, stack_maps: self.stack_maps, } } From 2f0abc3d74d0e8f7e9e2374bfdce6dfe5a348fac Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Wed, 2 Dec 2020 11:20:08 -0800 Subject: [PATCH 28/63] Avoid removing wasi-nn temp directory when specified (#2465) Since downloading the wasi-nn artifacts take a bit of time, the example script's first argument serves as a directory to reuse for running this script. This change cleans up temporary directories only when a directory was not specified. --- ci/run-wasi-nn-example.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ci/run-wasi-nn-example.sh b/ci/run-wasi-nn-example.sh index 4d82e84943..5a77c06fa5 100755 --- a/ci/run-wasi-nn-example.sh +++ b/ci/run-wasi-nn-example.sh @@ -8,6 +8,15 @@ set -e WASMTIME_DIR=$(dirname "$0" | xargs dirname) FIXTURE=https://gist.github.com/abrown/c7847bf3701f9efbb2070da1878542c1/raw/07a9f163994b0ff8f0d7c5a5c9645ec3d8b24024 +if [ -z "${1+x}" ]; then + # If no temporary directory is specified, create one. + TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX) + REMOVE_TMP_DIR=1 +else + # If a directory was specified, use it and avoid removing it. + TMP_DIR=$(realpath $1) + REMOVE_TMP_DIR=0 +fi # Inform the environment of OpenVINO library locations. Then we use OPENVINO_INSTALL_DIR below to avoid building all of # OpenVINO from source (quite slow). @@ -17,7 +26,6 @@ source /opt/intel/openvino/bin/setupvars.sh OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo build -p wasmtime-cli --features wasi-nn # Download all necessary test fixtures to the temporary directory. -TMP_DIR=${1:-$(mktemp -d -t ci-XXXXXXXXXX)} wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/frozen_inference_graph.bin wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/frozen_inference_graph.xml wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/tensor-1x3x300x300-f32.bgr @@ -31,5 +39,7 @@ popd # Run the example in Wasmtime (note that the example uses `fixture` as the expected location of the model/tensor files). OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo run --features wasi-nn -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm -# Clean up. -rm -rf $TMP_DIR \ No newline at end of file +# Clean up the temporary directory only if it was not specified (users may want to keep the directory around). +if [[ $REMOVE_TMP_DIR -eq 1 ]]; then + rm -rf $TMP_DIR +fi \ No newline at end of file From a33e755cb2db90e7ca7f4b093e008d10b1f2a86c Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sat, 28 Nov 2020 19:57:03 -0800 Subject: [PATCH 29/63] Adds x86 SIMD support for Ceil, Floor, Trunc, and Nearest --- cranelift/codegen/src/isa/x64/inst/args.rs | 20 +++++++ cranelift/codegen/src/isa/x64/inst/emit.rs | 2 + .../codegen/src/isa/x64/inst/emit_tests.rs | 21 +++++++ cranelift/codegen/src/isa/x64/lower.rs | 55 +++++++++++++------ 4 files changed, 82 insertions(+), 16 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 4542f3386d..817e7f830c 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -550,6 +550,8 @@ pub enum SseOpcode { Punpcklbw, Pxor, Rcpss, + Roundps, + Roundpd, Roundss, Roundsd, Rsqrtss, @@ -729,6 +731,8 @@ impl SseOpcode { | SseOpcode::Pmovzxdq | SseOpcode::Pmulld | SseOpcode::Ptest + | SseOpcode::Roundps + | SseOpcode::Roundpd | SseOpcode::Roundss | SseOpcode::Roundsd => SSE41, @@ -890,6 +894,8 @@ impl fmt::Debug for SseOpcode { SseOpcode::Punpcklbw => "punpcklbw", SseOpcode::Pxor => "pxor", SseOpcode::Rcpss => "rcpss", + SseOpcode::Roundps => "roundps", + SseOpcode::Roundpd => "roundpd", SseOpcode::Roundss => "roundss", SseOpcode::Roundsd => "roundsd", SseOpcode::Rsqrtss => "rsqrtss", @@ -1238,6 +1244,20 @@ impl From for FcmpImm { } } +/// Encode the rounding modes used as part of the Rounding Control field. +pub(crate) enum RoundImm { + RoundNearest = 0x00, + RoundDown = 0x01, + RoundUp = 0x02, + RoundZero = 0x03, +} + +impl RoundImm { + pub(crate) fn encode(self) -> u8 { + self as u8 + } +} + /// An operand's size in bits. #[derive(Clone, Copy, PartialEq)] pub enum OperandSize { diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 56ecc0e843..c0d94d2ab6 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1981,6 +1981,8 @@ pub(crate) fn emit( SseOpcode::Pextrw => (LegacyPrefixes::_66, 0x0FC5, 2), SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3), SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2), + SseOpcode::Roundps => (LegacyPrefixes::_66, 0x0F3A08, 3), + SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3), _ => unimplemented!("Opcode {:?} not implemented", op), }; let rex = if *is64 { diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index fb9f0c1c07..bda26e3f27 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3505,6 +3505,27 @@ fn test_x64_emit() { "palignr $3, %xmm1, %xmm9", )); + insns.push(( + Inst::xmm_rm_r_imm(SseOpcode::Roundps, RegMem::reg(xmm7), w_xmm8, 3, false), + "66440F3A08C703", + "roundps $3, %xmm7, %xmm8", + )); + insns.push(( + Inst::xmm_rm_r_imm(SseOpcode::Roundpd, RegMem::reg(xmm10), w_xmm7, 2, false), + "66410F3A09FA02", + "roundpd $2, %xmm10, %xmm7", + )); + insns.push(( + Inst::xmm_rm_r_imm(SseOpcode::Roundps, RegMem::reg(xmm4), w_xmm8, 1, false), + "66440F3A08C401", + "roundps $1, %xmm4, %xmm8", + )); + insns.push(( + Inst::xmm_rm_r_imm(SseOpcode::Roundpd, RegMem::reg(xmm15), w_xmm15, 0, false), + "66450F3A09FF00", + "roundpd $0, %xmm15, %xmm15", + )); + // ======================================================== // Pertaining to atomics. let am1: SyntheticAmode = Amode::imm_reg_reg_shift(321, r10, rdx, 2).into(); diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 09ded3c948..30cd1b4d2d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -3207,22 +3207,45 @@ fn lower_insn_to_regs>( // Lower to VM calls when there's no access to SSE4.1. let ty = ty.unwrap(); - let libcall = match (ty, op) { - (types::F32, Opcode::Ceil) => LibCall::CeilF32, - (types::F64, Opcode::Ceil) => LibCall::CeilF64, - (types::F32, Opcode::Floor) => LibCall::FloorF32, - (types::F64, Opcode::Floor) => LibCall::FloorF64, - (types::F32, Opcode::Nearest) => LibCall::NearestF32, - (types::F64, Opcode::Nearest) => LibCall::NearestF64, - (types::F32, Opcode::Trunc) => LibCall::TruncF32, - (types::F64, Opcode::Trunc) => LibCall::TruncF64, - _ => panic!( - "unexpected type/opcode {:?}/{:?} in Ceil/Floor/Nearest/Trunc", - ty, op - ), - }; - - emit_vm_call(ctx, flags, triple, libcall, insn, inputs, outputs)?; + if !ty.is_vector() { + let libcall = match (op, ty) { + (Opcode::Ceil, types::F32) => LibCall::CeilF32, + (Opcode::Ceil, types::F64) => LibCall::CeilF64, + (Opcode::Floor, types::F32) => LibCall::FloorF32, + (Opcode::Floor, types::F64) => LibCall::FloorF64, + (Opcode::Nearest, types::F32) => LibCall::NearestF32, + (Opcode::Nearest, types::F64) => LibCall::NearestF64, + (Opcode::Trunc, types::F32) => LibCall::TruncF32, + (Opcode::Trunc, types::F64) => LibCall::TruncF64, + _ => panic!( + "unexpected type/opcode {:?}/{:?} in Ceil/Floor/Nearest/Trunc", + ty, op + ), + }; + emit_vm_call(ctx, flags, triple, libcall, insn, inputs, outputs)?; + } else { + let (op, mode) = match (op, ty) { + (Opcode::Ceil, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundUp), + (Opcode::Ceil, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundUp), + (Opcode::Floor, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundDown), + (Opcode::Floor, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundDown), + (Opcode::Trunc, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundZero), + (Opcode::Trunc, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundZero), + (Opcode::Nearest, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundNearest), + (Opcode::Nearest, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundNearest), + _ => panic!("Unknown op/ty combination (vector){:?}", ty), + }; + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]); + ctx.emit(Inst::gen_move(dst, src, ty)); + ctx.emit(Inst::xmm_rm_r_imm( + op, + RegMem::reg(dst.to_reg()), + dst, + mode.encode(), + false, + )); + } } Opcode::Load From a548516f97ce0e62fedcf7a22ebe9b0106343bc4 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Sun, 29 Nov 2020 15:06:05 -0800 Subject: [PATCH 30/63] Enable SIMD spec tests for f32x4_rounding and f64x4_rounding. Also address some review comments pointing out minor issues. --- build.rs | 13 +++++-------- cranelift/codegen/src/isa/x64/inst/args.rs | 5 +++++ cranelift/codegen/src/isa/x64/lower.rs | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/build.rs b/build.rs index 4937f6bf12..aa791aeda1 100644 --- a/build.rs +++ b/build.rs @@ -212,6 +212,8 @@ fn experimental_x64_should_panic(testsuite: &str, testname: &str, strategy: &str ("simd", "simd_splat") => return false, ("simd", "simd_store") => return false, ("simd", "simd_conversions") => return false, + ("simd", "simd_f32x4_rounding") => return false, + ("simd", "simd_f64x2_rounding") => return false, ("simd", _) => return true, _ => {} } @@ -240,18 +242,13 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { // These are only implemented on aarch64 and x64. ("simd", "simd_boolean") | ("simd", "simd_f32x4_pmin_pmax") - | ("simd", "simd_f64x2_pmin_pmax") => { + | ("simd", "simd_f64x2_pmin_pmax") + | ("simd", "simd_f32x4_rounding") + | ("simd", "simd_f64x2_rounding") => { return !(cfg!(feature = "experimental_x64") || env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "aarch64") } - // These are only implemented on aarch64. - ("simd", "simd_f32x4_rounding") | ("simd", "simd_f64x2_rounding") => { - return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "aarch64"; - } - - // These tests have simd operators which aren't implemented yet. - // (currently none) _ => {} }, _ => panic!("unrecognized strategy"), diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 817e7f830c..7e3b3f22a2 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -1245,6 +1245,11 @@ impl From for FcmpImm { } /// Encode the rounding modes used as part of the Rounding Control field. +/// Note, these rounding immediates only consider the rounding control field +/// (i.e. the rounding mode) which only take up the first two bits when encoded. +/// However the rounding immediate which this field helps make up, also includes +/// bits 3 and 4 which define the rounding select and precision mask respectively. +/// These two bits are not defined here and are implictly set to zero when encoded. pub(crate) enum RoundImm { RoundNearest = 0x00, RoundDown = 0x01, diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 30cd1b4d2d..4b09681548 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -3240,7 +3240,7 @@ fn lower_insn_to_regs>( ctx.emit(Inst::gen_move(dst, src, ty)); ctx.emit(Inst::xmm_rm_r_imm( op, - RegMem::reg(dst.to_reg()), + RegMem::from(dst), dst, mode.encode(), false, From bbdea06e2d22b85cdacf5406171bccf65166031b Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Wed, 2 Dec 2020 14:52:44 -0800 Subject: [PATCH 31/63] Add differential fuzzing against wasmi (a Wasm interpreter). This PR adds a new fuzz target, `differential_wasmi`, that runs a Cranelift-based Wasm backend alongside a simple third-party Wasm interpeter crate (`wasmi`). The fuzzing runs the first function in a given module to completion on each side, and then diffs the return value and linear memory contents. This strategy should provide end-to-end coverage including both the Wasm translation to CLIF (which has seen some subtle and scary bugs at times), the lowering from CLIF to VCode, the register allocation, and the final code emission. This PR also adds a feature `experimental_x64` to the fuzzing crate (and the chain of dependencies down to `cranelift-codegen`) so that we can fuzz the new x86-64 backend as well as the current one. --- Cargo.lock | 82 +++++++++- crates/fuzzing/Cargo.toml | 6 +- crates/fuzzing/src/oracles.rs | 202 +++++++++++++++++++++--- crates/wasmtime/Cargo.toml | 3 + fuzz/Cargo.toml | 11 +- fuzz/fuzz_targets/differential_wasmi.rs | 13 ++ 6 files changed, 294 insertions(+), 23 deletions(-) create mode 100644 fuzz/fuzz_targets/differential_wasmi.rs diff --git a/Cargo.lock b/Cargo.lock index 40c4034ef9..2044cd3a6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -766,6 +766,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + [[package]] name = "dynasm" version = "1.0.0" @@ -1238,6 +1244,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -1264,6 +1276,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1274,6 +1297,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1352,6 +1387,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "parity-wasm" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -2313,13 +2354,47 @@ dependencies = [ ] [[package]] -name = "wasm-smith" -version = "0.1.10" +name = "wasm-encoder" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd5e4720bb44dc5e46a917139dc9dfa4bb6fee023bf9f77d2c55ec12eeaf9930" +checksum = "49891b7c581cf9e0090b25cd274e6498ad478f0a7819319ea96da2f253caaacb" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-smith" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fdf8c9ba2fdc0d8ffe3f7e5b23b5aac377eeca817a9885c058f27c8de5c500" dependencies = [ "arbitrary", "leb128", + "wasm-encoder", +] + +[[package]] +name = "wasmi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6825d9b2147105789adb4c2d84b9b568719713f3ac39618b637b4dafc86c4" +dependencies = [ + "downcast-rs", + "libc", + "memory_units", + "num-rational", + "num-traits", + "parity-wasm", + "wasmi-validation", +] + +[[package]] +name = "wasmi-validation" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" +dependencies = [ + "parity-wasm", ] [[package]] @@ -2522,6 +2597,7 @@ dependencies = [ "log", "rayon", "wasm-smith", + "wasmi", "wasmparser 0.68.0", "wasmprinter", "wasmtime", diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index c5625d293b..4526982823 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -16,7 +16,11 @@ wasmparser = "0.68.0" wasmprinter = "0.2.15" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } -wasm-smith = "0.1.10" +wasm-smith = "0.1.12" +wasmi = "0.7.0" [dev-dependencies] wat = "1.0.28" + +[features] +experimental_x64 = ["wasmtime/experimental_x64"] diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index cbd0d97107..2fb6c58682 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -12,7 +12,9 @@ pub mod dummy; +use arbitrary::Arbitrary; use dummy::dummy_imports; +use log::debug; use std::cell::Cell; use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; @@ -249,24 +251,8 @@ pub fn differential_execution( (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue, (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue, (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue, - (Val::F32(lhs), Val::F32(rhs)) => { - let lhs = f32::from_bits(*lhs); - let rhs = f32::from_bits(*rhs); - if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { - continue; - } else { - fail() - } - } - (Val::F64(lhs), Val::F64(rhs)) => { - let lhs = f64::from_bits(*lhs); - let rhs = f64::from_bits(*rhs); - if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { - continue; - } else { - fail() - } - } + (Val::F32(lhs), Val::F32(rhs)) if f32_equal(*lhs, *rhs) => continue, + (Val::F64(lhs), Val::F64(rhs)) if f64_equal(*lhs, *rhs) => continue, (Val::ExternRef(_), Val::ExternRef(_)) | (Val::FuncRef(_), Val::FuncRef(_)) => continue, _ => fail(), @@ -278,6 +264,18 @@ pub fn differential_execution( } } +fn f32_equal(a: u32, b: u32) -> bool { + let a = f32::from_bits(a); + let b = f32::from_bits(b); + a == b || (a.is_nan() && b.is_nan()) +} + +fn f64_equal(a: u64, b: u64) -> bool { + let a = f64::from_bits(a); + let b = f64::from_bits(b); + a == b || (a.is_nan() && b.is_nan()) +} + /// Invoke the given API calls. pub fn make_api_calls(api: crate::generators::api::ApiCalls) { use crate::generators::api::ApiCall; @@ -479,3 +477,171 @@ pub fn table_ops(config: crate::generators::Config, ops: crate::generators::tabl } } } + +/// Configuration options for wasm-smith such that generated modules always +/// conform to certain specifications. +#[derive(Default, Debug, Arbitrary)] +pub struct DifferentialWasmiModuleConfig; + +impl wasm_smith::Config for DifferentialWasmiModuleConfig { + fn allow_start_export(&self) -> bool { + false + } + + fn min_funcs(&self) -> usize { + 1 + } + + fn max_funcs(&self) -> usize { + 1 + } + + fn min_memories(&self) -> u32 { + 1 + } + + fn max_memories(&self) -> u32 { + 1 + } + + fn max_imports(&self) -> usize { + 0 + } + + fn min_exports(&self) -> usize { + 2 + } + + fn max_memory_pages(&self) -> u32 { + 1 + } + + fn memory_max_size_required(&self) -> bool { + true + } +} + +/// Perform differential execution between Cranelift and wasmi, diffing the +/// resulting memory image when execution terminates. This relies on the +/// module-under-test to be instrumented to bound the execution time. Invoke +/// with a module generated by `wasm-smith` using the +/// `DiferentialWasmiModuleConfig` configuration type for best results. +/// +/// May return `None` if we early-out due to a rejected fuzz config; these +/// should be rare if modules are generated appropriately. +pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> { + crate::init_fuzzing(); + + // Instantiate wasmi module and instance. + let wasmi_module = wasmi::Module::from_buffer(&wasm[..]).ok()?; + let wasmi_instance = + wasmi::ModuleInstance::new(&wasmi_module, &wasmi::ImportsBuilder::default()).ok()?; + let wasmi_instance = wasmi_instance.assert_no_start(); + + // TODO(paritytech/wasmi#19): wasmi does not currently canonicalize NaNs. To avoid spurious + // fuzz failures, for now let's fuzz only integer Wasm programs. + if wasmi_module.deny_floating_point().is_err() { + return None; + } + + // Instantiate wasmtime module and instance. + let mut wasmtime_config = config.to_wasmtime(); + wasmtime_config.cranelift_nan_canonicalization(true); + let wasmtime_engine = Engine::new(&wasmtime_config); + let wasmtime_store = Store::new(&wasmtime_engine); + let wasmtime_module = + Module::new(&wasmtime_engine, &wasm).expect("Wasmtime can compile module"); + let wasmtime_instance = Instance::new(&wasmtime_store, &wasmtime_module, &[]) + .expect("Wasmtime can instantiate module"); + + // Introspect wasmtime module to find name of an exported function and of an + // exported memory. Stop when we have one of each. (According to the config + // above, there should be at most one of each.) + let (func_name, memory_name) = { + let mut func_name = None; + let mut memory_name = None; + for e in wasmtime_module.exports() { + match e.ty() { + wasmtime::ExternType::Func(..) => func_name = Some(e.name().to_string()), + wasmtime::ExternType::Memory(..) => memory_name = Some(e.name().to_string()), + _ => {} + } + if func_name.is_some() && memory_name.is_some() { + break; + } + } + (func_name?, memory_name?) + }; + + let wasmi_mem_export = wasmi_instance.export_by_name(&memory_name[..]).unwrap(); + let wasmi_mem = wasmi_mem_export.as_memory().unwrap(); + let wasmi_main_export = wasmi_instance.export_by_name(&func_name[..]).unwrap(); + let wasmi_main = wasmi_main_export.as_func().unwrap(); + let wasmi_val = wasmi::FuncInstance::invoke(&wasmi_main, &[], &mut wasmi::NopExternals); + + let wasmtime_mem = wasmtime_instance + .get_memory(&memory_name[..]) + .expect("memory export is present"); + let wasmtime_main = wasmtime_instance + .get_func(&func_name[..]) + .expect("function export is present"); + let wasmtime_vals = wasmtime_main.call(&[]); + let wasmtime_val = wasmtime_vals.map(|v| v.iter().next().cloned()); + + debug!( + "Successful execution: wasmi returned {:?}, wasmtime returned {:?}", + wasmi_val, wasmtime_val + ); + + let show_wat = || { + if let Ok(s) = wasmprinter::print_bytes(&wasm[..]) { + eprintln!("wat:\n{}\n", s); + } + }; + + match (&wasmi_val, &wasmtime_val) { + (&Ok(Some(wasmi::RuntimeValue::I32(a))), &Ok(Some(Val::I32(b)))) if a == b => {} + (&Ok(Some(wasmi::RuntimeValue::F32(a))), &Ok(Some(Val::F32(b)))) + if f32_equal(a.to_bits(), b) => {} + (&Ok(Some(wasmi::RuntimeValue::I64(a))), &Ok(Some(Val::I64(b)))) if a == b => {} + (&Ok(Some(wasmi::RuntimeValue::F64(a))), &Ok(Some(Val::F64(b)))) + if f64_equal(a.to_bits(), b) => {} + (&Ok(None), &Ok(None)) => {} + (&Err(_), &Err(_)) => {} + _ => { + show_wat(); + panic!( + "Values do not match: wasmi returned {:?}; wasmtime returned {:?}", + wasmi_val, wasmtime_val + ); + } + } + + if wasmi_mem.current_size().0 != wasmtime_mem.size() as usize { + show_wat(); + panic!("resulting memories are not the same size"); + } + + // Wasmi memory may be stored non-contiguously; copy it out to a contiguous chunk. + let mut wasmi_buf: Vec = vec![0; wasmtime_mem.data_size()]; + wasmi_mem + .get_into(0, &mut wasmi_buf[..]) + .expect("can access wasmi memory"); + + let wasmtime_slice = unsafe { wasmtime_mem.data_unchecked() }; + + if wasmi_buf.len() >= 64 { + debug!("-> First 64 bytes of wasmi heap: {:?}", &wasmi_buf[0..64]); + debug!( + "-> First 64 bytes of Wasmtime heap: {:?}", + &wasmtime_slice[0..64] + ); + } + + if &wasmi_buf[..] != &wasmtime_slice[..] { + show_wat(); + panic!("memory contents are not equal"); + } + + Some(()) +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 28de26001e..57cdec42a5 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -59,3 +59,6 @@ parallel-compilation = ["wasmtime-jit/parallel-compilation"] # Enables support for automatic cache configuration to be enabled in `Config`. cache = ["wasmtime-cache"] + +# Enables support for new x64 backend. +experimental_x64 = ["wasmtime-jit/experimental_x64"] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index f295655755..32726a7940 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -17,7 +17,10 @@ target-lexicon = "0.11" peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true } wasmtime = { path = "../crates/wasmtime" } wasmtime-fuzzing = { path = "../crates/fuzzing" } -wasm-smith = "0.1.5" +wasm-smith = "0.1.12" + +[features] +experimental_x64 = ["wasmtime-fuzzing/experimental_x64"] [[bin]] name = "compile" @@ -43,6 +46,12 @@ path = "fuzz_targets/differential.rs" test = false doc = false +[[bin]] +name = "differential_wasmi" +path = "fuzz_targets/differential_wasmi.rs" +test = false +doc = false + [[bin]] name = "spectests" path = "fuzz_targets/spectests.rs" diff --git a/fuzz/fuzz_targets/differential_wasmi.rs b/fuzz/fuzz_targets/differential_wasmi.rs new file mode 100644 index 0000000000..e4e8fffead --- /dev/null +++ b/fuzz/fuzz_targets/differential_wasmi.rs @@ -0,0 +1,13 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use wasmtime_fuzzing::{generators, oracles}; + +fuzz_target!(|data: ( + generators::Config, + wasm_smith::ConfiguredModule +)| { + let (config, mut wasm) = data; + wasm.ensure_termination(1000); + oracles::differential_wasmi_execution(&wasm.to_bytes()[..], &config); +}); From 9ac7d01288ee8248a75c9e5510bab1db6d235c7c Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 2 Dec 2020 17:24:06 -0600 Subject: [PATCH 32/63] Implement the module linking alias section (#2451) This commit is intended to do almost everything necessary for processing the alias section of module linking. Most of this is internal refactoring, the highlights being: * Type contents are now stored separately from a `wasmtime_env::Module`. Given that modules can freely alias types and have them used all over the place, it seemed best to have one canonical location to type storage which everywhere else points to (with indices). A new `TypeTables` structure is produced during compilation which is shared amongst all member modules in a wasm blob. * Instantiation is heavily refactored to account for module linking. The main gotcha here is that imports are now listed as "initializers". We have a sort of pseudo-bytecode-interpreter which interprets the initialization of a module. This is more complicated than just matching imports at this point because in the module linking proposal the module, alias, import, and instance sections may all be interleaved. This means that imports aren't guaranteed to show up at the beginning of the address space for modules/instances. Otherwise most of the changes here largely fell out from these two design points. Aliases are recorded as initializers in this scheme. Copying around type information and/or just knowing type information during compilation is also pretty easy since everything is just a pointer into a `TypeTables` and we don't have to actually copy any types themselves. Lots of various refactorings were necessary to accomodate these changes. Tests are hoped to cover a breadth of functionality here, but not necessarily a depth. There's still one more piece of the module linking proposal missing which is exporting instances/modules, which will come in a future PR. It's also worth nothing that there's one large TODO which isn't implemented in this change that I plan on opening an issue for. With module linking when a set of modules comes back from compilation each modules has all the trampolines for the entire set of modules. This is quite a lot of duplicate trampolines across module-linking modules. We'll want to refactor this at some point to instead have only one set of trampolines per set of module linking modules and have them shared from there. I figured it was best to separate out this change, however, since it's purely related to resource usage, and doesn't impact non-module-linking modules at all. cc #2094 --- cranelift/wasm/src/environ/mod.rs | 4 +- cranelift/wasm/src/environ/spec.rs | 80 +++- cranelift/wasm/src/lib.rs | 2 +- cranelift/wasm/src/module_translator.rs | 12 +- cranelift/wasm/src/sections_translator.rs | 105 ++++- cranelift/wasm/src/translation_utils.rs | 18 +- crates/cranelift/src/func_environ.rs | 3 +- crates/cranelift/src/lib.rs | 7 +- crates/environ/src/compilation.rs | 3 +- crates/environ/src/module.rs | 172 ++++---- crates/environ/src/module_environ.rs | 369 ++++++++++++++---- crates/environ/src/vmoffsets.rs | 6 +- crates/jit/src/compiler.rs | 16 +- crates/jit/src/instantiate.rs | 50 ++- crates/jit/src/lib.rs | 2 +- crates/jit/src/object.rs | 13 +- crates/lightbeam/wasmtime/src/lib.rs | 17 +- crates/obj/src/context.rs | 23 +- crates/runtime/src/instance.rs | 17 +- crates/wasmtime/src/externals.rs | 9 + crates/wasmtime/src/instance.rs | 368 ++++++++++------- crates/wasmtime/src/module.rs | 66 +++- crates/wasmtime/src/store.rs | 15 +- .../wasmtime/src/trampoline/create_handle.rs | 6 +- crates/wasmtime/src/trampoline/func.rs | 23 +- crates/wasmtime/src/trampoline/global.rs | 25 +- crates/wasmtime/src/trampoline/memory.rs | 2 +- crates/wasmtime/src/trampoline/table.rs | 2 +- crates/wasmtime/src/types.rs | 102 ++--- crates/wast/src/wast.rs | 26 +- src/obj.rs | 4 +- tests/all/module_linking.rs | 123 ++++++ .../misc_testsuite/module-linking/alias.wast | 114 ++++++ .../module-linking/instantiate.wast | 39 +- 34 files changed, 1322 insertions(+), 521 deletions(-) create mode 100644 tests/misc_testsuite/module-linking/alias.wast diff --git a/cranelift/wasm/src/environ/mod.rs b/cranelift/wasm/src/environ/mod.rs index bb4b7cc34e..cf4672beb3 100644 --- a/cranelift/wasm/src/environ/mod.rs +++ b/cranelift/wasm/src/environ/mod.rs @@ -6,6 +6,6 @@ mod spec; pub use crate::environ::dummy::DummyEnvironment; pub use crate::environ::spec::{ - FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode, TargetEnvironment, WasmError, - WasmFuncType, WasmResult, WasmType, + Alias, FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode, TargetEnvironment, + WasmError, WasmFuncType, WasmResult, WasmType, }; diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 6d448f8d6d..f7577de5d3 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -9,7 +9,8 @@ use crate::state::FuncTranslationState; use crate::translation_utils::{ DataIndex, ElemIndex, EntityIndex, EntityType, Event, EventIndex, FuncIndex, Global, - GlobalIndex, Memory, MemoryIndex, ModuleIndex, Table, TableIndex, TypeIndex, + GlobalIndex, InstanceIndex, InstanceTypeIndex, Memory, MemoryIndex, ModuleIndex, + ModuleTypeIndex, SignatureIndex, Table, TableIndex, TypeIndex, }; use core::convert::From; use core::convert::TryFrom; @@ -202,6 +203,30 @@ pub enum ReturnMode { FallthroughReturn, } +/// An entry in the alias section of a wasm module (from the module linking +/// proposal) +pub enum Alias { + /// A parent's module is being aliased into our own index space. + /// + /// Note that the index here is in the parent's index space, not our own. + ParentModule(ModuleIndex), + + /// A parent's type is being aliased into our own index space + /// + /// Note that the index here is in the parent's index space, not our own. + ParentType(TypeIndex), + + /// A previously created instance is having one of its exports aliased into + /// our index space. + Child { + /// The index we're aliasing. + instance: InstanceIndex, + /// The nth export that we're inserting into our own index space + /// locally. + export: usize, + }, +} + /// Environment affecting the translation of a WebAssembly. pub trait TargetEnvironment { /// Get the information needed to produce Cranelift IR for the given target. @@ -684,6 +709,27 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { Err(WasmError::Unsupported("module linking".to_string())) } + /// Translates a type index to its signature index, only called for type + /// indices which point to functions. + fn type_to_signature(&self, index: TypeIndex) -> WasmResult { + drop(index); + Err(WasmError::Unsupported("module linking".to_string())) + } + + /// Translates a type index to its module type index, only called for type + /// indices which point to modules. + fn type_to_module_type(&self, index: TypeIndex) -> WasmResult { + drop(index); + Err(WasmError::Unsupported("module linking".to_string())) + } + + /// Translates a type index to its instance type index, only called for type + /// indices which point to instances. + fn type_to_instance_type(&self, index: TypeIndex) -> WasmResult { + drop(index); + Err(WasmError::Unsupported("module linking".to_string())) + } + /// Provides the number of imports up front. By default this does nothing, but /// implementations can use this to preallocate memory if desired. fn reserve_imports(&mut self, _num: u32) -> WasmResult<()> { @@ -845,6 +891,22 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { name: &'data str, ) -> WasmResult<()>; + /// Declares an instance export to the environment. + fn declare_instance_export( + &mut self, + index: InstanceIndex, + name: &'data str, + ) -> WasmResult<()> { + drop((index, name)); + Err(WasmError::Unsupported("module linking".to_string())) + } + + /// Declares an instance export to the environment. + fn declare_module_export(&mut self, index: ModuleIndex, name: &'data str) -> WasmResult<()> { + drop((index, name)); + Err(WasmError::Unsupported("module linking".to_string())) + } + /// Notifies the implementation that all exports have been declared. fn finish_exports(&mut self) -> WasmResult<()> { Ok(()) @@ -952,6 +1014,12 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { drop(amount); } + /// Declares that a module will come later with the type signature provided. + fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> { + drop(ty); + Err(WasmError::Unsupported("module linking".to_string())) + } + /// Called at the beginning of translating a module. /// /// The `index` argument is a monotonically increasing index which @@ -982,4 +1050,14 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { drop((module, args)); Err(WasmError::Unsupported("wasm instance".to_string())) } + + /// Declares a new alias being added to this module. + /// + /// The alias comes from the `instance` specified (or the parent if `None` + /// is supplied) and the index is either in the module's own index spaces + /// for the parent or an index into the exports for nested instances. + fn declare_alias(&mut self, alias: Alias) -> WasmResult<()> { + drop(alias); + Err(WasmError::Unsupported("wasm alias".to_string())) + } } diff --git a/cranelift/wasm/src/lib.rs b/cranelift/wasm/src/lib.rs index 3e6d4401a1..4e06680385 100644 --- a/cranelift/wasm/src/lib.rs +++ b/cranelift/wasm/src/lib.rs @@ -57,7 +57,7 @@ mod state; mod translation_utils; pub use crate::environ::{ - DummyEnvironment, FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode, + Alias, DummyEnvironment, FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode, TargetEnvironment, WasmError, WasmFuncType, WasmResult, WasmType, }; pub use crate::func_translator::FuncTranslator; diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs index 5fc60ccbdd..c3e27dd820 100644 --- a/cranelift/wasm/src/module_translator.rs +++ b/cranelift/wasm/src/module_translator.rs @@ -2,10 +2,10 @@ //! to deal with each part of it. use crate::environ::{ModuleEnvironment, WasmResult}; use crate::sections_translator::{ - parse_data_section, parse_element_section, parse_event_section, parse_export_section, - parse_function_section, parse_global_section, parse_import_section, parse_instance_section, - parse_memory_section, parse_name_section, parse_start_section, parse_table_section, - parse_type_section, + parse_alias_section, parse_data_section, parse_element_section, parse_event_section, + parse_export_section, parse_function_section, parse_global_section, parse_import_section, + parse_instance_section, parse_memory_section, parse_module_section, parse_name_section, + parse_start_section, parse_table_section, parse_type_section, }; use crate::state::ModuleTranslationState; use cranelift_codegen::timing; @@ -113,7 +113,7 @@ pub fn translate_module<'data>( Payload::ModuleSection(s) => { validator.module_section(&s)?; - environ.reserve_modules(s.get_count()); + parse_module_section(s, environ)?; } Payload::InstanceSection(s) => { validator.instance_section(&s)?; @@ -121,7 +121,7 @@ pub fn translate_module<'data>( } Payload::AliasSection(s) => { validator.alias_section(&s)?; - unimplemented!("module linking not implemented yet") + parse_alias_section(s, environ)?; } Payload::ModuleCodeSectionStart { count, diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index e9fc525542..0e791c1a43 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -7,7 +7,7 @@ //! The special case of the initialize expressions for table elements offsets or global variables //! is handled, according to the semantics of WebAssembly, to only specific expressions that are //! interpreted on the fly. -use crate::environ::{ModuleEnvironment, WasmError, WasmResult}; +use crate::environ::{Alias, ModuleEnvironment, WasmError, WasmResult}; use crate::state::ModuleTranslationState; use crate::translation_utils::{ tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityIndex, EntityType, Event, @@ -36,9 +36,15 @@ fn entity_type( environ: &mut dyn ModuleEnvironment<'_>, ) -> WasmResult { Ok(match ty { - ImportSectionEntryType::Function(sig) => EntityType::Function(TypeIndex::from_u32(sig)), - ImportSectionEntryType::Module(sig) => EntityType::Module(TypeIndex::from_u32(sig)), - ImportSectionEntryType::Instance(sig) => EntityType::Instance(TypeIndex::from_u32(sig)), + ImportSectionEntryType::Function(sig) => { + EntityType::Function(environ.type_to_signature(TypeIndex::from_u32(sig))?) + } + ImportSectionEntryType::Module(sig) => { + EntityType::Module(environ.type_to_module_type(TypeIndex::from_u32(sig))?) + } + ImportSectionEntryType::Instance(sig) => { + EntityType::Instance(environ.type_to_instance_type(TypeIndex::from_u32(sig))?) + } ImportSectionEntryType::Memory(ty) => EntityType::Memory(memory(ty)), ImportSectionEntryType::Event(evt) => EntityType::Event(event(evt)), ImportSectionEntryType::Global(ty) => { @@ -156,24 +162,40 @@ pub fn parse_import_section<'data>( for entry in imports { let import = entry?; - match entity_type(import.ty, environ)? { - EntityType::Function(idx) => { - environ.declare_func_import(idx, import.module, import.field)?; + match import.ty { + ImportSectionEntryType::Function(sig) => { + environ.declare_func_import( + TypeIndex::from_u32(sig), + import.module, + import.field, + )?; } - EntityType::Module(idx) => { - environ.declare_module_import(idx, import.module, import.field)?; + ImportSectionEntryType::Module(sig) => { + environ.declare_module_import( + TypeIndex::from_u32(sig), + import.module, + import.field, + )?; } - EntityType::Instance(idx) => { - environ.declare_instance_import(idx, import.module, import.field)?; + ImportSectionEntryType::Instance(sig) => { + environ.declare_instance_import( + TypeIndex::from_u32(sig), + import.module, + import.field, + )?; } - EntityType::Memory(ty) => { - environ.declare_memory_import(ty, import.module, import.field)?; + ImportSectionEntryType::Memory(ty) => { + environ.declare_memory_import(memory(ty), import.module, import.field)?; } - EntityType::Event(e) => environ.declare_event_import(e, import.module, import.field)?, - EntityType::Global(ty) => { + ImportSectionEntryType::Event(e) => { + environ.declare_event_import(event(e), import.module, import.field)?; + } + ImportSectionEntryType::Global(ty) => { + let ty = global(ty, environ, GlobalInit::Import)?; environ.declare_global_import(ty, import.module, import.field)?; } - EntityType::Table(ty) => { + ImportSectionEntryType::Table(ty) => { + let ty = table(ty, environ)?; environ.declare_table_import(ty, import.module, import.field)?; } } @@ -316,9 +338,15 @@ pub fn parse_export_section<'data>( ExternalKind::Global => { environ.declare_global_export(GlobalIndex::new(index), field)? } - ExternalKind::Type | ExternalKind::Module | ExternalKind::Instance => { - unimplemented!("module linking not implemented yet") + ExternalKind::Module => { + environ.declare_module_export(ModuleIndex::new(index), field)? } + ExternalKind::Instance => { + environ.declare_instance_export(InstanceIndex::new(index), field)? + } + + // this never gets past validation + ExternalKind::Type => unreachable!(), } } @@ -476,12 +504,25 @@ pub fn parse_name_section<'data>( Ok(()) } +/// Parses the Module section of the wasm module. +pub fn parse_module_section<'data>( + section: wasmparser::ModuleSectionReader<'data>, + environ: &mut dyn ModuleEnvironment<'data>, +) -> WasmResult<()> { + environ.reserve_modules(section.get_count()); + + for module_ty in section { + environ.declare_module(TypeIndex::from_u32(module_ty?))?; + } + Ok(()) +} + /// Parses the Instance section of the wasm module. pub fn parse_instance_section<'data>( section: wasmparser::InstanceSectionReader<'data>, environ: &mut dyn ModuleEnvironment<'data>, ) -> WasmResult<()> { - environ.reserve_types(section.get_count())?; + environ.reserve_instances(section.get_count()); for instance in section { let instance = instance?; @@ -509,3 +550,29 @@ pub fn parse_instance_section<'data>( } Ok(()) } + +/// Parses the Alias section of the wasm module. +pub fn parse_alias_section<'data>( + section: wasmparser::AliasSectionReader<'data>, + environ: &mut dyn ModuleEnvironment<'data>, +) -> WasmResult<()> { + for alias in section { + let alias = alias?; + let alias = match alias.instance { + wasmparser::AliasedInstance::Parent => { + match alias.kind { + ExternalKind::Module => Alias::ParentModule(ModuleIndex::from_u32(alias.index)), + ExternalKind::Type => Alias::ParentType(TypeIndex::from_u32(alias.index)), + // shouldn't get past validation + _ => unreachable!(), + } + } + wasmparser::AliasedInstance::Child(i) => Alias::Child { + instance: InstanceIndex::from_u32(i), + export: alias.index as usize, + }, + }; + environ.declare_alias(alias)?; + } + Ok(()) +} diff --git a/cranelift/wasm/src/translation_utils.rs b/cranelift/wasm/src/translation_utils.rs index 613e50ac73..e161033238 100644 --- a/cranelift/wasm/src/translation_utils.rs +++ b/cranelift/wasm/src/translation_utils.rs @@ -97,6 +97,18 @@ entity_impl!(InstanceIndex); pub struct EventIndex(u32); entity_impl!(EventIndex); +/// Specialized index for just module types. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ModuleTypeIndex(u32); +entity_impl!(ModuleTypeIndex); + +/// Specialized index for just instance types. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct InstanceTypeIndex(u32); +entity_impl!(InstanceTypeIndex); + /// An index of an entity. #[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] @@ -131,13 +143,13 @@ pub enum EntityType { Table(Table), /// A function type where the index points to the type section and records a /// function signature. - Function(TypeIndex), + Function(SignatureIndex), /// An instance where the index points to the type section and records a /// instance's exports. - Instance(TypeIndex), + Instance(InstanceTypeIndex), /// A module where the index points to the type section and records a /// module's imports and exports. - Module(TypeIndex), + Module(ModuleTypeIndex), } /// A WebAssembly global. diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index ad95c0395a..aa89f2cfc4 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1039,7 +1039,6 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m callee: ir::Value, call_args: &[ir::Value], ) -> WasmResult { - let sig_index = self.module.types[ty_index].unwrap_function(); let pointer_type = self.pointer_type(); let table_entry_addr = pos.ins().table_addr(pointer_type, table, callee, 0); @@ -1071,7 +1070,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m let vmctx = self.vmctx(pos.func); let base = pos.ins().global_value(pointer_type, vmctx); let offset = - i32::try_from(self.offsets.vmctx_vmshared_signature_id(sig_index)).unwrap(); + i32::try_from(self.offsets.vmctx_vmshared_signature_id(ty_index)).unwrap(); // Load the caller ID. let mut mem_flags = ir::MemFlags::trusted(); diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 1837ba02d1..ee330ee6a7 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -99,7 +99,7 @@ use std::sync::Mutex; use wasmtime_environ::{ CompileError, CompiledFunction, Compiler, FunctionAddressMap, FunctionBodyData, InstructionAddressMap, ModuleTranslation, Relocation, RelocationTarget, StackMapInformation, - TrapInformation, Tunables, + TrapInformation, Tunables, TypeTables, }; mod func_environ; @@ -348,13 +348,14 @@ impl Compiler for Cranelift { mut input: FunctionBodyData<'_>, isa: &dyn isa::TargetIsa, tunables: &Tunables, + types: &TypeTables, ) -> Result { let module = &translation.module; let func_index = module.func_index(func_index); let mut context = Context::new(); context.func.name = get_func_name(func_index); let sig_index = module.functions[func_index]; - context.func.signature = translation.native_signatures[sig_index].clone(); + context.func.signature = types.native_signatures[sig_index].clone(); if tunables.generate_native_debuginfo { context.func.collect_debug_info(); } @@ -362,7 +363,7 @@ impl Compiler for Cranelift { let mut func_env = FuncEnvironment::new( isa.frontend_config(), module, - &translation.native_signatures, + &types.native_signatures, tunables, ); diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index 5008273bc7..cd2f8b31a9 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -1,7 +1,7 @@ //! A `Compilation` contains the compiled function bodies for a WebAssembly //! module. -use crate::{FunctionAddressMap, FunctionBodyData, ModuleTranslation, Tunables}; +use crate::{FunctionAddressMap, FunctionBodyData, ModuleTranslation, Tunables, TypeTables}; use cranelift_codegen::{binemit, ir, isa, isa::unwind::UnwindInfo}; use cranelift_entity::PrimaryMap; use cranelift_wasm::{DefinedFuncIndex, FuncIndex, WasmError}; @@ -104,5 +104,6 @@ pub trait Compiler: Send + Sync { data: FunctionBodyData<'_>, isa: &dyn isa::TargetIsa, tunables: &Tunables, + types: &TypeTables, ) -> Result; } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 8cd04647f6..d78efadf0f 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -2,20 +2,14 @@ use crate::tunables::Tunables; use crate::WASM_MAX_PAGES; +use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; -use cranelift_wasm::{ - DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, - ElemIndex, EntityIndex, EntityType, FuncIndex, Global, GlobalIndex, InstanceIndex, Memory, - MemoryIndex, ModuleIndex, SignatureIndex, Table, TableIndex, TypeIndex, WasmFuncType, -}; +use cranelift_wasm::*; use indexmap::IndexMap; use more_asserts::assert_ge; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::sync::{ - atomic::{AtomicUsize, Ordering::SeqCst}, - Arc, -}; +use std::sync::Arc; /// A WebAssembly table initializer. #[derive(Clone, Debug, Hash, Serialize, Deserialize)] @@ -121,23 +115,16 @@ impl TablePlan { } } -/// Different types that can appear in a module -#[derive(Debug, Clone, Serialize, Deserialize)] +/// Different types that can appear in a module. +/// +/// Note that each of these variants are intended to index further into a +/// separate table. +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +#[allow(missing_docs)] pub enum ModuleType { - /// A function type, indexed further into the `signatures` table. Function(SignatureIndex), - /// A module type - Module { - /// The module's imports - imports: Vec<(String, Option, EntityType)>, - /// The module's exports - exports: Vec<(String, EntityType)>, - }, - /// An instance type - Instance { - /// the instance's exports - exports: Vec<(String, EntityType)>, - }, + Module(ModuleTypeIndex), + Instance(InstanceTypeIndex), } impl ModuleType { @@ -153,17 +140,19 @@ impl ModuleType { /// A translated WebAssembly module, excluding the function bodies and /// memory initializers. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, Serialize, Deserialize)] pub struct Module { - /// A unique identifier (within this process) for this module. - #[serde(skip_serializing, skip_deserializing, default = "Module::next_id")] - pub id: usize, + /// The parent index of this module, used for the module linking proposal. + /// + /// This index is into the list of modules returned from compilation of a + /// single wasm file with nested modules. + pub parent: Option, /// The name of this wasm module, often found in the wasm file. pub name: Option, /// All import records, in the order they are declared in the module. - pub imports: Vec<(String, Option, EntityIndex)>, + pub initializers: Vec, /// Exported entities. pub exports: IndexMap, @@ -184,22 +173,19 @@ pub struct Module { /// WebAssembly table initializers. pub func_names: HashMap, - /// Unprocessed signatures exactly as provided by `declare_signature()`. - pub signatures: PrimaryMap, - /// Types declared in the wasm module. pub types: PrimaryMap, - /// Number of imported functions in the module. + /// Number of imported or aliased functions in the module. pub num_imported_funcs: usize, - /// Number of imported tables in the module. + /// Number of imported or aliased tables in the module. pub num_imported_tables: usize, - /// Number of imported memories in the module. + /// Number of imported or aliased memories in the module. pub num_imported_memories: usize, - /// Number of imported globals in the module. + /// Number of imported or aliased globals in the module. pub num_imported_globals: usize, /// Types of functions, imported and local. @@ -214,54 +200,58 @@ pub struct Module { /// WebAssembly global variables. pub globals: PrimaryMap, - /// WebAssembly instances. - pub instances: PrimaryMap, + /// The type of each wasm instance this module defines. + pub instances: PrimaryMap, - /// WebAssembly modules. - pub modules: PrimaryMap, + /// The type of each nested wasm module this module contains. + pub modules: PrimaryMap, } -/// Different forms an instance can take in a wasm module +/// Initialization routines for creating an instance, encompassing imports, +/// modules, instances, aliases, etc. #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum Instance { - /// This is an imported instance with the specified type - Import(TypeIndex), - /// This is a locally created instance which instantiates the specified - /// module with the given list of entities. +pub enum Initializer { + /// An imported item is required to be provided. + Import { + /// Module name of this import + module: String, + /// Optional field name of this import + field: Option, + /// Where this import will be placed, which also has type information + /// about the import. + index: EntityIndex, + }, + + /// A module from the parent's declared modules is inserted into our own + /// index space. + AliasParentModule(ModuleIndex), + + /// A module from the parent's declared modules is inserted into our own + /// index space. + #[allow(missing_docs)] + AliasInstanceExport { + instance: InstanceIndex, + export: usize, + }, + + /// A module is being instantiated with previously configured intializers + /// as arguments. Instantiate { /// The module that this instance is instantiating. module: ModuleIndex, /// The arguments provided to instantiation. args: Vec, }, + + /// A module is defined into the module index space, and which module is + /// being defined is specified by the index payload. + DefineModule(usize), } impl Module { /// Allocates the module data structures. pub fn new() -> Self { - Self { - id: Self::next_id(), - name: None, - imports: Vec::new(), - exports: IndexMap::new(), - start_func: None, - table_elements: Vec::new(), - passive_elements: HashMap::new(), - passive_data: HashMap::new(), - func_names: HashMap::new(), - num_imported_funcs: 0, - num_imported_tables: 0, - num_imported_memories: 0, - num_imported_globals: 0, - signatures: PrimaryMap::new(), - functions: PrimaryMap::new(), - table_plans: PrimaryMap::new(), - memory_plans: PrimaryMap::new(), - globals: PrimaryMap::new(), - instances: PrimaryMap::new(), - modules: PrimaryMap::new(), - types: PrimaryMap::new(), - } + Module::default() } /// Get the given passive element, if it exists. @@ -269,11 +259,6 @@ impl Module { self.passive_elements.get(&index).map(|es| &**es) } - fn next_id() -> usize { - static NEXT_ID: AtomicUsize = AtomicUsize::new(0); - NEXT_ID.fetch_add(1, SeqCst) - } - /// Convert a `DefinedFuncIndex` into a `FuncIndex`. pub fn func_index(&self, defined_func: DefinedFuncIndex) -> FuncIndex { FuncIndex::new(self.num_imported_funcs + defined_func.index()) @@ -361,18 +346,37 @@ impl Module { pub fn is_imported_global(&self, index: GlobalIndex) -> bool { index.index() < self.num_imported_globals } - - /// Convenience method for looking up the original Wasm signature of a - /// function. - pub fn wasm_func_type(&self, func_index: FuncIndex) -> &WasmFuncType { - &self.signatures[self.functions[func_index]] - } } -impl Default for Module { - fn default() -> Module { - Module::new() - } +/// All types which are recorded for the entirety of a translation. +/// +/// Note that this is shared amongst all modules coming out of a translation +/// in the case of nested modules and the module linking proposal. +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct TypeTables { + pub wasm_signatures: PrimaryMap, + pub native_signatures: PrimaryMap, + pub module_signatures: PrimaryMap, + pub instance_signatures: PrimaryMap, +} + +/// The type signature of known modules. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModuleSignature { + /// All imports in this module, listed in order with their module/name and + /// what type they're importing. + pub imports: Vec<(String, Option, EntityType)>, + /// Exports are what an instance type conveys, so we go through an + /// indirection over there. + pub exports: InstanceTypeIndex, +} + +/// The type signature of known instances. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstanceSignature { + /// The name of what's being exported as well as its type signature. + pub exports: Vec<(String, EntityType)>, } mod passive_data_serde { diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 128cc71b31..f1af5c4ea0 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -1,13 +1,17 @@ -use crate::module::{Instance, MemoryPlan, Module, ModuleType, TableElements, TablePlan}; +use crate::module::{ + Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, TableElements, + TablePlan, TypeTables, +}; use crate::tunables::Tunables; use cranelift_codegen::ir; use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_entity::PrimaryMap; use cranelift_wasm::{ - self, translate_module, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, - FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, ModuleIndex, SignatureIndex, Table, - TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, + self, translate_module, Alias, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, + FuncIndex, Global, GlobalIndex, InstanceIndex, InstanceTypeIndex, Memory, MemoryIndex, + ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, TargetEnvironment, TypeIndex, + WasmError, WasmFuncType, WasmResult, }; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -27,10 +31,11 @@ pub struct ModuleEnvironment<'data> { /// the module linking proposal. results: Vec>, - /// Modules which are in-progress for being translated (our parents) and - /// we'll resume once we finish the current module. This is only applicable - /// with the module linking proposal. - in_progress: Vec>, + /// Intern'd types for this entire translation, shared by all modules. + types: TypeTables, + + /// Where our module will get pushed into `results` after it's finished. + cur: usize, // Various bits and pieces of configuration features: WasmFeatures, @@ -46,9 +51,6 @@ pub struct ModuleTranslation<'data> { /// Module information. pub module: Module, - /// Map of native signatures - pub native_signatures: PrimaryMap, - /// References to the function bodies. pub function_body_inputs: PrimaryMap>, @@ -58,15 +60,23 @@ pub struct ModuleTranslation<'data> { /// DWARF debug information, if enabled, parsed from the module. pub debuginfo: DebugInfoData<'data>, - /// Indexes into the returned list of translations that are submodules of - /// this module. - pub submodules: PrimaryMap, - /// Set if debuginfo was found but it was not parsed due to `Tunables` /// configuration. pub has_unparsed_debuginfo: bool, + /// When we're parsing the code section this will be incremented so we know + /// which function is currently being defined. code_index: u32, + + /// When local modules are declared an entry is pushed onto this list which + /// indicates that the initializer at the specified position needs to be + /// rewritten with the module's final index in the global list of compiled + /// modules. + module_initializer_indexes: Vec, + + /// Used as a pointer into the above list as the module code section is + /// parsed. + num_modules_defined: usize, } /// Contains function data: byte code and its offset in the module. @@ -128,7 +138,8 @@ impl<'data> ModuleEnvironment<'data> { Self { result: ModuleTranslation::default(), results: Vec::with_capacity(1), - in_progress: Vec::new(), + cur: 0, + types: Default::default(), target_config, tunables: tunables.clone(), features: *features, @@ -139,12 +150,28 @@ impl<'data> ModuleEnvironment<'data> { self.target_config.pointer_type() } - /// Translate a wasm module using this environment. This consumes the - /// `ModuleEnvironment` and produces a `ModuleTranslation`. - pub fn translate(mut self, data: &'data [u8]) -> WasmResult>> { + /// Translate a wasm module using this environment. + /// + /// This consumes the `ModuleEnvironment` and produces a list of + /// `ModuleTranslation`s as well as a `TypeTables`. The list of module + /// translations corresponds to all wasm modules found in the input `data`. + /// Note that for MVP modules this will always be a list with one element, + /// but with the module linking proposal this may have many elements. + /// + /// For the module linking proposal the top-level module is at index 0. + /// + /// The `TypeTables` structure returned contains intern'd versions of types + /// referenced from each module translation. This primarily serves as the + /// source of truth for module-linking use cases where modules can refer to + /// other module's types. All `SignatureIndex`, `ModuleTypeIndex`, and + /// `InstanceTypeIndex` values are resolved through the returned tables. + pub fn translate( + mut self, + data: &'data [u8], + ) -> WasmResult<(Vec>, TypeTables)> { translate_module(data, &mut self)?; assert!(self.results.len() > 0); - Ok(self.results) + Ok((self.results, self.types)) } fn declare_export(&mut self, export: EntityIndex, name: &str) -> WasmResult<()> { @@ -209,16 +236,23 @@ impl<'data> TargetEnvironment for ModuleEnvironment<'data> { impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data> { fn reserve_types(&mut self, num: u32) -> WasmResult<()> { let num = usize::try_from(num).unwrap(); - self.result.module.types.reserve_exact(num); - self.result.native_signatures.reserve_exact(num); + self.result.module.types.reserve(num); + self.types.native_signatures.reserve(num); + self.types.wasm_signatures.reserve(num); Ok(()) } fn declare_type_func(&mut self, wasm: WasmFuncType, sig: ir::Signature) -> WasmResult<()> { let sig = translate_signature(sig, self.pointer_type()); - // TODO: Deduplicate signatures. - self.result.native_signatures.push(sig); - let sig_index = self.result.module.signatures.push(wasm); + + // FIXME(#2469): Signatures should be deduplicated in these two tables + // since `SignatureIndex` is already a index space separate from the + // module's index space. Note that this may get more urgent with + // module-linking modules where types are more likely to get repeated + // (across modules). + let sig_index = self.types.native_signatures.push(sig); + let sig_index2 = self.types.wasm_signatures.push(wasm); + debug_assert_eq!(sig_index, sig_index2); self.result .module .types @@ -239,10 +273,19 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data .iter() .map(|e| (e.0.to_string(), e.1.clone())) .collect(); - self.result - .module + + // FIXME(#2469): Like signatures above we should probably deduplicate + // the listings of module types since with module linking it's possible + // you'll need to write down the module type in multiple locations. + let exports = self .types - .push(ModuleType::Module { imports, exports }); + .instance_signatures + .push(InstanceSignature { exports }); + let idx = self + .types + .module_signatures + .push(ModuleSignature { imports, exports }); + self.result.module.types.push(ModuleType::Module(idx)); Ok(()) } @@ -251,19 +294,46 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data .iter() .map(|e| (e.0.to_string(), e.1.clone())) .collect(); - self.result - .module + + // FIXME(#2469): Like signatures above we should probably deduplicate + // the listings of instance types since with module linking it's + // possible you'll need to write down the module type in multiple + // locations. + let idx = self .types - .push(ModuleType::Instance { exports }); + .instance_signatures + .push(InstanceSignature { exports }); + self.result.module.types.push(ModuleType::Instance(idx)); Ok(()) } + fn type_to_signature(&self, index: TypeIndex) -> WasmResult { + match self.result.module.types[index] { + ModuleType::Function(sig) => Ok(sig), + _ => unreachable!(), + } + } + + fn type_to_module_type(&self, index: TypeIndex) -> WasmResult { + match self.result.module.types[index] { + ModuleType::Module(sig) => Ok(sig), + _ => unreachable!(), + } + } + + fn type_to_instance_type(&self, index: TypeIndex) -> WasmResult { + match self.result.module.types[index] { + ModuleType::Instance(sig) => Ok(sig), + _ => unreachable!(), + } + } + fn reserve_imports(&mut self, num: u32) -> WasmResult<()> { Ok(self .result .module - .imports - .reserve_exact(usize::try_from(num).unwrap())) + .initializers + .reserve(usize::try_from(num).unwrap())) } fn declare_func_import( @@ -279,11 +349,11 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data ); let sig_index = self.result.module.types[index].unwrap_function(); let func_index = self.result.module.functions.push(sig_index); - self.result.module.imports.push(( - module.to_owned(), - field.map(|s| s.to_owned()), - EntityIndex::Function(func_index), - )); + self.result.module.initializers.push(Initializer::Import { + module: module.to_owned(), + field: field.map(|s| s.to_owned()), + index: EntityIndex::Function(func_index), + }); self.result.module.num_imported_funcs += 1; self.result.debuginfo.wasm_file.imported_func_count += 1; Ok(()) @@ -302,11 +372,11 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data ); let plan = TablePlan::for_table(table, &self.tunables); let table_index = self.result.module.table_plans.push(plan); - self.result.module.imports.push(( - module.to_owned(), - field.map(|s| s.to_owned()), - EntityIndex::Table(table_index), - )); + self.result.module.initializers.push(Initializer::Import { + module: module.to_owned(), + field: field.map(|s| s.to_owned()), + index: EntityIndex::Table(table_index), + }); self.result.module.num_imported_tables += 1; Ok(()) } @@ -327,11 +397,11 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } let plan = MemoryPlan::for_memory(memory, &self.tunables); let memory_index = self.result.module.memory_plans.push(plan); - self.result.module.imports.push(( - module.to_owned(), - field.map(|s| s.to_owned()), - EntityIndex::Memory(memory_index), - )); + self.result.module.initializers.push(Initializer::Import { + module: module.to_owned(), + field: field.map(|s| s.to_owned()), + index: EntityIndex::Memory(memory_index), + }); self.result.module.num_imported_memories += 1; Ok(()) } @@ -348,15 +418,47 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data "Imported globals must be declared first" ); let global_index = self.result.module.globals.push(global); - self.result.module.imports.push(( - module.to_owned(), - field.map(|s| s.to_owned()), - EntityIndex::Global(global_index), - )); + self.result.module.initializers.push(Initializer::Import { + module: module.to_owned(), + field: field.map(|s| s.to_owned()), + index: EntityIndex::Global(global_index), + }); self.result.module.num_imported_globals += 1; Ok(()) } + fn declare_module_import( + &mut self, + ty_index: TypeIndex, + module: &'data str, + field: Option<&'data str>, + ) -> WasmResult<()> { + let signature = self.type_to_module_type(ty_index)?; + let module_index = self.result.module.modules.push(signature); + self.result.module.initializers.push(Initializer::Import { + module: module.to_owned(), + field: field.map(|s| s.to_owned()), + index: EntityIndex::Module(module_index), + }); + Ok(()) + } + + fn declare_instance_import( + &mut self, + ty_index: TypeIndex, + module: &'data str, + field: Option<&'data str>, + ) -> WasmResult<()> { + let signature = self.type_to_instance_type(ty_index)?; + let instance_index = self.result.module.instances.push(signature); + self.result.module.initializers.push(Initializer::Import { + module: module.to_owned(), + field: field.map(|s| s.to_owned()), + index: EntityIndex::Instance(instance_index), + }); + Ok(()) + } + fn reserve_func_types(&mut self, num: u32) -> WasmResult<()> { self.result .module @@ -442,6 +544,14 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data self.declare_export(EntityIndex::Global(global_index), name) } + fn declare_module_export(&mut self, index: ModuleIndex, name: &str) -> WasmResult<()> { + self.declare_export(EntityIndex::Module(index), name) + } + + fn declare_instance_export(&mut self, index: InstanceIndex, name: &str) -> WasmResult<()> { + self.declare_export(EntityIndex::Instance(index), name) + } + fn declare_start_func(&mut self, func_index: FuncIndex) -> WasmResult<()> { debug_assert!(self.result.module.start_func.is_none()); self.result.module.start_func = Some(func_index); @@ -503,7 +613,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data let func_index = self.result.code_index + self.result.module.num_imported_funcs as u32; let func_index = FuncIndex::from_u32(func_index); let sig_index = self.result.module.functions[func_index]; - let sig = &self.result.module.signatures[sig_index]; + let sig = &self.types.wasm_signatures[sig_index]; let mut locals = Vec::new(); for pair in body.get_locals_reader()? { locals.push(pair?); @@ -629,42 +739,159 @@ and for re-adding support for interface types you can see this issue: } fn reserve_modules(&mut self, amount: u32) { + // Go ahead and reserve space in the final `results` array for `amount` + // more modules. let extra = self.results.capacity() + (amount as usize) - self.results.len(); self.results.reserve(extra); - self.result.submodules.reserve(amount as usize); + + // Then also reserve space in our own local module's metadata fields + // we'll be adding to. + self.result.module.modules.reserve(amount as usize); + self.result.module.initializers.reserve(amount as usize); + } + + fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> { + // Record the type signature of this module ... + let signature = self.type_to_module_type(ty)?; + self.result.module.modules.push(signature); + + // ... and then record that in the initialization steps of this module + // we're inserting this module into the module index space. At this + // point we don't know the final index of the module we're defining, so + // we leave a placeholder to get rewritten later. + let loc = self.result.module.initializers.len(); + self.result + .module + .initializers + .push(Initializer::DefineModule(usize::max_value())); + self.result.module_initializer_indexes.push(loc); + Ok(()) } fn module_start(&mut self, index: usize) { - // skip the first module since `self.result` is already empty and we'll - // be translating into that. + // Reset the contents of `self.result` for a new module that's getting + // translataed. + let mut prev = mem::replace(&mut self.result, ModuleTranslation::default()); + + // If this is a nested submodule then we record the final destination of + // the child in parent (we store `index` into `prev`) in the appropriate + // initialization slot as dicated by `num_modules_defined` (our index of + // iteration through the code section). + // Record that the `num_modules_defined`-th module is defined at index + // by updating the initializer entry. if index > 0 { - let in_progress = mem::replace(&mut self.result, ModuleTranslation::default()); - self.in_progress.push(in_progress); + let initializer_idx = prev.module_initializer_indexes[prev.num_modules_defined]; + prev.num_modules_defined += 1; + debug_assert!(match &prev.module.initializers[initializer_idx] { + Initializer::DefineModule(usize::MAX) => true, + _ => false, + }); + prev.module.initializers[initializer_idx] = Initializer::DefineModule(index); + self.result.module.parent = Some(self.cur); } + + // Update our current index counter and save our parent's translation + // where this current translation will end up, which we'll swap back as + // part of `module_end`. + self.cur = index; + assert_eq!(index, self.results.len()); + self.results.push(prev); } fn module_end(&mut self, index: usize) { - let to_continue = match self.in_progress.pop() { - Some(m) => m, - None => { - assert_eq!(index, 0); - ModuleTranslation::default() - } - }; - let finished = mem::replace(&mut self.result, to_continue); - self.result.submodules.push(self.results.len()); - self.results.push(finished); + assert!(self.result.num_modules_defined == self.result.module_initializer_indexes.len()); + + // Move our finished module into its final location, swapping it with + // what was this module's parent. + self.cur = self.result.module.parent.unwrap_or(0); + mem::swap(&mut self.result, &mut self.results[index]); } fn reserve_instances(&mut self, amt: u32) { self.result.module.instances.reserve(amt as usize); + self.result.module.initializers.reserve(amt as usize); } fn declare_instance(&mut self, module: ModuleIndex, args: Vec) -> WasmResult<()> { + // Record the type of this instance with the type signature of the + // module we're instantiating and then also add an initializer which + // records that we'll be adding to the instance index space here. + let module_ty = self.result.module.modules[module]; + let instance_ty = self.types.module_signatures[module_ty].exports; + self.result.module.instances.push(instance_ty); self.result .module - .instances - .push(Instance::Instantiate { module, args }); + .initializers + .push(Initializer::Instantiate { module, args }); + Ok(()) + } + + fn declare_alias(&mut self, alias: Alias) -> WasmResult<()> { + match alias { + // Types are easy, we statically know everything so we're just + // copying some pointers from our parent module to our own module. + // + // Note that we don't add an initializer for this alias because + // we statically know where all types point to. + Alias::ParentType(parent_idx) => { + let ty = self.results[self.cur].module.types[parent_idx]; + self.result.module.types.push(ty); + } + + // This is similar to types in that it's easy for us to record the + // type of the module that's being aliased, but we also need to add + // an initializer so during instantiation we can prepare the index + // space appropriately. + Alias::ParentModule(parent_idx) => { + let module_idx = self.results[self.cur].module.modules[parent_idx]; + self.result.module.modules.push(module_idx); + self.result + .module + .initializers + .push(Initializer::AliasParentModule(parent_idx)); + } + + // This case is slightly more involved, we'll be recording all the + // type information for each kind of entity, and then we also need + // to record an initialization step to get the export from the + // instance. + Alias::Child { instance, export } => { + let ty = self.result.module.instances[instance]; + match &self.types.instance_signatures[ty].exports[export].1 { + EntityType::Global(g) => { + self.result.module.globals.push(g.clone()); + self.result.module.num_imported_globals += 1; + } + EntityType::Memory(mem) => { + let plan = MemoryPlan::for_memory(*mem, &self.tunables); + self.result.module.memory_plans.push(plan); + self.result.module.num_imported_memories += 1; + } + EntityType::Table(t) => { + let plan = TablePlan::for_table(*t, &self.tunables); + self.result.module.table_plans.push(plan); + self.result.module.num_imported_tables += 1; + } + EntityType::Function(sig) => { + self.result.module.functions.push(*sig); + self.result.module.num_imported_funcs += 1; + self.result.debuginfo.wasm_file.imported_func_count += 1; + } + EntityType::Instance(sig) => { + self.result.module.instances.push(*sig); + } + EntityType::Module(sig) => { + self.result.module.modules.push(*sig); + } + EntityType::Event(_) => unimplemented!(), + } + self.result + .module + .initializers + .push(Initializer::AliasInstanceExport { instance, export }) + } + } + Ok(()) } } diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index aa5e00d0ac..8673a38d3c 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -24,7 +24,7 @@ use crate::BuiltinFunctionIndex; use cranelift_codegen::ir; use cranelift_wasm::{ DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalIndex, MemoryIndex, - SignatureIndex, TableIndex, + TableIndex, TypeIndex, }; use more_asserts::assert_lt; use std::convert::TryFrom; @@ -78,7 +78,7 @@ impl VMOffsets { pub fn new(pointer_size: u8, module: &Module) -> Self { Self { pointer_size, - num_signature_ids: cast_to_u32(module.signatures.len()), + num_signature_ids: cast_to_u32(module.types.len()), num_imported_functions: cast_to_u32(module.num_imported_funcs), num_imported_tables: cast_to_u32(module.num_imported_tables), num_imported_memories: cast_to_u32(module.num_imported_memories), @@ -430,7 +430,7 @@ impl VMOffsets { } /// Return the offset to `VMSharedSignatureId` index `index`. - pub fn vmctx_vmshared_signature_id(&self, index: SignatureIndex) -> u32 { + pub fn vmctx_vmshared_signature_id(&self, index: TypeIndex) -> u32 { assert_lt!(index.as_u32(), self.num_signature_ids); self.vmctx_signature_ids_begin() .checked_add( diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index a9a2587c84..5adb0e8312 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -14,7 +14,7 @@ use wasmtime_environ::isa::{TargetFrontendConfig, TargetIsa}; use wasmtime_environ::wasm::{DefinedMemoryIndex, MemoryIndex}; use wasmtime_environ::{ CompiledFunctions, Compiler as EnvCompiler, DebugInfoData, Module, ModuleMemoryOffset, - ModuleTranslation, Tunables, VMOffsets, + ModuleTranslation, Tunables, TypeTables, VMOffsets, }; /// Select which kind of compilation to use. @@ -127,13 +127,20 @@ impl Compiler { pub fn compile<'data>( &self, translation: &mut ModuleTranslation, + types: &TypeTables, ) -> Result { let functions = mem::take(&mut translation.function_body_inputs); let functions = functions.into_iter().collect::>(); let funcs = maybe_parallel!(functions.(into_iter | into_par_iter)) .map(|(index, func)| { - self.compiler - .compile_function(translation, index, func, &*self.isa, &self.tunables) + self.compiler.compile_function( + translation, + index, + func, + &*self.isa, + &self.tunables, + types, + ) }) .collect::, _>>()? .into_iter() @@ -150,7 +157,8 @@ impl Compiler { vec![] }; - let (obj, unwind_info) = build_object(&*self.isa, &translation, &funcs, dwarf_sections)?; + let (obj, unwind_info) = + build_object(&*self.isa, &translation, types, &funcs, dwarf_sections)?; Ok(Compilation { obj, diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 4bff020519..f739b79d1a 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -18,10 +18,13 @@ use thiserror::Error; use wasmtime_debug::create_gdbjit_image; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::wasm::{DefinedFuncIndex, ModuleIndex, SignatureIndex}; +use wasmtime_environ::wasm::{ + DefinedFuncIndex, InstanceTypeIndex, ModuleTypeIndex, SignatureIndex, WasmFuncType, +}; use wasmtime_environ::{ CompileError, DataInitializer, DataInitializerLocation, DebugInfoData, FunctionAddressMap, - Module, ModuleEnvironment, ModuleTranslation, StackMapInformation, TrapInformation, + InstanceSignature, Module, ModuleEnvironment, ModuleSignature, ModuleTranslation, + StackMapInformation, TrapInformation, }; use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::{ @@ -70,10 +73,6 @@ pub struct CompilationArtifacts { /// Descriptions of compiled functions funcs: PrimaryMap, - /// Where to find this module's submodule code in the top-level list of - /// modules. - submodules: PrimaryMap, - /// Whether or not native debug information is available in `obj` native_debug_info_present: bool, @@ -106,8 +105,8 @@ impl CompilationArtifacts { pub fn build( compiler: &Compiler, data: &[u8], - ) -> Result, SetupError> { - let translations = ModuleEnvironment::new( + ) -> Result<(Vec, TypeTables), SetupError> { + let (translations, types) = ModuleEnvironment::new( compiler.frontend_config(), compiler.tunables(), compiler.features(), @@ -115,18 +114,17 @@ impl CompilationArtifacts { .translate(data) .map_err(|error| SetupError::Compile(CompileError::Wasm(error)))?; - maybe_parallel!(translations.(into_iter | into_par_iter)) + let list = maybe_parallel!(translations.(into_iter | into_par_iter)) .map(|mut translation| { let Compilation { obj, unwind_info, funcs, - } = compiler.compile(&mut translation)?; + } = compiler.compile(&mut translation, &types)?; let ModuleTranslation { module, data_initializers, - submodules, debuginfo, has_unparsed_debuginfo, .. @@ -149,7 +147,6 @@ impl CompilationArtifacts { obj: obj.into_boxed_slice(), unwind_info: unwind_info.into_boxed_slice(), data_initializers, - submodules, funcs: funcs .into_iter() .map(|(_, func)| FunctionInfo { @@ -167,11 +164,21 @@ impl CompilationArtifacts { has_unparsed_debuginfo, }) }) - .collect::, SetupError>>() + .collect::, SetupError>>()?; + Ok(( + list, + TypeTables { + wasm_signatures: types.wasm_signatures, + module_signatures: types.module_signatures, + instance_signatures: types.instance_signatures, + }, + )) } } struct FinishedFunctions(PrimaryMap); +unsafe impl Send for FinishedFunctions {} +unsafe impl Sync for FinishedFunctions {} #[derive(Serialize, Deserialize, Clone)] struct FunctionInfo { @@ -180,8 +187,15 @@ struct FunctionInfo { stack_maps: Vec, } -unsafe impl Send for FinishedFunctions {} -unsafe impl Sync for FinishedFunctions {} +/// This is intended to mirror the type tables in `wasmtime_environ`, except that +/// it doesn't store the native signatures which are no longer needed past compilation. +#[derive(Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct TypeTables { + pub wasm_signatures: PrimaryMap, + pub module_signatures: PrimaryMap, + pub instance_signatures: PrimaryMap, +} /// Container for data needed for an Instance function to exist. pub struct ModuleCode { @@ -375,12 +389,6 @@ impl CompiledModule { &self.code } - /// Returns where the specified submodule lives in this module's - /// array-of-modules (store at the top-level) - pub fn submodule_idx(&self, idx: ModuleIndex) -> usize { - self.artifacts.submodules[idx] - } - /// Creates a new symbolication context which can be used to further /// symbolicate stack traces. /// diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 1421ed0544..17e6294250 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -47,7 +47,7 @@ pub mod trampoline; pub use crate::code_memory::CodeMemory; pub use crate::compiler::{Compilation, CompilationStrategy, Compiler}; pub use crate::instantiate::{ - CompilationArtifacts, CompiledModule, ModuleCode, SetupError, SymbolizeContext, + CompilationArtifacts, CompiledModule, ModuleCode, SetupError, SymbolizeContext, TypeTables, }; pub use crate::link::link_module; diff --git a/crates/jit/src/object.rs b/crates/jit/src/object.rs index f492d645fa..24b431e597 100644 --- a/crates/jit/src/object.rs +++ b/crates/jit/src/object.rs @@ -8,7 +8,7 @@ use wasmtime_debug::DwarfSection; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; -use wasmtime_environ::{CompiledFunctions, ModuleTranslation}; +use wasmtime_environ::{CompiledFunctions, ModuleTranslation, TypeTables}; use wasmtime_obj::{ObjectBuilder, ObjectBuilderTarget}; pub use wasmtime_obj::utils; @@ -24,6 +24,7 @@ pub enum ObjectUnwindInfo { pub(crate) fn build_object( isa: &dyn TargetIsa, translation: &ModuleTranslation, + types: &TypeTables, funcs: &CompiledFunctions, dwarf_sections: Vec, ) -> Result<(Object, Vec), anyhow::Error> { @@ -38,10 +39,16 @@ pub(crate) fn build_object( .map(|info| ObjectUnwindInfo::Func(translation.module.func_index(index), info.clone())) })); - let mut trampolines = PrimaryMap::with_capacity(translation.module.signatures.len()); + let mut trampolines = PrimaryMap::with_capacity(types.native_signatures.len()); let mut cx = FunctionBuilderContext::new(); // Build trampolines for every signature. - for (i, native_sig) in translation.native_signatures.iter() { + // + // TODO: for the module linking proposal this builds too many native + // signatures. This builds trampolines for all signatures for all modules + // for each module. That's a lot of trampolines! We should instead figure + // out a way to share trampolines amongst all modules when compiling + // module-linking modules. + for (i, native_sig) in types.native_signatures.iter() { let func = build_trampoline(isa, &mut cx, native_sig, std::mem::size_of::())?; // Preserve trampoline function unwind info. if let Some(info) = &func.unwind_info { diff --git a/crates/lightbeam/wasmtime/src/lib.rs b/crates/lightbeam/wasmtime/src/lib.rs index e38e48f20d..a798eee4d2 100644 --- a/crates/lightbeam/wasmtime/src/lib.rs +++ b/crates/lightbeam/wasmtime/src/lib.rs @@ -9,12 +9,12 @@ use cranelift_codegen::isa; use lightbeam::{CodeGenSession, NullOffsetSink, Sinks}; use wasmtime_environ::wasm::{ DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, - GlobalIndex, MemoryIndex, SignatureIndex, TableIndex, + GlobalIndex, MemoryIndex, SignatureIndex, TableIndex, TypeIndex, }; use wasmtime_environ::{ entity::PrimaryMap, BuiltinFunctionIndex, CompileError, CompiledFunction, Compiler, FunctionBodyData, Module, ModuleTranslation, Relocation, RelocationTarget, TrapInformation, - Tunables, VMOffsets, + Tunables, TypeTables, VMOffsets, }; /// A compiler that compiles a WebAssembly module with Lightbeam, directly translating the Wasm file. @@ -28,13 +28,14 @@ impl Compiler for Lightbeam { function_body: FunctionBodyData<'_>, isa: &dyn isa::TargetIsa, tunables: &Tunables, + types: &TypeTables, ) -> Result { if tunables.generate_native_debuginfo { return Err(CompileError::DebugInfoNotSupported); } let func_index = translation.module.func_index(i); - let env = FuncEnvironment::new(isa.frontend_config().pointer_bytes(), translation); + let env = FuncEnvironment::new(isa.frontend_config().pointer_bytes(), translation, types); let mut codegen_session: CodeGenSession<_> = CodeGenSession::new( translation.function_body_inputs.len() as u32, &env, @@ -180,11 +181,15 @@ struct FuncEnvironment<'module_environment> { } impl<'module_environment> FuncEnvironment<'module_environment> { - fn new(pointer_bytes: u8, translation: &'module_environment ModuleTranslation<'_>) -> Self { + fn new( + pointer_bytes: u8, + translation: &'module_environment ModuleTranslation<'_>, + types: &'module_environment TypeTables, + ) -> Self { Self { module: &translation.module, offsets: VMOffsets::new(pointer_bytes, &translation.module), - native_signatures: &translation.native_signatures, + native_signatures: &types.native_signatures, } } } @@ -322,7 +327,7 @@ impl lightbeam::ModuleContext for FuncEnvironment<'_> { } fn vmctx_vmshared_signature_id(&self, signature_idx: u32) -> u32 { self.offsets - .vmctx_vmshared_signature_id(SignatureIndex::from_u32(signature_idx)) + .vmctx_vmshared_signature_id(TypeIndex::from_u32(signature_idx)) } // TODO: type of a global diff --git a/crates/obj/src/context.rs b/crates/obj/src/context.rs index e306f6f36b..2f737dac8a 100644 --- a/crates/obj/src/context.rs +++ b/crates/obj/src/context.rs @@ -7,7 +7,7 @@ use std::ptr; use wasmtime_environ::entity::EntityRef; use wasmtime_environ::isa::TargetFrontendConfig; use wasmtime_environ::wasm::GlobalInit; -use wasmtime_environ::{Module, TargetSharedSignatureIndex, VMOffsets}; +use wasmtime_environ::{Module, ModuleType, TargetSharedSignatureIndex, VMOffsets}; pub struct TableRelocation { pub index: usize, @@ -25,16 +25,19 @@ pub fn layout_vmcontext( // Assign unique indices to unique signatures. let mut signature_registry = HashMap::new(); let mut signature_registry_len = signature_registry.len(); - for (index, sig) in module.signatures.iter() { + for (index, sig) in module.types.iter() { let offset = ofs.vmctx_vmshared_signature_id(index) as usize; - let target_index = match signature_registry.entry(sig) { - Entry::Occupied(o) => *o.get(), - Entry::Vacant(v) => { - assert_le!(signature_registry_len, std::u32::MAX as usize); - let id = TargetSharedSignatureIndex::new(signature_registry_len as u32); - signature_registry_len += 1; - *v.insert(id) - } + let target_index = match sig { + ModuleType::Function(sig) => match signature_registry.entry(sig) { + Entry::Occupied(o) => *o.get(), + Entry::Vacant(v) => { + assert_le!(signature_registry_len, std::u32::MAX as usize); + let id = TargetSharedSignatureIndex::new(signature_registry_len as u32); + signature_registry_len += 1; + *v.insert(id) + } + }, + _ => TargetSharedSignatureIndex::new(u32::max_value()), }; unsafe { let to = out.as_mut_ptr().add(offset) as *mut TargetSharedSignatureIndex; diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index a95a4b594b..1f39b171a0 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -31,7 +31,7 @@ use wasmtime_environ::wasm::{ ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, TableElementType, TableIndex, WasmType, }; -use wasmtime_environ::{ir, DataInitializer, Module, TableElements, VMOffsets}; +use wasmtime_environ::{ir, DataInitializer, Module, ModuleType, TableElements, VMOffsets}; /// A WebAssembly instance. /// @@ -78,12 +78,6 @@ impl Instance { .cast() } - /// Return the indexed `VMSharedSignatureIndex`. - fn signature_id(&self, index: SignatureIndex) -> VMSharedSignatureIndex { - let index = usize::try_from(index.as_u32()).unwrap(); - unsafe { *self.signature_ids_ptr().add(index) } - } - pub(crate) fn module(&self) -> &Module { &self.module } @@ -868,8 +862,11 @@ impl InstanceHandle { let instance = handle.instance(); let mut ptr = instance.signature_ids_ptr(); - for (signature, _) in handle.module().signatures.iter() { - *ptr = lookup_shared_signature(signature); + for sig in handle.module().types.values() { + *ptr = match sig { + ModuleType::Function(sig) => lookup_shared_signature(*sig), + _ => VMSharedSignatureIndex::new(u32::max_value()), + }; ptr = ptr.add(1); } @@ -924,7 +921,7 @@ impl InstanceHandle { *instance.stack_map_registry() = stack_map_registry; for (index, sig) in instance.module.functions.iter() { - let type_index = instance.signature_id(*sig); + let type_index = lookup_shared_signature(*sig); let (func_ptr, vmctx) = if let Some(def_index) = instance.module.defined_func_index(index) { diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index bc8d2daffc..63f2e2678f 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -115,6 +115,15 @@ impl Extern { }; Store::same(my_store, store) } + + pub(crate) fn desc(&self) -> &'static str { + match self { + Extern::Func(_) => "function", + Extern::Table(_) => "table", + Extern::Memory(_) => "memory", + Extern::Global(_) => "global", + } + } } impl From for Extern { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index b375b48848..066a618640 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -1,74 +1,158 @@ use crate::trampoline::StoreInstanceHandle; use crate::{Engine, Export, Extern, Func, Global, Memory, Module, Store, Table, Trap}; -use anyhow::{bail, Error, Result}; +use anyhow::{bail, Context, Error, Result}; use std::mem; use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::{EntityIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex}; -use wasmtime_jit::CompiledModule; +use wasmtime_environ::wasm::{ + EntityIndex, FuncIndex, GlobalIndex, InstanceIndex, MemoryIndex, ModuleIndex, TableIndex, +}; +use wasmtime_environ::Initializer; +use wasmtime_jit::{CompiledModule, TypeTables}; use wasmtime_runtime::{ Imports, InstantiationError, StackMapRegistry, VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, }; -fn instantiate( - store: &Store, - compiled_module: &CompiledModule, - all_modules: &[CompiledModule], - imports: &mut ImportsBuilder<'_>, +/// Performs all low-level steps necessary for instantiation. +/// +/// This function will take all the arguments and attempt to do everything +/// necessary to instantiate the referenced instance. The trickiness of this +/// function stems from the implementation of the module-linking proposal where +/// we're handling nested instances, interleaved imports/aliases, etc. That's +/// all an internal implementation here ideally though! +/// +/// * `store` - the store we're instantiating into +/// * `compiled_module` - the module that we're instantiating +/// * `all_modules` - the list of all modules that were part of the compilation +/// of `compiled_module`. This is only applicable in the module linking +/// proposal, otherwise this will just be a list containing `compiled_module` +/// itself. +/// * `type` - the type tables produced during compilation which +/// `compiled_module`'s metadata references. +/// * `parent_modules` - this is the list of compiled modules the parent has. +/// This is only applicable on recursive instantiations. +/// * `define_import` - this function, like the name implies, defines an import +/// into the provided builder. The expected entity that it's defining is also +/// passed in for the top-level case where type-checking is performed. This is +/// fallible because type checks may fail. +fn instantiate<'a>( + store: &'a Store, + compiled_module: &'a CompiledModule, + all_modules: &'a [CompiledModule], + types: &'a TypeTables, + parent_modules: &PrimaryMap, + define_import: &mut dyn FnMut(&EntityIndex, &mut ImportsBuilder<'a>) -> Result<()>, ) -> Result { let env_module = compiled_module.module(); - // The first part of instantiating any module is to first follow any - // `instantiate` instructions it has as part of the module linking - // proposal. Here we iterate overall those instructions and create the - // instances as necessary. - for instance in env_module.instances.values() { - let (module_idx, args) = match instance { - wasmtime_environ::Instance::Instantiate { module, args } => (*module, args), - wasmtime_environ::Instance::Import(_) => continue, - }; - // Translate the `module_idx` to a top-level module `usize` and then - // use that to extract the child `&CompiledModule` itself. Then we can - // iterate over each of the arguments provided to satisfy its imports. - // - // Note that we directly reach into `imports` below based on indexes - // and push raw value into how to instantiate our submodule. This should - // be safe due to wasm validation ensuring that all our indices are - // in-bounds and all the expected types and such line up. - let module_idx = compiled_module.submodule_idx(module_idx); - let compiled_module = &all_modules[module_idx]; - let mut builder = ImportsBuilder::new(compiled_module.module(), store); - for arg in args { - match *arg { - EntityIndex::Global(i) => { - builder.globals.push(imports.globals[i]); - } - EntityIndex::Table(i) => { - builder.tables.push(imports.tables[i]); - } - EntityIndex::Function(i) => { - builder.functions.push(imports.functions[i]); - } - EntityIndex::Memory(i) => { - builder.memories.push(imports.memories[i]); - } - EntityIndex::Module(_) => unimplemented!(), - EntityIndex::Instance(_) => unimplemented!(), + let mut imports = ImportsBuilder::new(env_module, types, store); + for initializer in env_module.initializers.iter() { + match initializer { + // Definition of an import depends on how our parent is providing + // imports, so we delegate to our custom closure. This will resolve + // to fetching from the import list for the top-level module and + // otherwise fetching from each nested instance's argument list for + // submodules. + Initializer::Import { + index, + module, + field, + } => { + define_import(index, &mut imports).with_context(|| match field { + Some(name) => format!("incompatible import type for `{}::{}`", module, name), + None => format!("incompatible import type for `{}`", module), + })?; + } + + // This one's pretty easy, we're just picking up our parent's module + // and putting it into our own index space. + Initializer::AliasParentModule(idx) => { + imports.modules.push(parent_modules[*idx]); + } + + // Turns out defining any kind of module is pretty easy, we're just + // slinging around pointers. + Initializer::DefineModule(idx) => { + imports.modules.push(&all_modules[*idx]); + } + + // Here we lookup our instance handle, ask it for the nth export, + // and then push that item into our own index space. We eschew + // type-checking since only valid modules reach this point. + Initializer::AliasInstanceExport { instance, export } => { + let handle = &imports.instances[*instance]; + let export_index = &handle.module().exports[*export]; + let item = Extern::from_wasmtime_export( + handle.lookup_by_declaration(export_index), + handle.clone(), + ); + imports.push_extern(&item); + } + + // Oh boy a recursive instantiation! The recursive arguments here + // are pretty simple, and the only slightly-meaty one is how + // arguments are pulled from `args` and pushed directly into the + // builder specified, which should be an easy enough + // copy-the-pointer operation in all cases. + // + // Note that this recursive call shouldn't result in an infinite + // loop because of wasm module validation which requires everything + // to be a DAG. Additionally the recursion should also be bounded + // due to validation. We may one day need to make this an iterative + // loop, however. + Initializer::Instantiate { module, args } => { + let module_to_instantiate = imports.modules[*module]; + let mut args = args.iter(); + let handle = instantiate( + store, + module_to_instantiate, + all_modules, + types, + &imports.modules, + &mut |_, builder| { + match *args.next().unwrap() { + EntityIndex::Global(i) => { + builder.globals.push(imports.globals[i]); + } + EntityIndex::Function(i) => { + builder.functions.push(imports.functions[i]); + } + EntityIndex::Table(i) => { + builder.tables.push(imports.tables[i]); + } + EntityIndex::Memory(i) => { + builder.memories.push(imports.memories[i]); + } + EntityIndex::Module(i) => { + builder.modules.push(imports.modules[i]); + } + EntityIndex::Instance(i) => { + builder.instances.push(imports.instances[i].clone()); + } + } + Ok(()) + }, + )?; + imports.instances.push(handle); } } - instantiate(store, compiled_module, all_modules, &mut builder)?; } + // With the above initialization done we've now acquired the final set of + // imports in all the right index spaces and everything. Time to carry on + // with the creation of our own instance. + let imports = imports.imports(); + // Register the module just before instantiation to ensure we have a // trampoline registered for every signature and to preserve the module's // compiled JIT code within the `Store`. - store.register_module(compiled_module); + store.register_module(compiled_module, types); let config = store.engine().config(); let instance = unsafe { let instance = compiled_module.instantiate( - imports.imports(), - &store.lookup_shared_signature(compiled_module.module()), + imports, + &store.lookup_shared_signature(types), config.memory_creator.as_ref().map(|a| a as _), store.interrupts(), Box::new(()), @@ -208,26 +292,38 @@ impl Instance { bail!("cross-`Engine` instantiation is not currently supported"); } - let mut builder = ImportsBuilder::new(module.compiled_module().module(), store); + // Perform some pre-flight checks before we get into the meat of + // instantiation. + let expected = module + .compiled_module() + .module() + .initializers + .iter() + .filter(|e| match e { + Initializer::Import { .. } => true, + _ => false, + }) + .count(); + if expected != imports.len() { + bail!("expected {} imports, found {}", expected, imports.len()); + } for import in imports { - // For now we have a restriction that the `Store` that we're working - // with is the same for everything involved here. if !import.comes_from_same_store(store) { bail!("cross-`Store` instantiation is not currently supported"); } - match import { - Extern::Global(e) => builder.global(e)?, - Extern::Func(e) => builder.func(e)?, - Extern::Table(e) => builder.table(e)?, - Extern::Memory(e) => builder.memory(e)?, - } } - builder.validate_all_imports_provided()?; + + let mut imports = imports.iter(); let handle = instantiate( store, module.compiled_module(), - &module.compiled, - &mut builder, + module.all_compiled_modules(), + module.types(), + &PrimaryMap::new(), + &mut |idx, builder| { + let import = imports.next().expect("already checked the length"); + builder.define_extern(idx, import) + }, )?; Ok(Instance { @@ -304,113 +400,105 @@ struct ImportsBuilder<'a> { tables: PrimaryMap, memories: PrimaryMap, globals: PrimaryMap, + instances: PrimaryMap, + modules: PrimaryMap, + module: &'a wasmtime_environ::Module, - imports: std::slice::Iter<'a, (String, Option, EntityIndex)>, store: &'a Store, + types: &'a TypeTables, } impl<'a> ImportsBuilder<'a> { - fn new(module: &'a wasmtime_environ::Module, store: &'a Store) -> ImportsBuilder<'a> { + fn new( + module: &'a wasmtime_environ::Module, + types: &'a TypeTables, + store: &'a Store, + ) -> ImportsBuilder<'a> { ImportsBuilder { - imports: module.imports.iter(), module, store, + types, functions: PrimaryMap::with_capacity(module.num_imported_funcs), tables: PrimaryMap::with_capacity(module.num_imported_tables), memories: PrimaryMap::with_capacity(module.num_imported_memories), globals: PrimaryMap::with_capacity(module.num_imported_globals), + instances: PrimaryMap::with_capacity(module.instances.len()), + modules: PrimaryMap::with_capacity(module.modules.len()), } } - fn next_import( - &mut self, - found: &str, - get: impl FnOnce(&wasmtime_environ::Module, &EntityIndex) -> Option, - ) -> Result<()> { - match self.imports.next() { - Some((module, field, idx)) => { - let error = match get(self.module, idx) { - Some(true) => return Ok(()), - Some(false) => { - anyhow::anyhow!("{} types incompatible", found) + fn define_extern(&mut self, expected: &EntityIndex, actual: &Extern) -> Result<()> { + match *expected { + EntityIndex::Table(i) => { + self.tables.push(match actual { + Extern::Table(e) if e.matches_expected(&self.module.table_plans[i]) => { + e.vmimport() } - None => { - let desc = match idx { - EntityIndex::Table(_) => "table", - EntityIndex::Function(_) => "func", - EntityIndex::Memory(_) => "memory", - EntityIndex::Global(_) => "global", - EntityIndex::Instance(_) => "instance", - EntityIndex::Module(_) => "module", - }; - anyhow::anyhow!("expected {}, but found {}", desc, found) - } - }; - let import_name = match field { - Some(name) => format!("{}/{}", module, name), - None => module.to_string(), - }; - Err(error.context(format!("incompatible import type for {}", import_name))) + Extern::Table(_) => bail!("table types incompatible"), + _ => bail!("expected table, but found {}", actual.desc()), + }); } - None => bail!("too many imports provided"), - } - } - - fn global(&mut self, global: &Global) -> Result<()> { - self.next_import("global", |m, e| match e { - EntityIndex::Global(i) => Some(global.matches_expected(&m.globals[*i])), - _ => None, - })?; - self.globals.push(global.vmimport()); - Ok(()) - } - - fn memory(&mut self, mem: &Memory) -> Result<()> { - self.next_import("memory", |m, e| match e { - EntityIndex::Memory(i) => Some(mem.matches_expected(&m.memory_plans[*i])), - _ => None, - })?; - self.memories.push(mem.vmimport()); - Ok(()) - } - - fn table(&mut self, table: &Table) -> Result<()> { - self.next_import("table", |m, e| match e { - EntityIndex::Table(i) => Some(table.matches_expected(&m.table_plans[*i])), - _ => None, - })?; - self.tables.push(table.vmimport()); - Ok(()) - } - - fn func(&mut self, func: &Func) -> Result<()> { - let store = self.store; - self.next_import("func", |m, e| match e { - EntityIndex::Function(i) => Some( + EntityIndex::Memory(i) => { + self.memories.push(match actual { + Extern::Memory(e) if e.matches_expected(&self.module.memory_plans[i]) => { + e.vmimport() + } + Extern::Memory(_) => bail!("memory types incompatible"), + _ => bail!("expected memory, but found {}", actual.desc()), + }); + } + EntityIndex::Global(i) => { + self.globals.push(match actual { + Extern::Global(e) if e.matches_expected(&self.module.globals[i]) => { + e.vmimport() + } + Extern::Global(_) => bail!("global types incompatible"), + _ => bail!("expected global, but found {}", actual.desc()), + }); + } + EntityIndex::Function(i) => { + let func = match actual { + Extern::Func(e) => e, + _ => bail!("expected function, but found {}", actual.desc()), + }; // Look up the `i`th function's type from the module in our // signature registry. If it's not present then we have no // functions registered with that type, so `func` is guaranteed // to not match. - match store + let ty = self + .store .signatures() .borrow() - .lookup(&m.signatures[m.functions[*i]]) - { - Some(ty) => func.matches_expected(ty), - None => false, - }, - ), - _ => None, - })?; - self.functions.push(func.vmimport()); + .lookup(&self.types.wasm_signatures[self.module.functions[i]]) + .ok_or_else(|| anyhow::format_err!("function types incompatible"))?; + if !func.matches_expected(ty) { + bail!("function types incompatible"); + } + self.functions.push(func.vmimport()); + } + + // FIXME(#2094) + EntityIndex::Module(_i) => unimplemented!(), + EntityIndex::Instance(_i) => unimplemented!(), + } Ok(()) } - fn validate_all_imports_provided(&mut self) -> Result<()> { - if self.imports.next().is_some() { - bail!("not enough imports provided"); + fn push_extern(&mut self, item: &Extern) { + match item { + Extern::Func(i) => { + self.functions.push(i.vmimport()); + } + Extern::Global(i) => { + self.globals.push(i.vmimport()); + } + Extern::Table(i) => { + self.tables.push(i.vmimport()); + } + Extern::Memory(i) => { + self.memories.push(i.vmimport()); + } } - Ok(()) } fn imports(&self) -> Imports<'_> { diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 875a1896d3..37f068af6c 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use wasmparser::Validator; #[cfg(feature = "cache")] use wasmtime_cache::ModuleCacheEntry; -use wasmtime_jit::{CompilationArtifacts, CompiledModule}; +use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables}; /// A compiled WebAssembly module, ready to be instantiated. /// @@ -81,10 +81,15 @@ use wasmtime_jit::{CompilationArtifacts, CompiledModule}; #[derive(Clone)] pub struct Module { engine: Engine, - pub(crate) compiled: Arc<[CompiledModule]>, + data: Arc, index: usize, } +pub(crate) struct ModuleData { + pub(crate) types: TypeTables, + pub(crate) modules: Vec, +} + impl Module { /// Creates a new WebAssembly `Module` from the given in-memory `bytes`. /// @@ -164,7 +169,7 @@ impl Module { /// See [`Module::new`] for other details. pub fn new_with_name(engine: &Engine, bytes: impl AsRef<[u8]>, name: &str) -> Result { let mut module = Module::new(engine, bytes.as_ref())?; - Arc::get_mut(&mut module.compiled).unwrap()[module.index] + Arc::get_mut(&mut module.data).unwrap().modules[module.index] .module_mut() .expect("mutable module") .name = Some(name.to_string()); @@ -240,14 +245,14 @@ impl Module { /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { #[cfg(feature = "cache")] - let artifacts = ModuleCacheEntry::new("wasmtime", engine.cache_config()) + let (artifacts, types) = ModuleCacheEntry::new("wasmtime", engine.cache_config()) .get_data((engine.compiler(), binary), |(compiler, binary)| { CompilationArtifacts::build(compiler, binary) })?; #[cfg(not(feature = "cache"))] - let artifacts = CompilationArtifacts::build(engine.compiler(), binary)?; + let (artifacts, types) = CompilationArtifacts::build(engine.compiler(), binary)?; - let compiled = CompiledModule::from_artifacts_list( + let modules = CompiledModule::from_artifacts_list( artifacts, engine.compiler().isa(), &*engine.config().profiler, @@ -255,8 +260,8 @@ impl Module { Ok(Module { engine: engine.clone(), - index: compiled.len() - 1, - compiled: compiled.into(), + index: 0, + data: Arc::new(ModuleData { types, modules }), }) } @@ -290,10 +295,12 @@ impl Module { pub fn serialize(&self) -> Result> { let artifacts = ( compiler_fingerprint(&self.engine), - self.compiled + self.data + .modules .iter() .map(|i| i.compilation_artifacts()) .collect::>(), + &self.data.types, self.index, ); @@ -313,14 +320,14 @@ impl Module { pub fn deserialize(engine: &Engine, serialized: &[u8]) -> Result { let expected_fingerprint = compiler_fingerprint(engine); - let (fingerprint, artifacts, index) = bincode_options() - .deserialize::<(u64, _, _)>(serialized) + let (fingerprint, artifacts, types, index) = bincode_options() + .deserialize::<(u64, _, _, _)>(serialized) .context("Deserialize compilation artifacts")?; if fingerprint != expected_fingerprint { bail!("Incompatible compilation artifact"); } - let compiled = CompiledModule::from_artifacts_list( + let modules = CompiledModule::from_artifacts_list( artifacts, engine.compiler().isa(), &*engine.config().profiler, @@ -329,12 +336,20 @@ impl Module { Ok(Module { engine: engine.clone(), index, - compiled: compiled.into(), + data: Arc::new(ModuleData { modules, types }), }) } pub(crate) fn compiled_module(&self) -> &CompiledModule { - &self.compiled[self.index] + &self.data.modules[self.index] + } + + pub(crate) fn all_compiled_modules(&self) -> &[CompiledModule] { + &self.data.modules + } + + pub(crate) fn types(&self) -> &TypeTables { + &self.data.types } /// Returns identifier/name that this [`Module`] has. This name @@ -419,12 +434,21 @@ impl Module { ) -> impl ExactSizeIterator> + 'module { let module = self.compiled_module().module(); module - .imports + .initializers .iter() - .map(move |(module_name, name, entity_index)| { - let r#type = EntityType::new(entity_index, module); - ImportType::new(module_name, name.as_deref(), r#type) + .filter_map(move |initializer| match initializer { + wasmtime_environ::Initializer::Import { + module, + field, + index, + } => { + let ty = EntityType::new(index, self); + Some(ImportType::new(module, field.as_deref(), ty)) + } + _ => None, }) + .collect::>() + .into_iter() } /// Returns the list of exports that this [`Module`] has and will be @@ -486,8 +510,8 @@ impl Module { ) -> impl ExactSizeIterator> + 'module { let module = self.compiled_module().module(); module.exports.iter().map(move |(name, entity_index)| { - let r#type = EntityType::new(entity_index, module); - ExportType::new(name, r#type) + let ty = EntityType::new(entity_index, self); + ExportType::new(name, ty) }) } @@ -537,7 +561,7 @@ impl Module { pub fn get_export<'module>(&'module self, name: &'module str) -> Option { let module = self.compiled_module().module(); let entity_index = module.exports.get(name)?; - Some(EntityType::new(entity_index, module).extern_type()) + Some(EntityType::new(entity_index, self).extern_type()) } /// Returns the [`Engine`] that this [`Module`] was compiled by. diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 5317631cc4..9761b40b91 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -11,7 +11,7 @@ use std::hash::{Hash, Hasher}; use std::rc::{Rc, Weak}; use std::sync::Arc; use wasmtime_environ::wasm; -use wasmtime_jit::{CompiledModule, ModuleCode}; +use wasmtime_jit::{CompiledModule, ModuleCode, TypeTables}; use wasmtime_runtime::{ InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, @@ -137,17 +137,17 @@ impl Store { pub(crate) fn lookup_shared_signature<'a>( &'a self, - module: &'a wasmtime_environ::Module, + types: &'a TypeTables, ) -> impl Fn(wasm::SignatureIndex) -> VMSharedSignatureIndex + 'a { move |index| { self.signatures() .borrow() - .lookup(&module.signatures[index]) + .lookup(&types.wasm_signatures[index]) .expect("signature not previously registered") } } - pub(crate) fn register_module(&self, module: &CompiledModule) { + pub(crate) fn register_module(&self, module: &CompiledModule, types: &TypeTables) { // All modules register their JIT code in a store for two reasons // currently: // @@ -169,7 +169,7 @@ impl Store { // once-per-module (and once-per-signature). This allows us to create // a `Func` wrapper for any function in the module, which requires that // we know about the signature and trampoline for all instances. - self.register_signatures(module); + self.register_signatures(module, types); // And finally with a module being instantiated into this `Store` we // need to preserve its jit-code. References to this module's code and @@ -205,11 +205,10 @@ impl Store { })); } - fn register_signatures(&self, module: &CompiledModule) { + fn register_signatures(&self, module: &CompiledModule, types: &TypeTables) { let trampolines = module.trampolines(); - let module = module.module(); let mut signatures = self.signatures().borrow_mut(); - for (index, wasm) in module.signatures.iter() { + for (index, wasm) in types.wasm_signatures.iter() { signatures.register(wasm, trampolines[index]); } } diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index 874431861c..f597987e1b 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -10,7 +10,7 @@ use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; use wasmtime_runtime::{ Imports, InstanceHandle, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, - VMFunctionImport, + VMFunctionImport, VMSharedSignatureIndex, }; pub(crate) fn create_handle( @@ -19,11 +19,11 @@ pub(crate) fn create_handle( finished_functions: PrimaryMap, state: Box, func_imports: &[VMFunctionImport], + shared_signature_id: Option, ) -> Result { let mut imports = Imports::default(); imports.functions = func_imports; let module = Arc::new(module); - let module2 = module.clone(); unsafe { let handle = InstanceHandle::new( @@ -31,7 +31,7 @@ pub(crate) fn create_handle( &finished_functions, imports, store.memory_creator(), - &store.lookup_shared_signature(&module2), + &|_| shared_signature_id.unwrap(), state, store.interrupts(), store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 0050f73330..1855a178eb 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -10,7 +10,8 @@ use std::mem; use std::panic::{self, AssertUnwindSafe}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::{ir, wasm, CompiledFunction, Module}; +use wasmtime_environ::wasm::SignatureIndex; +use wasmtime_environ::{ir, wasm, CompiledFunction, Module, ModuleType}; use wasmtime_jit::trampoline::ir::{ ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, }; @@ -223,7 +224,8 @@ pub fn create_handle_with_function( // First up we manufacture a trampoline which has the ABI specified by `ft` // and calls into `stub_fn`... - let sig_id = module.signatures.push(wft.clone()); + let sig_id = SignatureIndex::from_u32(u32::max_value() - 1); + module.types.push(ModuleType::Function(sig_id)); let func_id = module.functions.push(sig_id); module .exports @@ -241,7 +243,7 @@ pub fn create_handle_with_function( &sig, mem::size_of::(), )?; - store.signatures().borrow_mut().register(wft, trampoline); + let shared_signature_id = store.signatures().borrow_mut().register(wft, trampoline); // Next up we wrap everything up into an `InstanceHandle` by publishing our // code memory (makes it executable) and ensuring all our various bits of @@ -254,6 +256,7 @@ pub fn create_handle_with_function( finished_functions, Box::new(trampoline_state), &[], + Some(shared_signature_id), ) .map(|instance| (instance, trampoline)) } @@ -270,13 +273,21 @@ pub unsafe fn create_handle_with_raw_function( let mut module = Module::new(); let mut finished_functions = PrimaryMap::new(); - let sig_id = module.signatures.push(wft.clone()); + let sig_id = SignatureIndex::from_u32(u32::max_value() - 1); + module.types.push(ModuleType::Function(sig_id)); let func_id = module.functions.push(sig_id); module .exports .insert(String::new(), wasm::EntityIndex::Function(func_id)); finished_functions.push(func); - store.signatures().borrow_mut().register(wft, trampoline); + let shared_signature_id = store.signatures().borrow_mut().register(wft, trampoline); - create_handle(module, store, finished_functions, state, &[]) + create_handle( + module, + store, + finished_functions, + state, + &[], + Some(shared_signature_id), + ) } diff --git a/crates/wasmtime/src/trampoline/global.rs b/crates/wasmtime/src/trampoline/global.rs index dbaf7b3419..00a91bdb91 100644 --- a/crates/wasmtime/src/trampoline/global.rs +++ b/crates/wasmtime/src/trampoline/global.rs @@ -3,13 +3,17 @@ use crate::trampoline::StoreInstanceHandle; use crate::{GlobalType, Mutability, Store, Val}; use anyhow::Result; use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::{wasm, Module}; +use wasmtime_environ::{ + wasm::{self, SignatureIndex}, + Module, ModuleType, +}; use wasmtime_runtime::VMFunctionImport; pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result { let mut module = Module::new(); let mut func_imports = Vec::new(); let mut externref_init = None; + let mut shared_signature_id = None; let global = wasm::Global { wasm_ty: gt.content().to_wasm_type(), @@ -35,17 +39,19 @@ pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result { // Add a function import to the stub module, and then initialize // our global with a `ref.func` to grab that imported function. - let signatures = store.signatures().borrow(); let shared_sig_index = f.sig_index(); - let (wasm, _) = signatures - .lookup_shared(shared_sig_index) - .expect("signature not registered"); - let local_sig_index = module.signatures.push(wasm.clone()); - let func_index = module.functions.push(local_sig_index); + shared_signature_id = Some(shared_sig_index); + let sig_id = SignatureIndex::from_u32(u32::max_value() - 1); + module.types.push(ModuleType::Function(sig_id)); + let func_index = module.functions.push(sig_id); module.num_imported_funcs = 1; module - .imports - .push(("".into(), None, wasm::EntityIndex::Function(func_index))); + .initializers + .push(wasmtime_environ::Initializer::Import { + module: "".into(), + field: None, + index: wasm::EntityIndex::Function(func_index), + }); let f = f.caller_checked_anyfunc(); let f = unsafe { f.as_ref() }; @@ -70,6 +76,7 @@ pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result Result ExternType { + fn from_wasmtime(module: &Module, ty: &wasmtime_environ::wasm::EntityType) -> ExternType { use wasmtime_environ::wasm::EntityType; match ty { EntityType::Function(idx) => { - let sig = module.types[*idx].unwrap_function(); - let sig = &module.signatures[sig]; + let sig = &module.types().wasm_signatures[*idx]; FuncType::from_wasm_func_type(sig).into() } EntityType::Global(ty) => GlobalType::from_wasmtime_global(ty).into(), EntityType::Memory(ty) => MemoryType::from_wasmtime_memory(ty).into(), EntityType::Table(ty) => TableType::from_wasmtime_table(ty).into(), EntityType::Module(ty) => { - let (imports, exports) = match &module.types[*ty] { - wasmtime_environ::ModuleType::Module { imports, exports } => (imports, exports), - _ => unreachable!("not possible in valid wasm modules"), - }; - ModuleType::from_wasmtime(module, imports, exports).into() + let ty = &module.types().module_signatures[*ty]; + ModuleType::from_wasmtime(module, ty).into() } EntityType::Instance(ty) => { - let exports = match &module.types[*ty] { - wasmtime_environ::ModuleType::Instance { exports } => exports, - _ => unreachable!("not possible in valid wasm modules"), - }; - InstanceType::from_wasmtime(module, exports).into() + let ty = &module.types().instance_signatures[*ty]; + InstanceType::from_wasmtime(module, ty).into() } EntityType::Event(_) => unimplemented!("wasm event support"), } @@ -499,16 +490,17 @@ impl ModuleType { } pub(crate) fn from_wasmtime( - module: &wasmtime_environ::Module, - imports: &[(String, Option, wasmtime_environ::wasm::EntityType)], - exports: &[(String, wasmtime_environ::wasm::EntityType)], + module: &Module, + ty: &wasmtime_environ::ModuleSignature, ) -> ModuleType { + let exports = &module.types().instance_signatures[ty.exports].exports; ModuleType { exports: exports .iter() .map(|(name, ty)| (name.to_string(), ExternType::from_wasmtime(module, ty))) .collect(), - imports: imports + imports: ty + .imports .iter() .map(|(m, name, ty)| { ( @@ -556,11 +548,12 @@ impl InstanceType { } pub(crate) fn from_wasmtime( - module: &wasmtime_environ::Module, - exports: &[(String, wasmtime_environ::wasm::EntityType)], + module: &Module, + ty: &wasmtime_environ::InstanceSignature, ) -> InstanceType { InstanceType { - exports: exports + exports: ty + .exports .iter() .map(|(name, ty)| (name.to_string(), ExternType::from_wasmtime(module, ty))) .collect(), @@ -577,13 +570,12 @@ pub(crate) enum EntityType<'module> { Memory(&'module wasm::Memory), Global(&'module wasm::Global), Module { - imports: &'module [(String, Option, wasmtime_environ::wasm::EntityType)], - exports: &'module [(String, wasmtime_environ::wasm::EntityType)], - module: &'module wasmtime_environ::Module, + ty: &'module wasmtime_environ::ModuleSignature, + module: &'module Module, }, Instance { - exports: &'module [(String, wasmtime_environ::wasm::EntityType)], - module: &'module wasmtime_environ::Module, + ty: &'module wasmtime_environ::InstanceSignature, + module: &'module Module, }, } @@ -591,51 +583,31 @@ impl<'module> EntityType<'module> { /// Translate from a `EntityIndex` into an `ExternType`. pub(crate) fn new( entity_index: &wasm::EntityIndex, - module: &'module wasmtime_environ::Module, + module: &'module Module, ) -> EntityType<'module> { + let env_module = module.compiled_module().module(); match entity_index { wasm::EntityIndex::Function(func_index) => { - let sig = module.wasm_func_type(*func_index); - EntityType::Function(&sig) + let sig_index = env_module.functions[*func_index]; + let sig = &module.types().wasm_signatures[sig_index]; + EntityType::Function(sig) } wasm::EntityIndex::Table(table_index) => { - EntityType::Table(&module.table_plans[*table_index].table) + EntityType::Table(&env_module.table_plans[*table_index].table) } wasm::EntityIndex::Memory(memory_index) => { - EntityType::Memory(&module.memory_plans[*memory_index].memory) + EntityType::Memory(&env_module.memory_plans[*memory_index].memory) } wasm::EntityIndex::Global(global_index) => { - EntityType::Global(&module.globals[*global_index]) + EntityType::Global(&env_module.globals[*global_index]) } wasm::EntityIndex::Module(idx) => { - let (imports, exports) = match &module.types[module.modules[*idx]] { - wasmtime_environ::ModuleType::Module { imports, exports } => (imports, exports), - _ => unreachable!("valid modules should never hit this"), - }; - EntityType::Module { - imports, - exports, - module, - } + let ty = &module.types().module_signatures[env_module.modules[*idx]]; + EntityType::Module { ty, module } } wasm::EntityIndex::Instance(idx) => { - // Get the type, either a pointer to an instance for an import - // or a module for an instantiation. - let ty = match module.instances[*idx] { - wasmtime_environ::Instance::Import(ty) => ty, - wasmtime_environ::Instance::Instantiate { module: idx, .. } => { - module.modules[idx] - } - }; - // Get the exports of whatever our type specifies, ignoring - // imports in the module case since we're instantiating the - // module. - let exports = match &module.types[ty] { - wasmtime_environ::ModuleType::Instance { exports } => exports, - wasmtime_environ::ModuleType::Module { exports, .. } => exports, - _ => unreachable!("valid modules should never hit this"), - }; - EntityType::Instance { exports, module } + let ty = &module.types().instance_signatures[env_module.instances[*idx]]; + EntityType::Instance { ty, module } } } } @@ -647,14 +619,8 @@ impl<'module> EntityType<'module> { EntityType::Table(table) => TableType::from_wasmtime_table(table).into(), EntityType::Memory(memory) => MemoryType::from_wasmtime_memory(memory).into(), EntityType::Global(global) => GlobalType::from_wasmtime_global(global).into(), - EntityType::Instance { exports, module } => { - InstanceType::from_wasmtime(module, exports).into() - } - EntityType::Module { - imports, - exports, - module, - } => ModuleType::from_wasmtime(module, imports, exports).into(), + EntityType::Instance { module, ty } => InstanceType::from_wasmtime(module, ty).into(), + EntityType::Module { module, ty } => ModuleType::from_wasmtime(module, ty).into(), } } } diff --git a/crates/wast/src/wast.rs b/crates/wast/src/wast.rs index 1009a32537..692e2f583a 100644 --- a/crates/wast/src/wast.rs +++ b/crates/wast/src/wast.rs @@ -226,20 +226,25 @@ impl WastContext { for directive in ast.directives { let sp = directive.span(); - self.run_directive(directive).with_context(|| { - let (line, col) = sp.linecol_in(wast); - format!("failed directive on {}:{}:{}", filename, line + 1, col) - })?; + self.run_directive(directive, &adjust_wast) + .with_context(|| { + let (line, col) = sp.linecol_in(wast); + format!("failed directive on {}:{}:{}", filename, line + 1, col) + })?; } Ok(()) } - fn run_directive(&mut self, directive: wast::WastDirective) -> Result<()> { + fn run_directive( + &mut self, + directive: wast::WastDirective, + adjust: impl Fn(wast::Error) -> wast::Error, + ) -> Result<()> { use wast::WastDirective::*; match directive { Module(mut module) => { - let binary = module.encode()?; + let binary = module.encode().map_err(adjust)?; self.module(module.id.map(|s| s.name()), &binary)?; } QuoteModule { span: _, source } => { @@ -249,7 +254,10 @@ impl WastContext { module.push_str(" "); } let buf = ParseBuffer::new(&module)?; - let mut wat = parser::parse::(&buf)?; + let mut wat = parser::parse::(&buf).map_err(|mut e| { + e.set_text(&module); + e + })?; let binary = wat.module.encode()?; self.module(wat.module.id.map(|s| s.name()), &binary)?; } @@ -317,7 +325,7 @@ impl WastContext { // interested in. wast::QuoteModule::Quote(_) => return Ok(()), }; - let bytes = module.encode()?; + let bytes = module.encode().map_err(adjust)?; if let Ok(_) = self.module(None, &bytes) { bail!("expected malformed module to fail to instantiate"); } @@ -327,7 +335,7 @@ impl WastContext { mut module, message, } => { - let bytes = module.encode()?; + let bytes = module.encode().map_err(adjust)?; let err = match self.module(None, &bytes) { Ok(()) => bail!("expected module to fail to link"), Err(e) => e, diff --git a/src/obj.rs b/src/obj.rs index a14f79a8f8..e957fb6afd 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -65,10 +65,10 @@ pub fn compile_to_obj( ); let environ = ModuleEnvironment::new(compiler.isa().frontend_config(), &tunables, &features); - let mut translation = environ + let (mut translation, types) = environ .translate(wasm) .context("failed to translate module")?; assert_eq!(translation.len(), 1); - let compilation = compiler.compile(&mut translation[0])?; + let compilation = compiler.compile(&mut translation[0], &types)?; Ok(compilation.obj) } diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs index 30a4b27088..f352eb7f54 100644 --- a/tests/all/module_linking.rs +++ b/tests/all/module_linking.rs @@ -52,3 +52,126 @@ fn types() -> Result<()> { Module::new(&engine, "(module (type (instance)))")?; Ok(()) } + +#[test] +fn imports_exports() -> Result<()> { + let engine = engine(); + + // empty module type + let module = Module::new(&engine, "(module (module (export \"\")))")?; + let mut e = module.exports(); + assert_eq!(e.len(), 1); + let export = e.next().unwrap(); + assert_eq!(export.name(), ""); + let module_ty = match export.ty() { + ExternType::Module(m) => m, + _ => panic!("unexpected type"), + }; + assert_eq!(module_ty.imports().len(), 0); + assert_eq!(module_ty.exports().len(), 0); + + // empty instance type + let module = Module::new( + &engine, + " + (module + (module) + (instance (export \"\") (instantiate 0))) + ", + )?; + let mut e = module.exports(); + assert_eq!(e.len(), 1); + let export = e.next().unwrap(); + assert_eq!(export.name(), ""); + let instance_ty = match export.ty() { + ExternType::Instance(i) => i, + _ => panic!("unexpected type"), + }; + assert_eq!(instance_ty.exports().len(), 0); + + // full module type + let module = Module::new( + &engine, + " + (module + (import \"\" \"a\" (module + (import \"a\" (func)) + (export \"\" (global i32)) + )) + ) + ", + )?; + let mut i = module.imports(); + assert_eq!(i.len(), 1); + let import = i.next().unwrap(); + assert_eq!(import.module(), ""); + assert_eq!(import.name(), Some("a")); + let module_ty = match import.ty() { + ExternType::Module(m) => m, + _ => panic!("unexpected type"), + }; + assert_eq!(module_ty.imports().len(), 1); + assert_eq!(module_ty.exports().len(), 1); + let import = module_ty.imports().next().unwrap(); + assert_eq!(import.module(), "a"); + assert_eq!(import.name(), None); + match import.ty() { + ExternType::Func(f) => { + assert_eq!(f.results().len(), 0); + assert_eq!(f.params().len(), 0); + } + _ => panic!("unexpected type"), + } + let export = module_ty.exports().next().unwrap(); + assert_eq!(export.name(), ""); + match export.ty() { + ExternType::Global(g) => { + assert_eq!(*g.content(), ValType::I32); + assert_eq!(g.mutability(), Mutability::Const); + } + _ => panic!("unexpected type"), + } + + // full instance type + let module = Module::new( + &engine, + " + (module + (import \"\" \"b\" (instance + (export \"m\" (memory 1)) + (export \"t\" (table 1 funcref)) + )) + ) + ", + )?; + let mut i = module.imports(); + assert_eq!(i.len(), 1); + let import = i.next().unwrap(); + assert_eq!(import.module(), ""); + assert_eq!(import.name(), Some("b")); + let instance_ty = match import.ty() { + ExternType::Instance(m) => m, + _ => panic!("unexpected type"), + }; + assert_eq!(instance_ty.exports().len(), 2); + let mem_export = instance_ty.exports().nth(0).unwrap(); + assert_eq!(mem_export.name(), "m"); + match mem_export.ty() { + ExternType::Memory(m) => { + assert_eq!(m.limits().min(), 1); + assert_eq!(m.limits().max(), None); + } + _ => panic!("unexpected type"), + } + let table_export = instance_ty.exports().nth(1).unwrap(); + assert_eq!(table_export.name(), "t"); + match table_export.ty() { + ExternType::Table(t) => { + assert_eq!(t.limits().min(), 1); + assert_eq!(t.limits().max(), None); + assert_eq!(*t.element(), ValType::FuncRef); + } + _ => panic!("unexpected type"), + } + Ok(()) +} diff --git a/tests/misc_testsuite/module-linking/alias.wast b/tests/misc_testsuite/module-linking/alias.wast new file mode 100644 index 0000000000..73324b251b --- /dev/null +++ b/tests/misc_testsuite/module-linking/alias.wast @@ -0,0 +1,114 @@ +;; functions +(module + (module $m + (func $foo (export "foo") (result i32) + i32.const 1) + ) + (instance $a (instantiate $m)) + + (func (export "get") (result i32) + call $a.$foo) +) +(assert_return (invoke "get") (i32.const 1)) + +;; globals +(module + (module $m + (global $g (export "g") (mut i32) (i32.const 2)) + ) + (instance $a (instantiate $m)) + + (func (export "get") (result i32) + global.get $a.$g) +) +(assert_return (invoke "get") (i32.const 2)) + +;; memories +(module + (module $m + (memory $m (export "m") 1) + (data (i32.const 0) "\03\00\00\00") + ) + (instance $a (instantiate $m)) + (alias (instance $a) (memory $m)) + + (func (export "get") (result i32) + i32.const 0 + i32.load) +) +(assert_return (invoke "get") (i32.const 3)) + +;; tables +(module + (module $m + (table $t (export "t") 1 funcref) + (func (result i32) + i32.const 4) + (elem (i32.const 0) 0) + ) + (instance $a (instantiate $m)) + + (func (export "get") (result i32) + i32.const 0 + call_indirect $a.$t (result i32)) +) +(assert_return (invoke "get") (i32.const 4)) + +;; TODO instances/modules -- needs import/export of modules/instances to work + +;; alias parent -- type +(module + (type $t (func)) + (module $m + (func $f (type $t)) + ) + (instance $a (instantiate $m)) +) + +;; alias parent -- module +(module + (module $a) + (module $m + (instance (instantiate $a)) + ) + (instance (instantiate $m)) +) + +;; The alias, import, type, module, and instance sections can all be interleaved +(module + (module $a) + (type $t (func)) + (module $m + ;; alias + (alias $thunk parent (type $t)) + ;; import + (import "" "" (func (type $thunk))) + ;; module (referencing parent type) + (module + (func (type $thunk)) + ) + ;; type + (type $thunk2 (func)) + ;; module (referencing previous alias) + (module $m2 + (func (export "") (type $thunk2)) + ) + ;; instance + (instance $i (instantiate $m2)) + ;; alias that instance + (alias $my_f (instance $i) (func 0)) + ;; module + (module $m3 + (import "" (func))) + ;; use our aliased function to create the module + (instance $i2 (instantiate $m3 (func $my_f))) + ;; module + (module $m4 + (import "" (func))) + ) + + ;; instantiate the above module + (module $smol (func $f (export ""))) + (instance $smol (instantiate $smol)) + (instance (instantiate $m (func $smol.$f))) +) diff --git a/tests/misc_testsuite/module-linking/instantiate.wast b/tests/misc_testsuite/module-linking/instantiate.wast index 9afe725f52..a0e24c7a7b 100644 --- a/tests/misc_testsuite/module-linking/instantiate.wast +++ b/tests/misc_testsuite/module-linking/instantiate.wast @@ -74,13 +74,48 @@ (import "" (memory 1)) (func i32.const 0 - i32.const 4 + i32.const 100 i32.store) (start 0)) (instance $a (instantiate 0 (memory $m))) ) -(assert_return (invoke $a "load") (i32.const 4)) +(assert_return (invoke $a "load") (i32.const 100)) + +;; Imported instances work +(module + (import "a" "inc" (func $set)) + + (module $m1 + (import "" (instance (export "" (func)))) + (alias (instance 0) (func 0)) + (start 0)) + + (module $m2 + (func (export "") (import ""))) + (instance $i (instantiate $m2 (func $set))) + (instance (instantiate $m1 (instance $i))) +) +(assert_return (invoke $a "get") (i32.const 4)) + +;; Imported modules work +(module + (import "a" "inc" (func $set)) + + (module $m1 + (import "" (module $m (export "" (func $f (result i32))))) + (instance $i (instantiate $m)) + (func $get (export "") (result i32) + call $i.$f)) + + (module $m2 + (func (export "") (result i32) + i32.const 5)) + (instance $i (instantiate $m1 (module $m2))) + (func (export "get") (result i32) + call $i.$get) +) +(assert_return (invoke "get") (i32.const 5)) ;; all at once (module From f59b274d22016f008ed0ccb98c24c7551e0e349b Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Thu, 29 Oct 2020 13:29:03 +0000 Subject: [PATCH 33/63] Cranelift AArch64: Further vector constant improvements Introduce support for MOVI/MVNI with 16-, 32-, and 64-bit elements, and the vector variant of FMOV. Copyright (c) 2020, Arm Limited. --- .../codegen/src/isa/aarch64/inst/emit.rs | 18 ++ .../src/isa/aarch64/inst/emit_tests.rs | 110 +++++++- .../codegen/src/isa/aarch64/inst/imms.rs | 238 +++++++++++++++++- cranelift/codegen/src/isa/aarch64/inst/mod.rs | 86 ++++++- cranelift/codegen/src/isa/aarch64/lower.rs | 2 +- .../codegen/src/isa/aarch64/lower_inst.rs | 9 +- .../filetests/isa/aarch64/floating-point.clif | 16 +- .../filetests/filetests/isa/aarch64/simd.clif | 43 ++++ 8 files changed, 498 insertions(+), 24 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 0654711353..432bbc19dd 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -1312,6 +1312,13 @@ impl MachInstEmit for Inst { | machreg_to_vec(rd.to_reg()), ); } + &Inst::FpuExtend { rd, rn, size } => { + sink.put4(enc_fpurr( + 0b000_11110_00_1_000000_10000 | (size.ftype() << 13), + rd, + rn, + )); + } &Inst::FpuRR { fpu_op, rd, rn } => { let top22 = match fpu_op { FPUOp1::Abs32 => 0b000_11110_00_1_000001_10000, @@ -1746,6 +1753,17 @@ impl MachInstEmit for Inst { | machreg_to_vec(rd.to_reg()), ); } + &Inst::VecDupFPImm { rd, imm, size } => { + let imm = imm.enc_bits(); + let op = match size.lane_size() { + ScalarSize::Size32 => 0, + ScalarSize::Size64 => 1, + _ => unimplemented!(), + }; + let q_op = op | ((size.is_128bits() as u32) << 1); + + sink.put4(enc_asimd_mod_imm(rd, q_op, 0b1111, imm)); + } &Inst::VecDupImm { rd, imm, diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index cd0fbf9020..f01fbf43f0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -2072,6 +2072,24 @@ fn test_aarch64_binemit() { "5205084E", "dup v18.2d, v10.d[0]", )); + insns.push(( + Inst::VecDupFPImm { + rd: writable_vreg(31), + imm: ASIMDFPModImm::maybe_from_u64(1_f32.to_bits() as u64, ScalarSize::Size32).unwrap(), + size: VectorSize::Size32x2, + }, + "1FF6030F", + "fmov v31.2s, #1", + )); + insns.push(( + Inst::VecDupFPImm { + rd: writable_vreg(0), + imm: ASIMDFPModImm::maybe_from_u64(2_f64.to_bits(), ScalarSize::Size64).unwrap(), + size: VectorSize::Size64x2, + }, + "00F4006F", + "fmov v0.2d, #2", + )); insns.push(( Inst::VecDupImm { rd: writable_vreg(31), @@ -2082,16 +2100,96 @@ fn test_aarch64_binemit() { "FFE7074F", "movi v31.16b, #255", )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(30), + imm: ASIMDMovModImm::maybe_from_u64(0, ScalarSize::Size16).unwrap(), + invert: false, + size: VectorSize::Size16x8, + }, + "1E84004F", + "movi v30.8h, #0", + )); insns.push(( Inst::VecDupImm { rd: writable_vreg(0), - imm: ASIMDMovModImm::zero(), + imm: ASIMDMovModImm::zero(ScalarSize::Size16), invert: true, size: VectorSize::Size16x4, }, "0084002F", "mvni v0.4h, #0", )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(0), + imm: ASIMDMovModImm::maybe_from_u64(256, ScalarSize::Size16).unwrap(), + invert: false, + size: VectorSize::Size16x8, + }, + "20A4004F", + "movi v0.8h, #1, LSL #8", + )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(8), + imm: ASIMDMovModImm::maybe_from_u64(2228223, ScalarSize::Size32).unwrap(), + invert: false, + size: VectorSize::Size32x4, + }, + "28D4014F", + "movi v8.4s, #33, MSL #16", + )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(16), + imm: ASIMDMovModImm::maybe_from_u64(35071, ScalarSize::Size32).unwrap(), + invert: true, + size: VectorSize::Size32x2, + }, + "10C5042F", + "mvni v16.2s, #136, MSL #8", + )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(1), + imm: ASIMDMovModImm::maybe_from_u64(0, ScalarSize::Size32).unwrap(), + invert: false, + size: VectorSize::Size32x2, + }, + "0104000F", + "movi v1.2s, #0", + )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(24), + imm: ASIMDMovModImm::maybe_from_u64(1107296256, ScalarSize::Size32).unwrap(), + invert: false, + size: VectorSize::Size32x4, + }, + "5864024F", + "movi v24.4s, #66, LSL #24", + )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(8), + imm: ASIMDMovModImm::zero(ScalarSize::Size64), + invert: false, + size: VectorSize::Size64x2, + }, + "08E4006F", + "movi v8.2d, #0", + )); + insns.push(( + Inst::VecDupImm { + rd: writable_vreg(7), + imm: ASIMDMovModImm::maybe_from_u64(18374687574904995840, ScalarSize::Size64).unwrap(), + invert: false, + size: VectorSize::Size64x2, + }, + "87E6046F", + "movi v7.2d, #18374687574904995840", + )); insns.push(( Inst::VecExtend { t: VecExtendOp::Sxtl8, @@ -4376,6 +4474,16 @@ fn test_aarch64_binemit() { "mov d23, v11.d[0]", )); + insns.push(( + Inst::FpuExtend { + rd: writable_vreg(31), + rn: vreg(0), + size: ScalarSize::Size32, + }, + "1F40201E", + "fmov s31, s0", + )); + insns.push(( Inst::FpuRR { fpu_op: FPUOp1::Abs32, diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rs index b6da0402bc..34c2946db0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs @@ -668,39 +668,208 @@ impl MoveWideConst { } /// Advanced SIMD modified immediate as used by MOVI/MVNI. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct ASIMDMovModImm { imm: u8, shift: u8, + is_64bit: bool, shift_ones: bool, } impl ASIMDMovModImm { + /// Construct an ASIMDMovModImm from an arbitrary 64-bit constant, if possible. + /// Note that the bits in `value` outside of the range specified by `size` are + /// ignored; for example, in the case of `ScalarSize::Size8` all bits above the + /// lowest 8 are ignored. pub fn maybe_from_u64(value: u64, size: ScalarSize) -> Option { match size { ScalarSize::Size8 => Some(ASIMDMovModImm { imm: value as u8, shift: 0, + is_64bit: false, shift_ones: false, }), + ScalarSize::Size16 => { + let value = value as u16; + + if value >> 8 == 0 { + Some(ASIMDMovModImm { + imm: value as u8, + shift: 0, + is_64bit: false, + shift_ones: false, + }) + } else if value as u8 == 0 { + Some(ASIMDMovModImm { + imm: (value >> 8) as u8, + shift: 8, + is_64bit: false, + shift_ones: false, + }) + } else { + None + } + } + ScalarSize::Size32 => { + let value = value as u32; + + // Value is of the form 0x00MMFFFF. + if value & 0xFF00FFFF == 0x0000FFFF { + let imm = (value >> 16) as u8; + + Some(ASIMDMovModImm { + imm, + shift: 16, + is_64bit: false, + shift_ones: true, + }) + // Value is of the form 0x0000MMFF. + } else if value & 0xFFFF00FF == 0x000000FF { + let imm = (value >> 8) as u8; + + Some(ASIMDMovModImm { + imm, + shift: 8, + is_64bit: false, + shift_ones: true, + }) + } else { + // Of the 4 bytes, at most one is non-zero. + for shift in (0..32).step_by(8) { + if value & (0xFF << shift) == value { + return Some(ASIMDMovModImm { + imm: (value >> shift) as u8, + shift, + is_64bit: false, + shift_ones: false, + }); + } + } + + None + } + } + ScalarSize::Size64 => { + let mut imm = 0u8; + + // Check if all bytes are either 0 or 0xFF. + for i in 0..8 { + let b = (value >> (i * 8)) as u8; + + if b == 0 || b == 0xFF { + imm |= (b & 1) << i; + } else { + return None; + } + } + + Some(ASIMDMovModImm { + imm, + shift: 0, + is_64bit: true, + shift_ones: false, + }) + } _ => None, } } /// Create a zero immediate of this format. - pub fn zero() -> Self { + pub fn zero(size: ScalarSize) -> Self { ASIMDMovModImm { imm: 0, shift: 0, + is_64bit: size == ScalarSize::Size64, shift_ones: false, } } + /// Returns the value that this immediate represents. pub fn value(&self) -> (u8, u32, bool) { (self.imm, self.shift as u32, self.shift_ones) } } +/// Advanced SIMD modified immediate as used by the vector variant of FMOV. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ASIMDFPModImm { + imm: u8, + is_64bit: bool, +} + +impl ASIMDFPModImm { + /// Construct an ASIMDFPModImm from an arbitrary 64-bit constant, if possible. + pub fn maybe_from_u64(value: u64, size: ScalarSize) -> Option { + // In all cases immediates are encoded as an 8-bit number 0b_abcdefgh; + // let `D` be the inverse of the digit `d`. + match size { + ScalarSize::Size32 => { + // In this case the representable immediates are 32-bit numbers of the form + // 0b_aBbb_bbbc_defg_h000 shifted to the left by 16. + let value = value as u32; + let b0_5 = (value >> 19) & 0b111111; + let b6 = (value >> 19) & (1 << 6); + let b7 = (value >> 24) & (1 << 7); + let imm = (b0_5 | b6 | b7) as u8; + + if value == Self::value32(imm) { + Some(ASIMDFPModImm { + imm, + is_64bit: false, + }) + } else { + None + } + } + ScalarSize::Size64 => { + // In this case the representable immediates are 64-bit numbers of the form + // 0b_aBbb_bbbb_bbcd_efgh shifted to the left by 48. + let b0_5 = (value >> 48) & 0b111111; + let b6 = (value >> 48) & (1 << 6); + let b7 = (value >> 56) & (1 << 7); + let imm = (b0_5 | b6 | b7) as u8; + + if value == Self::value64(imm) { + Some(ASIMDFPModImm { + imm, + is_64bit: true, + }) + } else { + None + } + } + _ => None, + } + } + + /// Returns bits ready for encoding. + pub fn enc_bits(&self) -> u8 { + self.imm + } + + /// Returns the 32-bit value that corresponds to an 8-bit encoding. + fn value32(imm: u8) -> u32 { + let imm = imm as u32; + let b0_5 = imm & 0b111111; + let b6 = (imm >> 6) & 1; + let b6_inv = b6 ^ 1; + let b7 = (imm >> 7) & 1; + + b0_5 << 19 | (b6 * 0b11111) << 25 | b6_inv << 30 | b7 << 31 + } + + /// Returns the 64-bit value that corresponds to an 8-bit encoding. + fn value64(imm: u8) -> u64 { + let imm = imm as u64; + let b0_5 = imm & 0b111111; + let b6 = (imm >> 6) & 1; + let b6_inv = b6 ^ 1; + let b7 = (imm >> 7) & 1; + + b0_5 << 48 | (b6 * 0b11111111) << 54 | b6_inv << 62 | b7 << 63 + } +} + impl PrettyPrint for NZCV { fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String { let fmt = |c: char, v| if v { c.to_ascii_uppercase() } else { c }; @@ -782,7 +951,20 @@ impl PrettyPrint for MoveWideConst { impl PrettyPrint for ASIMDMovModImm { fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String { - if self.shift == 0 { + if self.is_64bit { + debug_assert_eq!(self.shift, 0); + + let enc_imm = self.imm as i8; + let mut imm = 0u64; + + for i in 0..8 { + let b = (enc_imm >> i) & 1; + + imm |= (-b as u8 as u64) << (i * 8); + } + + format!("#{}", imm) + } else if self.shift == 0 { format!("#{}", self.imm) } else { let shift_type = if self.shift_ones { "MSL" } else { "LSL" }; @@ -791,6 +973,16 @@ impl PrettyPrint for ASIMDMovModImm { } } +impl PrettyPrint for ASIMDFPModImm { + fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String { + if self.is_64bit { + format!("#{}", f64::from_bits(Self::value64(self.imm))) + } else { + format!("#{}", f32::from_bits(Self::value32(self.imm))) + } + } +} + #[cfg(test)] mod test { use super::*; @@ -1022,4 +1214,44 @@ mod test { unreachable!(); } } + + #[test] + fn asimd_fp_mod_imm_test() { + assert_eq!(None, ASIMDFPModImm::maybe_from_u64(0, ScalarSize::Size32)); + assert_eq!( + None, + ASIMDFPModImm::maybe_from_u64(0.013671875_f32.to_bits() as u64, ScalarSize::Size32) + ); + assert_eq!(None, ASIMDFPModImm::maybe_from_u64(0, ScalarSize::Size64)); + assert_eq!( + None, + ASIMDFPModImm::maybe_from_u64(10000_f64.to_bits(), ScalarSize::Size64) + ); + } + + #[test] + fn asimd_mov_mod_imm_test() { + assert_eq!( + None, + ASIMDMovModImm::maybe_from_u64(513, ScalarSize::Size16) + ); + assert_eq!( + None, + ASIMDMovModImm::maybe_from_u64(4278190335, ScalarSize::Size32) + ); + assert_eq!( + None, + ASIMDMovModImm::maybe_from_u64(8388608, ScalarSize::Size64) + ); + + assert_eq!( + Some(ASIMDMovModImm { + imm: 66, + shift: 16, + is_64bit: false, + shift_ones: true, + }), + ASIMDMovModImm::maybe_from_u64(4390911, ScalarSize::Size32) + ); + } } diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 676dff88e4..a8aa47c2a7 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -755,6 +755,13 @@ pub enum Inst { size: VectorSize, }, + /// Zero-extend a SIMD & FP scalar to the full width of a vector register. + FpuExtend { + rd: Writable, + rn: Reg, + size: ScalarSize, + }, + /// 1-op FPU instruction. FpuRR { fpu_op: FPUOp1, @@ -928,6 +935,13 @@ pub enum Inst { size: VectorSize, }, + /// Duplicate FP immediate to vector. + VecDupFPImm { + rd: Writable, + imm: ASIMDFPModImm, + size: VectorSize, + }, + /// Duplicate immediate to vector. VecDupImm { rd: Writable, @@ -1295,12 +1309,15 @@ impl Inst { value: u32, mut alloc_tmp: F, ) -> SmallVec<[Inst; 4]> { + // Note that we must make sure that all bits outside the lowest 32 are set to 0 + // because this function is also used to load wider constants (that have zeros + // in their most significant bits). if value == 0 { smallvec![Inst::VecDupImm { rd, - imm: ASIMDMovModImm::zero(), + imm: ASIMDMovModImm::zero(ScalarSize::Size32), invert: false, - size: VectorSize::Size8x8 + size: VectorSize::Size32x2 }] } else { // TODO: use FMOV immediate form when `value` has sufficiently few mantissa/exponent @@ -1324,6 +1341,9 @@ impl Inst { const_data: u64, mut alloc_tmp: F, ) -> SmallVec<[Inst; 4]> { + // Note that we must make sure that all bits outside the lowest 64 are set to 0 + // because this function is also used to load wider constants (that have zeros + // in their most significant bits). if let Ok(const_data) = u32::try_from(const_data) { Inst::load_fp_constant32(rd, const_data, alloc_tmp) // TODO: use FMOV immediate form when `const_data` has sufficiently few mantissa/exponent @@ -1394,7 +1414,7 @@ impl Inst { r } - /// Create instructions that load a 128-bit vector constant consisting of elements with + /// Create instructions that load a vector constant consisting of elements with /// the same value. pub fn load_replicated_vector_pattern Writable>( rd: Writable, @@ -1403,6 +1423,15 @@ impl Inst { mut alloc_tmp: F, ) -> SmallVec<[Inst; 5]> { let lane_size = size.lane_size(); + let widen_32_bit_pattern = |pattern, lane_size| { + if lane_size == ScalarSize::Size32 { + let pattern = pattern as u32 as u64; + + ASIMDMovModImm::maybe_from_u64(pattern | (pattern << 32), ScalarSize::Size64) + } else { + None + } + }; if let Some(imm) = ASIMDMovModImm::maybe_from_u64(pattern, lane_size) { smallvec![Inst::VecDupImm { @@ -1421,6 +1450,27 @@ impl Inst { invert: true, size }] + } else if let Some(imm) = widen_32_bit_pattern(pattern, lane_size) { + let mut insts = smallvec![Inst::VecDupImm { + rd, + imm, + invert: false, + size: VectorSize::Size64x2, + }]; + + // TODO: Implement support for 64-bit scalar MOVI; we zero-extend the + // lower 64 bits instead. + if !size.is_128bits() { + insts.push(Inst::FpuExtend { + rd, + rn: rd.to_reg(), + size: ScalarSize::Size64, + }); + } + + insts + } else if let Some(imm) = ASIMDFPModImm::maybe_from_u64(pattern, lane_size) { + smallvec![Inst::VecDupFPImm { rd, imm, size }] } else { let tmp = alloc_tmp(RegClass::I64, I64); let mut insts = SmallVec::from(&Inst::load_constant(tmp, pattern)[..]); @@ -1721,6 +1771,10 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(rd); collector.add_use(rn); } + &Inst::FpuExtend { rd, rn, .. } => { + collector.add_def(rd); + collector.add_use(rn); + } &Inst::FpuRR { rd, rn, .. } => { collector.add_def(rd); collector.add_use(rn); @@ -1870,6 +1924,9 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(rd); collector.add_use(rn); } + &Inst::VecDupFPImm { rd, .. } => { + collector.add_def(rd); + } &Inst::VecDupImm { rd, .. } => { collector.add_def(rd); } @@ -2299,6 +2356,14 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { map_def(mapper, rd); map_use(mapper, rn); } + &mut Inst::FpuExtend { + ref mut rd, + ref mut rn, + .. + } => { + map_def(mapper, rd); + map_use(mapper, rn); + } &mut Inst::FpuRR { ref mut rd, ref mut rn, @@ -2582,6 +2647,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { map_def(mapper, rd); map_use(mapper, rn); } + &mut Inst::VecDupFPImm { ref mut rd, .. } => { + map_def(mapper, rd); + } &mut Inst::VecDupImm { ref mut rd, .. } => { map_def(mapper, rd); } @@ -3229,6 +3297,12 @@ impl Inst { let rn = show_vreg_element(rn, mb_rru, idx, size); format!("mov {}, {}", rd, rn) } + &Inst::FpuExtend { rd, rn, size } => { + let rd = show_vreg_scalar(rd.to_reg(), mb_rru, size); + let rn = show_vreg_scalar(rn, mb_rru, size); + + format!("fmov {}, {}", rd, rn) + } &Inst::FpuRR { fpu_op, rd, rn } => { let (op, sizesrc, sizedest) = match fpu_op { FPUOp1::Abs32 => ("fabs", ScalarSize::Size32, ScalarSize::Size32), @@ -3465,6 +3539,12 @@ impl Inst { let rn = show_vreg_element(rn, mb_rru, 0, size); format!("dup {}, {}", rd, rn) } + &Inst::VecDupFPImm { rd, imm, size } => { + let imm = imm.show_rru(mb_rru); + let rd = show_vreg_vector(rd.to_reg(), mb_rru, size); + + format!("fmov {}, {}", rd, imm) + } &Inst::VecDupImm { rd, imm, diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 086b9a3a20..75b4cbe727 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -853,7 +853,7 @@ pub(crate) fn lower_constant_f128>( // is potentially expensive. ctx.emit(Inst::VecDupImm { rd, - imm: ASIMDMovModImm::zero(), + imm: ASIMDMovModImm::zero(ScalarSize::Size8), invert: false, size: VectorSize::Size8x16, }); diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 4d3d4bfb1d..89bcd517f4 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -2075,8 +2075,6 @@ pub(crate) fn lower_insn_to_regs>( // derivation of these sequences. Alternative sequences are discussed in // https://github.com/bytecodealliance/wasmtime/issues/2296, although they are not // used here. - // Also .. FIXME: when https://github.com/bytecodealliance/wasmtime/pull/2310 is - // merged, use `lower_splat_constant` instead to generate the constants. let tmp_r0 = ctx.alloc_tmp(RegClass::I64, I64); let tmp_v0 = ctx.alloc_tmp(RegClass::V128, I8X16); let tmp_v1 = ctx.alloc_tmp(RegClass::V128, I8X16); @@ -2100,12 +2098,7 @@ pub(crate) fn lower_insn_to_regs>( size: VectorSize::Size8x16, imm: 7, }); - lower_constant_u64(ctx, tmp_r0, 0x8040201008040201u64); - ctx.emit(Inst::VecDup { - rd: tmp_v0, - rn: tmp_r0.to_reg(), - size: VectorSize::Size64x2, - }); + lower_splat_const(ctx, tmp_v0, 0x8040201008040201u64, VectorSize::Size64x2); ctx.emit(Inst::VecRRR { alu_op: VecALUOp::And, rd: tmp_v1, diff --git a/cranelift/filetests/filetests/isa/aarch64/floating-point.clif b/cranelift/filetests/filetests/isa/aarch64/floating-point.clif index 25f53ff4b1..a3fa2c48c6 100644 --- a/cranelift/filetests/filetests/isa/aarch64/floating-point.clif +++ b/cranelift/filetests/filetests/isa/aarch64/floating-point.clif @@ -715,7 +715,7 @@ block0(v0: f32): ; nextln: movz x0, #20352, LSL #16 ; nextln: fmov d1, x0 ; nextln: fmin s2, s0, s1 -; nextln: movi v1.8b, #0 +; nextln: movi v1.2s, #0 ; nextln: fmax s2, s2, s1 ; nextln: fcmp s0, s0 ; nextln: fcsel s0, s1, s2, ne @@ -738,7 +738,7 @@ block0(v0: f32): ; nextln: movz x0, #52992, LSL #16 ; nextln: fmov d2, x0 ; nextln: fmax s1, s1, s2 -; nextln: movi v2.8b, #0 +; nextln: movi v2.2s, #0 ; nextln: fcmp s0, s0 ; nextln: fcsel s0, s2, s1, ne ; nextln: fcvtzs w0, s0 @@ -757,7 +757,7 @@ block0(v0: f32): ; nextln: movz x0, #24448, LSL #16 ; nextln: fmov d1, x0 ; nextln: fmin s2, s0, s1 -; nextln: movi v1.8b, #0 +; nextln: movi v1.2s, #0 ; nextln: fmax s2, s2, s1 ; nextln: fcmp s0, s0 ; nextln: fcsel s0, s1, s2, ne @@ -780,7 +780,7 @@ block0(v0: f32): ; nextln: movz x0, #57088, LSL #16 ; nextln: fmov d2, x0 ; nextln: fmax s1, s1, s2 -; nextln: movi v2.8b, #0 +; nextln: movi v2.2s, #0 ; nextln: fcmp s0, s0 ; nextln: fcsel s0, s2, s1, ne ; nextln: fcvtzs x0, s0 @@ -798,7 +798,7 @@ block0(v0: f64): ; nextln: mov fp, sp ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 4294967295 ; nextln: fmin d2, d0, d1 -; nextln: movi v1.8b, #0 +; nextln: movi v1.2s, #0 ; nextln: fmax d2, d2, d1 ; nextln: fcmp d0, d0 ; nextln: fcsel d0, d1, d2, ne @@ -820,7 +820,7 @@ block0(v0: f64): ; nextln: movz x0, #49632, LSL #48 ; nextln: fmov d2, x0 ; nextln: fmax d1, d1, d2 -; nextln: movi v2.8b, #0 +; nextln: movi v2.2s, #0 ; nextln: fcmp d0, d0 ; nextln: fcsel d0, d2, d1, ne ; nextln: fcvtzs w0, d0 @@ -839,7 +839,7 @@ block0(v0: f64): ; nextln: movz x0, #17392, LSL #48 ; nextln: fmov d1, x0 ; nextln: fmin d2, d0, d1 -; nextln: movi v1.8b, #0 +; nextln: movi v1.2s, #0 ; nextln: fmax d2, d2, d1 ; nextln: fcmp d0, d0 ; nextln: fcsel d0, d1, d2, ne @@ -862,7 +862,7 @@ block0(v0: f64): ; nextln: movz x0, #50144, LSL #48 ; nextln: fmov d2, x0 ; nextln: fmax d1, d1, d2 -; nextln: movi v2.8b, #0 +; nextln: movi v2.2s, #0 ; nextln: fcmp d0, d0 ; nextln: fcsel d0, d2, d1, ne ; nextln: fcvtzs x0, d0 diff --git a/cranelift/filetests/filetests/isa/aarch64/simd.clif b/cranelift/filetests/filetests/isa/aarch64/simd.clif index ac6fb7d6ef..ea9ad30d53 100644 --- a/cranelift/filetests/filetests/isa/aarch64/simd.clif +++ b/cranelift/filetests/filetests/isa/aarch64/simd.clif @@ -127,3 +127,46 @@ block0(v0: i64, v1: i64): ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret + +function %f9() -> i32x2 { +block0: + v0 = iconst.i32 4278190335 + v1 = splat.i32x2 v0 + return v1 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: movi v0.2d, #18374687579166474495 +; nextln: fmov d0, d0 +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f10() -> i32x4 { +block0: + v0 = iconst.i32 4293918720 + v1 = splat.i32x4 v0 + return v1 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: mvni v0.4s, #15, MSL #16 +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f11() -> f32x4 { +block0: + v0 = f32const 0x1.5 + v1 = splat.f32x4 v0 + return v1 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: fmov v0.4s, #1.3125 +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret From f003388ec7d881b963e37517f21b5548c8357972 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 3 Dec 2020 10:15:42 -0600 Subject: [PATCH 34/63] Implement imported/exported modules/instances (#2461) * Implement imported/exported modules/instances This commit implements the final piece of the module linking proposal which is to flesh out the support for importing/exporting instances and modules. This ended up having a few changes: * Two more `PrimaryMap` instances are now stored in an `Instance`. The value for instances is `InstanceHandle` (pretty easy) and for modules it's `Box` (less easy). * The custom host state for `InstanceHandle` for `wasmtime` is now `Arc wasm_externkind_t { Extern::Global(_) => crate::WASM_EXTERN_GLOBAL, Extern::Table(_) => crate::WASM_EXTERN_TABLE, Extern::Memory(_) => crate::WASM_EXTERN_MEMORY, + + // FIXME(#2094) + Extern::Instance(_) => unimplemented!(), + Extern::Module(_) => unimplemented!(), } } diff --git a/crates/debug/Cargo.toml b/crates/debug/Cargo.toml index 31caf0777c..401c443618 100644 --- a/crates/debug/Cargo.toml +++ b/crates/debug/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] gimli = "0.23.0" -wasmparser = "0.68.0" +wasmparser = "0.69.0" object = { version = "0.22.0", default-features = false, features = ["read", "write"] } wasmtime-environ = { path = "../environ", version = "0.21.0" } target-lexicon = { version = "0.11.0", default-features = false } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 847b749c71..1b944282e1 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.68.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.68.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.68.0", features = ["enable-serde"] } -wasmparser = "0.68.0" +wasmparser = "0.69.0" indexmap = { version = "1.0.2", features = ["serde-1"] } thiserror = "1.0.4" serde = { version = "1.0.94", features = ["derive"] } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index d78efadf0f..226d96134d 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -346,6 +346,31 @@ impl Module { pub fn is_imported_global(&self, index: GlobalIndex) -> bool { index.index() < self.num_imported_globals } + + /// Returns an iterator of all the imports in this module, along with their + /// module name, field name, and type that's being imported. + pub fn imports(&self) -> impl Iterator, EntityType)> { + self.initializers.iter().filter_map(move |i| match i { + Initializer::Import { + module, + field, + index, + } => Some((module.as_str(), field.as_deref(), self.type_of(*index))), + _ => None, + }) + } + + /// Returns the type of an item based on its index + pub fn type_of(&self, index: EntityIndex) -> EntityType { + match index { + EntityIndex::Global(i) => EntityType::Global(self.globals[i]), + EntityIndex::Table(i) => EntityType::Table(self.table_plans[i].table), + EntityIndex::Memory(i) => EntityType::Memory(self.memory_plans[i].memory), + EntityIndex::Function(i) => EntityType::Function(self.functions[i]), + EntityIndex::Instance(i) => EntityType::Instance(self.instances[i]), + EntityIndex::Module(i) => EntityType::Module(self.modules[i]), + } + } } /// All types which are recorded for the entirety of a translation. @@ -376,7 +401,7 @@ pub struct ModuleSignature { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InstanceSignature { /// The name of what's being exported as well as its type signature. - pub exports: Vec<(String, EntityType)>, + pub exports: IndexMap, } mod passive_data_serde { diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index f1af5c4ea0..6d36b4e7a9 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -857,7 +857,7 @@ and for re-adding support for interface types you can see this issue: // instance. Alias::Child { instance, export } => { let ty = self.result.module.instances[instance]; - match &self.types.instance_signatures[ty].exports[export].1 { + match &self.types.instance_signatures[ty].exports[export] { EntityType::Global(g) => { self.result.module.globals.push(g.clone()); self.result.module.num_imported_globals += 1; diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 4526982823..12a9842c00 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -12,8 +12,8 @@ arbitrary = { version = "0.4.1", features = ["derive"] } env_logger = "0.8.1" log = "0.4.8" rayon = "1.2.1" -wasmparser = "0.68.0" -wasmprinter = "0.2.15" +wasmparser = "0.69.0" +wasmprinter = "0.2.16" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } wasm-smith = "0.1.12" diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 4f4ceba8e9..4a3b9101d5 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -28,7 +28,7 @@ rayon = { version = "1.0", optional = true } region = "2.1.0" thiserror = "1.0.4" target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.68.0" +wasmparser = "0.69.0" more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "1.0" diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml index 9308b8918a..ff7d095e10 100644 --- a/crates/lightbeam/Cargo.toml +++ b/crates/lightbeam/Cargo.toml @@ -24,7 +24,7 @@ more-asserts = "0.2.1" smallvec = "1.0.0" thiserror = "1.0.9" typemap = "0.3" -wasmparser = "0.68.0" +wasmparser = "0.69.0" [dev-dependencies] lazy_static = "1.2" diff --git a/crates/lightbeam/wasmtime/Cargo.toml b/crates/lightbeam/wasmtime/Cargo.toml index f636577df0..2d3163ce06 100644 --- a/crates/lightbeam/wasmtime/Cargo.toml +++ b/crates/lightbeam/wasmtime/Cargo.toml @@ -13,6 +13,6 @@ edition = "2018" [dependencies] lightbeam = { path = "..", version = "0.21.0" } -wasmparser = "0.68" +wasmparser = "0.69" cranelift-codegen = { path = "../../../cranelift/codegen", version = "0.68.0" } wasmtime-environ = { path = "../../environ", version = "0.21.0" } diff --git a/crates/runtime/src/export.rs b/crates/runtime/src/export.rs index edfa51e1f2..0161e56a68 100644 --- a/crates/runtime/src/export.rs +++ b/crates/runtime/src/export.rs @@ -1,13 +1,14 @@ use crate::vmcontext::{ VMCallerCheckedAnyfunc, VMContext, VMGlobalDefinition, VMMemoryDefinition, VMTableDefinition, }; +use crate::InstanceHandle; +use std::any::Any; use std::ptr::NonNull; use wasmtime_environ::wasm::Global; use wasmtime_environ::{MemoryPlan, TablePlan}; /// The value of an export passed from one instance to another. -#[derive(Debug, Clone)] -pub enum Export { +pub enum Export<'a> { /// A function export value. Function(ExportFunction), @@ -19,6 +20,12 @@ pub enum Export { /// A global export value. Global(ExportGlobal), + + /// An instance + Instance(&'a InstanceHandle), + + /// A module + Module(&'a dyn Any), } /// A function export value. @@ -31,8 +38,8 @@ pub struct ExportFunction { pub anyfunc: NonNull, } -impl From for Export { - fn from(func: ExportFunction) -> Export { +impl<'a> From for Export<'a> { + fn from(func: ExportFunction) -> Export<'a> { Export::Function(func) } } @@ -48,8 +55,8 @@ pub struct ExportTable { pub table: TablePlan, } -impl From for Export { - fn from(func: ExportTable) -> Export { +impl<'a> From for Export<'a> { + fn from(func: ExportTable) -> Export<'a> { Export::Table(func) } } @@ -65,8 +72,8 @@ pub struct ExportMemory { pub memory: MemoryPlan, } -impl From for Export { - fn from(func: ExportMemory) -> Export { +impl<'a> From for Export<'a> { + fn from(func: ExportMemory) -> Export<'a> { Export::Memory(func) } } @@ -82,8 +89,8 @@ pub struct ExportGlobal { pub global: Global, } -impl From for Export { - fn from(func: ExportGlobal) -> Export { +impl<'a> From for Export<'a> { + fn from(func: ExportGlobal) -> Export<'a> { Export::Global(func) } } diff --git a/crates/runtime/src/imports.rs b/crates/runtime/src/imports.rs index 2f85ba2201..1969630750 100644 --- a/crates/runtime/src/imports.rs +++ b/crates/runtime/src/imports.rs @@ -1,12 +1,21 @@ use crate::vmcontext::{VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport}; +use crate::InstanceHandle; +use std::any::Any; +use wasmtime_environ::entity::PrimaryMap; +use wasmtime_environ::wasm::{InstanceIndex, ModuleIndex}; /// Resolved import pointers. /// -/// Note that each of these fields are slices, not `PrimaryMap`. They should be +/// Note that some of these fields are slices, not `PrimaryMap`. They should be /// stored in index-order as with the module that we're providing the imports /// for, and indexing is all done the same way as the main module's index /// spaces. -#[derive(Clone, Default)] +/// +/// Also note that the way we compile modules means that for the module linking +/// proposal all `alias` directives should map to imported items. This means +/// that each of these items aren't necessarily directly imported, but may be +/// aliased. +#[derive(Default)] pub struct Imports<'a> { /// Resolved addresses for imported functions. pub functions: &'a [VMFunctionImport], @@ -19,4 +28,15 @@ pub struct Imports<'a> { /// Resolved addresses for imported globals. pub globals: &'a [VMGlobalImport], + + /// Resolved imported instances. + pub instances: PrimaryMap, + + /// Resolved imported modules. + /// + /// Note that `Box` here is chosen to allow the embedder of this crate + /// to pick an appropriate representation of what module type should be. For + /// example for the `wasmtime` crate it's `wasmtime::Module` but that's not + /// defined way down here in this low crate. + pub modules: PrimaryMap>, } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 1f39b171a0..9ff7c4ee6d 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -28,8 +28,8 @@ use thiserror::Error; use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; use wasmtime_environ::wasm::{ DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, - ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, - TableElementType, TableIndex, WasmType, + ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, InstanceIndex, MemoryIndex, + ModuleIndex, SignatureIndex, TableElementType, TableIndex, WasmType, }; use wasmtime_environ::{ir, DataInitializer, Module, ModuleType, TableElements, VMOffsets}; @@ -50,6 +50,15 @@ pub(crate) struct Instance { /// WebAssembly table data. tables: BoxedSlice, + /// Instances our module defined and their handles. + instances: PrimaryMap, + + /// Modules that are located in our index space. + /// + /// For now these are `Box` so the caller can define the type of what a + /// module looks like. + modules: PrimaryMap>, + /// Passive elements in this instantiation. As `elem.drop`s happen, these /// entries get removed. A missing entry is considered equivalent to an /// empty slice. @@ -268,7 +277,7 @@ impl Instance { } /// Lookup an export with the given export declaration. - pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export { + pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export<'_> { match export { EntityIndex::Function(index) => { let anyfunc = self.get_caller_checked_anyfunc(*index).unwrap(); @@ -317,9 +326,8 @@ impl Instance { } .into(), - // FIXME(#2094) - EntityIndex::Instance(_index) => unimplemented!(), - EntityIndex::Module(_index) => unimplemented!(), + EntityIndex::Instance(index) => Export::Instance(&self.instances[*index]), + EntityIndex::Module(index) => Export::Module(&*self.modules[*index]), } } @@ -847,6 +855,8 @@ impl InstanceHandle { passive_elements: Default::default(), passive_data, host_state, + instances: imports.instances, + modules: imports.modules, vmctx: VMContext {}, }; let layout = instance.alloc_layout(); diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 57cdec42a5..881f1f8724 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -16,7 +16,7 @@ wasmtime-jit = { path = "../jit", version = "0.21.0" } wasmtime-cache = { path = "../cache", version = "0.21.0", optional = true } wasmtime-profiling = { path = "../profiling", version = "0.21.0" } target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.68.0" +wasmparser = "0.69.0" anyhow = "1.0.19" region = "2.2.0" libc = "0.2" @@ -29,6 +29,7 @@ wat = { version = "1.0.18", optional = true } smallvec = "1.4.0" serde = { version = "1.0.94", features = ["derive"] } bincode = "1.2.1" +indexmap = "1.6" [target.'cfg(target_os = "windows")'.dependencies] winapi = "0.3.7" diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index 63f2e2678f..9a075e308e 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -3,8 +3,8 @@ use crate::trampoline::{ }; use crate::values::{from_checked_anyfunc, into_checked_anyfunc, Val}; use crate::{ - ExternRef, ExternType, Func, GlobalType, MemoryType, Mutability, Store, TableType, Trap, - ValType, + ExternRef, ExternType, Func, GlobalType, Instance, MemoryType, Module, Mutability, Store, + TableType, Trap, ValType, }; use anyhow::{anyhow, bail, Result}; use std::mem; @@ -33,6 +33,10 @@ pub enum Extern { Table(Table), /// A WebAssembly linear memory. Memory(Memory), + /// A WebAssembly instance. + Instance(Instance), + /// A WebAssembly module. + Module(Module), } impl Extern { @@ -76,6 +80,26 @@ impl Extern { } } + /// Returns the underlying `Instance`, if this external is a instance. + /// + /// Returns `None` if this is not a instance. + pub fn into_instance(self) -> Option { + match self { + Extern::Instance(instance) => Some(instance), + _ => None, + } + } + + /// Returns the underlying `Module`, if this external is a module. + /// + /// Returns `None` if this is not a module. + pub fn into_module(self) -> Option { + match self { + Extern::Module(module) => Some(module), + _ => None, + } + } + /// Returns the type associated with this `Extern`. pub fn ty(&self) -> ExternType { match self { @@ -83,6 +107,8 @@ impl Extern { Extern::Memory(ft) => ExternType::Memory(ft.ty()), Extern::Table(tt) => ExternType::Table(tt.ty()), Extern::Global(gt) => ExternType::Global(gt.ty()), + Extern::Instance(i) => ExternType::Instance(i.ty()), + Extern::Module(m) => ExternType::Module(m.ty()), } } @@ -103,6 +129,13 @@ impl Extern { wasmtime_runtime::Export::Table(t) => { Extern::Table(Table::from_wasmtime_table(t, instance)) } + wasmtime_runtime::Export::Instance(i) => { + let handle = unsafe { instance.store.existing_instance_handle(i.clone()) }; + Extern::Instance(Instance::from_wasmtime(handle)) + } + wasmtime_runtime::Export::Module(m) => { + Extern::Module(m.downcast_ref::().unwrap().clone()) + } } } @@ -112,6 +145,10 @@ impl Extern { Extern::Global(g) => &g.instance.store, Extern::Memory(m) => &m.instance.store, Extern::Table(t) => &t.instance.store, + Extern::Instance(i) => i.store(), + // Modules don't live in stores right now, so they're compatible + // with all stores. + Extern::Module(_) => return true, }; Store::same(my_store, store) } @@ -122,6 +159,8 @@ impl Extern { Extern::Table(_) => "table", Extern::Memory(_) => "memory", Extern::Global(_) => "global", + Extern::Instance(_) => "instance", + Extern::Module(_) => "module", } } } @@ -150,6 +189,18 @@ impl From for Extern { } } +impl From for Extern { + fn from(r: Instance) -> Self { + Extern::Instance(r) + } +} + +impl From for Extern { + fn from(r: Module) -> Self { + Extern::Module(r) + } +} + /// A WebAssembly `global` value which can be read and written to. /// /// A `global` in WebAssembly is sort of like a global variable within an @@ -294,11 +345,8 @@ impl Global { } } - pub(crate) fn matches_expected(&self, expected: &wasmtime_environ::wasm::Global) -> bool { - let actual = &self.wasmtime_export.global; - expected.ty == actual.ty - && expected.wasm_ty == actual.wasm_ty - && expected.mutability == actual.mutability + pub(crate) fn wasmtime_ty(&self) -> &wasmtime_environ::wasm::Global { + &self.wasmtime_export.global } pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMGlobalImport { @@ -538,19 +586,8 @@ impl Table { } } - pub(crate) fn matches_expected(&self, ty: &wasmtime_environ::TablePlan) -> bool { - let expected = &ty.table; - let actual = &self.wasmtime_export.table.table; - expected.wasm_ty == actual.wasm_ty - && expected.ty == actual.ty - && expected.minimum <= actual.minimum - && match expected.maximum { - Some(expected) => match actual.maximum { - Some(actual) => expected >= actual, - None => false, - }, - None => true, - } + pub(crate) fn wasmtime_ty(&self) -> &wasmtime_environ::wasm::Table { + &self.wasmtime_export.table.table } pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMTableImport { @@ -960,18 +997,8 @@ impl Memory { } } - pub(crate) fn matches_expected(&self, ty: &wasmtime_environ::MemoryPlan) -> bool { - let expected = &ty.memory; - let actual = &self.wasmtime_export.memory.memory; - expected.shared == actual.shared - && expected.minimum <= actual.minimum - && match expected.maximum { - Some(expected) => match actual.maximum { - Some(actual) => expected >= actual, - None => false, - }, - None => true, - } + pub(crate) fn wasmtime_ty(&self) -> &wasmtime_environ::wasm::Memory { + &self.wasmtime_export.memory.memory } pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMMemoryImport { diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index b57936d67d..630c5f736d 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -798,10 +798,6 @@ impl Func { &self.instance.store } - pub(crate) fn matches_expected(&self, expected: VMSharedSignatureIndex) -> bool { - self.sig_index() == expected - } - pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMFunctionImport { unsafe { let f = self.caller_checked_anyfunc(); @@ -1503,7 +1499,6 @@ impl Caller<'_> { return None; } let instance = InstanceHandle::from_vmctx(self.caller_vmctx); - let export = instance.lookup(name)?; // Our `Weak` pointer is used only to break a cycle where `Store` // stores instance handles which have this weak pointer as their // custom host data. This function should only be invoke-able while @@ -1511,6 +1506,7 @@ impl Caller<'_> { debug_assert!(self.store.upgrade().is_some()); let handle = Store::from_inner(self.store.upgrade()?).existing_instance_handle(instance); + let export = handle.lookup(name)?; match export { Export::Memory(m) => Some(Extern::Memory(Memory::from_wasmtime_memory(m, handle))), Export::Function(f) => Some(Extern::Func(Func::from_wasmtime_function(f, handle))), diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 066a618640..6f69b66a36 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -1,16 +1,23 @@ use crate::trampoline::StoreInstanceHandle; -use crate::{Engine, Export, Extern, Func, Global, Memory, Module, Store, Table, Trap}; +use crate::types::matching; +use crate::{ + Engine, Export, Extern, ExternType, Func, Global, InstanceType, Memory, Module, Store, Table, + Trap, +}; use anyhow::{bail, Context, Error, Result}; use std::mem; +use std::sync::Arc; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::{ - EntityIndex, FuncIndex, GlobalIndex, InstanceIndex, MemoryIndex, ModuleIndex, TableIndex, + EntityIndex, EntityType, FuncIndex, GlobalIndex, InstanceIndex, MemoryIndex, ModuleIndex, + TableIndex, }; use wasmtime_environ::Initializer; -use wasmtime_jit::{CompiledModule, TypeTables}; +use wasmtime_jit::TypeTables; use wasmtime_runtime::{ - Imports, InstantiationError, StackMapRegistry, VMContext, VMExternRefActivationsTable, - VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, + Imports, InstanceHandle, InstantiationError, StackMapRegistry, VMContext, + VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, + VMTableImport, }; /// Performs all low-level steps necessary for instantiation. @@ -35,17 +42,16 @@ use wasmtime_runtime::{ /// into the provided builder. The expected entity that it's defining is also /// passed in for the top-level case where type-checking is performed. This is /// fallible because type checks may fail. -fn instantiate<'a>( - store: &'a Store, - compiled_module: &'a CompiledModule, - all_modules: &'a [CompiledModule], - types: &'a TypeTables, - parent_modules: &PrimaryMap, - define_import: &mut dyn FnMut(&EntityIndex, &mut ImportsBuilder<'a>) -> Result<()>, +fn instantiate( + store: &Store, + module: &Module, + parent_modules: &PrimaryMap, + define_import: &mut dyn FnMut(&EntityIndex, &mut ImportsBuilder<'_>) -> Result<()>, ) -> Result { + let compiled_module = module.compiled_module(); let env_module = compiled_module.module(); - let mut imports = ImportsBuilder::new(env_module, types, store); + let mut imports = ImportsBuilder::new(store, module); for initializer in env_module.initializers.iter() { match initializer { // Definition of an import depends on how our parent is providing @@ -67,24 +73,28 @@ fn instantiate<'a>( // This one's pretty easy, we're just picking up our parent's module // and putting it into our own index space. Initializer::AliasParentModule(idx) => { - imports.modules.push(parent_modules[*idx]); + imports.modules.push(parent_modules[*idx].clone()); } // Turns out defining any kind of module is pretty easy, we're just // slinging around pointers. Initializer::DefineModule(idx) => { - imports.modules.push(&all_modules[*idx]); + imports.modules.push(module.submodule(*idx)); } // Here we lookup our instance handle, ask it for the nth export, // and then push that item into our own index space. We eschew // type-checking since only valid modules reach this point. + // + // Note that the unsafety here is because we're asserting that the + // handle comes from our same store, but this should be true because + // we acquired the handle from an instance in the store. Initializer::AliasInstanceExport { instance, export } => { let handle = &imports.instances[*instance]; let export_index = &handle.module().exports[*export]; let item = Extern::from_wasmtime_export( handle.lookup_by_declaration(export_index), - handle.clone(), + unsafe { store.existing_instance_handle(handle.clone()) }, ); imports.push_extern(&item); } @@ -100,14 +110,16 @@ fn instantiate<'a>( // to be a DAG. Additionally the recursion should also be bounded // due to validation. We may one day need to make this an iterative // loop, however. + // + // Also note that there's some unsafety here around cloning + // `InstanceHandle` because the handle may not live long enough, but + // we're doing all of this in the context of our `Store` argument + // above so we should be safe here. Initializer::Instantiate { module, args } => { - let module_to_instantiate = imports.modules[*module]; let mut args = args.iter(); let handle = instantiate( store, - module_to_instantiate, - all_modules, - types, + &imports.modules[*module], &imports.modules, &mut |_, builder| { match *args.next().unwrap() { @@ -124,16 +136,18 @@ fn instantiate<'a>( builder.memories.push(imports.memories[i]); } EntityIndex::Module(i) => { - builder.modules.push(imports.modules[i]); + builder.modules.push(imports.modules[i].clone()); } EntityIndex::Instance(i) => { - builder.instances.push(imports.instances[i].clone()); + builder + .instances + .push(unsafe { imports.instances[i].clone() }); } } Ok(()) }, )?; - imports.instances.push(handle); + imports.instances.push(unsafe { (*handle).clone() }); } } } @@ -141,21 +155,21 @@ fn instantiate<'a>( // With the above initialization done we've now acquired the final set of // imports in all the right index spaces and everything. Time to carry on // with the creation of our own instance. - let imports = imports.imports(); + let imports = imports.build(); // Register the module just before instantiation to ensure we have a // trampoline registered for every signature and to preserve the module's // compiled JIT code within the `Store`. - store.register_module(compiled_module, types); + store.register_module(module); let config = store.engine().config(); let instance = unsafe { let instance = compiled_module.instantiate( imports, - &store.lookup_shared_signature(types), + &store.lookup_shared_signature(module.types()), config.memory_creator.as_ref().map(|a| a as _), store.interrupts(), - Box::new(()), + Box::new(module.types().clone()), store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, store.stack_map_registry() as *const StackMapRegistry as *mut _, )?; @@ -230,7 +244,6 @@ fn instantiate<'a>( #[derive(Clone)] pub struct Instance { pub(crate) handle: StoreInstanceHandle, - module: Module, } impl Instance { @@ -294,16 +307,7 @@ impl Instance { // Perform some pre-flight checks before we get into the meat of // instantiation. - let expected = module - .compiled_module() - .module() - .initializers - .iter() - .filter(|e| match e { - Initializer::Import { .. } => true, - _ => false, - }) - .count(); + let expected = module.compiled_module().module().imports().count(); if expected != imports.len() { bail!("expected {} imports, found {}", expected, imports.len()); } @@ -314,22 +318,34 @@ impl Instance { } let mut imports = imports.iter(); - let handle = instantiate( - store, - module.compiled_module(), - module.all_compiled_modules(), - module.types(), - &PrimaryMap::new(), - &mut |idx, builder| { - let import = imports.next().expect("already checked the length"); - builder.define_extern(idx, import) - }, - )?; + let handle = instantiate(store, module, &PrimaryMap::new(), &mut |idx, builder| { + let import = imports.next().expect("already checked the length"); + builder.define_extern(idx, import) + })?; - Ok(Instance { - handle, - module: module.clone(), - }) + Ok(Instance { handle }) + } + + pub(crate) fn from_wasmtime(handle: StoreInstanceHandle) -> Instance { + Instance { handle } + } + + /// Returns the type signature of this instance. + pub fn ty(&self) -> InstanceType { + let mut ty = InstanceType::new(); + let module = self.handle.module(); + let types = self + .handle + .host_state() + .downcast_ref::>() + .unwrap(); + for (name, index) in module.exports.iter() { + ty.add_named_export( + name, + ExternType::from_wasmtime(types, &module.type_of(*index)), + ); + } + ty } /// Returns the associated [`Store`] that this `Instance` is compiled into. @@ -400,24 +416,20 @@ struct ImportsBuilder<'a> { tables: PrimaryMap, memories: PrimaryMap, globals: PrimaryMap, - instances: PrimaryMap, - modules: PrimaryMap, + instances: PrimaryMap, + modules: PrimaryMap, module: &'a wasmtime_environ::Module, - store: &'a Store, - types: &'a TypeTables, + matcher: matching::MatchCx<'a>, } impl<'a> ImportsBuilder<'a> { - fn new( - module: &'a wasmtime_environ::Module, - types: &'a TypeTables, - store: &'a Store, - ) -> ImportsBuilder<'a> { + fn new(store: &'a Store, module: &'a Module) -> ImportsBuilder<'a> { + let types = module.types(); + let module = module.compiled_module().module(); ImportsBuilder { module, - store, - types, + matcher: matching::MatchCx { store, types }, functions: PrimaryMap::with_capacity(module.num_imported_funcs), tables: PrimaryMap::with_capacity(module.num_imported_tables), memories: PrimaryMap::with_capacity(module.num_imported_memories), @@ -428,59 +440,38 @@ impl<'a> ImportsBuilder<'a> { } fn define_extern(&mut self, expected: &EntityIndex, actual: &Extern) -> Result<()> { - match *expected { - EntityIndex::Table(i) => { - self.tables.push(match actual { - Extern::Table(e) if e.matches_expected(&self.module.table_plans[i]) => { - e.vmimport() - } - Extern::Table(_) => bail!("table types incompatible"), - _ => bail!("expected table, but found {}", actual.desc()), - }); - } - EntityIndex::Memory(i) => { - self.memories.push(match actual { - Extern::Memory(e) if e.matches_expected(&self.module.memory_plans[i]) => { - e.vmimport() - } - Extern::Memory(_) => bail!("memory types incompatible"), - _ => bail!("expected memory, but found {}", actual.desc()), - }); - } - EntityIndex::Global(i) => { - self.globals.push(match actual { - Extern::Global(e) if e.matches_expected(&self.module.globals[i]) => { - e.vmimport() - } - Extern::Global(_) => bail!("global types incompatible"), - _ => bail!("expected global, but found {}", actual.desc()), - }); - } - EntityIndex::Function(i) => { - let func = match actual { - Extern::Func(e) => e, - _ => bail!("expected function, but found {}", actual.desc()), - }; - // Look up the `i`th function's type from the module in our - // signature registry. If it's not present then we have no - // functions registered with that type, so `func` is guaranteed - // to not match. - let ty = self - .store - .signatures() - .borrow() - .lookup(&self.types.wasm_signatures[self.module.functions[i]]) - .ok_or_else(|| anyhow::format_err!("function types incompatible"))?; - if !func.matches_expected(ty) { - bail!("function types incompatible"); - } - self.functions.push(func.vmimport()); - } - - // FIXME(#2094) - EntityIndex::Module(_i) => unimplemented!(), - EntityIndex::Instance(_i) => unimplemented!(), + let expected_ty = self.module.type_of(*expected); + let compatible = match &expected_ty { + EntityType::Table(i) => match actual { + Extern::Table(e) => self.matcher.table(i, e), + _ => bail!("expected table, but found {}", actual.desc()), + }, + EntityType::Memory(i) => match actual { + Extern::Memory(e) => self.matcher.memory(i, e), + _ => bail!("expected memory, but found {}", actual.desc()), + }, + EntityType::Global(i) => match actual { + Extern::Global(e) => self.matcher.global(i, e), + _ => bail!("expected global, but found {}", actual.desc()), + }, + EntityType::Function(i) => match actual { + Extern::Func(e) => self.matcher.func(*i, e), + _ => bail!("expected func, but found {}", actual.desc()), + }, + EntityType::Instance(i) => match actual { + Extern::Instance(e) => self.matcher.instance(*i, e), + _ => bail!("expected instance, but found {}", actual.desc()), + }, + EntityType::Module(i) => match actual { + Extern::Module(e) => self.matcher.module(*i, e), + _ => bail!("expected module, but found {}", actual.desc()), + }, + EntityType::Event(_) => unimplemented!(), + }; + if !compatible { + bail!("{} types incompatible", actual.desc()); } + self.push_extern(actual); Ok(()) } @@ -498,15 +489,27 @@ impl<'a> ImportsBuilder<'a> { Extern::Memory(i) => { self.memories.push(i.vmimport()); } + Extern::Instance(i) => { + debug_assert!(Store::same(i.store(), self.matcher.store)); + self.instances.push(unsafe { (*i.handle).clone() }); + } + Extern::Module(m) => { + self.modules.push(m.clone()); + } } } - fn imports(&self) -> Imports<'_> { + fn build(&mut self) -> Imports<'_> { Imports { tables: self.tables.values().as_slice(), globals: self.globals.values().as_slice(), memories: self.memories.values().as_slice(), functions: self.functions.values().as_slice(), + instances: mem::take(&mut self.instances), + modules: mem::take(&mut self.modules) + .into_iter() + .map(|(_, m)| Box::new(m) as Box<_>) + .collect(), } } } diff --git a/crates/wasmtime/src/linker.rs b/crates/wasmtime/src/linker.rs index e040527898..763d0fa3cb 100644 --- a/crates/wasmtime/src/linker.rs +++ b/crates/wasmtime/src/linker.rs @@ -58,6 +58,8 @@ enum ImportKind { Global(GlobalType), Memory, Table, + Module, + Instance, } impl Linker { @@ -516,10 +518,8 @@ impl Linker { ExternType::Global(f) => ImportKind::Global(f), ExternType::Memory(_) => ImportKind::Memory, ExternType::Table(_) => ImportKind::Table, - - // FIXME(#2094) - ExternType::Module(_) => unimplemented!(), - ExternType::Instance(_) => unimplemented!(), + ExternType::Module(_) => ImportKind::Module, + ExternType::Instance(_) => ImportKind::Instance, } } diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 37f068af6c..d30309c5fc 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -1,5 +1,5 @@ -use crate::types::{EntityType, ExportType, ExternType, ImportType}; -use crate::Engine; +use crate::types::{ExportType, ExternType, ImportType}; +use crate::{Engine, ModuleType}; use anyhow::{bail, Context, Result}; use bincode::Options; use std::hash::Hash; @@ -86,7 +86,7 @@ pub struct Module { } pub(crate) struct ModuleData { - pub(crate) types: TypeTables, + pub(crate) types: Arc, pub(crate) modules: Vec, } @@ -258,6 +258,7 @@ impl Module { &*engine.config().profiler, )?; + let types = Arc::new(types); Ok(Module { engine: engine.clone(), index: 0, @@ -291,6 +292,23 @@ impl Module { Ok(()) } + /// Returns the type signature of this module. + pub fn ty(&self) -> ModuleType { + let mut sig = ModuleType::new(); + let env_module = self.compiled_module().module(); + let types = self.types(); + for (module, field, ty) in env_module.imports() { + sig.add_named_import(module, field, ExternType::from_wasmtime(types, &ty)); + } + for (name, index) in env_module.exports.iter() { + sig.add_named_export( + name, + ExternType::from_wasmtime(types, &env_module.type_of(*index)), + ); + } + sig + } + /// Serialize compilation artifacts to the buffer. See also `deseriaize`. pub fn serialize(&self) -> Result> { let artifacts = ( @@ -300,7 +318,7 @@ impl Module { .iter() .map(|i| i.compilation_artifacts()) .collect::>(), - &self.data.types, + &*self.data.types, self.index, ); @@ -333,6 +351,7 @@ impl Module { &*engine.config().profiler, )?; + let types = Arc::new(types); Ok(Module { engine: engine.clone(), index, @@ -344,11 +363,16 @@ impl Module { &self.data.modules[self.index] } - pub(crate) fn all_compiled_modules(&self) -> &[CompiledModule] { - &self.data.modules + pub(crate) fn submodule(&self, index: usize) -> Module { + assert!(index < self.data.modules.len()); + Module { + engine: self.engine.clone(), + data: self.data.clone(), + index, + } } - pub(crate) fn types(&self) -> &TypeTables { + pub(crate) fn types(&self) -> &Arc { &self.data.types } @@ -433,20 +457,10 @@ impl Module { &'module self, ) -> impl ExactSizeIterator> + 'module { let module = self.compiled_module().module(); + let types = self.types(); module - .initializers - .iter() - .filter_map(move |initializer| match initializer { - wasmtime_environ::Initializer::Import { - module, - field, - index, - } => { - let ty = EntityType::new(index, self); - Some(ImportType::new(module, field.as_deref(), ty)) - } - _ => None, - }) + .imports() + .map(move |(module, field, ty)| ImportType::new(module, field, ty, types)) .collect::>() .into_iter() } @@ -509,9 +523,9 @@ impl Module { &'module self, ) -> impl ExactSizeIterator> + 'module { let module = self.compiled_module().module(); + let types = self.types(); module.exports.iter().map(move |(name, entity_index)| { - let ty = EntityType::new(entity_index, self); - ExportType::new(name, ty) + ExportType::new(name, module.type_of(*entity_index), types) }) } @@ -561,7 +575,10 @@ impl Module { pub fn get_export<'module>(&'module self, name: &'module str) -> Option { let module = self.compiled_module().module(); let entity_index = module.exports.get(name)?; - Some(EntityType::new(entity_index, self).extern_type()) + Some(ExternType::from_wasmtime( + self.types(), + &module.type_of(*entity_index), + )) } /// Returns the [`Engine`] that this [`Module`] was compiled by. diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 9761b40b91..6b314f6ac1 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -1,7 +1,7 @@ use crate::frame_info::StoreFrameInfo; use crate::sig_registry::SignatureRegistry; use crate::trampoline::StoreInstanceHandle; -use crate::Engine; +use crate::{Engine, Module}; use anyhow::{bail, Result}; use std::any::Any; use std::cell::RefCell; @@ -147,7 +147,7 @@ impl Store { } } - pub(crate) fn register_module(&self, module: &CompiledModule, types: &TypeTables) { + pub(crate) fn register_module(&self, module: &Module) { // All modules register their JIT code in a store for two reasons // currently: // @@ -158,18 +158,18 @@ impl Store { // * Second when generating a backtrace we'll use this mapping to // only generate wasm frames for instruction pointers that fall // within jit code. - self.register_jit_code(module); + self.register_jit_code(module.compiled_module()); // We need to know about all the stack maps of all instantiated modules // so when performing a GC we know about all wasm frames that we find // on the stack. - self.register_stack_maps(module); + self.register_stack_maps(module.compiled_module()); // Signatures are loaded into our `SignatureRegistry` here // once-per-module (and once-per-signature). This allows us to create // a `Func` wrapper for any function in the module, which requires that // we know about the signature and trampoline for all instances. - self.register_signatures(module, types); + self.register_signatures(module); // And finally with a module being instantiated into this `Store` we // need to preserve its jit-code. References to this module's code and @@ -178,7 +178,7 @@ impl Store { self.inner .modules .borrow_mut() - .insert(ArcModuleCode(module.code().clone())); + .insert(ArcModuleCode(module.compiled_module().code().clone())); } fn register_jit_code(&self, module: &CompiledModule) { @@ -205,10 +205,10 @@ impl Store { })); } - fn register_signatures(&self, module: &CompiledModule, types: &TypeTables) { - let trampolines = module.trampolines(); + fn register_signatures(&self, module: &Module) { + let trampolines = module.compiled_module().trampolines(); let mut signatures = self.signatures().borrow_mut(); - for (index, wasm) in types.wasm_signatures.iter() { + for (index, wasm) in module.types().wasm_signatures.iter() { signatures.register(wasm, trampolines[index]); } } diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index 9e992aa8e8..3a7076a764 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -1,7 +1,9 @@ -use crate::Module; use std::fmt; -use wasmtime_environ::wasm::WasmFuncType; +use wasmtime_environ::wasm::{EntityType, WasmFuncType}; use wasmtime_environ::{ir, wasm}; +use wasmtime_jit::TypeTables; + +pub(crate) mod matching; // Type Representations @@ -196,23 +198,25 @@ impl ExternType { (Instance(InstanceType) instance unwrap_instance) } - fn from_wasmtime(module: &Module, ty: &wasmtime_environ::wasm::EntityType) -> ExternType { - use wasmtime_environ::wasm::EntityType; + pub(crate) fn from_wasmtime( + types: &TypeTables, + ty: &wasmtime_environ::wasm::EntityType, + ) -> ExternType { match ty { EntityType::Function(idx) => { - let sig = &module.types().wasm_signatures[*idx]; + let sig = &types.wasm_signatures[*idx]; FuncType::from_wasm_func_type(sig).into() } EntityType::Global(ty) => GlobalType::from_wasmtime_global(ty).into(), EntityType::Memory(ty) => MemoryType::from_wasmtime_memory(ty).into(), EntityType::Table(ty) => TableType::from_wasmtime_table(ty).into(), EntityType::Module(ty) => { - let ty = &module.types().module_signatures[*ty]; - ModuleType::from_wasmtime(module, ty).into() + let ty = &types.module_signatures[*ty]; + ModuleType::from_wasmtime(types, ty).into() } EntityType::Instance(ty) => { - let ty = &module.types().instance_signatures[*ty]; - InstanceType::from_wasmtime(module, ty).into() + let ty = &types.instance_signatures[*ty]; + InstanceType::from_wasmtime(types, ty).into() } EntityType::Event(_) => unimplemented!("wasm event support"), } @@ -490,14 +494,14 @@ impl ModuleType { } pub(crate) fn from_wasmtime( - module: &Module, + types: &TypeTables, ty: &wasmtime_environ::ModuleSignature, ) -> ModuleType { - let exports = &module.types().instance_signatures[ty.exports].exports; + let exports = &types.instance_signatures[ty.exports].exports; ModuleType { exports: exports .iter() - .map(|(name, ty)| (name.to_string(), ExternType::from_wasmtime(module, ty))) + .map(|(name, ty)| (name.to_string(), ExternType::from_wasmtime(types, ty))) .collect(), imports: ty .imports @@ -506,7 +510,7 @@ impl ModuleType { ( m.to_string(), name.as_ref().map(|n| n.to_string()), - ExternType::from_wasmtime(module, ty), + ExternType::from_wasmtime(types, ty), ) }) .collect(), @@ -548,83 +552,19 @@ impl InstanceType { } pub(crate) fn from_wasmtime( - module: &Module, + types: &TypeTables, ty: &wasmtime_environ::InstanceSignature, ) -> InstanceType { InstanceType { exports: ty .exports .iter() - .map(|(name, ty)| (name.to_string(), ExternType::from_wasmtime(module, ty))) + .map(|(name, ty)| (name.to_string(), ExternType::from_wasmtime(types, ty))) .collect(), } } } -// Entity Types - -#[derive(Clone)] -pub(crate) enum EntityType<'module> { - Function(&'module wasm::WasmFuncType), - Table(&'module wasm::Table), - Memory(&'module wasm::Memory), - Global(&'module wasm::Global), - Module { - ty: &'module wasmtime_environ::ModuleSignature, - module: &'module Module, - }, - Instance { - ty: &'module wasmtime_environ::InstanceSignature, - module: &'module Module, - }, -} - -impl<'module> EntityType<'module> { - /// Translate from a `EntityIndex` into an `ExternType`. - pub(crate) fn new( - entity_index: &wasm::EntityIndex, - module: &'module Module, - ) -> EntityType<'module> { - let env_module = module.compiled_module().module(); - match entity_index { - wasm::EntityIndex::Function(func_index) => { - let sig_index = env_module.functions[*func_index]; - let sig = &module.types().wasm_signatures[sig_index]; - EntityType::Function(sig) - } - wasm::EntityIndex::Table(table_index) => { - EntityType::Table(&env_module.table_plans[*table_index].table) - } - wasm::EntityIndex::Memory(memory_index) => { - EntityType::Memory(&env_module.memory_plans[*memory_index].memory) - } - wasm::EntityIndex::Global(global_index) => { - EntityType::Global(&env_module.globals[*global_index]) - } - wasm::EntityIndex::Module(idx) => { - let ty = &module.types().module_signatures[env_module.modules[*idx]]; - EntityType::Module { ty, module } - } - wasm::EntityIndex::Instance(idx) => { - let ty = &module.types().instance_signatures[env_module.instances[*idx]]; - EntityType::Instance { ty, module } - } - } - } - - /// Convert this `EntityType` to an `ExternType`. - pub(crate) fn extern_type(&self) -> ExternType { - match self { - EntityType::Function(sig) => FuncType::from_wasm_func_type(sig).into(), - EntityType::Table(table) => TableType::from_wasmtime_table(table).into(), - EntityType::Memory(memory) => MemoryType::from_wasmtime_memory(memory).into(), - EntityType::Global(global) => GlobalType::from_wasmtime_global(global).into(), - EntityType::Instance { module, ty } => InstanceType::from_wasmtime(module, ty).into(), - EntityType::Module { module, ty } => ModuleType::from_wasmtime(module, ty).into(), - } - } -} - // Import Types /// A descriptor for an imported value into a wasm module. @@ -647,7 +587,7 @@ pub struct ImportType<'module> { #[derive(Clone)] enum EntityOrExtern<'a> { - Entity(EntityType<'a>), + Entity(EntityType, &'a TypeTables), Extern(&'a ExternType), } @@ -657,12 +597,13 @@ impl<'module> ImportType<'module> { pub(crate) fn new( module: &'module str, name: Option<&'module str>, - ty: EntityType<'module>, + ty: EntityType, + types: &'module TypeTables, ) -> ImportType<'module> { ImportType { module, name, - ty: EntityOrExtern::Entity(ty), + ty: EntityOrExtern::Entity(ty, types), } } @@ -683,7 +624,7 @@ impl<'module> ImportType<'module> { /// Returns the expected type of this import. pub fn ty(&self) -> ExternType { match &self.ty { - EntityOrExtern::Entity(e) => e.extern_type(), + EntityOrExtern::Entity(e, types) => ExternType::from_wasmtime(types, e), EntityOrExtern::Extern(e) => (*e).clone(), } } @@ -719,10 +660,14 @@ pub struct ExportType<'module> { impl<'module> ExportType<'module> { /// Creates a new export which is exported with the given `name` and has the /// given `ty`. - pub(crate) fn new(name: &'module str, ty: EntityType<'module>) -> ExportType<'module> { + pub(crate) fn new( + name: &'module str, + ty: EntityType, + types: &'module TypeTables, + ) -> ExportType<'module> { ExportType { name, - ty: EntityOrExtern::Entity(ty), + ty: EntityOrExtern::Entity(ty, types), } } @@ -734,7 +679,7 @@ impl<'module> ExportType<'module> { /// Returns the type of this export. pub fn ty(&self) -> ExternType { match &self.ty { - EntityOrExtern::Entity(e) => e.extern_type(), + EntityOrExtern::Entity(e, types) => ExternType::from_wasmtime(types, e), EntityOrExtern::Extern(e) => (*e).clone(), } } diff --git a/crates/wasmtime/src/types/matching.rs b/crates/wasmtime/src/types/matching.rs new file mode 100644 index 0000000000..018ab35fd9 --- /dev/null +++ b/crates/wasmtime/src/types/matching.rs @@ -0,0 +1,195 @@ +use crate::Store; +use std::sync::Arc; +use wasmtime_environ::wasm::{ + EntityType, Global, InstanceTypeIndex, Memory, ModuleTypeIndex, SignatureIndex, Table, +}; +use wasmtime_jit::TypeTables; + +pub struct MatchCx<'a> { + pub types: &'a TypeTables, + pub store: &'a Store, +} + +impl MatchCx<'_> { + pub fn global(&self, expected: &Global, actual: &crate::Global) -> bool { + self.global_ty(expected, actual.wasmtime_ty()) + } + + fn global_ty(&self, expected: &Global, actual: &Global) -> bool { + expected.ty == actual.ty + && expected.wasm_ty == actual.wasm_ty + && expected.mutability == actual.mutability + } + + pub fn table(&self, expected: &Table, actual: &crate::Table) -> bool { + self.table_ty(expected, actual.wasmtime_ty()) + } + + fn table_ty(&self, expected: &Table, actual: &Table) -> bool { + expected.wasm_ty == actual.wasm_ty + && expected.ty == actual.ty + && expected.minimum <= actual.minimum + && match expected.maximum { + Some(expected) => match actual.maximum { + Some(actual) => expected >= actual, + None => false, + }, + None => true, + } + } + + pub fn memory(&self, expected: &Memory, actual: &crate::Memory) -> bool { + self.memory_ty(expected, actual.wasmtime_ty()) + } + + fn memory_ty(&self, expected: &Memory, actual: &Memory) -> bool { + expected.shared == actual.shared + && expected.minimum <= actual.minimum + && match expected.maximum { + Some(expected) => match actual.maximum { + Some(actual) => expected >= actual, + None => false, + }, + None => true, + } + } + + pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> bool { + match self + .store + .signatures() + .borrow() + .lookup(&self.types.wasm_signatures[expected]) + { + Some(idx) => actual.sig_index() == idx, + // If our expected signature isn't registered, then there's no way + // that `actual` can match it. + None => false, + } + } + + pub fn instance(&self, expected: InstanceTypeIndex, actual: &crate::Instance) -> bool { + let module = actual.handle.module(); + self.exports_match( + expected, + actual + .handle + .host_state() + .downcast_ref::>() + .unwrap(), + |name| module.exports.get(name).map(|idx| module.type_of(*idx)), + ) + } + + /// Validates that the type signature of `actual` matches the `expected` + /// module type signature. + pub fn module(&self, expected: ModuleTypeIndex, actual: &crate::Module) -> bool { + let expected_sig = &self.types.module_signatures[expected]; + let module = actual.compiled_module().module(); + self.imports_match(expected, actual.types(), module.imports()) + && self.exports_match(expected_sig.exports, actual.types(), |name| { + module.exports.get(name).map(|idx| module.type_of(*idx)) + }) + } + + /// Validates that the `actual_imports` list of module imports matches the + /// `expected` module type signature. + /// + /// Types specified in `actual_imports` are relative to `actual_types`. + fn imports_match<'a>( + &self, + expected: ModuleTypeIndex, + actual_types: &TypeTables, + mut actual_imports: impl Iterator, EntityType)>, + ) -> bool { + let expected_sig = &self.types.module_signatures[expected]; + for (_, _, expected) in expected_sig.imports.iter() { + let (_, _, ty) = match actual_imports.next() { + Some(e) => e, + None => return false, + }; + if !self.extern_ty_matches(expected, &ty, actual_types) { + return false; + } + } + actual_imports.next().is_none() + } + + /// Validates that all exports in `expected` are defined by `lookup` within + /// `actual_types`. + fn exports_match( + &self, + expected: InstanceTypeIndex, + actual_types: &TypeTables, + lookup: impl Fn(&str) -> Option, + ) -> bool { + // The `expected` type must be a subset of `actual`, meaning that all + // names in `expected` must be present in `actual`. Note that we do + // name-based lookup here instead of index-based lookup. + self.types.instance_signatures[expected].exports.iter().all( + |(name, expected)| match lookup(name) { + Some(ty) => self.extern_ty_matches(expected, &ty, actual_types), + None => false, + }, + ) + } + + /// Validates that the `expected` entity matches the `actual_ty` defined + /// within `actual_types`. + fn extern_ty_matches( + &self, + expected: &EntityType, + actual_ty: &EntityType, + actual_types: &TypeTables, + ) -> bool { + match expected { + EntityType::Global(expected) => match actual_ty { + EntityType::Global(actual) => self.global_ty(expected, actual), + _ => false, + }, + EntityType::Table(expected) => match actual_ty { + EntityType::Table(actual) => self.table_ty(expected, actual), + _ => false, + }, + EntityType::Memory(expected) => match actual_ty { + EntityType::Memory(actual) => self.memory_ty(expected, actual), + _ => false, + }, + EntityType::Function(expected) => match *actual_ty { + EntityType::Function(actual) => { + self.types.wasm_signatures[*expected] == actual_types.wasm_signatures[actual] + } + _ => false, + }, + EntityType::Instance(expected) => match actual_ty { + EntityType::Instance(actual) => { + let sig = &actual_types.instance_signatures[*actual]; + self.exports_match(*expected, actual_types, |name| { + sig.exports.get(name).cloned() + }) + } + _ => false, + }, + EntityType::Module(expected) => match actual_ty { + EntityType::Module(actual) => { + let expected_module_sig = &self.types.module_signatures[*expected]; + let actual_module_sig = &actual_types.module_signatures[*actual]; + let actual_instance_sig = + &actual_types.instance_signatures[actual_module_sig.exports]; + + self.imports_match( + *expected, + actual_types, + actual_module_sig.imports.iter().map(|(module, field, ty)| { + (module.as_str(), field.as_deref(), ty.clone()) + }), + ) && self.exports_match(expected_module_sig.exports, actual_types, |name| { + actual_instance_sig.exports.get(name).cloned() + }) + } + _ => false, + }, + EntityType::Event(_) => unimplemented!(), + } + } +} diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml index 02d48e87c4..aaa130acec 100644 --- a/crates/wast/Cargo.toml +++ b/crates/wast/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] anyhow = "1.0.19" wasmtime = { path = "../wasmtime", version = "0.21.0", default-features = false } -wast = "27.0.0" +wast = "28.0.0" [badges] maintenance = { status = "actively-developed" } diff --git a/scripts/publish.rs b/scripts/publish.rs index 9cff25a4a0..a56e03b351 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -293,7 +293,10 @@ fn verify(crates: &[Crate]) { // Vendor witx which wasn't vendored because it's a path dependency, but // it'll need to be in our directory registry for crates that depend on it. - let witx = crates.iter().find(|c| c.name == "witx").unwrap(); + let witx = crates + .iter() + .find(|c| c.name == "witx" && c.manifest.iter().any(|p| p == "wasi-common")) + .unwrap(); verify_and_vendor(&witx); for krate in crates { diff --git a/tests/all/wast.rs b/tests/all/wast.rs index 9b069c2ebe..9227079a0f 100644 --- a/tests/all/wast.rs +++ b/tests/all/wast.rs @@ -23,8 +23,8 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { let mut cfg = Config::new(); cfg.wasm_simd(simd) .wasm_bulk_memory(bulk_mem) - .wasm_reference_types(reftypes) - .wasm_multi_memory(multi_memory) + .wasm_reference_types(reftypes || module_linking) + .wasm_multi_memory(multi_memory || module_linking) .wasm_module_linking(module_linking) .strategy(strategy)? .cranelift_debug_verifier(true); diff --git a/tests/misc_testsuite/module-linking/alias.wast b/tests/misc_testsuite/module-linking/alias.wast index 73324b251b..0c69b4084c 100644 --- a/tests/misc_testsuite/module-linking/alias.wast +++ b/tests/misc_testsuite/module-linking/alias.wast @@ -54,7 +54,38 @@ ) (assert_return (invoke "get") (i32.const 4)) -;; TODO instances/modules -- needs import/export of modules/instances to work +;; modules +(module + (module $m + (module $sub (export "module") + (func $f (export "") (result i32) + i32.const 5)) + ) + (instance $a (instantiate $m)) + (instance $b (instantiate $a.$sub)) + (alias $b.$f (instance $b) (func 0)) + + (func (export "get") (result i32) + call $b.$f) +) +(assert_return (invoke "get") (i32.const 5)) + +;; instances +(module + (module $m + (module $sub + (func $f (export "") (result i32) + i32.const 6)) + (instance $i (export "") (instantiate $sub)) + ) + (instance $a (instantiate $m)) + (alias $a.$i (instance $a) (instance 0)) + (alias $a.$i.$f (instance $a.$i) (func 0)) + + (func (export "get") (result i32) + call $a.$i.$f) +) +(assert_return (invoke "get") (i32.const 6)) ;; alias parent -- type (module diff --git a/tests/misc_testsuite/module-linking/import-subtyping.wast b/tests/misc_testsuite/module-linking/import-subtyping.wast new file mode 100644 index 0000000000..4ac1658070 --- /dev/null +++ b/tests/misc_testsuite/module-linking/import-subtyping.wast @@ -0,0 +1,348 @@ +;; subsets of imports +(module $a + (module (export "m") + (func (export "")) + (func (export "a")) + (global (export "b") i32 (i32.const 0)) + ) +) + +(module + (import "a" "m" (module)) + (import "a" "m" (module (export "" (func)))) + (import "a" "m" (module (export "a" (func)))) + (import "a" "m" (module (export "b" (global i32)))) + (import "a" "m" (module + (export "" (func)) + (export "a" (func)) + )) + (import "a" "m" (module + (export "a" (func)) + (export "" (func)) + )) + (import "a" "m" (module + (export "a" (func)) + (export "" (func)) + (export "b" (global i32)) + )) + (import "a" "m" (module + (export "b" (global i32)) + (export "a" (func)) + (export "" (func)) + )) +) + +;; functions +(module $a + (module (export "m") + (func (export "")))) + +(module + (import "a" "m" (module)) + (import "a" "m" (module (export "" (func)))) +) +(assert_unlinkable + (module (import "a" "m" (module (export "" (func (param i32)))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (func (result i32)))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (global i32))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (table 1 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 1))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (module))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (instance))))) + "module types incompatible") + +(module $a + (module (export "m") + (global (export "") i32 (i32.const 0)))) + +;; globals +(module + (import "a" "m" (module)) + (import "a" "m" (module (export "" (global i32)))) +) +(assert_unlinkable + (module + (import "a" "m" (module (export "" (global (mut i32))))) + ) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (global f32))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (func))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (table 1 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 1))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (module))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (instance))))) + "module types incompatible") + +;; tables +(module $a + (module (export "m") + (table (export "") 1 funcref) + (table (export "max") 1 10 funcref) + ) +) +(module + (import "a" "m" (module)) + (import "a" "m" (module (export "" (table 1 funcref)))) + (import "a" "m" (module (export "" (table 0 funcref)))) + (import "a" "m" (module (export "max" (table 1 10 funcref)))) + (import "a" "m" (module (export "max" (table 0 10 funcref)))) + (import "a" "m" (module (export "max" (table 0 11 funcref)))) + (import "a" "m" (module (export "max" (table 0 funcref)))) +) +(assert_unlinkable + (module (import "a" "m" (module (export "" (global f32))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (func))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (table 2 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (table 1 10 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "max" (table 2 10 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "max" (table 1 9 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 1))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (module))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (instance))))) + "module types incompatible") + +;; memories +(module $a + (module (export "m") + (memory (export "") 1) + (memory (export "max") 1 10) + ) +) +(module + (import "a" "m" (module)) + (import "a" "m" (module (export "" (memory 1)))) + (import "a" "m" (module (export "" (memory 0)))) + (import "a" "m" (module (export "max" (memory 1 10)))) + (import "a" "m" (module (export "max" (memory 0 10)))) + (import "a" "m" (module (export "max" (memory 0 11)))) + (import "a" "m" (module (export "max" (memory 0)))) +) +(assert_unlinkable + (module (import "a" "m" (module (export "" (global f32))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (func))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (table 1 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 2))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 1 10))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "max" (memory 2 10))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "max" (memory 2))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (module))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (instance))))) + "module types incompatible") + +;; modules +(module $a + (module (export "m") + ;; export nothing + (module (export "a")) + ;; export one thing + (module (export "b") + (func (export "")) + ) + ;; export a mixture + (module (export "c") + (func (export "a")) + (func (export "b") (result i32) + i32.const 0) + (global (export "c") i32 (i32.const 0)) + ) + ;; import one thing + (module (export "d") + (import "" (func)) + ) + ;; import a mixture + (module (export "e") + (import "" (func)) + (import "" (func)) + (import "" (global i32)) + ) + ) +) +(module + (import "a" "m" (module)) + (import "a" "m" (module (export "a" (module)))) + (import "a" "m" (module (export "b" (module)))) + (import "a" "m" (module (export "b" (module (export "" (func)))))) + (import "a" "m" (module (export "c" (module)))) + (import "a" "m" (module (export "c" (module + (export "a" (func)) + )))) + (import "a" "m" (module (export "c" (module + (export "a" (func)) + (export "b" (func (result i32))) + )))) + (import "a" "m" (module (export "c" (module + (export "c" (global i32)) + )))) + (import "a" "m" (module (export "c" (module + (export "c" (global i32)) + (export "a" (func)) + )))) + + ;; for now import strings aren't matched at all, imports must simply pairwise + ;; line up + (import "a" "m" (module (export "d" (module (import "" (func)))))) + (import "a" "m" (module (export "d" (module (import "x" (func)))))) + (import "a" "m" (module (export "d" (module (import "x" "y" (func)))))) + + (import "a" "m" (module (export "e" (module + (import "x" "y" (func)) + (import "a" (func)) + (import "z" (global i32)) + )))) +) +(assert_unlinkable + (module (import "a" "m" (module (export "" (module (export "a" (func))))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "d" (module))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "d" (module (import "" (module))))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (global f32))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (func))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (table 1 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 2))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (module (export "foo" (func))))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (instance))))) + "module types incompatible") + +;; instances +(module $a + ;; export nothing + (module $m1) + (instance (export "a") (instantiate $m1)) + ;; export one thing + (module $m2 + (func (export "")) + ) + (instance (export "b") (instantiate $m2)) + ;; export a mixture + (module $m3 + (func (export "a")) + (func (export "b") (result i32) + i32.const 0) + (global (export "c") i32 (i32.const 0)) + ) + (instance (export "c") (instantiate $m3)) + + (module (export "m") + ;; export one thing + (module $m2 + (func (export "")) + ) + (instance (export "i") (instantiate $m2)) + ) + +) +(module + (import "a" "a" (instance)) + (import "a" "b" (instance)) + (import "a" "b" (instance (export "" (func)))) + (import "a" "c" (instance)) + (import "a" "c" (instance (export "a" (func)))) + (import "a" "c" (instance (export "b" (func (result i32))))) + (import "a" "c" (instance (export "c" (global i32)))) + (import "a" "c" (instance + (export "a" (func)) + (export "b" (func (result i32))) + (export "c" (global i32)) + )) + (import "a" "c" (instance + (export "c" (global i32)) + (export "a" (func)) + )) + + (import "a" "m" (module (export "i" (instance)))) + (import "a" "m" (module (export "i" (instance (export "" (func)))))) +) +(assert_unlinkable + (module (import "a" "a" (instance (export "" (global f32))))) + "instance types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "i" (instance (export "x" (func))))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (func))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (table 1 funcref))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 2))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (memory 1 10))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "max" (memory 2 10))))) + "module types incompatible") +(assert_unlinkable + (module (import "a" "m" (module (export "" (module))))) + "module types incompatible") diff --git a/tests/misc_testsuite/module-linking/instantiate.wast b/tests/misc_testsuite/module-linking/instantiate.wast index a0e24c7a7b..c04929257f 100644 --- a/tests/misc_testsuite/module-linking/instantiate.wast +++ b/tests/misc_testsuite/module-linking/instantiate.wast @@ -117,6 +117,24 @@ ) (assert_return (invoke "get") (i32.const 5)) +;; imported modules again +(module + (module $m + (import "" (module $m (export "get" (func (result i32))))) + (instance $i (instantiate $m)) + (alias $f (instance $i) (func 0)) + (export "" (func $f)) + ) + (module $m2 + (func (export "get") (result i32) + i32.const 6)) + (instance $a (instantiate $m (module $m2))) + + (func (export "get") (result i32) + call $a.$f) +) +(assert_return (invoke "get") (i32.const 6)) + ;; all at once (module (import "a" "inc" (func $f)) @@ -195,3 +213,23 @@ (instance (instantiate 0 (func 0))) ) (assert_return (invoke $a "get") (i32.const 1)) + +;; module/instance top-level imports work +(module $b + (module (export "m")) + (instance (export "i") (instantiate 0)) +) +(module + (import "b" "m" (module)) + (import "b" "i" (instance)) +) +(assert_unlinkable + (module + (import "b" "m" (module (import "" (func)))) + ) + "module types incompatible") +(assert_unlinkable + (module + (import "b" "i" (instance (export "" (func)))) + ) + "instance types incompatible") From 69d041faf17db7c396a928704388e5a557770e34 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 3 Dec 2020 18:49:18 +0100 Subject: [PATCH 35/63] Restore support for non-pic code in SimpleJIT --- cranelift/simplejit/src/backend.rs | 157 +++++++++++++++++++++-------- 1 file changed, 116 insertions(+), 41 deletions(-) diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/simplejit/src/backend.rs index eb6610907c..67ba0f8a37 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/simplejit/src/backend.rs @@ -36,6 +36,7 @@ pub struct SimpleJITBuilder { isa: Box, symbols: HashMap, libcall_names: Box String + Send + Sync>, + hotswap_enabled: bool, } impl SimpleJITBuilder { @@ -75,12 +76,12 @@ impl SimpleJITBuilder { isa: Box, libcall_names: Box String + Send + Sync>, ) -> Self { - assert!(isa.flags().is_pic(), "SimpleJIT requires PIC code"); let symbols = HashMap::new(); Self { isa, symbols, libcall_names, + hotswap_enabled: false, } } @@ -119,6 +120,13 @@ impl SimpleJITBuilder { } self } + + /// Enable or disable hotswap support. See [`SimpleJITModule::prepare_for_function_redefine`] + /// for more information. + pub fn hotswap(&mut self, enabled: bool) -> &mut Self { + self.hotswap_enabled = enabled; + self + } } /// A `SimpleJITModule` implements `Module` and emits code and data into memory where it can be @@ -127,6 +135,7 @@ impl SimpleJITBuilder { /// See the `SimpleJITBuilder` for a convenient way to construct `SimpleJITModule` instances. pub struct SimpleJITModule { isa: Box, + hotswap_enabled: bool, symbols: HashMap, libcall_names: Box String>, memory: MemoryHandle, @@ -138,6 +147,7 @@ pub struct SimpleJITModule { libcall_plt_entries: HashMap>, compiled_functions: SecondaryMap>, compiled_data_objects: SecondaryMap>, + functions_to_finalize: Vec, data_objects_to_finalize: Vec, } @@ -189,7 +199,18 @@ impl SimpleJITModule { match *name { ir::ExternalName::User { .. } => { let (name, linkage) = if ModuleDeclarations::is_function(name) { - return self.get_plt_address(name); + if self.hotswap_enabled { + return self.get_plt_address(name); + } else { + let func_id = FuncId::from_name(name); + match &self.compiled_functions[func_id] { + Some(compiled) => return compiled.ptr, + None => { + let decl = self.declarations.get_function_decl(func_id); + (&decl.name, decl.linkage) + } + } + } } else { let data_id = DataId::from_name(name); match &self.compiled_data_objects[data_id] { @@ -270,6 +291,10 @@ impl SimpleJITModule { /// Returns the address of a finalized function. pub fn get_finalized_function(&self, func_id: FuncId) -> *const u8 { let info = &self.compiled_functions[func_id]; + assert!( + !self.functions_to_finalize.iter().any(|x| *x == func_id), + "function not yet finalized" + ); info.as_ref() .expect("function must be compiled before it can be finalized") .ptr @@ -313,6 +338,19 @@ impl SimpleJITModule { /// Use `get_finalized_function` and `get_finalized_data` to obtain the final /// artifacts. pub fn finalize_definitions(&mut self) { + for func in std::mem::take(&mut self.functions_to_finalize) { + let decl = self.declarations.get_function_decl(func); + assert!(decl.linkage.is_definable()); + let func = self.compiled_functions[func] + .as_ref() + .expect("function must be compiled before it can be finalized"); + func.perform_relocations( + |name| self.get_address(name), + |name| self.get_got_address(name), + |name| self.get_plt_address(name), + ); + } + for data in std::mem::take(&mut self.data_objects_to_finalize) { let decl = self.declarations.get_data_decl(data); assert!(decl.linkage.is_definable()); @@ -333,6 +371,13 @@ impl SimpleJITModule { /// Create a new `SimpleJITModule`. pub fn new(builder: SimpleJITBuilder) -> Self { + if builder.hotswap_enabled { + assert!( + builder.isa.flags().is_pic(), + "Hotswapping requires PIC code" + ); + } + let mut memory = MemoryHandle { code: Memory::new(), readonly: Memory::new(), @@ -342,8 +387,13 @@ impl SimpleJITModule { let mut libcall_got_entries = HashMap::new(); let mut libcall_plt_entries = HashMap::new(); - // Pre-create a GOT entry for each libcall. - for &libcall in ir::LibCall::all_libcalls() { + // Pre-create a GOT and PLT entry for each libcall. + let all_libcalls = if builder.isa.flags().is_pic() { + ir::LibCall::all_libcalls() + } else { + &[] // Not PIC, so no GOT and PLT entries necessary + }; + for &libcall in all_libcalls { let got_entry = memory .writable .allocate( @@ -380,6 +430,7 @@ impl SimpleJITModule { Self { isa: builder.isa, + hotswap_enabled: builder.hotswap_enabled, symbols: builder.symbols, libcall_names: builder.libcall_names, memory, @@ -391,13 +442,17 @@ impl SimpleJITModule { libcall_plt_entries, compiled_functions: SecondaryMap::new(), compiled_data_objects: SecondaryMap::new(), + functions_to_finalize: Vec::new(), data_objects_to_finalize: Vec::new(), } } /// Allow a single future `define_function` on a previously defined function. This allows for /// hot code swapping and lazy compilation of functions. + /// + /// This requires hotswap support to be enabled first using [`SimpleJITBuilder::hotswap`]. pub fn prepare_for_function_redefine(&mut self, func_id: FuncId) -> ModuleResult<()> { + assert!(self.hotswap_enabled, "Hotswap support is not enabled"); let decl = self.declarations.get_function_decl(func_id); if !decl.linkage.is_definable() { return Err(ModuleError::InvalidImportDefinition(decl.name.clone())); @@ -436,7 +491,7 @@ impl<'simple_jit_backend> Module for SimpleJITModule { let (id, _decl) = self .declarations .declare_function(name, linkage, signature)?; - if self.function_got_entries[id].is_none() { + if self.function_got_entries[id].is_none() && self.isa.flags().is_pic() { let got_entry = self .memory .writable @@ -482,7 +537,7 @@ impl<'simple_jit_backend> Module for SimpleJITModule { let (id, _decl) = self .declarations .declare_data(name, linkage, writable, tls)?; - if self.data_object_got_entries[id].is_none() { + if self.data_object_got_entries[id].is_none() && self.isa.flags().is_pic() { let got_entry = self .memory .writable @@ -509,10 +564,11 @@ impl<'simple_jit_backend> Module for SimpleJITModule { fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { let decl = self.declarations.get_function_decl(func); let signature = in_func.import_signature(decl.signature.clone()); + let colocated = !self.hotswap_enabled && decl.linkage.is_final(); in_func.import_function(ir::ExtFuncData { name: ir::ExternalName::user(0, func.as_u32()), signature, - colocated: false, + colocated, }) } @@ -521,10 +577,11 @@ impl<'simple_jit_backend> Module for SimpleJITModule { /// TODO: Same as above. fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue { let decl = self.declarations.get_data_decl(data); + let colocated = !self.hotswap_enabled && decl.linkage.is_final(); func.create_global_value(ir::GlobalValueData::Symbol { name: ir::ExternalName::user(1, data.as_u32()), offset: ir::immediates::Imm64::new(0), - colocated: false, + colocated, tls: decl.tls, }) } @@ -588,28 +645,36 @@ impl<'simple_jit_backend> Module for SimpleJITModule { size, relocs: reloc_sink.relocs, }); - unsafe { - std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); + + if self.isa.flags().is_pic() { + unsafe { + std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); + } + } + + if self.hotswap_enabled { + self.compiled_functions[id] + .as_ref() + .unwrap() + .perform_relocations( + |name| match *name { + ir::ExternalName::User { .. } => { + unreachable!("non GOT or PLT relocation in function {} to {}", id, name) + } + ir::ExternalName::LibCall(ref libcall) => self + .libcall_plt_entries + .get(libcall) + .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) + .as_ptr() + .cast::(), + _ => panic!("invalid ExternalName {}", name), + }, + |name| self.get_got_address(name), + |name| self.get_plt_address(name), + ); + } else { + self.functions_to_finalize.push(id); } - self.compiled_functions[id] - .as_ref() - .unwrap() - .perform_relocations( - |name| match *name { - ir::ExternalName::User { .. } => { - unreachable!("non GOT or PLT relocation in function {} to {}", id, name) - } - ir::ExternalName::LibCall(ref libcall) => self - .libcall_plt_entries - .get(libcall) - .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) - .as_ptr() - .cast::(), - _ => panic!("invalid ExternalName {}", name), - }, - |name| self.get_got_address(name), - |name| self.get_plt_address(name), - ); Ok(ModuleCompiledFunction { size: code_size }) } @@ -652,17 +717,25 @@ impl<'simple_jit_backend> Module for SimpleJITModule { size, relocs: relocs.to_vec(), }); - unsafe { - std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); + + if self.isa.flags().is_pic() { + unsafe { + std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr); + } + } + + if self.hotswap_enabled { + self.compiled_functions[id] + .as_ref() + .unwrap() + .perform_relocations( + |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), + |name| self.get_got_address(name), + |name| self.get_plt_address(name), + ); + } else { + self.functions_to_finalize.push(id); } - self.compiled_functions[id] - .as_ref() - .unwrap() - .perform_relocations( - |name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name), - |name| self.get_got_address(name), - |name| self.get_plt_address(name), - ); Ok(ModuleCompiledFunction { size: total_size }) } @@ -727,8 +800,10 @@ impl<'simple_jit_backend> Module for SimpleJITModule { self.compiled_data_objects[id] = Some(CompiledBlob { ptr, size, relocs }); self.data_objects_to_finalize.push(id); - unsafe { - std::ptr::write(self.data_object_got_entries[id].unwrap().as_ptr(), ptr); + if self.isa.flags().is_pic() { + unsafe { + std::ptr::write(self.data_object_got_entries[id].unwrap().as_ptr(), ptr); + } } Ok(()) From d2721064dfae09b1256c53e36be6193a66a153b3 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 3 Dec 2020 09:38:38 -0800 Subject: [PATCH 36/63] Disable all x64 SIMD tests, as some seem to be nondeterministic. We are seeing some repeated but nondeterministic failures of x64 SIMD tests, as tracked in #2470. In order to err on the side of not inadvertently breaking/blocking CI for others, we will disable all SIMD tests for now while we investigate. --- build.rs | 52 +++++++++------------------------------------------- 1 file changed, 9 insertions(+), 43 deletions(-) diff --git a/build.rs b/build.rs index aa791aeda1..c766521678 100644 --- a/build.rs +++ b/build.rs @@ -175,49 +175,10 @@ fn write_testsuite_tests( /// For experimental_x64 backend features that are not supported yet, mark tests as panicking, so /// they stop "passing" once the features are properly implemented. -fn experimental_x64_should_panic(testsuite: &str, testname: &str, strategy: &str) -> bool { - if !cfg!(feature = "experimental_x64") || strategy != "Cranelift" { - return false; - } - - match (testsuite, testname) { - ("simd", "simd_address") => return false, - ("simd", "simd_bitwise") => return false, - ("simd", "simd_bit_shift") => return false, - ("simd", "simd_boolean") => return false, - ("simd", "simd_const") => return false, - ("simd", "simd_i8x16_arith") => return false, - ("simd", "simd_i8x16_arith2") => return false, - ("simd", "simd_i8x16_cmp") => return false, - ("simd", "simd_i8x16_sat_arith") => return false, - ("simd", "simd_i16x8_arith") => return false, - ("simd", "simd_i16x8_arith2") => return false, - ("simd", "simd_i16x8_cmp") => return false, - ("simd", "simd_i16x8_sat_arith") => return false, - ("simd", "simd_i32x4_arith") => return false, - ("simd", "simd_i32x4_arith2") => return false, - ("simd", "simd_i32x4_cmp") => return false, - ("simd", "simd_i64x2_arith") => return false, - ("simd", "simd_f32x4") => return false, - ("simd", "simd_f32x4_arith") => return false, - ("simd", "simd_f32x4_cmp") => return false, - ("simd", "simd_f32x4_pmin_pmax") => return false, - ("simd", "simd_f64x2") => return false, - ("simd", "simd_f64x2_arith") => return false, - ("simd", "simd_f64x2_cmp") => return false, - ("simd", "simd_f64x2_pmin_pmax") => return false, - ("simd", "simd_lane") => return false, - ("simd", "simd_load") => return false, - ("simd", "simd_load_splat") => return false, - ("simd", "simd_splat") => return false, - ("simd", "simd_store") => return false, - ("simd", "simd_conversions") => return false, - ("simd", "simd_f32x4_rounding") => return false, - ("simd", "simd_f64x2_rounding") => return false, - ("simd", _) => return true, - _ => {} - } - +/// +/// TODO(#2470): removed all tests from this set as we are disabling x64 SIMD tests unconditionally +/// instead until we resolve a nondeterminism bug. Restore when fixed. +fn experimental_x64_should_panic(_testsuite: &str, _testname: &str, _strategy: &str) -> bool { false } @@ -239,6 +200,11 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64"; } + // Ignore all x64 SIMD tests for now (#2470). + ("simd", _) if cfg!(feature = "experimental_x64") => { + return env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64"; + } + // These are only implemented on aarch64 and x64. ("simd", "simd_boolean") | ("simd", "simd_f32x4_pmin_pmax") From 937a3fde4000aa69e7f8c9bfff1261da4220d452 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 3 Dec 2020 18:57:03 +0100 Subject: [PATCH 37/63] Fix simplejit tests for x64 by disabling is_pic --- .../simplejit/examples/simplejit-minimal.rs | 12 ++++++- cranelift/simplejit/tests/basic.rs | 32 +++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/cranelift/simplejit/examples/simplejit-minimal.rs b/cranelift/simplejit/examples/simplejit-minimal.rs index ae06c07b95..7274c4d5e7 100644 --- a/cranelift/simplejit/examples/simplejit-minimal.rs +++ b/cranelift/simplejit/examples/simplejit-minimal.rs @@ -1,12 +1,22 @@ use cranelift::prelude::*; use cranelift_codegen::binemit::NullTrapSink; +use cranelift_codegen::settings::{self, Configurable}; use cranelift_module::{default_libcall_names, Linkage, Module}; use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; use std::mem; fn main() { + let mut flag_builder = settings::builder(); + flag_builder.set("use_colocated_libcalls", "false").unwrap(); + // FIXME set back to true once the x64 backend supports it. + flag_builder.set("is_pic", "false").unwrap(); + let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { + panic!("host machine is not supported: {}", msg); + }); + let isa = isa_builder.finish(settings::Flags::new(flag_builder)); let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); + SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); + let mut ctx = module.make_context(); let mut func_ctx = FunctionBuilderContext::new(); diff --git a/cranelift/simplejit/tests/basic.rs b/cranelift/simplejit/tests/basic.rs index 0b91765433..fc3f5b0056 100644 --- a/cranelift/simplejit/tests/basic.rs +++ b/cranelift/simplejit/tests/basic.rs @@ -1,6 +1,7 @@ use cranelift_codegen::binemit::NullTrapSink; use cranelift_codegen::ir::*; use cranelift_codegen::isa::CallConv; +use cranelift_codegen::settings::{self, Configurable}; use cranelift_codegen::{ir::types::I16, Context}; use cranelift_entity::EntityRef; use cranelift_frontend::*; @@ -9,8 +10,17 @@ use cranelift_simplejit::*; #[test] fn error_on_incompatible_sig_in_declare_function() { + let mut flag_builder = settings::builder(); + flag_builder.set("use_colocated_libcalls", "false").unwrap(); + // FIXME set back to true once the x64 backend supports it. + flag_builder.set("is_pic", "false").unwrap(); + let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { + panic!("host machine is not supported: {}", msg); + }); + let isa = isa_builder.finish(settings::Flags::new(flag_builder)); let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); + SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); + let mut sig = Signature { params: vec![AbiParam::new(types::I64)], returns: vec![], @@ -58,8 +68,16 @@ fn define_simple_function(module: &mut SimpleJITModule) -> FuncId { #[test] #[should_panic(expected = "Result::unwrap()` on an `Err` value: DuplicateDefinition(\"abc\")")] fn panic_on_define_after_finalize() { + let mut flag_builder = settings::builder(); + flag_builder.set("use_colocated_libcalls", "false").unwrap(); + // FIXME set back to true once the x64 backend supports it. + flag_builder.set("is_pic", "false").unwrap(); + let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { + panic!("host machine is not supported: {}", msg); + }); + let isa = isa_builder.finish(settings::Flags::new(flag_builder)); let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); + SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); define_simple_function(&mut module); define_simple_function(&mut module); @@ -140,8 +158,16 @@ fn switch_error() { #[test] fn libcall_function() { + let mut flag_builder = settings::builder(); + flag_builder.set("use_colocated_libcalls", "false").unwrap(); + // FIXME set back to true once the x64 backend supports it. + flag_builder.set("is_pic", "false").unwrap(); + let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { + panic!("host machine is not supported: {}", msg); + }); + let isa = isa_builder.finish(settings::Flags::new(flag_builder)); let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); + SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); let sig = Signature { params: vec![], From 09662fa7162579b63dd883e30c97c10e0b85f825 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 3 Dec 2020 13:41:32 -0600 Subject: [PATCH 38/63] Fix module-linking handling of instance subtypes (#2466) * Fix module-linking handling of instance subtypes When we alias the nth export of an instance, due to subtyping the nth export may not actually be what we want. Instead we need to look at our local type definition's nth export's name, and lookup that name off the export. * Update crates/wasmtime/src/instance.rs Co-authored-by: Peter Huene Co-authored-by: Peter Huene --- crates/wasmtime/src/instance.rs | 17 +++++-- .../module-linking/instantiate.wast | 49 +++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 6f69b66a36..0dff161052 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -82,18 +82,29 @@ fn instantiate( imports.modules.push(module.submodule(*idx)); } - // Here we lookup our instance handle, ask it for the nth export, + // Here we lookup our instance handle, find the right export, // and then push that item into our own index space. We eschew // type-checking since only valid modules reach this point. // + // Note that export lookup here needs to happen by name. The + // `export` index is an index into our local type definition of the + // type of the instance to figure out what name it was assigned. + // This is where the subtyping happens! + // // Note that the unsafety here is because we're asserting that the // handle comes from our same store, but this should be true because // we acquired the handle from an instance in the store. Initializer::AliasInstanceExport { instance, export } => { + let instance_ty = env_module.instances[*instance]; + let export_name = module.types().instance_signatures[instance_ty] + .exports + .get_index(*export) + .expect("validation bug - should be valid") + .0; let handle = &imports.instances[*instance]; - let export_index = &handle.module().exports[*export]; + let entity_index = &handle.module().exports[export_name]; let item = Extern::from_wasmtime_export( - handle.lookup_by_declaration(export_index), + handle.lookup_by_declaration(entity_index), unsafe { store.existing_instance_handle(handle.clone()) }, ); imports.push_extern(&item); diff --git a/tests/misc_testsuite/module-linking/instantiate.wast b/tests/misc_testsuite/module-linking/instantiate.wast index c04929257f..8a61684438 100644 --- a/tests/misc_testsuite/module-linking/instantiate.wast +++ b/tests/misc_testsuite/module-linking/instantiate.wast @@ -233,3 +233,52 @@ (import "b" "i" (instance (export "" (func)))) ) "instance types incompatible") + +;; ensure we ignore other exported items +(module $b + (module $m + (func (export "f") (result i32) + i32.const 300) + (global (export "g") i32 (i32.const 0xfeed)) + ) + + (instance (export "i") (instantiate 0)) +) +(module + (import "b" "i" (instance $i + (export "g" (global $g i32)) + )) + + (func (export "get") (result i32) + global.get $i.$g) +) +(assert_return (invoke "get") (i32.const 0xfeed)) + +;; ensure the right export is used even when subtyping comes into play +(module $b + (module $m + (func (export "f") (result i32) + i32.const 300) + (func (export "g") (param i32) (result i32) + i32.const 100 + local.get 0 + i32.add) + ) + + (instance (export "i") (instantiate 0)) +) +(module + (import "b" "i" (instance $i + ;; notice that this order is swapped + (export "g" (func $g (param i32) (result i32))) + (export "f" (func $f (result i32))) + )) + + (func (export "f") (result i32) + call $i.$f) + (func (export "g") (param i32) (result i32) + local.get 0 + call $i.$g) +) +(assert_return (invoke "f") (i32.const 300)) +(assert_return (invoke "g" (i32.const 3000)) (i32.const 3100)) From 41caf67af3ba7c7c14c771e3e32be4ecb187447d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 3 Dec 2020 15:51:38 -0600 Subject: [PATCH 39/63] Update the C API with module linking support (#2472) * Update the C API with module linking support This commit does everything necessary (ideally) to support the module linking proposal in the C API. The changes here are: * New `wasm_{module,instance}type_t` types and accessors * New `wasm_{module,instance}_type` functions * Conversions between `wasm_extern_t` and `wasm_{instance,module}_t`, as well as `wasm_externtype_t` and the new types. * Addition of `WASM_EXTERN_{MODULE,INSTANCE}` constants * New `wasm_config_t` modifier to enable/disable module linking With these functions it should be possible to pass instances/modules to instances and also acquire them from exports. Altogether this should enable everything for module linking. An important point for this is that I've opted to add all these items under the `wasm_*` name prefix instead of `wasmtime_*`. I've done this since they're all following the idioms of existing APIs and while not standard the intention would be to standardize them (unlike many other Wasmtime-specific APIs). cc #2094 * Appease doxygen --- crates/c-api/include/doc-wasm.h | 7 + crates/c-api/include/wasmtime.h | 293 +++++++++++++++++++++++++++++ crates/c-api/src/config.rs | 5 + crates/c-api/src/extern.rs | 32 +++- crates/c-api/src/instance.rs | 44 ++++- crates/c-api/src/linker.rs | 6 +- crates/c-api/src/module.rs | 140 +++++++------- crates/c-api/src/types/extern.rs | 8 +- crates/c-api/src/types/instance.rs | 41 +++- crates/c-api/src/types/module.rs | 64 ++++++- crates/c-api/src/vec.rs | 27 ++- 11 files changed, 553 insertions(+), 114 deletions(-) diff --git a/crates/c-api/include/doc-wasm.h b/crates/c-api/include/doc-wasm.h index 301989437d..73d57a647b 100644 --- a/crates/c-api/include/doc-wasm.h +++ b/crates/c-api/include/doc-wasm.h @@ -793,8 +793,15 @@ * \typedef wasm_externkind_t * \brief Classifier for #wasm_externtype_t, defined by #wasm_externkind_enum * + * This is returned from #wasm_extern_kind and #wasm_externtype_kind to + * determine what kind of type is wrapped. + * * \enum wasm_externkind_enum * \brief Kinds of external items for a wasm module. + * + * Note that this also includes #WASM_EXTERN_INSTANCE as well as + * #WASM_EXTERN_MODULE and is intended to be used when #wasm_externkind_t is + * used. */ /** diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index 75880ce020..769cfc1268 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -208,6 +208,14 @@ WASMTIME_CONFIG_PROP(void, wasm_bulk_memory, bool) */ WASMTIME_CONFIG_PROP(void, wasm_multi_value, bool) +/** + * \brief Configures whether the WebAssembly module linking proposal is + * enabled. + * + * This setting is `false` by default. + */ +WASMTIME_CONFIG_PROP(void, wasm_module_linking, bool) + /** * \brief Configures how JIT code will be compiled. * @@ -961,6 +969,291 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize( own wasm_module_t **ret ); +/** + * \struct wasm_instancetype_t + * \brief An opaque object representing the type of a function. + * + * \typedef wasm_instancetype_t + * \brief Convenience alias for #wasm_instancetype_t + * + * \struct wasm_instancetype_vec_t + * \brief A list of #wasm_instancetype_t values. + * + * \var wasm_instancetype_vec_t::size + * \brief Length of this vector. + * + * \var wasm_instancetype_vec_t::data + * \brief Pointer to the base of this vector + * + * \typedef wasm_instancetype_vec_t + * \brief Convenience alias for #wasm_instancetype_vec_t + * + * \fn void wasm_instancetype_delete(own wasm_instancetype_t *); + * \brief Deletes a type. + * + * \fn void wasm_instancetype_vec_new_empty(own wasm_instancetype_vec_t *out); + * \brief Creates an empty vector. + * + * See #wasm_byte_vec_new_empty for more information. + * + * \fn void wasm_instancetype_vec_new_uninitialized(own wasm_instancetype_vec_t *out, size_t); + * \brief Creates a vector with the given capacity. + * + * See #wasm_byte_vec_new_uninitialized for more information. + * + * \fn void wasm_instancetype_vec_new(own wasm_instancetype_vec_t *out, size_t, own wasm_instancetype_t *const[]); + * \brief Creates a vector with the provided contents. + * + * See #wasm_byte_vec_new for more information. + * + * \fn void wasm_instancetype_vec_copy(own wasm_instancetype_vec_t *out, const wasm_instancetype_vec_t *) + * \brief Copies one vector to another + * + * See #wasm_byte_vec_copy for more information. + * + * \fn void wasm_instancetype_vec_delete(own wasm_instancetype_vec_t *out) + * \brief Deallocates memory for a vector. + * + * See #wasm_byte_vec_delete for more information. + * + * \fn own wasm_instancetype_t* wasm_instancetype_copy(wasm_instancetype_t *) + * \brief Creates a new value which matches the provided one. + * + * The caller is responsible for deleting the returned value. + */ +WASM_DECLARE_TYPE(instancetype) + +/** + * \brief Returns the list of exports that this instance type provides. + * + * This function does not take ownership of the provided instance type but + * ownership of `out` is passed to the caller. Note that `out` is treated as + * uninitialized when passed to this function. + */ +WASM_API_EXTERN void wasm_instancetype_exports(const wasm_instancetype_t*, own wasm_exporttype_vec_t* out); + +/** + * \brief Converts a #wasm_instancetype_t to a #wasm_externtype_t + * + * The returned value is owned by the #wasm_instancetype_t argument and should not + * be deleted. + */ +WASM_API_EXTERN wasm_externtype_t* wasm_instancetype_as_externtype(wasm_instancetype_t*); + +/** + * \brief Attempts to convert a #wasm_externtype_t to a #wasm_instancetype_t + * + * The returned value is owned by the #wasm_instancetype_t argument and should not + * be deleted. Returns `NULL` if the provided argument is not a + * #wasm_instancetype_t. + */ +WASM_API_EXTERN wasm_instancetype_t* wasm_externtype_as_instancetype(wasm_externtype_t*); + +/** + * \brief Converts a #wasm_instancetype_t to a #wasm_externtype_t + * + * The returned value is owned by the #wasm_instancetype_t argument and should not + * be deleted. + */ +WASM_API_EXTERN const wasm_externtype_t* wasm_instancetype_as_externtype_const(const wasm_instancetype_t*); + +/** + * \brief Attempts to convert a #wasm_externtype_t to a #wasm_instancetype_t + * + * The returned value is owned by the #wasm_instancetype_t argument and should not + * be deleted. Returns `NULL` if the provided argument is not a + * #wasm_instancetype_t. + */ +WASM_API_EXTERN const wasm_instancetype_t* wasm_externtype_as_instancetype_const(const wasm_externtype_t*); + +/** + * \struct wasm_moduletype_t + * \brief An opaque object representing the type of a function. + * + * \typedef wasm_moduletype_t + * \brief Convenience alias for #wasm_moduletype_t + * + * \struct wasm_moduletype_vec_t + * \brief A list of #wasm_moduletype_t values. + * + * \var wasm_moduletype_vec_t::size + * \brief Length of this vector. + * + * \var wasm_moduletype_vec_t::data + * \brief Pointer to the base of this vector + * + * \typedef wasm_moduletype_vec_t + * \brief Convenience alias for #wasm_moduletype_vec_t + * + * \fn void wasm_moduletype_delete(own wasm_moduletype_t *); + * \brief Deletes a type. + * + * \fn void wasm_moduletype_vec_new_empty(own wasm_moduletype_vec_t *out); + * \brief Creates an empty vector. + * + * See #wasm_byte_vec_new_empty for more information. + * + * \fn void wasm_moduletype_vec_new_uninitialized(own wasm_moduletype_vec_t *out, size_t); + * \brief Creates a vector with the given capacity. + * + * See #wasm_byte_vec_new_uninitialized for more information. + * + * \fn void wasm_moduletype_vec_new(own wasm_moduletype_vec_t *out, size_t, own wasm_moduletype_t *const[]); + * \brief Creates a vector with the provided contents. + * + * See #wasm_byte_vec_new for more information. + * + * \fn void wasm_moduletype_vec_copy(own wasm_moduletype_vec_t *out, const wasm_moduletype_vec_t *) + * \brief Copies one vector to another + * + * See #wasm_byte_vec_copy for more information. + * + * \fn void wasm_moduletype_vec_delete(own wasm_moduletype_vec_t *out) + * \brief Deallocates memory for a vector. + * + * See #wasm_byte_vec_delete for more information. + * + * \fn own wasm_moduletype_t* wasm_moduletype_copy(wasm_moduletype_t *) + * \brief Creates a new value which matches the provided one. + * + * The caller is responsible for deleting the returned value. + */ +WASM_DECLARE_TYPE(moduletype) + +/** + * \brief Returns the list of imports that this module type requires. + * + * This function does not take ownership of the provided module type but + * ownership of `out` is passed to the caller. Note that `out` is treated as + * uninitialized when passed to this function. + */ +WASM_API_EXTERN void wasm_moduletype_imports(const wasm_moduletype_t*, own wasm_importtype_vec_t* out); + +/** + * \brief Returns the list of exports that this module type provides. + * + * This function does not take ownership of the provided module type but + * ownership of `out` is passed to the caller. Note that `out` is treated as + * uninitialized when passed to this function. + */ +WASM_API_EXTERN void wasm_moduletype_exports(const wasm_moduletype_t*, own wasm_exporttype_vec_t* out); + +/** + * \brief Converts a #wasm_moduletype_t to a #wasm_externtype_t + * + * The returned value is owned by the #wasm_moduletype_t argument and should not + * be deleted. + */ +WASM_API_EXTERN wasm_externtype_t* wasm_moduletype_as_externtype(wasm_moduletype_t*); + +/** + * \brief Attempts to convert a #wasm_externtype_t to a #wasm_moduletype_t + * + * The returned value is owned by the #wasm_moduletype_t argument and should not + * be deleted. Returns `NULL` if the provided argument is not a + * #wasm_moduletype_t. + */ +WASM_API_EXTERN wasm_moduletype_t* wasm_externtype_as_moduletype(wasm_externtype_t*); + +/** + * \brief Converts a #wasm_moduletype_t to a #wasm_externtype_t + * + * The returned value is owned by the #wasm_moduletype_t argument and should not + * be deleted. + */ +WASM_API_EXTERN const wasm_externtype_t* wasm_moduletype_as_externtype_const(const wasm_moduletype_t*); + +/** + * \brief Attempts to convert a #wasm_externtype_t to a #wasm_moduletype_t + * + * The returned value is owned by the #wasm_moduletype_t argument and should not + * be deleted. Returns `NULL` if the provided argument is not a + * #wasm_moduletype_t. + */ +WASM_API_EXTERN const wasm_moduletype_t* wasm_externtype_as_moduletype_const(const wasm_externtype_t*); + +/** + * \brief Converts a #wasm_module_t to #wasm_extern_t. + * + * The returned #wasm_extern_t is owned by the #wasm_module_t argument. Callers + * should not delete the returned value, and it only lives as long as the + * #wasm_module_t argument. + */ +WASM_API_EXTERN wasm_extern_t* wasm_module_as_extern(wasm_module_t*); + +/** + * \brief Converts a #wasm_extern_t to #wasm_module_t. + * + * The returned #wasm_module_t is owned by the #wasm_extern_t argument. Callers + * should not delete the returned value, and it only lives as long as the + * #wasm_extern_t argument. + * + * If the #wasm_extern_t argument isn't a #wasm_module_t then `NULL` is returned. + */ +WASM_API_EXTERN wasm_module_t* wasm_extern_as_module(wasm_extern_t*); + +/** + * \brief Converts a #wasm_extern_t to #wasm_instance_t. + * + * The returned #wasm_instance_t is owned by the #wasm_extern_t argument. Callers + * should not delete the returned value, and it only lives as long as the + * #wasm_extern_t argument. + */ +WASM_API_EXTERN const wasm_module_t* wasm_extern_as_module_const(const wasm_extern_t*); + +/** + * \brief Converts a #wasm_instance_t to #wasm_extern_t. + * + * The returned #wasm_extern_t is owned by the #wasm_instance_t argument. Callers + * should not delete the returned value, and it only lives as long as the + * #wasm_instance_t argument. + */ +WASM_API_EXTERN wasm_extern_t* wasm_instance_as_extern(wasm_instance_t*); + +/** + * \brief Converts a #wasm_extern_t to #wasm_instance_t. + * + * The returned #wasm_instance_t is owned by the #wasm_extern_t argument. Callers + * should not delete the returned value, and it only lives as long as the + * #wasm_extern_t argument. + * + * If the #wasm_extern_t argument isn't a #wasm_instance_t then `NULL` is returned. + */ +WASM_API_EXTERN wasm_instance_t* wasm_extern_as_instance(wasm_extern_t*); + +/** + * \brief Converts a #wasm_extern_t to #wasm_instance_t. + * + * The returned #wasm_instance_t is owned by the #wasm_extern_t argument. Callers + * should not delete the returned value, and it only lives as long as the + * #wasm_extern_t argument. + */ +WASM_API_EXTERN const wasm_instance_t* wasm_extern_as_instance_const(const wasm_extern_t*); + +/** + * \brief Returns the type of this instance. + * + * The returned #wasm_instancetype_t is expected to be deallocated by the caller. + */ +WASM_API_EXTERN own wasm_instancetype_t* wasm_instance_type(const wasm_instance_t*); + +/** + * \brief Returns the type of this module. + * + * The returned #wasm_moduletype_t is expected to be deallocated by the caller. + */ +WASM_API_EXTERN own wasm_moduletype_t* wasm_module_type(const wasm_module_t*); + +/** + * \brief Value of #wasm_externkind_enum corresponding to a wasm module. + */ +#define WASM_EXTERN_MODULE 4 + +/** + * \brief Value of #wasm_externkind_enum corresponding to a wasm instance. + */ +#define WASM_EXTERN_INSTANCE 5 + #undef own #ifdef __cplusplus diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index 49dce19841..08da598ec1 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -85,6 +85,11 @@ pub extern "C" fn wasmtime_config_wasm_multi_value_set(c: &mut wasm_config_t, en c.config.wasm_multi_value(enable); } +#[no_mangle] +pub extern "C" fn wasmtime_config_wasm_module_linking_set(c: &mut wasm_config_t, enable: bool) { + c.config.wasm_module_linking(enable); +} + #[no_mangle] pub extern "C" fn wasmtime_config_strategy_set( c: &mut wasm_config_t, diff --git a/crates/c-api/src/extern.rs b/crates/c-api/src/extern.rs index 6a08a6e13e..414946a293 100644 --- a/crates/c-api/src/extern.rs +++ b/crates/c-api/src/extern.rs @@ -1,5 +1,7 @@ -use crate::wasm_externkind_t; -use crate::{wasm_externtype_t, wasm_func_t, wasm_global_t, wasm_memory_t, wasm_table_t}; +use crate::{ + wasm_externkind_t, wasm_externtype_t, wasm_func_t, wasm_global_t, wasm_instance_t, + wasm_memory_t, wasm_module_t, wasm_table_t, +}; use wasmtime::Extern; #[derive(Clone)] @@ -16,10 +18,8 @@ pub extern "C" fn wasm_extern_kind(e: &wasm_extern_t) -> wasm_externkind_t { Extern::Global(_) => crate::WASM_EXTERN_GLOBAL, Extern::Table(_) => crate::WASM_EXTERN_TABLE, Extern::Memory(_) => crate::WASM_EXTERN_MEMORY, - - // FIXME(#2094) - Extern::Instance(_) => unimplemented!(), - Extern::Module(_) => unimplemented!(), + Extern::Instance(_) => crate::WASM_EXTERN_INSTANCE, + Extern::Module(_) => crate::WASM_EXTERN_MODULE, } } @@ -67,3 +67,23 @@ pub extern "C" fn wasm_extern_as_memory(e: &wasm_extern_t) -> Option<&wasm_memor pub extern "C" fn wasm_extern_as_memory_const(e: &wasm_extern_t) -> Option<&wasm_memory_t> { wasm_extern_as_memory(e) } + +#[no_mangle] +pub extern "C" fn wasm_extern_as_module(e: &wasm_extern_t) -> Option<&wasm_module_t> { + wasm_module_t::try_from(e) +} + +#[no_mangle] +pub extern "C" fn wasm_extern_as_module_const(e: &wasm_extern_t) -> Option<&wasm_module_t> { + wasm_extern_as_module(e) +} + +#[no_mangle] +pub extern "C" fn wasm_extern_as_instance(e: &wasm_extern_t) -> Option<&wasm_instance_t> { + wasm_instance_t::try_from(e) +} + +#[no_mangle] +pub extern "C" fn wasm_extern_as_instance_const(e: &wasm_extern_t) -> Option<&wasm_instance_t> { + wasm_extern_as_instance(e) +} diff --git a/crates/c-api/src/instance.rs b/crates/c-api/src/instance.rs index 6fa2088886..5a41d68a1c 100644 --- a/crates/c-api/src/instance.rs +++ b/crates/c-api/src/instance.rs @@ -1,20 +1,38 @@ use crate::{wasm_extern_t, wasm_extern_vec_t, wasm_module_t, wasm_trap_t}; -use crate::{wasm_store_t, wasmtime_error_t}; +use crate::{wasm_instancetype_t, wasm_store_t, wasmtime_error_t}; use anyhow::Result; use std::ptr; -use wasmtime::{Instance, Trap}; +use wasmtime::{Extern, Instance, Trap}; -#[repr(C)] #[derive(Clone)] +#[repr(transparent)] pub struct wasm_instance_t { - pub(crate) instance: Instance, + ext: wasm_extern_t, } wasmtime_c_api_macros::declare_ref!(wasm_instance_t); impl wasm_instance_t { pub(crate) fn new(instance: Instance) -> wasm_instance_t { - wasm_instance_t { instance: instance } + wasm_instance_t { + ext: wasm_extern_t { + which: instance.into(), + }, + } + } + + pub(crate) fn try_from(e: &wasm_extern_t) -> Option<&wasm_instance_t> { + match &e.which { + Extern::Instance(_) => Some(unsafe { &*(e as *const _ as *const _) }), + _ => None, + } + } + + pub(crate) fn instance(&self) -> &Instance { + match &self.ext.which { + Extern::Instance(i) => i, + _ => unreachable!(), + } } } @@ -31,7 +49,7 @@ pub unsafe extern "C" fn wasm_instance_new( store, wasm_module, imports, - wasm_module.imports.len(), + wasm_module.module().imports().len(), &mut instance, &mut trap, ); @@ -92,7 +110,7 @@ fn _wasmtime_instance_new( .map(|import| import.which.clone()) .collect::>(); handle_instantiate( - Instance::new(store, &module.module, &imports), + Instance::new(store, module.module(), &imports), instance_ptr, trap_ptr, ) @@ -122,11 +140,16 @@ pub fn handle_instantiate( } } +#[no_mangle] +pub extern "C" fn wasm_instance_as_extern(m: &wasm_instance_t) -> &wasm_extern_t { + &m.ext +} + #[no_mangle] pub extern "C" fn wasm_instance_exports(instance: &wasm_instance_t, out: &mut wasm_extern_vec_t) { out.set_buffer( instance - .instance + .instance() .exports() .map(|e| { Some(Box::new(wasm_extern_t { @@ -136,3 +159,8 @@ pub extern "C" fn wasm_instance_exports(instance: &wasm_instance_t, out: &mut wa .collect(), ); } + +#[no_mangle] +pub extern "C" fn wasm_instance_type(f: &wasm_instance_t) -> Box { + Box::new(wasm_instancetype_t::new(f.instance().ty())) +} diff --git a/crates/c-api/src/linker.rs b/crates/c-api/src/linker.rs index ee77001c50..80aa3fa945 100644 --- a/crates/c-api/src/linker.rs +++ b/crates/c-api/src/linker.rs @@ -68,7 +68,7 @@ pub extern "C" fn wasmtime_linker_define_instance( Ok(s) => s, Err(_) => return bad_utf8(), }; - handle_result(linker.instance(name, &instance.instance), |_linker| ()) + handle_result(linker.instance(name, instance.instance()), |_linker| ()) } #[no_mangle] @@ -78,7 +78,7 @@ pub extern "C" fn wasmtime_linker_instantiate( instance_ptr: &mut *mut wasm_instance_t, trap_ptr: &mut *mut wasm_trap_t, ) -> Option> { - let result = linker.linker.instantiate(&module.module); + let result = linker.linker.instantiate(module.module()); super::instance::handle_instantiate(result, instance_ptr, trap_ptr) } @@ -93,7 +93,7 @@ pub extern "C" fn wasmtime_linker_module( Ok(s) => s, Err(_) => return bad_utf8(), }; - handle_result(linker.module(name, &module.module), |_linker| ()) + handle_result(linker.module(name, module.module()), |_linker| ()) } #[no_mangle] diff --git a/crates/c-api/src/module.rs b/crates/c-api/src/module.rs index 52f5768ce2..c31f125b70 100644 --- a/crates/c-api/src/module.rs +++ b/crates/c-api/src/module.rs @@ -1,20 +1,43 @@ use crate::{ handle_result, wasm_byte_vec_t, wasm_engine_t, wasm_exporttype_t, wasm_exporttype_vec_t, - wasm_importtype_t, wasm_importtype_vec_t, wasm_store_t, wasmtime_error_t, + wasm_extern_t, wasm_importtype_t, wasm_importtype_vec_t, wasm_moduletype_t, wasm_store_t, + wasmtime_error_t, }; use std::ptr; -use wasmtime::{Engine, Module}; +use wasmtime::{Engine, Extern, Module}; -#[repr(C)] #[derive(Clone)] +#[repr(transparent)] pub struct wasm_module_t { - pub(crate) module: Module, - pub(crate) imports: Vec, - pub(crate) exports: Vec, + ext: wasm_extern_t, } wasmtime_c_api_macros::declare_ref!(wasm_module_t); +impl wasm_module_t { + pub(crate) fn new(module: Module) -> wasm_module_t { + wasm_module_t { + ext: wasm_extern_t { + which: module.into(), + }, + } + } + + pub(crate) fn try_from(e: &wasm_extern_t) -> Option<&wasm_module_t> { + match &e.which { + Extern::Module(_) => Some(unsafe { &*(e as *const _ as *const _) }), + _ => None, + } + } + + pub(crate) fn module(&self) -> &Module { + match &self.ext.which { + Extern::Module(i) => i, + _ => unreachable!(), + } + } +} + #[repr(C)] #[derive(Clone)] pub struct wasm_shared_module_t { @@ -49,25 +72,7 @@ pub extern "C" fn wasmtime_module_new( ) -> Option> { let binary = binary.as_slice(); handle_result(Module::from_binary(&engine.engine, binary), |module| { - let imports = module - .imports() - .map(|i| { - wasm_importtype_t::new( - i.module().to_owned(), - i.name().map(|s| s.to_owned()), - i.ty(), - ) - }) - .collect::>(); - let exports = module - .exports() - .map(|e| wasm_exporttype_t::new(e.name().to_owned(), e.ty())) - .collect::>(); - let module = Box::new(wasm_module_t { - module: module, - imports, - exports, - }); + let module = Box::new(wasm_module_t::new(module)); *ret = Box::into_raw(module); }) } @@ -86,30 +91,46 @@ pub extern "C" fn wasmtime_module_validate( handle_result(Module::validate(store.store.engine(), binary), |()| {}) } +#[no_mangle] +pub extern "C" fn wasm_module_as_extern(m: &wasm_module_t) -> &wasm_extern_t { + &m.ext +} + #[no_mangle] pub extern "C" fn wasm_module_exports(module: &wasm_module_t, out: &mut wasm_exporttype_vec_t) { - let buffer = module - .exports - .iter() - .map(|et| Some(Box::new(et.clone()))) + let exports = module + .module() + .exports() + .map(|e| { + Some(Box::new(wasm_exporttype_t::new( + e.name().to_owned(), + e.ty(), + ))) + }) .collect::>(); - out.set_buffer(buffer); + out.set_buffer(exports); } #[no_mangle] pub extern "C" fn wasm_module_imports(module: &wasm_module_t, out: &mut wasm_importtype_vec_t) { - let buffer = module - .imports - .iter() - .map(|it| Some(Box::new(it.clone()))) + let imports = module + .module() + .imports() + .map(|i| { + Some(Box::new(wasm_importtype_t::new( + i.module().to_owned(), + i.name().map(|s| s.to_owned()), + i.ty(), + ))) + }) .collect::>(); - out.set_buffer(buffer); + out.set_buffer(imports); } #[no_mangle] pub extern "C" fn wasm_module_share(module: &wasm_module_t) -> Box { Box::new(wasm_shared_module_t { - module: module.module.clone(), + module: module.module().clone(), }) } @@ -122,25 +143,7 @@ pub extern "C" fn wasm_module_obtain( if !Engine::same(store.store.engine(), module.engine()) { return None; } - let imports = module - .imports() - .map(|i| { - wasm_importtype_t::new( - i.module().to_owned(), - i.name().map(|s| s.to_owned()), - i.ty(), - ) - }) - .collect::>(); - let exports = module - .exports() - .map(|e| wasm_exporttype_t::new(e.name().to_owned(), e.ty())) - .collect::>(); - Some(Box::new(wasm_module_t { - module: module, - imports, - exports, - })) + Some(Box::new(wasm_module_t::new(module))) } #[no_mangle] @@ -171,7 +174,7 @@ pub extern "C" fn wasmtime_module_serialize( module: &wasm_module_t, ret: &mut wasm_byte_vec_t, ) -> Option> { - handle_result(module.module.serialize(), |buf| { + handle_result(module.module().serialize(), |buf| { ret.set_buffer(buf); }) } @@ -185,26 +188,13 @@ pub extern "C" fn wasmtime_module_deserialize( handle_result( Module::deserialize(&engine.engine, binary.as_slice()), |module| { - let imports = module - .imports() - .map(|i| { - wasm_importtype_t::new( - i.module().to_owned(), - i.name().map(|s| s.to_owned()), - i.ty(), - ) - }) - .collect::>(); - let exports = module - .exports() - .map(|e| wasm_exporttype_t::new(e.name().to_owned(), e.ty())) - .collect::>(); - let module = Box::new(wasm_module_t { - module: module, - imports, - exports, - }); + let module = Box::new(wasm_module_t::new(module)); *ret = Box::into_raw(module); }, ) } + +#[no_mangle] +pub extern "C" fn wasm_module_type(f: &wasm_module_t) -> Box { + Box::new(wasm_moduletype_t::new(f.module().ty())) +} diff --git a/crates/c-api/src/types/extern.rs b/crates/c-api/src/types/extern.rs index fefc62c511..d86a9e0810 100644 --- a/crates/c-api/src/types/extern.rs +++ b/crates/c-api/src/types/extern.rs @@ -27,8 +27,8 @@ pub const WASM_EXTERN_FUNC: wasm_externkind_t = 0; pub const WASM_EXTERN_GLOBAL: wasm_externkind_t = 1; pub const WASM_EXTERN_TABLE: wasm_externkind_t = 2; pub const WASM_EXTERN_MEMORY: wasm_externkind_t = 3; -pub const WASMTIME_EXTERN_MODULE: wasm_externkind_t = 4; -pub const WASMTIME_EXTERN_INSTANCE: wasm_externkind_t = 5; +pub const WASM_EXTERN_MODULE: wasm_externkind_t = 4; +pub const WASM_EXTERN_INSTANCE: wasm_externkind_t = 5; impl wasm_externtype_t { pub(crate) fn new(ty: ExternType) -> wasm_externtype_t { @@ -63,8 +63,8 @@ pub extern "C" fn wasm_externtype_kind(et: &wasm_externtype_t) -> wasm_externkin CExternType::Table(_) => WASM_EXTERN_TABLE, CExternType::Global(_) => WASM_EXTERN_GLOBAL, CExternType::Memory(_) => WASM_EXTERN_MEMORY, - CExternType::Instance(_) => WASMTIME_EXTERN_INSTANCE, - CExternType::Module(_) => WASMTIME_EXTERN_MODULE, + CExternType::Instance(_) => WASM_EXTERN_INSTANCE, + CExternType::Module(_) => WASM_EXTERN_MODULE, } } diff --git a/crates/c-api/src/types/instance.rs b/crates/c-api/src/types/instance.rs index 83ec12d893..373431c362 100644 --- a/crates/c-api/src/types/instance.rs +++ b/crates/c-api/src/types/instance.rs @@ -1,5 +1,4 @@ -use crate::{wasm_externtype_t, wasm_limits_t, CExternType}; -use once_cell::unsync::OnceCell; +use crate::{wasm_exporttype_t, wasm_exporttype_vec_t, wasm_externtype_t, CExternType}; use wasmtime::InstanceType; #[repr(transparent)] @@ -13,24 +12,33 @@ wasmtime_c_api_macros::declare_ty!(wasm_instancetype_t); #[derive(Clone)] pub(crate) struct CInstanceType { pub(crate) ty: InstanceType, - limits_cache: OnceCell, } impl wasm_instancetype_t { + pub(crate) fn new(ty: InstanceType) -> wasm_instancetype_t { + wasm_instancetype_t { + ext: wasm_externtype_t::new(ty.into()), + } + } + pub(crate) fn try_from(e: &wasm_externtype_t) -> Option<&wasm_instancetype_t> { match &e.which { CExternType::Instance(_) => Some(unsafe { &*(e as *const _ as *const _) }), _ => None, } } + + pub(crate) fn ty(&self) -> &CInstanceType { + match &self.ext.which { + CExternType::Instance(f) => &f, + _ => unreachable!(), + } + } } impl CInstanceType { pub(crate) fn new(ty: InstanceType) -> CInstanceType { - CInstanceType { - ty, - limits_cache: OnceCell::new(), - } + CInstanceType { ty } } } #[no_mangle] @@ -44,3 +52,22 @@ pub extern "C" fn wasm_instancetype_as_externtype_const( ) -> &wasm_externtype_t { &ty.ext } + +#[no_mangle] +pub extern "C" fn wasm_instancetype_exports( + instance: &wasm_instancetype_t, + out: &mut wasm_exporttype_vec_t, +) { + let exports = instance + .ty() + .ty + .exports() + .map(|e| { + Some(Box::new(wasm_exporttype_t::new( + e.name().to_owned(), + e.ty(), + ))) + }) + .collect::>(); + out.set_buffer(exports); +} diff --git a/crates/c-api/src/types/module.rs b/crates/c-api/src/types/module.rs index 3ac3b80757..eaa681c608 100644 --- a/crates/c-api/src/types/module.rs +++ b/crates/c-api/src/types/module.rs @@ -1,5 +1,7 @@ -use crate::{wasm_externtype_t, wasm_limits_t, CExternType}; -use once_cell::unsync::OnceCell; +use crate::{ + wasm_exporttype_t, wasm_exporttype_vec_t, wasm_externtype_t, wasm_importtype_t, + wasm_importtype_vec_t, CExternType, +}; use wasmtime::ModuleType; #[repr(transparent)] @@ -13,24 +15,33 @@ wasmtime_c_api_macros::declare_ty!(wasm_moduletype_t); #[derive(Clone)] pub(crate) struct CModuleType { pub(crate) ty: ModuleType, - limits_cache: OnceCell, } impl wasm_moduletype_t { + pub(crate) fn new(ty: ModuleType) -> wasm_moduletype_t { + wasm_moduletype_t { + ext: wasm_externtype_t::new(ty.into()), + } + } + pub(crate) fn try_from(e: &wasm_externtype_t) -> Option<&wasm_moduletype_t> { match &e.which { CExternType::Module(_) => Some(unsafe { &*(e as *const _ as *const _) }), _ => None, } } + + pub(crate) fn ty(&self) -> &CModuleType { + match &self.ext.which { + CExternType::Module(f) => &f, + _ => unreachable!(), + } + } } impl CModuleType { pub(crate) fn new(ty: ModuleType) -> CModuleType { - CModuleType { - ty, - limits_cache: OnceCell::new(), - } + CModuleType { ty } } } @@ -45,3 +56,42 @@ pub extern "C" fn wasm_moduletype_as_externtype_const( ) -> &wasm_externtype_t { &ty.ext } + +#[no_mangle] +pub extern "C" fn wasm_moduletype_exports( + module: &wasm_moduletype_t, + out: &mut wasm_exporttype_vec_t, +) { + let exports = module + .ty() + .ty + .exports() + .map(|e| { + Some(Box::new(wasm_exporttype_t::new( + e.name().to_owned(), + e.ty(), + ))) + }) + .collect::>(); + out.set_buffer(exports); +} + +#[no_mangle] +pub extern "C" fn wasm_moduletype_imports( + module: &wasm_moduletype_t, + out: &mut wasm_importtype_vec_t, +) { + let imports = module + .ty() + .ty + .imports() + .map(|i| { + Some(Box::new(wasm_importtype_t::new( + i.module().to_owned(), + i.name().map(|s| s.to_owned()), + i.ty(), + ))) + }) + .collect::>(); + out.set_buffer(imports); +} diff --git a/crates/c-api/src/vec.rs b/crates/c-api/src/vec.rs index 2952e6553e..21a47f0717 100644 --- a/crates/c-api/src/vec.rs +++ b/crates/c-api/src/vec.rs @@ -1,7 +1,8 @@ -use crate::wasm_valtype_t; -use crate::{wasm_exporttype_t, wasm_extern_t, wasm_frame_t, wasm_val_t}; -use crate::{wasm_externtype_t, wasm_importtype_t, wasm_memorytype_t}; -use crate::{wasm_functype_t, wasm_globaltype_t, wasm_tabletype_t}; +use crate::{ + wasm_exporttype_t, wasm_extern_t, wasm_externtype_t, wasm_frame_t, wasm_functype_t, + wasm_globaltype_t, wasm_importtype_t, wasm_instancetype_t, wasm_memorytype_t, + wasm_moduletype_t, wasm_tabletype_t, wasm_val_t, wasm_valtype_t, +}; use std::mem; use std::ptr; use std::slice; @@ -172,6 +173,24 @@ declare_vecs! { copy: wasm_memorytype_vec_copy, delete: wasm_memorytype_vec_delete, ) + ( + name: wasm_instancetype_vec_t, + ty: Option>, + new: wasm_instancetype_vec_new, + empty: wasm_instancetype_vec_new_empty, + uninit: wasm_instancetype_vec_new_uninitialized, + copy: wasm_instancetype_vec_copy, + delete: wasm_instancetype_vec_delete, + ) + ( + name: wasm_moduletype_vec_t, + ty: Option>, + new: wasm_moduletype_vec_new, + empty: wasm_moduletype_vec_new_empty, + uninit: wasm_moduletype_vec_new_uninitialized, + copy: wasm_moduletype_vec_copy, + delete: wasm_moduletype_vec_delete, + ) ( name: wasm_externtype_vec_t, ty: Option>, From f7cf771ee674c0315b0602449e65959a0937066d Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 15 Oct 2020 17:00:02 -0700 Subject: [PATCH 40/63] souper-harvest: Do not generate assignments of constants It turns out that Souper does not allow a constant to be assigned to a variable, they may only be used as operands. The 2.0.0 version of the `souper-ir` crate correctly reflects this. In the `cranelift_codegen::souper_harvest` module, we need to modify our Souper IR harvester so that it delays converting `iconst` and `bconst` into Souper IR until their values are used as operands. Finally, some unit tests in the `peepmatic-souper` crate need some small updates as well. --- Cargo.lock | 4 +- cranelift/codegen/Cargo.toml | 2 +- cranelift/codegen/src/souper_harvest.rs | 68 ++++++++++++++------ cranelift/peepmatic/crates/souper/Cargo.toml | 2 +- cranelift/peepmatic/crates/souper/src/lib.rs | 19 ++---- 5 files changed, 60 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5e646f2a6..f177b974ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1989,9 +1989,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "souper-ir" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "163cc2bdd8a66cbaccdf06a6b476689a97e928883e09bffbe06fd5945842a83f" +checksum = "1c7eaed476845a4755001a900e0412dabd356d312ddc35dd32ff91de266b1558" dependencies = [ "id-arena", ] diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index bdf7dfd080..3f6874778e 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -29,7 +29,7 @@ peepmatic = { path = "../peepmatic", optional = true, version = "0.68.0" } peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.68.0" } peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.68.0" } regalloc = { version = "0.0.31" } -souper-ir = { version = "1", optional = true } +souper-ir = { version = "2.0.0", optional = true } wast = { version = "28.0.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary diff --git a/cranelift/codegen/src/souper_harvest.rs b/cranelift/codegen/src/souper_harvest.rs index fcf53b0ed5..7f8c34d4a2 100644 --- a/cranelift/codegen/src/souper_harvest.rs +++ b/cranelift/codegen/src/souper_harvest.rs @@ -142,7 +142,49 @@ fn harvest_candidate_lhs( let souper_assignment_rhs = match func.dfg.value_def(val) { ir::ValueDef::Result(inst, 0) => { let args = func.dfg.inst_args(inst); - let arg = |allocs: &mut Allocs, n| allocs.ir_to_souper_val[&args[n]].into(); + + // Get the n^th argument as a souper operand. + let arg = |allocs: &mut Allocs, n| { + let arg = args[n]; + if let Some(a) = allocs.ir_to_souper_val.get(&arg).copied() { + a.into() + } else { + // The only arguments we get that we haven't already + // converted into a souper instruction are `iconst`s and + // `bconst`s. This is because souper only allows + // constants as operands, and it doesn't allow assigning + // constants to a variable name. So we lazily convert + // `iconst`s and `bconst`s into souper operands here, + // when they are actually used. + match func.dfg.value_def(arg) { + ir::ValueDef::Result(inst, 0) => match func.dfg[inst] { + ir::InstructionData::UnaryImm { opcode, imm } => { + debug_assert_eq!(opcode, ir::Opcode::Iconst); + let imm: i64 = imm.into(); + ast::Operand::Constant(ast::Constant { + value: imm.into(), + r#type: souper_type_of(&func.dfg, arg), + }) + } + ir::InstructionData::UnaryBool { opcode, imm } => { + debug_assert_eq!(opcode, ir::Opcode::Iconst); + ast::Operand::Constant(ast::Constant { + value: imm.into(), + r#type: souper_type_of(&func.dfg, arg), + }) + } + _ => unreachable!( + "only iconst and bconst instructions \ + aren't in `ir_to_souper_val`" + ), + }, + _ => unreachable!( + "only iconst and bconst instructions \ + aren't in `ir_to_souper_val`" + ), + } + } + }; match (func.dfg[inst].opcode(), &func.dfg[inst]) { (ir::Opcode::Iadd, _) => { @@ -421,23 +463,13 @@ fn harvest_candidate_lhs( let b = arg(allocs, 1); ast::Instruction::UsubSat { a, b }.into() } - (ir::Opcode::Iconst, ir::InstructionData::UnaryImm { imm, .. }) => { - let value: i64 = (*imm).into(); - let value: i128 = value.into(); - ast::Constant { - value, - r#type: souper_type_of(&func.dfg, val), - } - .into() - } - (ir::Opcode::Bconst, ir::InstructionData::UnaryBool { imm, .. }) => { - let value = *imm as i128; - ast::Constant { - value, - r#type: souper_type_of(&func.dfg, val), - } - .into() - } + // Because Souper doesn't allow constants to be on the right + // hand side of an assignment (i.e. `%0:i32 = 1234` is + // disallowed) we have to ignore `iconst` and `bconst` + // instructions until we process them as operands for some + // other instruction. See the `arg` closure above for + // details. + (ir::Opcode::Iconst, _) | (ir::Opcode::Bconst, _) => return, _ => ast::AssignmentRhs::Var, } } diff --git a/cranelift/peepmatic/crates/souper/Cargo.toml b/cranelift/peepmatic/crates/souper/Cargo.toml index 7792f5268a..e5717ecea8 100644 --- a/cranelift/peepmatic/crates/souper/Cargo.toml +++ b/cranelift/peepmatic/crates/souper/Cargo.toml @@ -10,7 +10,7 @@ description = "Converting Souper optimizations into Peepmatic DSL" [dependencies] anyhow = "1" -souper-ir = { version = "1", features = ["parse"] } +souper-ir = { version = "2.0.0", features = ["parse"] } log = "0.4.8" [dev-dependencies] diff --git a/cranelift/peepmatic/crates/souper/src/lib.rs b/cranelift/peepmatic/crates/souper/src/lib.rs index 705abe549f..fa1e4a2748 100644 --- a/cranelift/peepmatic/crates/souper/src/lib.rs +++ b/cranelift/peepmatic/crates/souper/src/lib.rs @@ -174,7 +174,6 @@ fn convert_operand( } Some(format!("{}", convert_name(&assn.name))) } - ast::AssignmentRhs::Constant(c) => Some(format!("{}", c.value)), ast::AssignmentRhs::Instruction(inst) => match inst { // Unsupported instructions. ast::Instruction::Bswap { .. } @@ -619,8 +618,7 @@ mod tests { " %0:i64 = var %1:i32 = trunc %0 - %2:i32 = 0 - cand %1 %2 + cand %1 0 ", "\ (=> (when (ireduce {i32} $v0) @@ -631,8 +629,7 @@ mod tests { " %0:i32 = var %1:i64 = sext %0 - %2:i64 = 0 - cand %1 %2 + cand %1 0 ", "\ (=> (when (sextend {i64} $v0) @@ -643,8 +640,7 @@ mod tests { " %0:i32 = var %1:i64 = zext %0 - %2:i64 = 0 - cand %1 %2 + cand %1 0 ", "\ (=> (when (uextend {i64} $v0) @@ -677,8 +673,7 @@ mod tests { %1:i32 = var %2:i1 = eq %0, %1 %3:i32 = zext %2 - %4:i32 = 0 - cand %3 %4 + cand %3 0 ", "\ (=> (when (bint (icmp eq $v0 $v1)) @@ -693,8 +688,7 @@ mod tests { " %0:i32 = var %1:i32 = add %0, 1 - %2:i32 = 0 - cand %1 %2 + cand %1 0 ", "\ (=> (when (iadd_imm 1 $v0) @@ -705,8 +699,7 @@ mod tests { " %0:i32 = var %1:i32 = add 1, %0 - %2:i32 = 0 - cand %1 %2 + cand %1 0 ", "\ (=> (when (iadd_imm 1 $v0) From 1efdf10ca79d7971e9c80603edb46777e2dd07f3 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 16 Oct 2020 11:18:43 -0700 Subject: [PATCH 41/63] souper-harvest: ensure that `select` conditions are of type `i1` --- Cargo.lock | 4 ++-- cranelift/codegen/Cargo.toml | 2 +- cranelift/codegen/src/souper_harvest.rs | 22 ++++++++++++++++++++ cranelift/peepmatic/crates/souper/Cargo.toml | 2 +- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f177b974ab..716f156bfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1989,9 +1989,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "souper-ir" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c7eaed476845a4755001a900e0412dabd356d312ddc35dd32ff91de266b1558" +checksum = "a50c18ce33988e1973003afbaa66e6a465ad7a614dc33f246879ccc209c2c044" dependencies = [ "id-arena", ] diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 3f6874778e..cc0a6fd62d 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -29,7 +29,7 @@ peepmatic = { path = "../peepmatic", optional = true, version = "0.68.0" } peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.68.0" } peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.68.0" } regalloc = { version = "0.0.31" } -souper-ir = { version = "2.0.0", optional = true } +souper-ir = { version = "2.1.0", optional = true } wast = { version = "28.0.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary diff --git a/cranelift/codegen/src/souper_harvest.rs b/cranelift/codegen/src/souper_harvest.rs index 7f8c34d4a2..4884ae5002 100644 --- a/cranelift/codegen/src/souper_harvest.rs +++ b/cranelift/codegen/src/souper_harvest.rs @@ -392,6 +392,28 @@ fn harvest_candidate_lhs( } (ir::Opcode::Select, _) => { let a = arg(allocs, 0); + + // While Cranelift allows any width condition for + // `select`, Souper requires an `i1`. + let a = match a { + ast::Operand::Value(id) => match lhs.get_value(id).r#type { + Some(ast::Type { width: 1 }) => a, + _ => lhs + .assignment( + None, + Some(ast::Type { width: 1 }), + ast::Instruction::Trunc { a }, + vec![], + ) + .into(), + }, + ast::Operand::Constant(ast::Constant { value, .. }) => ast::Constant { + value: (value != 0) as _, + r#type: Some(ast::Type { width: 1 }), + } + .into(), + }; + let b = arg(allocs, 1); let c = arg(allocs, 2); ast::Instruction::Select { a, b, c }.into() diff --git a/cranelift/peepmatic/crates/souper/Cargo.toml b/cranelift/peepmatic/crates/souper/Cargo.toml index e5717ecea8..84bf6f52e0 100644 --- a/cranelift/peepmatic/crates/souper/Cargo.toml +++ b/cranelift/peepmatic/crates/souper/Cargo.toml @@ -10,7 +10,7 @@ description = "Converting Souper optimizations into Peepmatic DSL" [dependencies] anyhow = "1" -souper-ir = { version = "2.0.0", features = ["parse"] } +souper-ir = { version = "2.1.0", features = ["parse"] } log = "0.4.8" [dev-dependencies] From 3e516e784b6e953fbb426eb38af57d5d72fdbcd0 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 3 Dec 2020 14:33:43 -0800 Subject: [PATCH 42/63] Fix lowering instruction-sinking (load-merging) bug. This fixes a subtle corner case exposed during fuzzing. If we have a bit of CLIF like: ``` v0 = load.i64 ... v1 = iadd.i64 v0, ... v2 = do_other_thing v1 v3 = load.i64 v1 ``` and if this is lowered using a machine backend that can merge loads into ALU ops, *and* that has an addressing mode that can look through add ops, then the following can happen: 1. We lower the load at `v3`. This looks backward at the address operand tree and finds that `v1` is `v0` plus other things; it has an addressing mode that can add `v0`'s register and the other things directly; so it calls `put_value_in_reg(v0)` and uses its register in the amode. At this point, the add producing `v1` has no references, so it will not (yet) be codegen'd. 2. We lower `do_other_thing`, which puts `v1` in a register and uses it. the `iadd` now has a reference. 3. We reach the `iadd` and, because it has a reference, lower it. Our machine has the ability to merge a load into an ALU operation. Crucially, *we think the load at `v0` is mergeable* because it has only one user, the add at `v1` (!). So we merge it. 4. We reach the `load` at `v0` and because it has been merged into the `iadd`, we do not separately codegen it. The register that holds `v0` is thus never written, and the use of this register by the final load (Step 1) will see an undefined value. The logic error here is that in the presence of pattern matching that looks through pure ops, we can end up with multiple uses of a value that originally had a single use (because we allow lookthrough of pure ops in all cases). In other words, the multiple-use-ness of `v1` "passes through" in some sense to `v0`. However, the load sinking logic is not aware of this. The fix, I think, is pretty simple: we disallow an effectful instruction from sinking/merging if it already has some other use when we look back at it. If we disallowed lookthrough of *any* op that had multiple uses, even pure ones, then we would avoid this scenario; but earlier experiments showed that to have a non-negligible performance impact, so (given that we've worked out the logic above) I think this complexity is worth it. --- cranelift/codegen/src/machinst/lower.rs | 1 + .../filetests/filetests/isa/x64/load-op.clif | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 32dfaa336c..6a42f3a2b6 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -913,6 +913,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { // the code-motion. if self.cur_scan_entry_color.is_some() && self.value_uses[val] == 1 + && self.value_lowered_uses[val] == 0 && self.num_outputs(src_inst) == 1 && self .side_effect_inst_entry_colors diff --git a/cranelift/filetests/filetests/isa/x64/load-op.clif b/cranelift/filetests/filetests/isa/x64/load-op.clif index 87069262ed..77e4b05420 100644 --- a/cranelift/filetests/filetests/isa/x64/load-op.clif +++ b/cranelift/filetests/filetests/isa/x64/load-op.clif @@ -44,3 +44,18 @@ block0(v0: i64, v1: i8): ; nextln: addl %esi, %r12d return v3 } + +function %no_merge_if_lookback_use(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = load.i64 v0 + v3 = iadd.i64 v2, v0 + store.i64 v3, v1 + v4 = load.i64 v3 + return v4 + ; check: movq 0(%rdi), %r12 + ; nextln: movq %r12, %r13 + ; nextln: addq %rdi, %r13 + ; nextln: movq %r13, 0(%rsi) + ; nextln: movq 0(%r12,%rdi,1), %r12 + ; nextln: movq %r12, %rax +} From d07fffeb411d13e5b43e6890f3e1e1cd544b8c0f Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Fri, 4 Dec 2020 07:07:19 -0800 Subject: [PATCH 43/63] Use AlexNet for wasi-nn example (#2474) --- ci/run-wasi-nn-example.sh | 8 ++-- .../classification-example/src/main.rs | 45 ++++++++++++++----- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/ci/run-wasi-nn-example.sh b/ci/run-wasi-nn-example.sh index 5a77c06fa5..e24ffa75ac 100755 --- a/ci/run-wasi-nn-example.sh +++ b/ci/run-wasi-nn-example.sh @@ -7,7 +7,7 @@ # executed with the Wasmtime CLI. set -e WASMTIME_DIR=$(dirname "$0" | xargs dirname) -FIXTURE=https://gist.github.com/abrown/c7847bf3701f9efbb2070da1878542c1/raw/07a9f163994b0ff8f0d7c5a5c9645ec3d8b24024 +FIXTURE=https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/alexnet if [ -z "${1+x}" ]; then # If no temporary directory is specified, create one. TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX) @@ -26,9 +26,9 @@ source /opt/intel/openvino/bin/setupvars.sh OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo build -p wasmtime-cli --features wasi-nn # Download all necessary test fixtures to the temporary directory. -wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/frozen_inference_graph.bin -wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/frozen_inference_graph.xml -wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/tensor-1x3x300x300-f32.bgr +wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/alexnet.bin +wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/alexnet.xml +wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/tensor-1x3x227x227-f32.bgr # Now build an example that uses the wasi-nn API. pushd $WASMTIME_DIR/crates/wasi-nn/examples/classification-example diff --git a/crates/wasi-nn/examples/classification-example/src/main.rs b/crates/wasi-nn/examples/classification-example/src/main.rs index 898a4bfff3..3465de5cae 100644 --- a/crates/wasi-nn/examples/classification-example/src/main.rs +++ b/crates/wasi-nn/examples/classification-example/src/main.rs @@ -3,11 +3,11 @@ use std::fs; use wasi_nn; pub fn main() { - let xml = fs::read_to_string("fixture/frozen_inference_graph.xml").unwrap(); - println!("First 50 characters of graph: {}", &xml[..50]); + let xml = fs::read_to_string("fixture/alexnet.xml").unwrap(); + println!("Read graph XML, first 50 characters: {}", &xml[..50]); - let weights = fs::read("fixture/frozen_inference_graph.bin").unwrap(); - println!("Size of weights: {}", weights.len()); + let weights = fs::read("fixture/alexnet.bin").unwrap(); + println!("Read graph weights, size in bytes: {}", weights.len()); let graph = unsafe { wasi_nn::load( @@ -17,17 +17,17 @@ pub fn main() { ) .unwrap() }; - println!("Graph handle ID: {}", graph); + println!("Loaded graph into wasi-nn with ID: {}", graph); let context = unsafe { wasi_nn::init_execution_context(graph).unwrap() }; - println!("Execution context ID: {}", context); + println!("Created wasi-nn execution context with ID: {}", context); // Load a tensor that precisely matches the graph input tensor (see // `fixture/frozen_inference_graph.xml`). - let tensor_data = fs::read("fixture/tensor-1x3x300x300-f32.bgr").unwrap(); - println!("Tensor bytes: {}", tensor_data.len()); + let tensor_data = fs::read("fixture/tensor-1x3x227x227-f32.bgr").unwrap(); + println!("Read input tensor, size in bytes: {}", tensor_data.len()); let tensor = wasi_nn::Tensor { - dimensions: &[1, 3, 300, 300], + dimensions: &[1, 3, 227, 227], r#type: wasi_nn::TENSOR_TYPE_F32, data: &tensor_data, }; @@ -39,9 +39,10 @@ pub fn main() { unsafe { wasi_nn::compute(context).unwrap(); } + println!("Executed graph inference"); - // Retrieve the output (TODO output looks incorrect). - let mut output_buffer = vec![0f32; 1 << 20]; + // Retrieve the output. + let mut output_buffer = vec![0f32; 1000]; unsafe { wasi_nn::get_output( context, @@ -50,5 +51,25 @@ pub fn main() { (output_buffer.len() * 4).try_into().unwrap(), ); } - println!("output tensor: {:?}", &output_buffer[..1000]) + println!( + "Found results, sorted top 5: {:?}", + &sort_results(&output_buffer)[..5] + ) } + +// Sort the buffer of probabilities. The graph places the match probability for each class at the +// index for that class (e.g. the probability of class 42 is placed at buffer[42]). Here we convert +// to a wrapping InferenceResult and sort the results. +fn sort_results(buffer: &[f32]) -> Vec { + let mut results: Vec = buffer + .iter() + .enumerate() + .map(|(c, p)| InferenceResult(c, *p)) + .collect(); + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); + results +} + +// A wrapper for class ID and match probabilities. +#[derive(Debug, PartialEq)] +struct InferenceResult(usize, f32); From 8f34d2dc59ef37d12bdeff0a3e2664134d3cd39d Mon Sep 17 00:00:00 2001 From: Julian Seward Date: Fri, 4 Dec 2020 18:29:39 +0100 Subject: [PATCH 44/63] aarch64 isel: collect_address_addends: correctly handle `ExtendOp::UXTW(negative immediate)`. The current code doesn't correctly handle the case where `ExtendOp::UXTW` has as source, a constant-producing insn that produces a negative (32-bit) value. Then the value is incorrectly sign-extended to 64 bits (in fact, this has already been done by `ctx.get_constant(insn)`), whereas it needs to be zero extended. The obvious fix, done here, is just to force bits 63:32 of the extension to zero, hence zero-extending it. --- cranelift/codegen/src/isa/aarch64/lower.rs | 2 +- .../filetests/isa/aarch64/amodes.clif | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 75b4cbe727..1c8f407b7b 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -616,7 +616,7 @@ fn collect_address_addends>( maybe_input_insn(ctx, extendee_input, Opcode::Iconst), extendop, ) { - let value = ctx.get_constant(insn).unwrap() as i64; + let value = (ctx.get_constant(insn).unwrap() & 0xFFFF_FFFF_u64) as i64; offset += value; } else { let reg = put_input_in_reg(ctx, extendee_input, NarrowValueMode::None); diff --git a/cranelift/filetests/filetests/isa/aarch64/amodes.clif b/cranelift/filetests/filetests/isa/aarch64/amodes.clif index ad109e340e..3f94a149f8 100644 --- a/cranelift/filetests/filetests/isa/aarch64/amodes.clif +++ b/cranelift/filetests/filetests/isa/aarch64/amodes.clif @@ -344,3 +344,69 @@ block0(v0: i64, v1: i32): ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret + +function %f18(i64, i64, i64) -> i32 { +block0(v0: i64, v1: i64, v2: i64): + v3 = iconst.i32 -4098 + v6 = uextend.i64 v3 + v5 = sload16.i32 v6+0 + return v5 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: movn w0, #4097 +; nextln: ldrsh x0, [x0] +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f19(i64, i64, i64) -> i32 { +block0(v0: i64, v1: i64, v2: i64): + v3 = iconst.i32 4098 + v6 = uextend.i64 v3 + v5 = sload16.i32 v6+0 + return v5 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: movz x0, #4098 +; nextln: ldrsh x0, [x0] +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f20(i64, i64, i64) -> i32 { +block0(v0: i64, v1: i64, v2: i64): + v3 = iconst.i32 -4098 + v6 = sextend.i64 v3 + v5 = sload16.i32 v6+0 + return v5 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: movn w0, #4097 +; nextln: sxtw x0, w0 +; nextln: ldrsh x0, [x0] +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f21(i64, i64, i64) -> i32 { +block0(v0: i64, v1: i64, v2: i64): + v3 = iconst.i32 4098 + v6 = sextend.i64 v3 + v5 = sload16.i32 v6+0 + return v5 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: movz x0, #4098 +; nextln: sxtw x0, w0 +; nextln: ldrsh x0, [x0] +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret From 411ec3a857651b845fa19cf859e74d71e1aed539 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 4 Dec 2020 19:28:31 +0100 Subject: [PATCH 45/63] Rename SimpleJIT to JIT as it isn't simple anymore --- Cargo.lock | 42 ++++++++--------- cranelift/Cargo.toml | 2 +- cranelift/README.md | 4 +- cranelift/docs/index.md | 4 +- cranelift/filetests/src/function_runner.rs | 2 +- cranelift/{simplejit => jit}/Cargo.toml | 6 +-- cranelift/{simplejit => jit}/LICENSE | 0 cranelift/{simplejit => jit}/README.md | 4 +- .../examples/jit-minimal.rs} | 5 +- cranelift/{simplejit => jit}/src/backend.rs | 46 +++++++++---------- .../{simplejit => jit}/src/compiled_blob.rs | 0 cranelift/{simplejit => jit}/src/lib.rs | 4 +- cranelift/{simplejit => jit}/src/memory.rs | 0 cranelift/{simplejit => jit}/tests/basic.rs | 13 ++---- cranelift/module/README.md | 8 ++-- scripts/publish.rs | 2 +- 16 files changed, 68 insertions(+), 74 deletions(-) rename cranelift/{simplejit => jit}/Cargo.toml (89%) rename cranelift/{simplejit => jit}/LICENSE (100%) rename cranelift/{simplejit => jit}/README.md (66%) rename cranelift/{simplejit/examples/simplejit-minimal.rs => jit/examples/jit-minimal.rs} (94%) rename cranelift/{simplejit => jit}/src/backend.rs (96%) rename cranelift/{simplejit => jit}/src/compiled_blob.rs (100%) rename cranelift/{simplejit => jit}/src/lib.rs (87%) rename cranelift/{simplejit => jit}/src/memory.rs (100%) rename cranelift/{simplejit => jit}/tests/basic.rs (93%) diff --git a/Cargo.lock b/Cargo.lock index 6f458b7096..844b60b4bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -462,6 +462,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cranelift-jit" +version = "0.68.0" +dependencies = [ + "anyhow", + "cranelift", + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "cranelift-module", + "cranelift-native", + "errno", + "libc", + "log", + "memmap", + "region", + "target-lexicon", + "winapi", +] + [[package]] name = "cranelift-module" version = "0.68.0" @@ -527,26 +547,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "cranelift-simplejit" -version = "0.68.0" -dependencies = [ - "anyhow", - "cranelift", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-module", - "cranelift-native", - "errno", - "libc", - "log", - "memmap", - "region", - "target-lexicon", - "winapi", -] - [[package]] name = "cranelift-tools" version = "0.66.0" @@ -560,13 +560,13 @@ dependencies = [ "cranelift-filetests", "cranelift-frontend", "cranelift-interpreter", + "cranelift-jit", "cranelift-module", "cranelift-native", "cranelift-object", "cranelift-preopt", "cranelift-reader", "cranelift-serde", - "cranelift-simplejit", "cranelift-wasm", "file-per-thread-logger", "filecheck", diff --git a/cranelift/Cargo.toml b/cranelift/Cargo.toml index 9594f22bf0..cd33d160a2 100644 --- a/cranelift/Cargo.toml +++ b/cranelift/Cargo.toml @@ -26,7 +26,7 @@ cranelift-native = { path = "native", version = "0.68.0" } cranelift-filetests = { path = "filetests", version = "0.66.0" } cranelift-module = { path = "module", version = "0.68.0" } cranelift-object = { path = "object", version = "0.68.0" } -cranelift-simplejit = { path = "simplejit", version = "0.68.0" } +cranelift-jit = { path = "jit", version = "0.68.0" } cranelift-preopt = { path = "preopt", version = "0.68.0" } cranelift = { path = "umbrella", version = "0.68.0" } filecheck = "0.5.0" diff --git a/cranelift/README.md b/cranelift/README.md index 7f35e570f7..52d4e0cf8f 100644 --- a/cranelift/README.md +++ b/cranelift/README.md @@ -16,10 +16,10 @@ into executable machine code. For more information, see [the documentation](docs/index.md). -For an example of how to use the JIT, see the [SimpleJIT Demo], which +For an example of how to use the JIT, see the [JIT Demo], which implements a toy language. -[SimpleJIT Demo]: https://github.com/bytecodealliance/simplejit-demo +[JIT Demo]: https://github.com/bytecodealliance/simplejit-demo For an example of how to use Cranelift to run WebAssembly code, see [Wasmtime], which implements a standalone, embeddable, VM using Cranelift. diff --git a/cranelift/docs/index.md b/cranelift/docs/index.md index 2334325f15..62c4e51bd8 100644 --- a/cranelift/docs/index.md +++ b/cranelift/docs/index.md @@ -52,6 +52,6 @@ emits native object files using the `object `_ library. - - [cranelift-simplejit](https://docs.rs/cranelift-simplejit) - This crate provides a simple JIT backend for `cranelift-module`, which + - [cranelift-jit](https://docs.rs/cranelift-jit) + This crate provides a JIT backend for `cranelift-module`, which emits code and data into memory. diff --git a/cranelift/filetests/src/function_runner.rs b/cranelift/filetests/src/function_runner.rs index ed8411f448..01ddc1f3dd 100644 --- a/cranelift/filetests/src/function_runner.rs +++ b/cranelift/filetests/src/function_runner.rs @@ -21,7 +21,7 @@ use thiserror::Error; /// `CompiledFunction`s and subsequently calling them through the use of a `Trampoline`. As its /// name indicates, this compiler is limited: any functionality that requires knowledge of things /// outside the [Function] will likely not work (e.g. global values, calls). For an example of this -/// "outside-of-function" functionality, see `cranelift_simplejit::backend::SimpleJITBackend`. +/// "outside-of-function" functionality, see `cranelift_jit::backend::JITBackend`. /// /// ``` /// use cranelift_filetests::SingleFunctionCompiler; diff --git a/cranelift/simplejit/Cargo.toml b/cranelift/jit/Cargo.toml similarity index 89% rename from cranelift/simplejit/Cargo.toml rename to cranelift/jit/Cargo.toml index ea8165035a..1b07e23c86 100644 --- a/cranelift/simplejit/Cargo.toml +++ b/cranelift/jit/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "cranelift-simplejit" +name = "cranelift-jit" version = "0.68.0" authors = ["The Cranelift Project Developers"] -description = "A simple JIT library backed by Cranelift" +description = "A JIT library backed by Cranelift" repository = "https://github.com/bytecodealliance/wasmtime" -documentation = "https://docs.rs/cranelift-simplejit" +documentation = "https://docs.rs/cranelift-jit" license = "Apache-2.0 WITH LLVM-exception" readme = "README.md" edition = "2018" diff --git a/cranelift/simplejit/LICENSE b/cranelift/jit/LICENSE similarity index 100% rename from cranelift/simplejit/LICENSE rename to cranelift/jit/LICENSE diff --git a/cranelift/simplejit/README.md b/cranelift/jit/README.md similarity index 66% rename from cranelift/simplejit/README.md rename to cranelift/jit/README.md index c3b0693b17..652f181fc8 100644 --- a/cranelift/simplejit/README.md +++ b/cranelift/jit/README.md @@ -1,8 +1,8 @@ -This crate provides a simple JIT library that uses +This crate provides a JIT library that uses [Cranelift](https://crates.io/crates/cranelift). This crate is extremely experimental. See the [example program] for a brief overview of how to use this. -[example program]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/simplejit/examples/simplejit-minimal.rs +[example program]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/jit/examples/jit-minimal.rs diff --git a/cranelift/simplejit/examples/simplejit-minimal.rs b/cranelift/jit/examples/jit-minimal.rs similarity index 94% rename from cranelift/simplejit/examples/simplejit-minimal.rs rename to cranelift/jit/examples/jit-minimal.rs index 7274c4d5e7..138c545831 100644 --- a/cranelift/simplejit/examples/simplejit-minimal.rs +++ b/cranelift/jit/examples/jit-minimal.rs @@ -1,8 +1,8 @@ use cranelift::prelude::*; use cranelift_codegen::binemit::NullTrapSink; use cranelift_codegen::settings::{self, Configurable}; +use cranelift_jit::{JITBuilder, JITModule}; use cranelift_module::{default_libcall_names, Linkage, Module}; -use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; use std::mem; fn main() { @@ -14,8 +14,7 @@ fn main() { panic!("host machine is not supported: {}", msg); }); let isa = isa_builder.finish(settings::Flags::new(flag_builder)); - let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); + let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names())); let mut ctx = module.make_context(); let mut func_ctx = FunctionBuilderContext::new(); diff --git a/cranelift/simplejit/src/backend.rs b/cranelift/jit/src/backend.rs similarity index 96% rename from cranelift/simplejit/src/backend.rs rename to cranelift/jit/src/backend.rs index 67ba0f8a37..418b26ef58 100644 --- a/cranelift/simplejit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -1,4 +1,4 @@ -//! Defines `SimpleJITModule`. +//! Defines `JITModule`. use crate::{compiled_blob::CompiledBlob, memory::Memory}; use cranelift_codegen::isa::TargetIsa; @@ -31,16 +31,16 @@ const EXECUTABLE_DATA_ALIGNMENT: u64 = 0x10; const WRITABLE_DATA_ALIGNMENT: u64 = 0x8; const READONLY_DATA_ALIGNMENT: u64 = 0x1; -/// A builder for `SimpleJITModule`. -pub struct SimpleJITBuilder { +/// A builder for `JITModule`. +pub struct JITBuilder { isa: Box, symbols: HashMap, libcall_names: Box String + Send + Sync>, hotswap_enabled: bool, } -impl SimpleJITBuilder { - /// Create a new `SimpleJITBuilder`. +impl JITBuilder { + /// Create a new `JITBuilder`. /// /// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall` /// enum to symbols. LibCalls are inserted in the IR as part of the legalization for certain @@ -60,12 +60,10 @@ impl SimpleJITBuilder { Self::with_isa(isa, libcall_names) } - /// Create a new `SimpleJITBuilder` with an arbitrary target. This is mainly + /// Create a new `JITBuilder` with an arbitrary target. This is mainly /// useful for testing. /// - /// SimpleJIT requires a `TargetIsa` configured for non-PIC. - /// - /// To create a `SimpleJITBuilder` for native use, use the `new` constructor + /// To create a `JITBuilder` for native use, use the `new` constructor /// instead. /// /// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall` @@ -121,19 +119,21 @@ impl SimpleJITBuilder { self } - /// Enable or disable hotswap support. See [`SimpleJITModule::prepare_for_function_redefine`] + /// Enable or disable hotswap support. See [`JITModule::prepare_for_function_redefine`] /// for more information. + /// + /// Enabling hotswap support requires PIC code. pub fn hotswap(&mut self, enabled: bool) -> &mut Self { self.hotswap_enabled = enabled; self } } -/// A `SimpleJITModule` implements `Module` and emits code and data into memory where it can be +/// A `JITModule` implements `Module` and emits code and data into memory where it can be /// directly called and accessed. /// -/// See the `SimpleJITBuilder` for a convenient way to construct `SimpleJITModule` instances. -pub struct SimpleJITModule { +/// See the `JITBuilder` for a convenient way to construct `JITModule` instances. +pub struct JITModule { isa: Box, hotswap_enabled: bool, symbols: HashMap, @@ -158,7 +158,7 @@ struct MemoryHandle { writable: Memory, } -impl SimpleJITModule { +impl JITModule { /// Free memory allocated for code and data segments of compiled functions. /// /// # Safety @@ -369,8 +369,8 @@ impl SimpleJITModule { self.memory.code.set_readable_and_executable(); } - /// Create a new `SimpleJITModule`. - pub fn new(builder: SimpleJITBuilder) -> Self { + /// Create a new `JITModule`. + pub fn new(builder: JITBuilder) -> Self { if builder.hotswap_enabled { assert!( builder.isa.flags().is_pic(), @@ -450,7 +450,7 @@ impl SimpleJITModule { /// Allow a single future `define_function` on a previously defined function. This allows for /// hot code swapping and lazy compilation of functions. /// - /// This requires hotswap support to be enabled first using [`SimpleJITBuilder::hotswap`]. + /// This requires hotswap support to be enabled first using [`JITBuilder::hotswap`]. pub fn prepare_for_function_redefine(&mut self, func_id: FuncId) -> ModuleResult<()> { assert!(self.hotswap_enabled, "Hotswap support is not enabled"); let decl = self.declarations.get_function_decl(func_id); @@ -473,7 +473,7 @@ impl SimpleJITModule { } } -impl<'simple_jit_backend> Module for SimpleJITModule { +impl Module for JITModule { fn isa(&self) -> &dyn TargetIsa { &*self.isa } @@ -533,7 +533,7 @@ impl<'simple_jit_backend> Module for SimpleJITModule { writable: bool, tls: bool, ) -> ModuleResult { - assert!(!tls, "SimpleJIT doesn't yet support TLS"); + assert!(!tls, "JIT doesn't yet support TLS"); let (id, _decl) = self .declarations .declare_data(name, linkage, writable, tls)?; @@ -627,7 +627,7 @@ impl<'simple_jit_backend> Module for SimpleJITModule { .allocate(size, EXECUTABLE_DATA_ALIGNMENT) .expect("TODO: handle OOM etc."); - let mut reloc_sink = SimpleJITRelocSink::default(); + let mut reloc_sink = JITRelocSink::default(); let mut stack_map_sink = binemit::NullStackMapSink {}; unsafe { ctx.emit_to_memory( @@ -750,7 +750,7 @@ impl<'simple_jit_backend> Module for SimpleJITModule { return Err(ModuleError::DuplicateDefinition(decl.name.to_owned())); } - assert!(!decl.tls, "SimpleJIT doesn't yet support TLS"); + assert!(!decl.tls, "JIT doesn't yet support TLS"); let &DataDescription { ref init, @@ -850,11 +850,11 @@ fn lookup_with_dlsym(name: &str) -> Option<*const u8> { } #[derive(Default)] -struct SimpleJITRelocSink { +struct JITRelocSink { relocs: Vec, } -impl RelocSink for SimpleJITRelocSink { +impl RelocSink for JITRelocSink { fn reloc_external( &mut self, offset: CodeOffset, diff --git a/cranelift/simplejit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs similarity index 100% rename from cranelift/simplejit/src/compiled_blob.rs rename to cranelift/jit/src/compiled_blob.rs diff --git a/cranelift/simplejit/src/lib.rs b/cranelift/jit/src/lib.rs similarity index 87% rename from cranelift/simplejit/src/lib.rs rename to cranelift/jit/src/lib.rs index d558b44396..6e66ca8dd6 100644 --- a/cranelift/simplejit/src/lib.rs +++ b/cranelift/jit/src/lib.rs @@ -1,4 +1,4 @@ -//! Top-level lib.rs for `cranelift_simplejit`. +//! Top-level lib.rs for `cranelift_jit`. #![deny( missing_docs, @@ -27,7 +27,7 @@ mod backend; mod compiled_blob; mod memory; -pub use crate::backend::{SimpleJITBuilder, SimpleJITModule}; +pub use crate::backend::{JITBuilder, JITModule}; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/cranelift/simplejit/src/memory.rs b/cranelift/jit/src/memory.rs similarity index 100% rename from cranelift/simplejit/src/memory.rs rename to cranelift/jit/src/memory.rs diff --git a/cranelift/simplejit/tests/basic.rs b/cranelift/jit/tests/basic.rs similarity index 93% rename from cranelift/simplejit/tests/basic.rs rename to cranelift/jit/tests/basic.rs index fc3f5b0056..9e7e3cbe52 100644 --- a/cranelift/simplejit/tests/basic.rs +++ b/cranelift/jit/tests/basic.rs @@ -5,8 +5,8 @@ use cranelift_codegen::settings::{self, Configurable}; use cranelift_codegen::{ir::types::I16, Context}; use cranelift_entity::EntityRef; use cranelift_frontend::*; +use cranelift_jit::*; use cranelift_module::*; -use cranelift_simplejit::*; #[test] fn error_on_incompatible_sig_in_declare_function() { @@ -18,8 +18,7 @@ fn error_on_incompatible_sig_in_declare_function() { panic!("host machine is not supported: {}", msg); }); let isa = isa_builder.finish(settings::Flags::new(flag_builder)); - let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); + let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names())); let mut sig = Signature { params: vec![AbiParam::new(types::I64)], @@ -36,7 +35,7 @@ fn error_on_incompatible_sig_in_declare_function() { .unwrap(); // Make sure this is an error } -fn define_simple_function(module: &mut SimpleJITModule) -> FuncId { +fn define_simple_function(module: &mut JITModule) -> FuncId { let sig = Signature { params: vec![], returns: vec![], @@ -76,8 +75,7 @@ fn panic_on_define_after_finalize() { panic!("host machine is not supported: {}", msg); }); let isa = isa_builder.finish(settings::Flags::new(flag_builder)); - let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); + let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names())); define_simple_function(&mut module); define_simple_function(&mut module); @@ -166,8 +164,7 @@ fn libcall_function() { panic!("host machine is not supported: {}", msg); }); let isa = isa_builder.finish(settings::Flags::new(flag_builder)); - let mut module: SimpleJITModule = - SimpleJITModule::new(SimpleJITBuilder::with_isa(isa, default_libcall_names())); + let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names())); let sig = Signature { params: vec![], diff --git a/cranelift/module/README.md b/cranelift/module/README.md index 1727975f96..09caf7e270 100644 --- a/cranelift/module/README.md +++ b/cranelift/module/README.md @@ -10,10 +10,8 @@ A module is a collection of functions and data objects that are linked together. The `Module` trait that defines a common interface for various kinds of modules. Most users will use one of the following `Module` implementations: - - `SimpleJITModule`, provided by [cranelift-simplejit], which JITs - code to memory for direct execution. - - `ObjectModule`, provided by [cranelift-object], which emits native - object files. + - `JITModule`, provided by [cranelift-jit], which JITs code to memory for direct execution. + - `ObjectModule`, provided by [cranelift-object], which emits native object files. -[cranelift-simplejit]: https://crates.io/crates/cranelift-simplejit +[cranelift-jit]: https://crates.io/crates/cranelift-jit [cranelift-object]: https://crates.io/crates/cranelift-object diff --git a/scripts/publish.rs b/scripts/publish.rs index a56e03b351..330a7e0dfb 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -39,7 +39,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "cranelift-object", "cranelift-interpreter", "cranelift", - "cranelift-simplejit", + "cranelift-jit", // wig/wiggle "wiggle-generate", "wiggle-macro", From 651f405220ad146608b80a9f8feff72ca14044a4 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Thu, 3 Dec 2020 15:18:50 -0800 Subject: [PATCH 46/63] Print WAST assertion failures in a hexadecimal pattern format Fixes #1681. --- crates/wast/src/wast.rs | 187 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 185 insertions(+), 2 deletions(-) diff --git a/crates/wast/src/wast.rs b/crates/wast/src/wast.rs index 692e2f583a..c0c5f61c12 100644 --- a/crates/wast/src/wast.rs +++ b/crates/wast/src/wast.rs @@ -1,7 +1,8 @@ use crate::spectest::link_spectest; use anyhow::{anyhow, bail, Context as _, Result}; -use std::path::Path; +use core::fmt; use std::str; +use std::{mem::size_of, path::Path}; use wasmtime::*; use wast::Wat; use wast::{ @@ -185,7 +186,13 @@ impl WastContext { if val_matches(v, e)? { continue; } - bail!("expected {:?}, got {:?}", e, v) + bail!( + "expected {:?} ({}), got {:?} ({})", + e, + e.as_hex_pattern(), + v, + v.as_hex_pattern() + ) } Ok(()) } @@ -482,3 +489,179 @@ fn v128_matches(actual: u128, expected: &wast::V128Pattern) -> bool { }), } } + +/// When troubleshooting a failure in a spec test, it is valuable to understand the bit-by-bit +/// difference. To do this, we print a hex-encoded version of Wasm values and assertion expressions +/// using this helper. +fn as_hex_pattern(bits: T) -> String +where + T: fmt::LowerHex, +{ + match size_of::() { + 1 => format!("{:#04x}", bits), + 2 => format!("{:#06x}", bits), + 4 => format!("{:#010x}", bits), + 8 => format!("{:#018x}", bits), + 16 => format!("{:#034x}", bits), + _ => unimplemented!(), + } +} + +/// The [AsHexPattern] allows us to extend `as_hex_pattern` to various structures. +trait AsHexPattern { + fn as_hex_pattern(&self) -> String; +} + +impl AsHexPattern for wast::AssertExpression<'_> { + fn as_hex_pattern(&self) -> String { + match self { + wast::AssertExpression::I32(i) => as_hex_pattern(*i), + wast::AssertExpression::I64(i) => as_hex_pattern(*i), + wast::AssertExpression::F32(f) => f.as_hex_pattern(), + wast::AssertExpression::F64(f) => f.as_hex_pattern(), + wast::AssertExpression::V128(v) => v.as_hex_pattern(), + wast::AssertExpression::RefNull(_) + | wast::AssertExpression::RefExtern(_) + | wast::AssertExpression::RefFunc(_) + | wast::AssertExpression::LegacyArithmeticNaN + | wast::AssertExpression::LegacyCanonicalNaN => "no hex representation".to_string(), + } + } +} + +impl AsHexPattern for wast::NanPattern { + fn as_hex_pattern(&self) -> String { + match self { + wast::NanPattern::CanonicalNan => "0x7fc00000".to_string(), + // Note that NaN patterns can have varying sign bits and payloads. Technically the first + // bit should be a `*` but it is impossible to show that in hex. + wast::NanPattern::ArithmeticNan => "0x7fc*****".to_string(), + wast::NanPattern::Value(wast::Float32 { bits }) => as_hex_pattern(*bits), + } + } +} + +impl AsHexPattern for wast::NanPattern { + fn as_hex_pattern(&self) -> String { + match self { + wast::NanPattern::CanonicalNan => "0x7ff8000000000000".to_string(), + // Note that NaN patterns can have varying sign bits and payloads. Technically the first + // bit should be a `*` but it is impossible to show that in hex. + wast::NanPattern::ArithmeticNan => "0x7ff8************".to_string(), + wast::NanPattern::Value(wast::Float64 { bits }) => as_hex_pattern(*bits), + } + } +} + +// This implementation reverses both the lanes and the lane bytes in order to match the Wasm SIMD +// little-endian order. This implementation must include special behavior for this reversal; other +// implementations do not because they deal with raw values (`u128`) or use big-endian order for +// display (scalars). +impl AsHexPattern for wast::V128Pattern { + fn as_hex_pattern(&self) -> String { + fn reverse_pattern(pattern: String) -> String { + let chars: Vec = pattern[2..].chars().collect(); + let reversed: Vec<&[char]> = chars.chunks(2).rev().collect(); + reversed.concat().iter().collect() + } + + fn as_hex_pattern(bits: &[u8]) -> String { + bits.iter() + .map(|b| format!("{:02x}", b)) + .collect::>() + .concat() + } + + fn reverse_lanes( + lanes: impl DoubleEndedIterator, + as_hex_pattern: F, + ) -> String + where + F: Fn(T) -> String, + { + lanes + .rev() + .map(|f| as_hex_pattern(f)) + .collect::>() + .concat() + } + + let lanes_as_hex = match self { + wast::V128Pattern::I8x16(v) => { + reverse_lanes(v.iter(), |b| as_hex_pattern(&b.to_le_bytes())) + } + wast::V128Pattern::I16x8(v) => { + reverse_lanes(v.iter(), |b| as_hex_pattern(&b.to_le_bytes())) + } + wast::V128Pattern::I32x4(v) => { + reverse_lanes(v.iter(), |b| as_hex_pattern(&b.to_le_bytes())) + } + wast::V128Pattern::I64x2(v) => { + reverse_lanes(v.iter(), |b| as_hex_pattern(&b.to_le_bytes())) + } + wast::V128Pattern::F32x4(v) => { + reverse_lanes(v.iter(), |b| reverse_pattern(b.as_hex_pattern())) + } + wast::V128Pattern::F64x2(v) => { + reverse_lanes(v.iter(), |b| reverse_pattern(b.as_hex_pattern())) + } + }; + + String::from("0x") + &lanes_as_hex + } +} + +impl AsHexPattern for Val { + fn as_hex_pattern(&self) -> String { + match self { + Val::I32(i) => as_hex_pattern(*i), + Val::I64(i) => as_hex_pattern(*i), + Val::F32(f) => as_hex_pattern(*f), + Val::F64(f) => as_hex_pattern(*f), + Val::V128(v) => as_hex_pattern(*v), + Val::ExternRef(_) | Val::FuncRef(_) => "no hex representation".to_string(), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn val_to_hex() { + assert_eq!(Val::I32(0x42).as_hex_pattern(), "0x00000042"); + assert_eq!(Val::F64(0x0).as_hex_pattern(), "0x0000000000000000"); + assert_eq!( + Val::V128(u128::from_le_bytes([ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf + ])) + .as_hex_pattern(), + "0x0f0e0d0c0b0a09080706050403020100" + ); + } + + #[test] + fn assert_expression_to_hex() { + assert_eq!( + wast::AssertExpression::F32(wast::NanPattern::ArithmeticNan).as_hex_pattern(), + "0x7fc*****" + ); + assert_eq!( + wast::AssertExpression::F64(wast::NanPattern::Value(wast::Float64 { bits: 0x42 })) + .as_hex_pattern(), + "0x0000000000000042" + ); + assert_eq!( + wast::AssertExpression::V128(wast::V128Pattern::I32x4([0, 1, 2, 3])).as_hex_pattern(), + "0x03000000020000000100000000000000" + ); + assert_eq!( + wast::AssertExpression::V128(wast::V128Pattern::F64x2([ + wast::NanPattern::CanonicalNan, + wast::NanPattern::ArithmeticNan + ])) + .as_hex_pattern(), + "0x************f87f000000000000f87f" + ); + } +} From fc752efa89e1c0c13cc2de900adc945c4e319e93 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Thu, 3 Dec 2020 15:55:00 -0800 Subject: [PATCH 47/63] Fix incorrect arithmetic NaN check and document --- crates/wast/src/wast.rs | 47 +++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/crates/wast/src/wast.rs b/crates/wast/src/wast.rs index c0c5f61c12..47b38d13e2 100644 --- a/crates/wast/src/wast.rs +++ b/crates/wast/src/wast.rs @@ -2,7 +2,7 @@ use crate::spectest::link_spectest; use anyhow::{anyhow, bail, Context as _, Result}; use core::fmt; use std::str; -use std::{mem::size_of, path::Path}; +use std::{mem::size_of_val, path::Path}; use wasmtime::*; use wast::Wat; use wast::{ @@ -397,22 +397,50 @@ fn extract_lane_as_i64(bytes: u128, lane: usize) -> i64 { (bytes >> (lane * 64)) as i64 } +/// Check if an f32 (as u32 bits to avoid possible quieting when moving values in registers, e.g. +/// https://developer.arm.com/documentation/ddi0344/i/neon-and-vfp-programmers-model/modes-of-operation/default-nan-mode?lang=en) +/// is a canonical NaN: +/// - the sign bit is unspecified, +/// - the 8-bit exponent is set to all 1s +/// - the MSB of the payload is set to 1 (a quieted NaN) and all others to 0. +/// See https://webassembly.github.io/spec/core/syntax/values.html#floating-point. fn is_canonical_f32_nan(bits: u32) -> bool { (bits & 0x7fff_ffff) == 0x7fc0_0000 } +/// Check if an f64 (as u64 bits to avoid possible quieting when moving values in registers, e.g. +/// https://developer.arm.com/documentation/ddi0344/i/neon-and-vfp-programmers-model/modes-of-operation/default-nan-mode?lang=en) +/// is a canonical NaN: +/// - the sign bit is unspecified, +/// - the 11-bit exponent is set to all 1s +/// - the MSB of the payload is set to 1 (a quieted NaN) and all others to 0. +/// See https://webassembly.github.io/spec/core/syntax/values.html#floating-point. fn is_canonical_f64_nan(bits: u64) -> bool { (bits & 0x7fff_ffff_ffff_ffff) == 0x7ff8_0000_0000_0000 } +/// Check if an f32 (as u32, see comments above) is an arithmetic NaN. This is the same as a +/// canonical NaN including that the payload MSB is set to 1, but one or more of the remaining +/// payload bits MAY BE set to 1 (a canonical NaN specifies all 0s). See +/// https://webassembly.github.io/spec/core/syntax/values.html#floating-point. fn is_arithmetic_f32_nan(bits: u32) -> bool { - const AF32_NAN: u32 = 0x0040_0000; - (bits & AF32_NAN) == AF32_NAN + const AF32_NAN: u32 = 0x7f80_0000; + let is_nan = bits & AF32_NAN == AF32_NAN; + const AF32_PAYLOAD_MSB: u32 = 0x0040_0000; + let is_msb_set = bits & AF32_PAYLOAD_MSB == AF32_PAYLOAD_MSB; + is_nan && is_msb_set } +/// Check if an f64 (as u64, see comments above) is an arithmetic NaN. This is the same as a +/// canonical NaN including that the payload MSB is set to 1, but one or more of the remaining +/// payload bits MAY BE set to 1 (a canonical NaN specifies all 0s). See +/// https://webassembly.github.io/spec/core/syntax/values.html#floating-point. fn is_arithmetic_f64_nan(bits: u64) -> bool { - const AF64_NAN: u64 = 0x0008_0000_0000_0000; - (bits & AF64_NAN) == AF64_NAN + const AF64_NAN: u64 = 0x7ff0_0000_0000_0000; + let is_nan = bits & AF64_NAN == AF64_NAN; + const AF64_PAYLOAD_MSB: u64 = 0x0008_0000_0000_0000; + let is_msb_set = bits & AF64_PAYLOAD_MSB == AF64_PAYLOAD_MSB; + is_nan && is_msb_set } fn val_matches(actual: &Val, expected: &wast::AssertExpression) -> Result { @@ -497,14 +525,7 @@ fn as_hex_pattern(bits: T) -> String where T: fmt::LowerHex, { - match size_of::() { - 1 => format!("{:#04x}", bits), - 2 => format!("{:#06x}", bits), - 4 => format!("{:#010x}", bits), - 8 => format!("{:#018x}", bits), - 16 => format!("{:#034x}", bits), - _ => unimplemented!(), - } + format!("{1:#00$x}", size_of_val(&bits) * 2 + 2, bits) } /// The [AsHexPattern] allows us to extend `as_hex_pattern` to various structures. From 1dddba649a92adda92f20ef191806338aa486118 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Sun, 6 Dec 2020 22:26:42 -0800 Subject: [PATCH 48/63] x64 regalloc register order: put caller-saves (volatiles) first. The x64 backend currently builds the `RealRegUniverse` in a way that is generating somewhat suboptimal code. In many blocks, we see uses of callee-save (non-volatile) registers (r12, r13, r14, rbx) first, even in very short leaf functions where there are plenty of volatiles to use. This is leading to unnecessary spills/reloads. On one (local) test program, a medium-sized C benchmark compiled to Wasm and run on Wasmtime, I am seeing a ~10% performance improvement with this change; it will be less pronounced in programs with high register pressure (there we are likely to use all registers regardless, so the prologue/epilogue will save/restore all callee-saves), or in programs with fewer calls, but this is a clear win for small functions and in many cases removes prologue/epilogue clobber-saves altogether. Separately, I think the RA's coalescing is tripping up a bit in some cases; see e.g. the filetest touched by this commit that loads a value into %rsi then moves to %rax and returns immediately. This is an orthogonal issue, though, and should be addressed (if worthwhile) in regalloc.rs. --- cranelift/codegen/src/isa/x64/inst/regs.rs | 78 ++++++++++--------- .../filetests/isa/x64/amode-opt.clif | 8 +- .../filetests/filetests/isa/x64/heap.clif | 8 +- .../filetests/filetests/isa/x64/load-op.clif | 24 +++--- .../isa/x64/simd-bitwise-compile.clif | 16 ++-- .../isa/x64/simd-lane-access-compile.clif | 4 +- .../isa/x64/simd-logical-compile.clif | 4 +- 7 files changed, 74 insertions(+), 68 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst/regs.rs b/cranelift/codegen/src/isa/x64/inst/regs.rs index 04bc1f09bf..1d9e30bd3a 100644 --- a/cranelift/codegen/src/isa/x64/inst/regs.rs +++ b/cranelift/codegen/src/isa/x64/inst/regs.rs @@ -1,14 +1,20 @@ //! Registers, the Universe thereof, and printing. //! -//! These are ordered by sequence number, as required in the Universe. The strange ordering is -//! intended to make callee-save registers available before caller-saved ones. This is a net win -//! provided that each function makes at least one onward call. It'll be a net loss for leaf -//! functions, and we should change the ordering in that case, so as to make caller-save regs -//! available first. +//! These are ordered by sequence number, as required in the Universe. //! -//! TODO Maybe have two different universes, one for leaf functions and one for non-leaf functions? -//! Also, they will have to be ABI dependent. Need to find a way to avoid constructing a universe -//! for each function we compile. +//! The caller-saved registers are placed first in order to prefer not to clobber (requiring +//! saves/restores in prologue/epilogue code) when possible. Note that there is no other heuristic +//! in the backend that will apply such pressure; the register allocator's cost heuristics are not +//! aware of the cost of clobber-save/restore code. +//! +//! One might worry that this pessimizes code with many callsites, where using caller-saves causes +//! us to have to save them (as we are the caller) frequently. However, the register allocator +//! *should be* aware of *this* cost, because it sees that the call instruction modifies all of the +//! caller-saved (i.e., callee-clobbered) registers. +//! +//! Hence, this ordering encodes pressure in one direction (prefer not to clobber registers that we +//! ourselves have to save) and this is balanaced against the RA's pressure in the other direction +//! at callsites. use crate::settings; use alloc::vec::Vec; @@ -31,44 +37,44 @@ fn gpr(enc: u8, index: u8) -> Reg { Reg::new_real(RegClass::I64, enc, index) } -pub(crate) fn r12() -> Reg { - gpr(ENC_R12, 16) -} -pub(crate) fn r13() -> Reg { - gpr(ENC_R13, 17) -} -pub(crate) fn r14() -> Reg { - gpr(ENC_R14, 18) -} -pub(crate) fn rbx() -> Reg { - gpr(ENC_RBX, 19) -} pub(crate) fn rsi() -> Reg { - gpr(6, 20) + gpr(6, 16) } pub(crate) fn rdi() -> Reg { - gpr(7, 21) + gpr(7, 17) } pub(crate) fn rax() -> Reg { - gpr(0, 22) + gpr(0, 18) } pub(crate) fn rcx() -> Reg { - gpr(1, 23) + gpr(1, 19) } pub(crate) fn rdx() -> Reg { - gpr(2, 24) + gpr(2, 20) } pub(crate) fn r8() -> Reg { - gpr(8, 25) + gpr(8, 21) } pub(crate) fn r9() -> Reg { - gpr(9, 26) + gpr(9, 22) } pub(crate) fn r10() -> Reg { - gpr(10, 27) + gpr(10, 23) } pub(crate) fn r11() -> Reg { - gpr(11, 28) + gpr(11, 24) +} +pub(crate) fn r12() -> Reg { + gpr(ENC_R12, 25) +} +pub(crate) fn r13() -> Reg { + gpr(ENC_R13, 26) +} +pub(crate) fn r14() -> Reg { + gpr(ENC_R14, 27) +} +pub(crate) fn rbx() -> Reg { + gpr(ENC_RBX, 28) } pub(crate) fn r15() -> Reg { @@ -176,13 +182,6 @@ pub(crate) fn create_reg_universe_systemv(flags: &settings::Flags) -> RealRegUni // Integer regs. let first_gpr = regs.len(); - // Callee-saved, in the SystemV x86_64 ABI. - regs.push((r12().to_real_reg(), "%r12".into())); - regs.push((r13().to_real_reg(), "%r13".into())); - regs.push((r14().to_real_reg(), "%r14".into())); - - regs.push((rbx().to_real_reg(), "%rbx".into())); - // Caller-saved, in the SystemV x86_64 ABI. regs.push((rsi().to_real_reg(), "%rsi".into())); regs.push((rdi().to_real_reg(), "%rdi".into())); @@ -194,6 +193,13 @@ pub(crate) fn create_reg_universe_systemv(flags: &settings::Flags) -> RealRegUni regs.push((r10().to_real_reg(), "%r10".into())); regs.push((r11().to_real_reg(), "%r11".into())); + // Callee-saved, in the SystemV x86_64 ABI. + regs.push((r12().to_real_reg(), "%r12".into())); + regs.push((r13().to_real_reg(), "%r13".into())); + regs.push((r14().to_real_reg(), "%r14".into())); + + regs.push((rbx().to_real_reg(), "%rbx".into())); + // Other regs, not available to the allocator. debug_assert_eq!(r15(), pinned_reg()); let allocable = if use_pinned_reg { diff --git a/cranelift/filetests/filetests/isa/x64/amode-opt.clif b/cranelift/filetests/filetests/isa/x64/amode-opt.clif index dbeed5475e..bfe0198753 100644 --- a/cranelift/filetests/filetests/isa/x64/amode-opt.clif +++ b/cranelift/filetests/filetests/isa/x64/amode-opt.clif @@ -7,7 +7,7 @@ block0(v0: i64, v1: i64): v2 = iadd v0, v1 v3 = load.i64 v2 return v3 - ; check: movq 0(%rdi,%rsi,1), %r12 + ; check: movq 0(%rdi,%rsi,1), %rsi } function %amode_add_imm(i64) -> i64 { @@ -16,7 +16,7 @@ block0(v0: i64): v2 = iadd v0, v1 v3 = load.i64 v2 return v3 - ; check: movq 42(%rdi), %r12 + ; check: movq 42(%rdi), %rsi } ;; Same as above, but add operands have been reversed. @@ -26,7 +26,7 @@ block0(v0: i64): v2 = iadd v1, v0 v3 = load.i64 v2 return v3 - ; check: movq 42(%rdi), %r12 + ; check: movq 42(%rdi), %rsi } ;; Make sure that uextend(cst) are ignored when the cst will naturally sign-extend. @@ -37,5 +37,5 @@ block0(v0: i64): v3 = iadd v2, v0 v4 = load.i64 v3 return v4 - ; check: movq 42(%rdi), %r12 + ; check: movq 42(%rdi), %rsi } diff --git a/cranelift/filetests/filetests/isa/x64/heap.clif b/cranelift/filetests/filetests/isa/x64/heap.clif index d9efb083c6..c547582008 100644 --- a/cranelift/filetests/filetests/isa/x64/heap.clif +++ b/cranelift/filetests/filetests/isa/x64/heap.clif @@ -11,11 +11,11 @@ function %f(i32, i64 vmctx) -> i64 { block0(v0: i32, v1: i64): v2 = heap_addr.i64 heap0, v0, 0x8000 - ; check: movl 8(%rsi), %r12d - ; nextln: movq %rdi, %r13 - ; nextln: addl $$32768, %r13d + ; check: movl 8(%rsi), %ecx + ; nextln: movq %rdi, %rax + ; nextln: addl $$32768, %eax ; nextln: jnb ; ud2 heap_oob ; - ; nextln: cmpl %r12d, %r13d + ; nextln: cmpl %ecx, %eax ; nextln: jbe label1; j label2 ; check: Block 1: diff --git a/cranelift/filetests/filetests/isa/x64/load-op.clif b/cranelift/filetests/filetests/isa/x64/load-op.clif index 77e4b05420..6570a798c3 100644 --- a/cranelift/filetests/filetests/isa/x64/load-op.clif +++ b/cranelift/filetests/filetests/isa/x64/load-op.clif @@ -6,7 +6,7 @@ function %add_from_mem_u32_1(i64, i32) -> i32 { block0(v0: i64, v1: i32): v2 = load.i32 v0 v3 = iadd.i32 v2, v1 - ; check: addl 0(%rdi), %r12d + ; check: addl 0(%rdi), %esi return v3 } @@ -14,7 +14,7 @@ function %add_from_mem_u32_2(i64, i32) -> i32 { block0(v0: i64, v1: i32): v2 = load.i32 v0 v3 = iadd.i32 v1, v2 - ; check: addl 0(%rdi), %r12d + ; check: addl 0(%rdi), %esi return v3 } @@ -22,7 +22,7 @@ function %add_from_mem_u64_1(i64, i64) -> i64 { block0(v0: i64, v1: i64): v2 = load.i64 v0 v3 = iadd.i64 v2, v1 - ; check: addq 0(%rdi), %r12 + ; check: addq 0(%rdi), %rsi return v3 } @@ -30,7 +30,7 @@ function %add_from_mem_u64_2(i64, i64) -> i64 { block0(v0: i64, v1: i64): v2 = load.i64 v0 v3 = iadd.i64 v1, v2 - ; check: addq 0(%rdi), %r12 + ; check: addq 0(%rdi), %rsi return v3 } @@ -40,8 +40,8 @@ function %add_from_mem_not_narrow(i64, i8) -> i8 { block0(v0: i64, v1: i8): v2 = load.i8 v0 v3 = iadd.i8 v2, v1 - ; check: movzbq 0(%rdi), %r12 - ; nextln: addl %esi, %r12d + ; check: movzbq 0(%rdi), %rdi + ; nextln: addl %esi, %edi return v3 } @@ -52,10 +52,10 @@ block0(v0: i64, v1: i64): store.i64 v3, v1 v4 = load.i64 v3 return v4 - ; check: movq 0(%rdi), %r12 - ; nextln: movq %r12, %r13 - ; nextln: addq %rdi, %r13 - ; nextln: movq %r13, 0(%rsi) - ; nextln: movq 0(%r12,%rdi,1), %r12 - ; nextln: movq %r12, %rax + ; check: movq 0(%rdi), %rax + ; nextln: movq %rax, %rcx + ; nextln: addq %rdi, %rcx + ; nextln: movq %rcx, 0(%rsi) + ; nextln: movq 0(%rax,%rdi,1), %rsi + ; nextln: movq %rsi, %rax } diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif index fe52e3c503..65e1b5df7e 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif @@ -28,9 +28,9 @@ block0(v0: i32): } ; check: movd %edi, %xmm1 ; nextln: psllw %xmm1, %xmm0 -; nextln: lea const(VCodeConstant(0)), %r12 +; nextln: lea const(VCodeConstant(0)), %rsi ; nextln: shlq $$4, %rdi -; nextln: movdqu 0(%r12,%rdi,1), %xmm1 +; nextln: movdqu 0(%rsi,%rdi,1), %xmm1 ; nextln: pand %xmm1, %xmm0 function %ushr_i8x16_imm() -> i8x16 { @@ -81,12 +81,12 @@ block0(v0: i64x2, v1: i32): v2 = sshr v0, v1 return v2 } -; check: pextrd.w $$0, %xmm0, %r12 -; nextln: pextrd.w $$1, %xmm0, %r13 +; check: pextrd.w $$0, %xmm0, %rsi +; nextln: pextrd.w $$1, %xmm0, %rax ; nextln: movq %rdi, %rcx -; nextln: sarq %cl, %r12 +; nextln: sarq %cl, %rsi ; nextln: movq %rdi, %rcx -; nextln: sarq %cl, %r13 -; nextln: pinsrd.w $$0, %r12, %xmm1 -; nextln: pinsrd.w $$1, %r13, %xmm1 +; nextln: sarq %cl, %rax +; nextln: pinsrd.w $$0, %rsi, %xmm1 +; nextln: pinsrd.w $$1, %rax, %xmm1 ; nextln: movdqa %xmm1, %xmm0 diff --git a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif index c9188a9514..f44dbd3b62 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif @@ -70,8 +70,8 @@ block0: return v1 } ; check: uninit %xmm0 -; nextln: pinsrw $$0, %r12, %xmm0 -; nextln: pinsrw $$1, %r12, %xmm0 +; nextln: pinsrw $$0, %rsi, %xmm0 +; nextln: pinsrw $$1, %rsi, %xmm0 ; nextln: pshufd $$0, %xmm0, %xmm0 function %splat_i32(i32) -> i32x4 { diff --git a/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif index 0e366db441..d03aa0b204 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif @@ -17,7 +17,7 @@ block0(v0: b32x4): return v1 } ; check: ptest %xmm0, %xmm0 -; nextln: setnz %r12b +; nextln: setnz %sil function %vall_true_i64x2(i64x2) -> b1 { block0(v0: i64x2): @@ -27,4 +27,4 @@ block0(v0: i64x2): ; check: pxor %xmm1, %xmm1 ; nextln: pcmpeqq %xmm0, %xmm1 ; nextln: ptest %xmm1, %xmm1 -; nextln: setz %r12b +; nextln: setz %sil From f1025322fa72acf3e48ff982b0706a7c007d4fd2 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 7 Dec 2020 09:20:06 -0800 Subject: [PATCH 49/63] Update wasmparser to 0.69.2 --- Cargo.lock | 24 ++++++++++++------------ Cargo.toml | 2 +- cranelift/wasm/Cargo.toml | 2 +- crates/debug/Cargo.toml | 2 +- crates/environ/Cargo.toml | 2 +- crates/fuzzing/Cargo.toml | 2 +- crates/jit/Cargo.toml | 2 +- crates/lightbeam/Cargo.toml | 2 +- crates/lightbeam/wasmtime/Cargo.toml | 2 +- crates/wasmtime/Cargo.toml | 2 +- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 844b60b4bc..13c13d89cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -597,7 +597,7 @@ dependencies = [ "smallvec", "target-lexicon", "thiserror", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wat", ] @@ -1167,7 +1167,7 @@ dependencies = [ "smallvec", "thiserror", "typemap", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wat", ] @@ -2406,9 +2406,9 @@ checksum = "32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6" [[package]] name = "wasmparser" -version = "0.69.1" +version = "0.69.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd19c6066bcf391a9d6f81db9b809f31d31723da2652c8416cb81cd5aabed944" +checksum = "fd2dd6dadf3a723971297bcc0ec103e0aa8118bf68e23f49cb575e21621894a8" [[package]] name = "wasmprinter" @@ -2417,7 +2417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dba006f5c5bf41a2a5c3b45e861ea6eb067382acb022b6a35a00a0390f9547f6" dependencies = [ "anyhow", - "wasmparser 0.69.1", + "wasmparser 0.69.2", ] [[package]] @@ -2438,7 +2438,7 @@ dependencies = [ "smallvec", "target-lexicon", "tempfile", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wasmtime-cache", "wasmtime-environ", "wasmtime-jit", @@ -2516,7 +2516,7 @@ dependencies = [ "test-programs", "tracing-subscriber", "wasi-common", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wasmtime", "wasmtime-cache", "wasmtime-debug", @@ -2552,7 +2552,7 @@ dependencies = [ "object", "target-lexicon", "thiserror", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wasmtime-environ", ] @@ -2571,7 +2571,7 @@ dependencies = [ "more-asserts", "serde", "thiserror", - "wasmparser 0.69.1", + "wasmparser 0.69.2", ] [[package]] @@ -2600,7 +2600,7 @@ dependencies = [ "rayon", "wasm-smith", "wasmi", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wasmprinter", "wasmtime", "wasmtime-wast", @@ -2628,7 +2628,7 @@ dependencies = [ "serde", "target-lexicon", "thiserror", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wasmtime-cranelift", "wasmtime-debug", "wasmtime-environ", @@ -2645,7 +2645,7 @@ version = "0.21.0" dependencies = [ "cranelift-codegen", "lightbeam", - "wasmparser 0.69.1", + "wasmparser 0.69.2", "wasmtime-environ", ] diff --git a/Cargo.toml b/Cargo.toml index aff7e98e33..7ea112be71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ libc = "0.2.60" log = "0.4.8" rayon = "1.2.1" humantime = "2.0.0" -wasmparser = "0.69" +wasmparser = "0.69.2" [dev-dependencies] env_logger = "0.8.1" diff --git a/cranelift/wasm/Cargo.toml b/cranelift/wasm/Cargo.toml index 515391382d..d6aa8b4ced 100644 --- a/cranelift/wasm/Cargo.toml +++ b/cranelift/wasm/Cargo.toml @@ -12,7 +12,7 @@ keywords = ["webassembly", "wasm"] edition = "2018" [dependencies] -wasmparser = { version = "0.69.1", default-features = false } +wasmparser = { version = "0.69.2", default-features = false } cranelift-codegen = { path = "../codegen", version = "0.68.0", default-features = false } cranelift-entity = { path = "../entity", version = "0.68.0" } cranelift-frontend = { path = "../frontend", version = "0.68.0", default-features = false } diff --git a/crates/debug/Cargo.toml b/crates/debug/Cargo.toml index 401c443618..62a07daf56 100644 --- a/crates/debug/Cargo.toml +++ b/crates/debug/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] gimli = "0.23.0" -wasmparser = "0.69.0" +wasmparser = "0.69.2" object = { version = "0.22.0", default-features = false, features = ["read", "write"] } wasmtime-environ = { path = "../environ", version = "0.21.0" } target-lexicon = { version = "0.11.0", default-features = false } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 1b944282e1..118e319ebd 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.68.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.68.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.68.0", features = ["enable-serde"] } -wasmparser = "0.69.0" +wasmparser = "0.69.2" indexmap = { version = "1.0.2", features = ["serde-1"] } thiserror = "1.0.4" serde = { version = "1.0.94", features = ["derive"] } diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 12a9842c00..8aa5768457 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -12,7 +12,7 @@ arbitrary = { version = "0.4.1", features = ["derive"] } env_logger = "0.8.1" log = "0.4.8" rayon = "1.2.1" -wasmparser = "0.69.0" +wasmparser = "0.69.2" wasmprinter = "0.2.16" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 4a3b9101d5..7b357d0461 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -28,7 +28,7 @@ rayon = { version = "1.0", optional = true } region = "2.1.0" thiserror = "1.0.4" target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.69.0" +wasmparser = "0.69.2" more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "1.0" diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml index ff7d095e10..baf518e642 100644 --- a/crates/lightbeam/Cargo.toml +++ b/crates/lightbeam/Cargo.toml @@ -24,7 +24,7 @@ more-asserts = "0.2.1" smallvec = "1.0.0" thiserror = "1.0.9" typemap = "0.3" -wasmparser = "0.69.0" +wasmparser = "0.69.2" [dev-dependencies] lazy_static = "1.2" diff --git a/crates/lightbeam/wasmtime/Cargo.toml b/crates/lightbeam/wasmtime/Cargo.toml index 2d3163ce06..cf0bab9442 100644 --- a/crates/lightbeam/wasmtime/Cargo.toml +++ b/crates/lightbeam/wasmtime/Cargo.toml @@ -13,6 +13,6 @@ edition = "2018" [dependencies] lightbeam = { path = "..", version = "0.21.0" } -wasmparser = "0.69" +wasmparser = "0.69.2" cranelift-codegen = { path = "../../../cranelift/codegen", version = "0.68.0" } wasmtime-environ = { path = "../../environ", version = "0.21.0" } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 881f1f8724..65ce5873c4 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -16,7 +16,7 @@ wasmtime-jit = { path = "../jit", version = "0.21.0" } wasmtime-cache = { path = "../cache", version = "0.21.0", optional = true } wasmtime-profiling = { path = "../profiling", version = "0.21.0" } target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.69.0" +wasmparser = "0.69.2" anyhow = "1.0.19" region = "2.2.0" libc = "0.2" From 2c765c18c2271ff528c875fb1e612170f4d772cd Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Fri, 4 Dec 2020 15:48:14 -0800 Subject: [PATCH 50/63] Update spec tests --- build.rs | 3 ++- crates/wast/src/wast.rs | 3 --- tests/spec_testsuite | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/build.rs b/build.rs index c766521678..4258c27ca0 100644 --- a/build.rs +++ b/build.rs @@ -210,7 +210,8 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { | ("simd", "simd_f32x4_pmin_pmax") | ("simd", "simd_f64x2_pmin_pmax") | ("simd", "simd_f32x4_rounding") - | ("simd", "simd_f64x2_rounding") => { + | ("simd", "simd_f64x2_rounding") + | ("simd", "simd_i32x4_dot_i16x8") => { return !(cfg!(feature = "experimental_x64") || env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "aarch64") } diff --git a/crates/wast/src/wast.rs b/crates/wast/src/wast.rs index 47b38d13e2..e18f5e543a 100644 --- a/crates/wast/src/wast.rs +++ b/crates/wast/src/wast.rs @@ -371,9 +371,6 @@ impl WastContext { fn is_matching_assert_invalid_error_message(expected: &str, actual: &str) -> bool { actual.contains(expected) - // Waiting on https://github.com/WebAssembly/bulk-memory-operations/pull/137 - // to propagate to WebAssembly/testsuite. - || (expected.contains("unknown table") && actual.contains("unknown elem")) // `elem.wast` and `proposals/bulk-memory-operations/elem.wast` disagree // on the expected error message for the same error. || (expected.contains("out of bounds") && actual.contains("does not fit")) diff --git a/tests/spec_testsuite b/tests/spec_testsuite index 18f83401a4..35c50bf6fb 160000 --- a/tests/spec_testsuite +++ b/tests/spec_testsuite @@ -1 +1 @@ -Subproject commit 18f83401a47a0e43772cf7d9f216e994bf7c7fa6 +Subproject commit 35c50bf6fbb002cfdc1227b0af731bdcaf877714 From 3a01d147129fa1a5273d39d57fc9303a632ba40e Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Mon, 7 Dec 2020 15:09:47 -0800 Subject: [PATCH 51/63] Two Lucet-related fixes to stack overflow handling. Lucet uses stack probes rather than explicit stack limit checks as Wasmtime does. In bytecodealliance/lucet#616, I have discovered that I previously was not running some Lucet runtime tests with the new backend, so was missing some test failures due to missing pieces in the new backend. This PR adds (i) calls to probestack, when enabled, in the prologue of every function with a stack frame larger than one page (configurable via flags); and (ii) trap metadata for every instruction on x86-64 that can access the stack, hence be the first point at which a stack overflow is detected when the stack pointer is decremented. --- .github/workflows/main.yml | 13 ++- cranelift/codegen/src/isa/aarch64/abi.rs | 8 +- cranelift/codegen/src/isa/arm32/abi.rs | 8 +- cranelift/codegen/src/isa/x64/abi.rs | 18 +++- cranelift/codegen/src/isa/x64/inst/emit.rs | 96 +++++++++++++++++-- cranelift/codegen/src/isa/x64/lower.rs | 4 +- cranelift/codegen/src/machinst/abi_impl.rs | 24 +++++ .../filetests/isa/x64/probestack.clif | 17 ++++ 8 files changed, 169 insertions(+), 19 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/probestack.clif diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3bd17dbdca..5e80781daf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -60,7 +60,10 @@ jobs: # nightly-only feature right now. - uses: ./.github/actions/install-rust with: - toolchain: nightly + # TODO (rust-lang/rust#79661): We are seeing an internal compiler error when + # building with the latest (2020-12-06) nightly; pin on a slightly older + # version for now. + toolchain: nightly-2020-11-29 - run: cargo doc --no-deps --all --exclude wasmtime-cli --exclude test-programs --exclude cranelift-codegen-meta - run: cargo doc --package cranelift-codegen-meta --document-private-items - uses: actions/upload-artifact@v1 @@ -145,7 +148,7 @@ jobs: # flags to rustc. - uses: ./.github/actions/install-rust with: - toolchain: nightly + toolchain: nightly-2020-11-29 - run: cargo install cargo-fuzz --vers "^0.8" - run: cargo fetch working-directory: ./fuzz @@ -202,7 +205,7 @@ jobs: rust: beta - build: nightly os: ubuntu-latest - rust: nightly + rust: nightly-2020-11-29 - build: macos os: macos-latest rust: stable @@ -292,7 +295,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - toolchain: nightly + toolchain: nightly-2020-11-29 - uses: ./.github/actions/define-llvm-env # Install wasm32 targets in order to build various tests throughout the @@ -303,7 +306,7 @@ jobs: # Run the x64 CI script. - run: ./ci/run-experimental-x64-ci.sh env: - CARGO_VERSION: "+nightly" + CARGO_VERSION: "+nightly-2020-11-29" RUST_BACKTRACE: 1 # Build and test the wasi-nn module. diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 8496e51711..69335c10cd 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -12,7 +12,7 @@ use crate::{CodegenError, CodegenResult}; use alloc::boxed::Box; use alloc::vec::Vec; use regalloc::{RealReg, Reg, RegClass, Set, Writable}; -use smallvec::SmallVec; +use smallvec::{smallvec, SmallVec}; // We use a generic implementation that factors out AArch64 and x64 ABI commonalities, because // these ABIs are very similar. @@ -508,6 +508,12 @@ impl ABIMachineSpec for AArch64MachineDeps { insts } + fn gen_probestack(_: u32) -> SmallVec<[Self::I; 2]> { + // TODO: implement if we ever require stack probes on an AArch64 host + // (unlikely unless Lucet is ported) + smallvec![] + } + // Returns stack bytes used as well as instructions. Does not adjust // nominal SP offset; abi_impl generic code will do that. fn gen_clobber_save( diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 4b69e1cf43..f09dd7dced 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -10,7 +10,7 @@ use crate::{CodegenError, CodegenResult}; use alloc::boxed::Box; use alloc::vec::Vec; use regalloc::{RealReg, Reg, RegClass, Set, Writable}; -use smallvec::SmallVec; +use smallvec::{smallvec, SmallVec}; /// Support for the ARM ABI from the callee side (within a function body). pub(crate) type Arm32ABICallee = ABICalleeImpl; @@ -305,6 +305,12 @@ impl ABIMachineSpec for Arm32MachineDeps { ret } + fn gen_probestack(_: u32) -> SmallVec<[Self::I; 2]> { + // TODO: implement if we ever require stack probes on ARM32 (unlikely + // unless Lucet is ported) + smallvec![] + } + /// Returns stack bytes used as well as instructions. Does not adjust /// nominal SP offset; caller will do that. fn gen_clobber_save( diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 4f84d75c1d..dc9592908b 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -1,7 +1,7 @@ //! Implementation of the standard x64 ABI. use crate::ir::types::*; -use crate::ir::{self, types, MemFlags, TrapCode, Type}; +use crate::ir::{self, types, ExternalName, LibCall, MemFlags, Opcode, TrapCode, Type}; use crate::isa; use crate::isa::{x64::inst::*, CallConv}; use crate::machinst::abi_impl::*; @@ -389,6 +389,22 @@ impl ABIMachineSpec for X64ABIMachineSpec { insts } + fn gen_probestack(frame_size: u32) -> SmallVec<[Self::I; 2]> { + let mut insts = SmallVec::new(); + insts.push(Inst::imm( + OperandSize::Size32, + frame_size as u64, + Writable::from_reg(regs::rax()), + )); + insts.push(Inst::CallKnown { + dest: ExternalName::LibCall(LibCall::Probestack), + uses: vec![regs::rax()], + defs: vec![], + opcode: Opcode::Call, + }); + insts + } + fn gen_clobber_save( call_conv: isa::CallConv, _: &settings::Flags, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 7d15063ad4..0289f2ea92 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -182,6 +182,7 @@ impl LegacyPrefixes { fn emit_std_enc_mem( sink: &mut MachBuffer, state: &EmitState, + info: &EmitInfo, prefixes: LegacyPrefixes, opcodes: u32, mut num_opcodes: usize, @@ -194,7 +195,8 @@ fn emit_std_enc_mem( // expression. But `enc_g` can be derived from a register of any class. let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && mem_e.can_trap() { + let can_trap = mem_e.can_trap(); + if srcloc != SourceLoc::default() && can_trap { sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); } @@ -202,6 +204,12 @@ fn emit_std_enc_mem( match mem_e { Amode::ImmReg { simm32, base, .. } => { + // If this is an access based off of RSP, it may trap with a stack overflow if it's the + // first touch of a new stack page. + if *base == regs::rsp() && !can_trap && info.flags().enable_probestack() { + sink.add_trap(srcloc, TrapCode::StackOverflow); + } + // First, the REX byte. let enc_e = int_reg_enc(*base); rex.emit_two_op(sink, enc_g, enc_e); @@ -262,6 +270,12 @@ fn emit_std_enc_mem( shift, .. } => { + // If this is an access based off of RSP, it may trap with a stack overflow if it's the + // first touch of a new stack page. + if *reg_base == regs::rsp() && !can_trap && info.flags().enable_probestack() { + sink.add_trap(srcloc, TrapCode::StackOverflow); + } + let enc_base = int_reg_enc(*reg_base); let enc_index = int_reg_enc(*reg_index); @@ -350,6 +364,7 @@ fn emit_std_enc_enc( fn emit_std_reg_mem( sink: &mut MachBuffer, state: &EmitState, + info: &EmitInfo, prefixes: LegacyPrefixes, opcodes: u32, num_opcodes: usize, @@ -361,6 +376,7 @@ fn emit_std_reg_mem( emit_std_enc_mem( sink, state, + info, prefixes, opcodes, num_opcodes, @@ -538,6 +554,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, 0x0FAF, 2, @@ -597,6 +614,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, opcode_m, 1, @@ -654,6 +672,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, prefix, opcode, num_opcodes, @@ -717,7 +736,9 @@ pub(crate) fn emit( } RegMem::Mem { addr: src } => { let amode = src.finalize(state, sink); - emit_std_enc_mem(sink, state, prefix, opcode, 1, subopcode, &amode, rex_flags); + emit_std_enc_mem( + sink, state, info, prefix, opcode, 1, subopcode, &amode, rex_flags, + ); } } } @@ -738,7 +759,9 @@ pub(crate) fn emit( } RegMem::Mem { addr: src } => { let amode = src.finalize(state, sink); - emit_std_enc_mem(sink, state, prefix, 0xF7, 1, subopcode, &amode, rex_flags); + emit_std_enc_mem( + sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags, + ); } } } @@ -987,6 +1010,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, opcodes, num_opcodes, @@ -1004,6 +1028,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, 0x8B, 1, @@ -1019,6 +1044,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, 0x8D, 1, @@ -1081,6 +1107,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, opcodes, num_opcodes, @@ -1108,7 +1135,17 @@ pub(crate) fn emit( }; // MOV r8, r/m8 is (REX.W==0) 88 /r - emit_std_reg_mem(sink, state, LegacyPrefixes::None, 0x88, 1, *src, dst, rex) + emit_std_reg_mem( + sink, + state, + info, + LegacyPrefixes::None, + 0x88, + 1, + *src, + dst, + rex, + ) } 2 => { @@ -1116,6 +1153,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::_66, 0x89, 1, @@ -1130,6 +1168,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, 0x89, 1, @@ -1144,6 +1183,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, LegacyPrefixes::None, 0x89, 1, @@ -1253,6 +1293,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, prefix, opcode_bytes, 2, @@ -1311,7 +1352,7 @@ pub(crate) fn emit( let addr = &addr.finalize(state, sink); // Whereas here we revert to the "normal" G-E ordering. let opcode = if *size == 1 { 0x3A } else { 0x3B }; - emit_std_reg_mem(sink, state, prefix, opcode, 1, *reg_g, addr, rex); + emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *reg_g, addr, rex); } RegMemImm::Imm { simm32 } => { @@ -1372,6 +1413,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, prefix, opcode, 2, @@ -1408,6 +1450,10 @@ pub(crate) fn emit( } Inst::Push64 { src } => { + if info.flags().enable_probestack() { + sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow); + } + match src { RegMemImm::Reg { reg } => { let enc_reg = int_reg_enc(*reg); @@ -1423,6 +1469,7 @@ pub(crate) fn emit( emit_std_enc_mem( sink, state, + info, LegacyPrefixes::None, 0xFF, 1, @@ -1454,6 +1501,9 @@ pub(crate) fn emit( } Inst::CallKnown { dest, opcode, .. } => { + if info.flags().enable_probestack() { + sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow); + } if let Some(s) = state.take_stack_map() { sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s); } @@ -1469,6 +1519,9 @@ pub(crate) fn emit( } Inst::CallUnknown { dest, opcode, .. } => { + if info.flags().enable_probestack() { + sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow); + } let start_offset = sink.cur_offset(); match dest { RegMem::Reg { reg } => { @@ -1489,6 +1542,7 @@ pub(crate) fn emit( emit_std_enc_mem( sink, state, + info, LegacyPrefixes::None, 0xFF, 1, @@ -1587,6 +1641,7 @@ pub(crate) fn emit( emit_std_enc_mem( sink, state, + info, LegacyPrefixes::None, 0xFF, 1, @@ -1733,6 +1788,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, prefix, opcode, num_opcodes, @@ -1848,6 +1904,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, prefix, opcode, length, @@ -1994,7 +2051,17 @@ pub(crate) fn emit( !regs_swapped, "No existing way to encode a mem argument in the ModRM r/m field." ); - emit_std_reg_mem(sink, state, prefix, opcode, len, dst.to_reg(), addr, rex); + emit_std_reg_mem( + sink, + state, + info, + prefix, + opcode, + len, + dst.to_reg(), + addr, + rex, + ); } } sink.put1(*imm); @@ -2027,6 +2094,7 @@ pub(crate) fn emit( emit_std_reg_mem( sink, state, + info, prefix, opcode, 2, @@ -2091,7 +2159,17 @@ pub(crate) fn emit( } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); - emit_std_reg_mem(sink, state, prefix, opcode, 2, reg_g.to_reg(), addr, rex); + emit_std_reg_mem( + sink, + state, + info, + prefix, + opcode, + 2, + reg_g.to_reg(), + addr, + rex, + ); } } } @@ -2111,7 +2189,7 @@ pub(crate) fn emit( } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); - emit_std_reg_mem(sink, state, prefix, opcode, len, *dst, addr, rex); + emit_std_reg_mem(sink, state, info, prefix, opcode, len, *dst, addr, rex); } } } @@ -2625,7 +2703,7 @@ pub(crate) fn emit( _ => unreachable!(), }; let amode = dst.finalize(state, sink); - emit_std_reg_mem(sink, state, prefix, opcodes, 2, *src, &amode, rex); + emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex); } Inst::AtomicRmwSeq { ty, op } => { diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index bbe886c24b..538e0e6dac 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -4079,8 +4079,8 @@ impl LowerBackend for X64Backend { let ty = ctx.input_ty(ifcmp_sp, 0); ctx.emit(Inst::cmp_rmi_r( ty.bytes() as u8, - RegMemImm::reg(operand), - regs::rsp(), + RegMemImm::reg(regs::rsp()), + operand, )); let cond_code = ctx.data(branches[0]).cond_code().unwrap(); let cc = CC::from_intcc(cond_code); diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index b9774faba9..4b21c2d946 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -314,6 +314,9 @@ pub trait ABIMachineSpec { /// Generate the usual frame-restore sequence for this architecture. fn gen_epilogue_frame_restore() -> SmallVec<[Self::I; 2]>; + /// Generate a probestack call. + fn gen_probestack(_frame_size: u32) -> SmallVec<[Self::I; 2]>; + /// Generate a clobber-save sequence. This takes the list of *all* registers /// written/modified by the function body. The implementation here is /// responsible for determining which of these are callee-saved according to @@ -481,6 +484,9 @@ pub struct ABICalleeImpl { /// manually register-allocated and carefully only use caller-saved /// registers and keep nothing live after this sequence of instructions. stack_limit: Option<(Reg, Vec)>, + /// Are we to invoke the probestack function in the prologue? If so, + /// what is the minimum size at which we must invoke it? + probestack_min_frame: Option, _mach: PhantomData, } @@ -536,6 +542,18 @@ impl ABICalleeImpl { .map(|reg| (reg, Vec::new())) .or_else(|| f.stack_limit.map(|gv| gen_stack_limit::(f, &sig, gv))); + // Determine whether a probestack call is required for large enough + // frames (and the minimum frame size if so). + let probestack_min_frame = if flags.enable_probestack() { + assert!( + !flags.probestack_func_adjusts_sp(), + "SP-adjusting probestack not supported in new backends" + ); + Some(1 << flags.probestack_size_log2()) + } else { + None + }; + Ok(Self { sig, stackslots, @@ -550,6 +568,7 @@ impl ABICalleeImpl { flags, is_leaf: f.is_leaf(), stack_limit, + probestack_min_frame, _mach: PhantomData, }) } @@ -978,6 +997,11 @@ impl ABICallee for ABICalleeImpl { insts.extend_from_slice(stack_limit_load); self.insert_stack_check(*reg, total_stacksize, &mut insts); } + if let Some(min_frame) = &self.probestack_min_frame { + if total_stacksize >= *min_frame { + insts.extend(M::gen_probestack(total_stacksize)); + } + } } if total_stacksize > 0 { self.fixed_frame_storage_size += total_stacksize; diff --git a/cranelift/filetests/filetests/isa/x64/probestack.clif b/cranelift/filetests/filetests/isa/x64/probestack.clif new file mode 100644 index 0000000000..135587d355 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/probestack.clif @@ -0,0 +1,17 @@ +test compile +set enable_probestack=true +target x86_64 +feature "experimental_x64" + +function %f1() -> i64 { +ss0 = explicit_slot 100000 + +block0: + v1 = stack_addr.i64 ss0 + return v1 +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: movl $$100000, %eax +; nextln: call LibCall(Probestack) From 6632c45c01fec9d30b62031e75e7c3e29b3fac04 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 8 Dec 2020 11:46:15 -0800 Subject: [PATCH 52/63] x64 lowering fix: i32.popcnt should not merge load and make it 64-bit. As a subtle consequence of the recent load-op fusion, popcnt of a value that came from a load.i32 was compiling into a 64-bit load. This is a result of the way in which x86 infers the width of loads: it is a consequence of the instruction containing the memory reference, not the memory reference itself. So the `input_to_reg_mem()` helper (convert an instruction input into a register or memory reference) was providing the appropriate memory reference for the result of a load.i32, but never encoded the assumption that it would only be used in a 32-bit instruction. It turns out that popcnt.i32 uses a 64-bit instruction to load this RM op, hence widening a 32-bit to 64-bit load (which is problematic when the offset is (memory_length - 4)). Separately, popcnt was using the RM operand twice, resulting in two loads if we merged a load. This isn't a correctness bug in practice because only a racy sequence (store interleaving between the loads) would produce incorrect results, but we decided earlier to treat loads as effectful for now, neither reordering nor duplicating them, to deliberately reduce complexity. Because of the second issue, the fix is just to force the operand into a register always, so any source load will not be merged. Discovered via fuzzing with oss-fuzz. --- cranelift/codegen/src/isa/x64/lower.rs | 5 +- .../filetests/filetests/isa/x64/popcnt.clif | 113 ++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 cranelift/filetests/filetests/isa/x64/popcnt.clif diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 16e7490a09..a01e35bc0d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1530,7 +1530,10 @@ fn lower_insn_to_regs>( let src = if let Some(ext_spec) = ext_spec { RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec)) } else { - input_to_reg_mem(ctx, inputs[0]) + // N.B.: explicitly put input in a reg here because the width of the instruction + // into which this RM op goes may not match the width of the input type (in fact, + // it won't for i32.popcnt), and we don't want a larger than necessary load. + RegMem::reg(put_input_in_reg(ctx, inputs[0])) }; let dst = get_output_reg(ctx, outputs[0]); diff --git a/cranelift/filetests/filetests/isa/x64/popcnt.clif b/cranelift/filetests/filetests/isa/x64/popcnt.clif new file mode 100644 index 0000000000..a06f5a27ce --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/popcnt.clif @@ -0,0 +1,113 @@ +test compile +target x86_64 +feature "experimental_x64" + +; TODO: test with popcnt feature available too, once new backend supports that. + +function %popcnt64(i64) -> i64 { +block0(v0: i64): + v1 = popcnt v0 +; check: movq %rdi, %rsi +; nextln: shrq $$1, %rsi +; nextln: movabsq $$8608480567731124087, %rax +; nextln: andq %rax, %rsi +; nextln: subq %rsi, %rdi +; nextln: shrq $$1, %rsi +; nextln: andq %rax, %rsi +; nextln: subq %rsi, %rdi +; nextln: shrq $$1, %rsi +; nextln: andq %rax, %rsi +; nextln: subq %rsi, %rdi +; nextln: movq %rdi, %rsi +; nextln: shrq $$4, %rsi +; nextln: addq %rdi, %rsi +; nextln: movabsq $$1085102592571150095, %rdi +; nextln: andq %rdi, %rsi +; nextln: movabsq $$72340172838076673, %rdi +; nextln: imulq %rdi, %rsi +; nextln: shrq $$56, %rsi +; nextln: movq %rsi, %rax + return v1 +} + +function %popcnt64load(i64) -> i64 { +block0(v0: i64): + v1 = load.i64 v0 + v2 = popcnt v1 + return v2 +; check: movq 0(%rdi), %rdi +; nextln: movq %rdi, %rsi +; nextln: shrq $$1, %rsi +; nextln: movabsq $$8608480567731124087, %rax +; nextln: andq %rax, %rsi +; nextln: subq %rsi, %rdi +; nextln: shrq $$1, %rsi +; nextln: andq %rax, %rsi +; nextln: subq %rsi, %rdi +; nextln: shrq $$1, %rsi +; nextln: andq %rax, %rsi +; nextln: subq %rsi, %rdi +; nextln: movq %rdi, %rsi +; nextln: shrq $$4, %rsi +; nextln: addq %rdi, %rsi +; nextln: movabsq $$1085102592571150095, %rdi +; nextln: andq %rdi, %rsi +; nextln: movabsq $$72340172838076673, %rdi +; nextln: imulq %rdi, %rsi +; nextln: shrq $$56, %rsi +; nextln: movq %rsi, %rax +} + +function %popcnt32(i32) -> i32 { +block0(v0: i32): + v1 = popcnt v0 + return v1 +; check: movq %rdi, %rsi +; nextln: shrl $$1, %esi +; nextln: andl $$2004318071, %esi +; nextln: subl %esi, %edi +; nextln: shrl $$1, %esi +; nextln: andl $$2004318071, %esi +; nextln: subl %esi, %edi +; nextln: shrl $$1, %esi +; nextln: andl $$2004318071, %esi +; nextln: subl %esi, %edi +; nextln: movq %rdi, %rsi +; nextln: shrl $$4, %esi +; nextln: addl %edi, %esi +; nextln: andl $$252645135, %esi +; nextln: imull $$16843009, %esi +; nextln: shrl $$24, %esi +; nextln: movq %rsi, %rax +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %popcnt32load(i64) -> i32 { +block0(v0: i64): + v1 = load.i32 v0 + v2 = popcnt v1 + return v2 +; check: movl 0(%rdi), %edi +; nextln: movq %rdi, %rsi +; nextln: shrl $$1, %esi +; nextln: andl $$2004318071, %esi +; nextln: subl %esi, %edi +; nextln: shrl $$1, %esi +; nextln: andl $$2004318071, %esi +; nextln: subl %esi, %edi +; nextln: shrl $$1, %esi +; nextln: andl $$2004318071, %esi +; nextln: subl %esi, %edi +; nextln: movq %rdi, %rsi +; nextln: shrl $$4, %esi +; nextln: addl %edi, %esi +; nextln: andl $$252645135, %esi +; nextln: imull $$16843009, %esi +; nextln: shrl $$24, %esi +; nextln: movq %rsi, %rax +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} From 855a6374dd744e1f61737e7c5bce09e367571c67 Mon Sep 17 00:00:00 2001 From: Y-Nak Date: Thu, 14 May 2020 22:20:24 +0900 Subject: [PATCH 53/63] Fix missing modification of jump table in licm --- cranelift/codegen/src/ir/function.rs | 44 ++++++++++++++++++- cranelift/codegen/src/ir/instructions.rs | 2 +- cranelift/codegen/src/licm.rs | 37 ++++++++-------- .../filetests/filetests/licm/br-table.clif | 19 ++++++++ .../filetests/licm/rewrite-jump-table.clif | 28 ++++++++++++ 5 files changed, 109 insertions(+), 21 deletions(-) create mode 100644 cranelift/filetests/filetests/licm/br-table.clif create mode 100644 cranelift/filetests/filetests/licm/rewrite-jump-table.clif diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 92911c8a59..1833af27f5 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -7,8 +7,9 @@ use crate::binemit::CodeOffset; use crate::entity::{PrimaryMap, SecondaryMap}; use crate::ir; use crate::ir::{ - Block, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, JumpTable, - JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, + instructions::BranchInfo, Block, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, + HeapData, Inst, InstructionData, JumpTable, JumpTableData, Opcode, SigRef, StackSlot, + StackSlotData, Table, TableData, }; use crate::ir::{BlockOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations}; use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature}; @@ -270,6 +271,8 @@ impl Function { /// Changes the destination of a jump or branch instruction. /// Does nothing if called with a non-jump or non-branch instruction. + /// + /// Note that this method ignores multi-destination branches like `br_table`. pub fn change_branch_destination(&mut self, inst: Inst, new_dest: Block) { match self.dfg[inst].branch_destination_mut() { None => (), @@ -277,6 +280,43 @@ impl Function { } } + /// Rewrite the branch destination to `new_dest` if the destination matches `old_dest`. + /// Does nothing if called with a non-jump or non-branch instruction. + /// + /// Unlike [change_branch_destination](Function::change_branch_destination), this method rewrite the destinations of + /// multi-destination branches like `br_table`. + pub fn rewrite_branch_destination(&mut self, inst: Inst, old_dest: Block, new_dest: Block) { + match self.dfg.analyze_branch(inst) { + BranchInfo::SingleDest(dest, ..) => { + if dest == old_dest { + self.change_branch_destination(inst, new_dest); + } + } + + BranchInfo::Table(table, default_dest) => { + self.jump_tables[table].iter_mut().for_each(|entry| { + if *entry == old_dest { + *entry = new_dest; + } + }); + + if default_dest == Some(old_dest) { + match &mut self.dfg[inst] { + InstructionData::BranchTable { destination, .. } => { + *destination = new_dest; + } + _ => panic!( + "Unexpected instruction {} having default destination", + self.dfg.display_inst(inst, None) + ), + } + } + } + + BranchInfo::NotABranch => {} + } + } + /// Checks that the specified block can be encoded as a basic block. /// /// On error, returns the first invalid instruction and an error message. diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs index 4a14b77f33..13310bc01c 100644 --- a/cranelift/codegen/src/ir/instructions.rs +++ b/cranelift/codegen/src/ir/instructions.rs @@ -277,7 +277,7 @@ impl InstructionData { ref mut destination, .. } => Some(destination), - Self::BranchTable { .. } => None, + Self::BranchTable { .. } | Self::IndirectJump { .. } => None, _ => { debug_assert!(!self.opcode().is_branch()); None diff --git a/cranelift/codegen/src/licm.rs b/cranelift/codegen/src/licm.rs index 75000b5297..5e9e0c1262 100644 --- a/cranelift/codegen/src/licm.rs +++ b/cranelift/codegen/src/licm.rs @@ -61,8 +61,8 @@ pub fn do_licm( domtree.compute(func, cfg); } -// Insert a pre-header before the header, modifying the function layout and CFG to reflect it. -// A jump instruction to the header is placed at the end of the pre-header. +/// Insert a pre-header before the header, modifying the function layout and CFG to reflect it. +/// A jump instruction to the header is placed at the end of the pre-header. fn create_pre_header( isa: &dyn TargetIsa, header: Block, @@ -81,30 +81,31 @@ fn create_pre_header( for typ in header_args_types { pre_header_args_value.push(func.dfg.append_block_param(pre_header, typ), pool); } + for BlockPredecessor { inst: last_inst, .. } in cfg.pred_iter(header) { // We only follow normal edges (not the back edges) if !domtree.dominates(header, last_inst, &func.layout) { - func.change_branch_destination(last_inst, pre_header); + func.rewrite_branch_destination(last_inst, header, pre_header); } } - { - let mut pos = EncCursor::new(func, isa).at_top(header); - // Inserts the pre-header at the right place in the layout. - pos.insert_block(pre_header); - pos.next_inst(); - pos.ins().jump(header, pre_header_args_value.as_slice(pool)); - } + + // Inserts the pre-header at the right place in the layout. + let mut pos = EncCursor::new(func, isa).at_top(header); + pos.insert_block(pre_header); + pos.next_inst(); + pos.ins().jump(header, pre_header_args_value.as_slice(pool)); + pre_header } -// Detects if a loop header has a natural pre-header. -// -// A loop header has a pre-header if there is only one predecessor that the header doesn't -// dominate. -// Returns the pre-header Block and the instruction jumping to the header. +/// Detects if a loop header has a natural pre-header. +/// +/// A loop header has a pre-header if there is only one predecessor that the header doesn't +/// dominate. +/// Returns the pre-header Block and the instruction jumping to the header. fn has_pre_header( layout: &Layout, cfg: &ControlFlowGraph, @@ -176,9 +177,9 @@ fn is_loop_invariant(inst: Inst, dfg: &DataFlowGraph, loop_values: &FxHashSet Date: Wed, 9 Dec 2020 10:34:37 -0600 Subject: [PATCH 54/63] Restrict threads spawned during fuzzing (#2485) I was having limited success fuzzing locally because apparently the fuzzer was spawning too many threads. Looking into it that indeed appears to be the case! The threads which time out runtime of wasm only exit after the sleep has completely finished, meaning that if we execute a ton of wasm that exits quickly each run will generate a sleeping thread. This commit fixes the issue by using some synchronization to ensure the sleeping thread exits when our fuzzed run also exits. --- crates/fuzzing/src/oracles.rs | 73 ++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 2fb6c58682..5a5ab3e68a 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -18,7 +18,8 @@ use log::debug; use std::cell::Cell; use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; -use std::time::Duration; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; use wasmtime::*; use wasmtime_wast::WastContext; @@ -73,12 +74,20 @@ pub fn instantiate_with_config(wasm: &[u8], mut config: Config, timeout: Option< let engine = Engine::new(&config); let store = Store::new(&engine); + // If a timeout is requested then we spawn a helper thread to wait for the + // requested time and then send us a signal to get interrupted. We also + // arrange for the thread's sleep to get interrupted if we return early (or + // the wasm returns within the time limit), which allows the thread to get + // torn down. + // + // This prevents us from creating a huge number of sleeping threads if this + // function is executed in a loop, like it does on nightly fuzzing + // infrastructure. + + let mut timeout_state = SignalOnDrop::default(); if let Some(timeout) = timeout { let handle = store.interrupt_handle().unwrap(); - std::thread::spawn(move || { - std::thread::sleep(timeout); - handle.interrupt(); - }); + timeout_state.spawn_timeout(timeout, move || handle.interrupt()); } log_wasm(wasm); @@ -645,3 +654,57 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Con Some(()) } + +#[derive(Default)] +struct SignalOnDrop { + state: Arc<(Mutex, Condvar)>, + thread: Option>, +} + +impl SignalOnDrop { + fn spawn_timeout(&mut self, dur: Duration, closure: impl FnOnce() + Send + 'static) { + let state = self.state.clone(); + let start = Instant::now(); + self.thread = Some(std::thread::spawn(move || { + // Using our mutex/condvar we wait here for the first of `dur` to + // pass or the `SignalOnDrop` instance to get dropped. + let (lock, cvar) = &*state; + let mut signaled = lock.lock().unwrap(); + while !*signaled { + // Adjust our requested `dur` based on how much time has passed. + let dur = match dur.checked_sub(start.elapsed()) { + Some(dur) => dur, + None => break, + }; + let (lock, result) = cvar.wait_timeout(signaled, dur).unwrap(); + signaled = lock; + // If we timed out for sure then there's no need to continue + // since we'll just abort on the next `checked_sub` anyway. + if result.timed_out() { + break; + } + } + drop(signaled); + + closure(); + })); + } +} + +impl Drop for SignalOnDrop { + fn drop(&mut self) { + if let Some(thread) = self.thread.take() { + let (lock, cvar) = &*self.state; + // Signal our thread that we've been dropped and wake it up if it's + // blocked. + let mut g = lock.lock().unwrap(); + *g = true; + cvar.notify_one(); + drop(g); + + // ... and then wait for the thread to exit to ensure we clean up + // after ourselves. + thread.join().unwrap(); + } + } +} From 25000afe69bd32ded541a41074b217defa03a912 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 7 Dec 2020 15:57:35 -0800 Subject: [PATCH 55/63] Enable fuzzing the module linking implementation This commit updates all the wasm-tools crates that we use and enables fuzzing of the module linking proposal in our various fuzz targets. This also refactors some of the dummy value generation logic to not be fallible and to always succeed, the thinking being that we don't want to accidentally hide errors while fuzzing. Additionally instantiation is only allowed to fail with a `Trap`, other failure reasons are unwrapped. --- Cargo.lock | 60 +-- Cargo.toml | 4 +- cranelift/codegen/Cargo.toml | 2 +- cranelift/peepmatic/Cargo.toml | 2 +- cranelift/peepmatic/crates/fuzzing/Cargo.toml | 2 +- cranelift/peepmatic/crates/runtime/Cargo.toml | 2 +- cranelift/peepmatic/crates/souper/Cargo.toml | 2 +- .../peepmatic/crates/test-operator/Cargo.toml | 2 +- cranelift/wasm/Cargo.toml | 2 +- cranelift/wasm/src/code_translator.rs | 6 +- crates/debug/Cargo.toml | 2 +- crates/environ/Cargo.toml | 2 +- crates/fuzzing/Cargo.toml | 6 +- crates/fuzzing/src/lib.rs | 1 + crates/fuzzing/src/oracles.rs | 71 +--- crates/fuzzing/src/oracles/dummy.rs | 351 +++++++++++++++--- crates/jit/Cargo.toml | 2 +- crates/lightbeam/Cargo.toml | 2 +- crates/lightbeam/wasmtime/Cargo.toml | 2 +- crates/wasmtime/Cargo.toml | 2 +- crates/wast/Cargo.toml | 2 +- fuzz/Cargo.toml | 2 +- 22 files changed, 377 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13c13d89cf..64237674ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -394,7 +394,7 @@ dependencies = [ "souper-ir", "target-lexicon", "thiserror", - "wast 28.0.0", + "wast 29.0.0", ] [[package]] @@ -597,7 +597,7 @@ dependencies = [ "smallvec", "target-lexicon", "thiserror", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wat", ] @@ -1167,7 +1167,7 @@ dependencies = [ "smallvec", "thiserror", "typemap", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wat", ] @@ -1411,7 +1411,7 @@ dependencies = [ "peepmatic-test-operator", "peepmatic-traits", "serde", - "wast 28.0.0", + "wast 29.0.0", "z3", ] @@ -1439,7 +1439,7 @@ dependencies = [ "peepmatic-traits", "rand", "serde", - "wast 28.0.0", + "wast 29.0.0", ] [[package]] @@ -1464,7 +1464,7 @@ dependencies = [ "serde", "serde_test", "thiserror", - "wast 28.0.0", + "wast 29.0.0", ] [[package]] @@ -1476,7 +1476,7 @@ dependencies = [ "peepmatic", "peepmatic-test-operator", "souper-ir", - "wast 28.0.0", + "wast 29.0.0", ] [[package]] @@ -1497,7 +1497,7 @@ version = "0.68.0" dependencies = [ "peepmatic-traits", "serde", - "wast 28.0.0", + "wast 29.0.0", ] [[package]] @@ -2356,18 +2356,18 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49891b7c581cf9e0090b25cd274e6498ad478f0a7819319ea96da2f253caaacb" +checksum = "ed89eaf99e08b84f96e477a16588a07dd3b51dc5f07291c3706782f62a10a5e1" dependencies = [ "leb128", ] [[package]] name = "wasm-smith" -version = "0.1.12" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fdf8c9ba2fdc0d8ffe3f7e5b23b5aac377eeca817a9885c058f27c8de5c500" +checksum = "cdd382e46cf44347f4d0c29122143cbab85cf248b9a8f680845c0239b6842a85" dependencies = [ "arbitrary", "leb128", @@ -2406,18 +2406,18 @@ checksum = "32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6" [[package]] name = "wasmparser" -version = "0.69.2" +version = "0.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd2dd6dadf3a723971297bcc0ec103e0aa8118bf68e23f49cb575e21621894a8" +checksum = "ed1b3f9e9cf01a580b9f3281214dfdb1922b5dfb8494ee312ca03ae10036c2a2" [[package]] name = "wasmprinter" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba006f5c5bf41a2a5c3b45e861ea6eb067382acb022b6a35a00a0390f9547f6" +checksum = "f89b2b24dce17e27fe9c09c28331cbd77067fcde5c6ea2508ac84bcbd5d3e018" dependencies = [ "anyhow", - "wasmparser 0.69.2", + "wasmparser 0.70.0", ] [[package]] @@ -2438,7 +2438,7 @@ dependencies = [ "smallvec", "target-lexicon", "tempfile", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wasmtime-cache", "wasmtime-environ", "wasmtime-jit", @@ -2516,7 +2516,7 @@ dependencies = [ "test-programs", "tracing-subscriber", "wasi-common", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wasmtime", "wasmtime-cache", "wasmtime-debug", @@ -2552,7 +2552,7 @@ dependencies = [ "object", "target-lexicon", "thiserror", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wasmtime-environ", ] @@ -2571,7 +2571,7 @@ dependencies = [ "more-asserts", "serde", "thiserror", - "wasmparser 0.69.2", + "wasmparser 0.70.0", ] [[package]] @@ -2600,7 +2600,7 @@ dependencies = [ "rayon", "wasm-smith", "wasmi", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wasmprinter", "wasmtime", "wasmtime-wast", @@ -2628,7 +2628,7 @@ dependencies = [ "serde", "target-lexicon", "thiserror", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wasmtime-cranelift", "wasmtime-debug", "wasmtime-environ", @@ -2645,7 +2645,7 @@ version = "0.21.0" dependencies = [ "cranelift-codegen", "lightbeam", - "wasmparser 0.69.2", + "wasmparser 0.70.0", "wasmtime-environ", ] @@ -2752,7 +2752,7 @@ version = "0.21.0" dependencies = [ "anyhow", "wasmtime", - "wast 28.0.0", + "wast 29.0.0", ] [[package]] @@ -2788,20 +2788,20 @@ dependencies = [ [[package]] name = "wast" -version = "28.0.0" +version = "29.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0586061bfacc035034672c8d760802b428ab4c80a92e2a392425c516df9be1" +checksum = "dcf2268937131d63c3d833242bf5e075406f9ed868b4265f3280e15dac29ac18" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.29" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d55b5ec4f9d9396fa99abaafa0688597395e57827dffd89731412ae90c9bf" +checksum = "0d11a88d953b298172d218d18f22853f4e6e12873b62755d05617b864d312c68" dependencies = [ - "wast 28.0.0", + "wast 29.0.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7ea112be71..42ed33eb90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,12 +38,12 @@ anyhow = "1.0.19" target-lexicon = { version = "0.11.0", default-features = false } pretty_env_logger = "0.4.0" file-per-thread-logger = "0.1.1" -wat = "1.0.29" +wat = "1.0.30" libc = "0.2.60" log = "0.4.8" rayon = "1.2.1" humantime = "2.0.0" -wasmparser = "0.69.2" +wasmparser = "0.70.0" [dev-dependencies] env_logger = "0.8.1" diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index cc0a6fd62d..c812c3a5cb 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -30,7 +30,7 @@ peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, versi peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.68.0" } regalloc = { version = "0.0.31" } souper-ir = { version = "2.1.0", optional = true } -wast = { version = "28.0.0", optional = true } +wast = { version = "29.0.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary # machine code. Integration tests that need external dependencies can be diff --git a/cranelift/peepmatic/Cargo.toml b/cranelift/peepmatic/Cargo.toml index 3691cf713e..f61e840b5d 100644 --- a/cranelift/peepmatic/Cargo.toml +++ b/cranelift/peepmatic/Cargo.toml @@ -15,7 +15,7 @@ peepmatic-macro = { version = "0.68.0", path = "crates/macro" } peepmatic-runtime = { version = "0.68.0", path = "crates/runtime", features = ["construct"] } peepmatic-traits = { version = "0.68.0", path = "crates/traits" } serde = { version = "1.0.105", features = ["derive"] } -wast = "28.0.0" +wast = "29.0.0" z3 = { version = "0.7.1", features = ["static-link-z3"] } [dev-dependencies] diff --git a/cranelift/peepmatic/crates/fuzzing/Cargo.toml b/cranelift/peepmatic/crates/fuzzing/Cargo.toml index 83a57c943f..1611b75c6d 100644 --- a/cranelift/peepmatic/crates/fuzzing/Cargo.toml +++ b/cranelift/peepmatic/crates/fuzzing/Cargo.toml @@ -21,4 +21,4 @@ peepmatic-test-operator = { path = "../test-operator" } peepmatic-traits = { path = "../traits" } rand = { version = "0.7.3", features = ["small_rng"] } serde = "1.0.106" -wast = "28.0.0" +wast = "29.0.0" diff --git a/cranelift/peepmatic/crates/runtime/Cargo.toml b/cranelift/peepmatic/crates/runtime/Cargo.toml index 2b60e7d557..deb8d9d0c3 100644 --- a/cranelift/peepmatic/crates/runtime/Cargo.toml +++ b/cranelift/peepmatic/crates/runtime/Cargo.toml @@ -16,7 +16,7 @@ peepmatic-automata = { version = "0.68.0", path = "../automata", features = ["se peepmatic-traits = { version = "0.68.0", path = "../traits" } serde = { version = "1.0.105", features = ["derive"] } thiserror = "1.0.15" -wast = { version = "28.0.0", optional = true } +wast = { version = "29.0.0", optional = true } [dev-dependencies] peepmatic-test-operator = { version = "0.68.0", path = "../test-operator" } diff --git a/cranelift/peepmatic/crates/souper/Cargo.toml b/cranelift/peepmatic/crates/souper/Cargo.toml index 84bf6f52e0..5970b40e73 100644 --- a/cranelift/peepmatic/crates/souper/Cargo.toml +++ b/cranelift/peepmatic/crates/souper/Cargo.toml @@ -16,4 +16,4 @@ log = "0.4.8" [dev-dependencies] peepmatic = { path = "../..", version = "0.68.0" } peepmatic-test-operator = { version = "0.68.0", path = "../test-operator" } -wast = "28.0.0" +wast = "29.0.0" diff --git a/cranelift/peepmatic/crates/test-operator/Cargo.toml b/cranelift/peepmatic/crates/test-operator/Cargo.toml index 59ce9a15c6..9b49020071 100644 --- a/cranelift/peepmatic/crates/test-operator/Cargo.toml +++ b/cranelift/peepmatic/crates/test-operator/Cargo.toml @@ -9,4 +9,4 @@ edition = "2018" [dependencies] peepmatic-traits = { version = "0.68.0", path = "../traits" } serde = { version = "1.0.105", features = ["derive"] } -wast = "28.0.0" +wast = "29.0.0" diff --git a/cranelift/wasm/Cargo.toml b/cranelift/wasm/Cargo.toml index d6aa8b4ced..48e19a9d57 100644 --- a/cranelift/wasm/Cargo.toml +++ b/cranelift/wasm/Cargo.toml @@ -12,7 +12,7 @@ keywords = ["webassembly", "wasm"] edition = "2018" [dependencies] -wasmparser = { version = "0.69.2", default-features = false } +wasmparser = { version = "0.70", default-features = false } cranelift-codegen = { path = "../codegen", version = "0.68.0", default-features = false } cranelift-entity = { path = "../entity", version = "0.68.0" } cranelift-frontend = { path = "../frontend", version = "0.68.0", default-features = false } diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs index c522feb059..cf15787c18 100644 --- a/cranelift/wasm/src/code_translator.rs +++ b/cranelift/wasm/src/code_translator.rs @@ -538,10 +538,10 @@ pub fn translate_operator( } /********************************** Exception handing **********************************/ Operator::Try { .. } - | Operator::Catch - | Operator::BrOnExn { .. } + | Operator::Catch { .. } | Operator::Throw { .. } - | Operator::Rethrow => { + | Operator::Unwind + | Operator::Rethrow { .. } => { return Err(wasm_unsupported!( "proposed exception handling operator {:?}", op diff --git a/crates/debug/Cargo.toml b/crates/debug/Cargo.toml index 62a07daf56..f3b5bdede7 100644 --- a/crates/debug/Cargo.toml +++ b/crates/debug/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] gimli = "0.23.0" -wasmparser = "0.69.2" +wasmparser = "0.70" object = { version = "0.22.0", default-features = false, features = ["read", "write"] } wasmtime-environ = { path = "../environ", version = "0.21.0" } target-lexicon = { version = "0.11.0", default-features = false } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 118e319ebd..26b0ec0eaf 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.68.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.68.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.68.0", features = ["enable-serde"] } -wasmparser = "0.69.2" +wasmparser = "0.70" indexmap = { version = "1.0.2", features = ["serde-1"] } thiserror = "1.0.4" serde = { version = "1.0.94", features = ["derive"] } diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 8aa5768457..b0dd625c02 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -12,11 +12,11 @@ arbitrary = { version = "0.4.1", features = ["derive"] } env_logger = "0.8.1" log = "0.4.8" rayon = "1.2.1" -wasmparser = "0.69.2" -wasmprinter = "0.2.16" +wasmparser = "0.70" +wasmprinter = "0.2.17" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } -wasm-smith = "0.1.12" +wasm-smith = "0.2.0" wasmi = "0.7.0" [dev-dependencies] diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs index 7e649272de..7ef3382411 100644 --- a/crates/fuzzing/src/lib.rs +++ b/crates/fuzzing/src/lib.rs @@ -38,6 +38,7 @@ pub fn fuzz_default_config(strategy: wasmtime::Strategy) -> anyhow::Result module, - Err(_) => return, - }; + let module = Module::new(&engine, wasm).unwrap(); + let imports = dummy_imports(&store, module.imports()); - let imports = match dummy_imports(&store, module.imports()) { - Ok(imps) => imps, - Err(_) => { - // There are some value types that we can't synthesize a - // dummy value for (e.g. externrefs) and for modules that - // import things of these types we skip instantiation. - return; - } - }; - - // Don't unwrap this: there can be instantiation-/link-time errors that - // aren't caught during validation or compilation. For example, an imported - // table might not have room for an element segment that we want to - // initialize into it. - let _result = Instance::new(&store, &module, &imports); + match Instance::new(&store, &module, &imports) { + Ok(_) => {} + // Allow traps which can happen normally with `unreachable` + Err(e) if e.downcast_ref::().is_some() => {} + Err(e) => panic!("failed to instantiate {}", e), + } } /// Compile the Wasm buffer, and implicitly fail if we have an unexpected @@ -162,31 +151,14 @@ pub fn differential_execution( let engine = Engine::new(config); let store = Store::new(&engine); - let module = match Module::new(&engine, &wasm) { - Ok(module) => module, - // The module might rely on some feature that our config didn't - // enable or something like that. - Err(e) => { - eprintln!("Warning: failed to compile `wasm-opt -ttf` module: {}", e); - continue; - } - }; + let module = Module::new(&engine, &wasm).unwrap(); // TODO: we should implement tracing versions of these dummy imports // that record a trace of the order that imported functions were called // in and with what values. Like the results of exported functions, // calls to imports should also yield the same values for each // configuration, and we should assert that. - let imports = match dummy_imports(&store, module.imports()) { - Ok(imps) => imps, - Err(e) => { - // There are some value types that we can't synthesize a - // dummy value for (e.g. externrefs) and for modules that - // import things of these types we skip instantiation. - eprintln!("Warning: failed to synthesize dummy imports: {}", e); - continue; - } - }; + let imports = dummy_imports(&store, module.imports()); // Don't unwrap this: there can be instantiation-/link-time errors that // aren't caught during validation or compilation. For example, an imported @@ -212,10 +184,7 @@ pub fn differential_execution( init_hang_limit(&instance); let ty = f.ty(); - let params = match dummy::dummy_values(ty.params()) { - Ok(p) => p, - Err(_) => continue, - }; + let params = dummy::dummy_values(ty.params()); let this_result = f.call(¶ms).map_err(|e| e.downcast::().unwrap()); let existing_result = export_func_results @@ -353,16 +322,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { }; let store = store.as_ref().unwrap(); - - let imports = match dummy_imports(store, module.imports()) { - Ok(imps) => imps, - Err(_) => { - // There are some value types that we can't synthesize a - // dummy value for (e.g. externrefs) and for modules that - // import things of these types we skip instantiation. - continue; - } - }; + let imports = dummy_imports(store, module.imports()); // Don't unwrap this: there can be instantiation-/link-time errors that // aren't caught during validation or compilation. For example, an imported @@ -408,10 +368,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { let nth = nth % funcs.len(); let f = &funcs[nth]; let ty = f.ty(); - let params = match dummy::dummy_values(ty.params()) { - Ok(p) => p, - Err(_) => continue, - }; + let params = dummy::dummy_values(ty.params()); let _ = f.call(¶ms); } } @@ -509,7 +466,7 @@ impl wasm_smith::Config for DifferentialWasmiModuleConfig { 1 } - fn max_memories(&self) -> u32 { + fn max_memories(&self) -> usize { 1 } diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs index f05a52e08e..561a8fb329 100644 --- a/crates/fuzzing/src/oracles/dummy.rs +++ b/crates/fuzzing/src/oracles/dummy.rs @@ -1,27 +1,23 @@ //! Dummy implementations of things that a Wasm module can import. -use wasmtime::{ - Extern, ExternType, Func, FuncType, Global, GlobalType, ImportType, Memory, MemoryType, Store, - Table, TableType, Trap, Val, ValType, -}; +use std::fmt::Write; +use wasmtime::*; /// Create a set of dummy functions/globals/etc for the given imports. pub fn dummy_imports<'module>( store: &Store, import_tys: impl Iterator>, -) -> Result, Trap> { +) -> Vec { import_tys - .map(|imp| { - Ok(match imp.ty() { - ExternType::Func(func_ty) => Extern::Func(dummy_func(&store, func_ty)), - ExternType::Global(global_ty) => Extern::Global(dummy_global(&store, global_ty)?), - ExternType::Table(table_ty) => Extern::Table(dummy_table(&store, table_ty)?), - ExternType::Memory(mem_ty) => Extern::Memory(dummy_memory(&store, mem_ty)), - - // FIXME(#2094) - ExternType::Instance(_) => unimplemented!(), - ExternType::Module(_) => unimplemented!(), - }) + .map(|imp| match imp.ty() { + ExternType::Func(func_ty) => Extern::Func(dummy_func(&store, func_ty)), + ExternType::Global(global_ty) => Extern::Global(dummy_global(&store, global_ty)), + ExternType::Table(table_ty) => Extern::Table(dummy_table(&store, table_ty)), + ExternType::Memory(mem_ty) => Extern::Memory(dummy_memory(&store, mem_ty)), + ExternType::Instance(instance_ty) => { + Extern::Instance(dummy_instance(&store, instance_ty)) + } + ExternType::Module(module_ty) => Extern::Module(dummy_module(&store, module_ty)), }) .collect() } @@ -30,55 +26,326 @@ pub fn dummy_imports<'module>( pub fn dummy_func(store: &Store, ty: FuncType) -> Func { Func::new(store, ty.clone(), move |_, _, results| { for (ret_ty, result) in ty.results().zip(results) { - *result = dummy_value(ret_ty)?; + *result = dummy_value(ret_ty); } Ok(()) }) } /// Construct a dummy value for the given value type. -pub fn dummy_value(val_ty: ValType) -> Result { - Ok(match val_ty { +pub fn dummy_value(val_ty: ValType) -> Val { + match val_ty { ValType::I32 => Val::I32(0), ValType::I64 => Val::I64(0), ValType::F32 => Val::F32(0), ValType::F64 => Val::F64(0), - ValType::V128 => { - return Err(Trap::new( - "dummy_value: unsupported function return type: v128".to_string(), - )) - } - ValType::ExternRef => { - return Err(Trap::new( - "dummy_value: unsupported function return type: externref".to_string(), - )) - } - ValType::FuncRef => { - return Err(Trap::new( - "dummy_value: unsupported function return type: funcref".to_string(), - )) - } - }) + ValType::V128 => Val::V128(0), + ValType::ExternRef => Val::ExternRef(None), + ValType::FuncRef => Val::FuncRef(None), + } } /// Construct a sequence of dummy values for the given types. -pub fn dummy_values(val_tys: impl IntoIterator) -> Result, Trap> { +pub fn dummy_values(val_tys: impl IntoIterator) -> Vec { val_tys.into_iter().map(dummy_value).collect() } /// Construct a dummy global for the given global type. -pub fn dummy_global(store: &Store, ty: GlobalType) -> Result { - let val = dummy_value(ty.content().clone())?; - Ok(Global::new(store, ty, val).unwrap()) +pub fn dummy_global(store: &Store, ty: GlobalType) -> Global { + let val = dummy_value(ty.content().clone()); + Global::new(store, ty, val).unwrap() } /// Construct a dummy table for the given table type. -pub fn dummy_table(store: &Store, ty: TableType) -> Result { - let init_val = dummy_value(ty.element().clone())?; - Ok(Table::new(store, ty, init_val).unwrap()) +pub fn dummy_table(store: &Store, ty: TableType) -> Table { + let init_val = dummy_value(ty.element().clone()); + Table::new(store, ty, init_val).unwrap() } /// Construct a dummy memory for the given memory type. pub fn dummy_memory(store: &Store, ty: MemoryType) -> Memory { Memory::new(store, ty) } + +/// Construct a dummy instance for the given instance type. +/// +/// This is done by using the expected type to generate a module on-the-fly +/// which we the instantiate. +pub fn dummy_instance(store: &Store, ty: InstanceType) -> Instance { + let mut wat = WatGenerator::new(); + for ty in ty.exports() { + wat.export(&ty); + } + let module = Module::new(store.engine(), &wat.finish()).unwrap(); + Instance::new(store, &module, &[]).unwrap() +} + +/// Construct a dummy module for the given module type. +/// +/// This is done by using the expected type to generate a module on-the-fly. +pub fn dummy_module(store: &Store, ty: ModuleType) -> Module { + let mut wat = WatGenerator::new(); + for ty in ty.imports() { + wat.import(&ty); + } + for ty in ty.exports() { + wat.export(&ty); + } + Module::new(store.engine(), &wat.finish()).unwrap() +} + +struct WatGenerator { + tmp: usize, + dst: String, +} + +impl WatGenerator { + fn new() -> WatGenerator { + WatGenerator { + tmp: 0, + dst: String::from("(module\n"), + } + } + + fn finish(mut self) -> String { + self.dst.push_str(")\n"); + self.dst + } + + fn import(&mut self, ty: &ImportType<'_>) { + write!(self.dst, "(import ").unwrap(); + self.str(ty.module()); + write!(self.dst, " ").unwrap(); + if let Some(field) = ty.name() { + self.str(field); + write!(self.dst, " ").unwrap(); + } + self.item_ty(&ty.ty()); + writeln!(self.dst, ")").unwrap(); + } + + fn item_ty(&mut self, ty: &ExternType) { + match ty { + ExternType::Memory(mem) => { + write!( + self.dst, + "(memory {} {})", + mem.limits().min(), + match mem.limits().max() { + Some(max) => max.to_string(), + None => String::new(), + } + ) + .unwrap(); + } + ExternType::Table(table) => { + write!( + self.dst, + "(table {} {} {})", + table.limits().min(), + match table.limits().max() { + Some(max) => max.to_string(), + None => String::new(), + }, + wat_ty(table.element()), + ) + .unwrap(); + } + ExternType::Global(ty) => { + if ty.mutability() == Mutability::Const { + write!(self.dst, "(global {})", wat_ty(ty.content())).unwrap(); + } else { + write!(self.dst, "(global (mut {}))", wat_ty(ty.content())).unwrap(); + } + } + ExternType::Func(ty) => { + write!(self.dst, "(func ").unwrap(); + self.func_sig(ty); + write!(self.dst, ")").unwrap(); + } + ExternType::Instance(ty) => { + writeln!(self.dst, "(instance").unwrap(); + for ty in ty.exports() { + write!(self.dst, "(export ").unwrap(); + self.str(ty.name()); + write!(self.dst, " ").unwrap(); + self.item_ty(&ty.ty()); + writeln!(self.dst, ")").unwrap(); + } + write!(self.dst, ")").unwrap(); + } + ExternType::Module(ty) => { + writeln!(self.dst, "(module").unwrap(); + for ty in ty.imports() { + self.import(&ty); + writeln!(self.dst, "").unwrap(); + } + for ty in ty.exports() { + write!(self.dst, "(export ").unwrap(); + self.str(ty.name()); + write!(self.dst, " ").unwrap(); + self.item_ty(&ty.ty()); + writeln!(self.dst, ")").unwrap(); + } + write!(self.dst, ")").unwrap(); + } + } + } + + fn export(&mut self, ty: &ExportType<'_>) { + let wat_name = format!("item{}", self.tmp); + self.tmp += 1; + let item_ty = ty.ty(); + self.item(&wat_name, &item_ty); + + write!(self.dst, "(export ").unwrap(); + self.str(ty.name()); + write!(self.dst, " (").unwrap(); + match item_ty { + ExternType::Memory(_) => write!(self.dst, "memory").unwrap(), + ExternType::Global(_) => write!(self.dst, "global").unwrap(), + ExternType::Func(_) => write!(self.dst, "func").unwrap(), + ExternType::Instance(_) => write!(self.dst, "instance").unwrap(), + ExternType::Table(_) => write!(self.dst, "table").unwrap(), + ExternType::Module(_) => write!(self.dst, "module").unwrap(), + } + writeln!(self.dst, " ${}))", wat_name).unwrap(); + } + + fn item(&mut self, name: &str, ty: &ExternType) { + match ty { + ExternType::Memory(mem) => { + write!( + self.dst, + "(memory ${} {} {})\n", + name, + mem.limits().min(), + match mem.limits().max() { + Some(max) => max.to_string(), + None => String::new(), + } + ) + .unwrap(); + } + ExternType::Table(table) => { + write!( + self.dst, + "(table ${} {} {} {})\n", + name, + table.limits().min(), + match table.limits().max() { + Some(max) => max.to_string(), + None => String::new(), + }, + wat_ty(table.element()), + ) + .unwrap(); + } + ExternType::Global(ty) => { + write!(self.dst, "(global ${} ", name).unwrap(); + if ty.mutability() == Mutability::Var { + write!(self.dst, "(mut ").unwrap(); + } + write!(self.dst, "{}", wat_ty(ty.content())).unwrap(); + if ty.mutability() == Mutability::Var { + write!(self.dst, ")").unwrap(); + } + write!(self.dst, " (").unwrap(); + self.value(ty.content()); + writeln!(self.dst, "))").unwrap(); + } + ExternType::Func(ty) => { + write!(self.dst, "(func ${} ", name).unwrap(); + self.func_sig(ty); + for ty in ty.results() { + writeln!(self.dst, "").unwrap(); + self.value(&ty); + } + writeln!(self.dst, ")").unwrap(); + } + ExternType::Module(ty) => { + writeln!(self.dst, "(module ${}", name).unwrap(); + for ty in ty.imports() { + self.import(&ty); + } + for ty in ty.exports() { + self.export(&ty); + } + self.dst.push_str(")\n"); + } + ExternType::Instance(ty) => { + writeln!(self.dst, "(module ${}_module", name).unwrap(); + for ty in ty.exports() { + self.export(&ty); + } + self.dst.push_str(")\n"); + writeln!(self.dst, "(instance ${} (instantiate ${0}_module))", name).unwrap(); + } + } + } + + fn func_sig(&mut self, ty: &FuncType) { + write!(self.dst, "(param ").unwrap(); + for ty in ty.params() { + write!(self.dst, "{} ", wat_ty(&ty)).unwrap(); + } + write!(self.dst, ") (result ").unwrap(); + for ty in ty.results() { + write!(self.dst, "{} ", wat_ty(&ty)).unwrap(); + } + write!(self.dst, ")").unwrap(); + } + + fn value(&mut self, ty: &ValType) { + match ty { + ValType::I32 => write!(self.dst, "i32.const 0").unwrap(), + ValType::I64 => write!(self.dst, "i64.const 0").unwrap(), + ValType::F32 => write!(self.dst, "f32.const 0").unwrap(), + ValType::F64 => write!(self.dst, "f64.const 0").unwrap(), + ValType::V128 => write!(self.dst, "v128.const i32x4 0 0 0 0").unwrap(), + ValType::ExternRef => write!(self.dst, "ref.null extern").unwrap(), + ValType::FuncRef => write!(self.dst, "ref.null func").unwrap(), + } + } + + fn str(&mut self, name: &str) { + let mut bytes = [0; 4]; + self.dst.push_str("\""); + for c in name.chars() { + let v = c as u32; + if v >= 0x20 && v < 0x7f && c != '"' && c != '\\' && v < 0xff { + self.dst.push(c); + } else { + for byte in c.encode_utf8(&mut bytes).as_bytes() { + self.hex_byte(*byte); + } + } + } + self.dst.push_str("\""); + } + + fn hex_byte(&mut self, byte: u8) { + fn to_hex(b: u8) -> char { + if b < 10 { + (b'0' + b) as char + } else { + (b'a' + b - 10) as char + } + } + self.dst.push('\\'); + self.dst.push(to_hex((byte >> 4) & 0xf)); + self.dst.push(to_hex(byte & 0xf)); + } +} + +fn wat_ty(ty: &ValType) -> &'static str { + match ty { + ValType::I32 => "i32", + ValType::I64 => "i64", + ValType::F32 => "f32", + ValType::F64 => "f64", + ValType::V128 => "v128", + ValType::ExternRef => "externref", + ValType::FuncRef => "funcref", + } +} diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 7b357d0461..c754126d92 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -28,7 +28,7 @@ rayon = { version = "1.0", optional = true } region = "2.1.0" thiserror = "1.0.4" target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.69.2" +wasmparser = "0.70" more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "1.0" diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml index baf518e642..d178090cfa 100644 --- a/crates/lightbeam/Cargo.toml +++ b/crates/lightbeam/Cargo.toml @@ -24,7 +24,7 @@ more-asserts = "0.2.1" smallvec = "1.0.0" thiserror = "1.0.9" typemap = "0.3" -wasmparser = "0.69.2" +wasmparser = "0.70" [dev-dependencies] lazy_static = "1.2" diff --git a/crates/lightbeam/wasmtime/Cargo.toml b/crates/lightbeam/wasmtime/Cargo.toml index cf0bab9442..41b47115ee 100644 --- a/crates/lightbeam/wasmtime/Cargo.toml +++ b/crates/lightbeam/wasmtime/Cargo.toml @@ -13,6 +13,6 @@ edition = "2018" [dependencies] lightbeam = { path = "..", version = "0.21.0" } -wasmparser = "0.69.2" +wasmparser = "0.70" cranelift-codegen = { path = "../../../cranelift/codegen", version = "0.68.0" } wasmtime-environ = { path = "../../environ", version = "0.21.0" } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 65ce5873c4..1fa4f012f4 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -16,7 +16,7 @@ wasmtime-jit = { path = "../jit", version = "0.21.0" } wasmtime-cache = { path = "../cache", version = "0.21.0", optional = true } wasmtime-profiling = { path = "../profiling", version = "0.21.0" } target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.69.2" +wasmparser = "0.70" anyhow = "1.0.19" region = "2.2.0" libc = "0.2" diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml index aaa130acec..25e2bccbf1 100644 --- a/crates/wast/Cargo.toml +++ b/crates/wast/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] anyhow = "1.0.19" wasmtime = { path = "../wasmtime", version = "0.21.0", default-features = false } -wast = "28.0.0" +wast = "29.0.0" [badges] maintenance = { status = "actively-developed" } diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 32726a7940..6220bdcb91 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -17,7 +17,7 @@ target-lexicon = "0.11" peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true } wasmtime = { path = "../crates/wasmtime" } wasmtime-fuzzing = { path = "../crates/fuzzing" } -wasm-smith = "0.1.12" +wasm-smith = "0.2.0" [features] experimental_x64 = ["wasmtime-fuzzing/experimental_x64"] From c3f0471ff2421eb110ccba28b0609a4a1d0339eb Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 9 Dec 2020 16:49:59 -0800 Subject: [PATCH 56/63] Fix the return value from wasi-common's `fd_readdir`. `fd_readdir` returns a "bufused" value, which indicates the number of bytes read into the buffer. WASI libc expects this value to be equal to the size of the buffer if the end of the directory has not yet been scanned. Previously, wasi-common's `fd_readdir` was writing as many complete entries as it could fit and then stopping, but this meant it was returning size less than the buffer size even when the directory had more entries. This patch makes it continue writing up until the end of the buffer, and return that number of bytes, to let WASI libc know that there's more to be read. Fixes #2493. --- .../wasi-tests/src/bin/fd_readdir.rs | 72 +++++++++++++++++-- .../src/snapshots/wasi_snapshot_preview1.rs | 34 ++++++--- 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs b/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs index 8790ea0218..6e4345107a 100644 --- a/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs +++ b/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs @@ -26,15 +26,20 @@ impl<'a> Iterator for ReadDir<'a> { fn next(&mut self) -> Option { unsafe { - if self.buf.is_empty() { + if self.buf.len() < mem::size_of::() { return None; } // Read the data let dirent_ptr = self.buf.as_ptr() as *const wasi::Dirent; let dirent = dirent_ptr.read_unaligned(); + + if self.buf.len() < mem::size_of::() + dirent.d_namlen as usize { + return None; + } + let name_ptr = dirent_ptr.offset(1) as *const u8; - // NOTE Linux syscall returns a NULL-terminated name, but WASI doesn't + // NOTE Linux syscall returns a NUL-terminated name, but WASI doesn't let namelen = dirent.d_namlen as usize; let slice = slice::from_raw_parts(name_ptr, namelen); let name = str::from_utf8(slice).expect("invalid utf8").to_owned(); @@ -48,21 +53,24 @@ impl<'a> Iterator for ReadDir<'a> { } } -unsafe fn exec_fd_readdir(fd: wasi::Fd, cookie: wasi::Dircookie) -> Vec { +/// Return the entries plus a bool indicating EOF. +unsafe fn exec_fd_readdir(fd: wasi::Fd, cookie: wasi::Dircookie) -> (Vec, bool) { let mut buf: [u8; BUF_LEN] = [0; BUF_LEN]; let bufused = wasi::fd_readdir(fd, buf.as_mut_ptr(), BUF_LEN, cookie).expect("failed fd_readdir"); let sl = slice::from_raw_parts(buf.as_ptr(), min(BUF_LEN, bufused)); let dirs: Vec<_> = ReadDir::from_slice(sl).collect(); - dirs + let eof = bufused < BUF_LEN; + (dirs, eof) } unsafe fn test_fd_readdir(dir_fd: wasi::Fd) { let stat = wasi::fd_filestat_get(dir_fd).expect("failed filestat"); // Check the behavior in an empty directory - let mut dirs = exec_fd_readdir(dir_fd, 0); + let (mut dirs, eof) = exec_fd_readdir(dir_fd, 0); + assert!(eof, "expected to read the entire directory"); dirs.sort_by_key(|d| d.name.clone()); assert_eq!(dirs.len(), 2, "expected two entries in an empty directory"); let mut dirs = dirs.into_iter(); @@ -105,9 +113,11 @@ unsafe fn test_fd_readdir(dir_fd: wasi::Fd) { ); let stat = wasi::fd_filestat_get(file_fd).expect("failed filestat"); + wasi::fd_close(file_fd).expect("closing a file"); // Execute another readdir - let mut dirs = exec_fd_readdir(dir_fd, 0); + let (mut dirs, eof) = exec_fd_readdir(dir_fd, 0); + assert!(eof, "expected to read the entire directory"); assert_eq!(dirs.len(), 3, "expected three entries"); // Save the data about the last entry. We need to do it before sorting. let lastfile_cookie = dirs[1].dirent.d_next; @@ -130,9 +140,56 @@ unsafe fn test_fd_readdir(dir_fd: wasi::Fd) { assert_eq!(dir.dirent.d_ino, stat.ino); // check if cookie works as expected - let dirs = exec_fd_readdir(dir_fd, lastfile_cookie); + let (dirs, eof) = exec_fd_readdir(dir_fd, lastfile_cookie); + assert!(eof, "expected to read the entire directory"); assert_eq!(dirs.len(), 1, "expected one entry"); assert_eq!(dirs[0].name, lastfile_name, "name of the only entry"); + + wasi::path_unlink_file(dir_fd, "file").expect("removing a file"); +} + +unsafe fn test_fd_readdir_lots(dir_fd: wasi::Fd) { + let stat = wasi::fd_filestat_get(dir_fd).expect("failed filestat"); + + // Add a file and check the behavior + for count in 0..1000 { + let file_fd = wasi::path_open( + dir_fd, + 0, + &format!("file.{}", count), + wasi::OFLAGS_CREAT, + wasi::RIGHTS_FD_READ + | wasi::RIGHTS_FD_WRITE + | wasi::RIGHTS_FD_READDIR + | wasi::RIGHTS_FD_FILESTAT_GET, + 0, + 0, + ) + .expect("failed to create file"); + assert_gt!( + file_fd, + libc::STDERR_FILENO as wasi::Fd, + "file descriptor range check", + ); + wasi::fd_close(file_fd).expect("closing a file"); + } + + // Count the entries to ensure that we see the correct number. + let mut total = 0; + let mut cookie = 0; + loop { + let (dirs, eof) = exec_fd_readdir(dir_fd, cookie); + total += dirs.len(); + if eof { + break; + } + cookie = dirs[dirs.len()-1].dirent.d_next; + } + assert_eq!(total, 1002, "expected 1000 entries plus . and .."); + + for count in 0..1000 { + wasi::path_unlink_file(dir_fd, &format!("file.{}", count)).expect("removing a file"); + } } fn main() { @@ -156,4 +213,5 @@ fn main() { // Run the tests. unsafe { test_fd_readdir(dir_fd) } + unsafe { test_fd_readdir_lots(dir_fd) } } diff --git a/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs b/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs index f5b80eb5a1..769fc3601b 100644 --- a/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs +++ b/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs @@ -7,6 +7,7 @@ use crate::{path, sched, Error, Result, WasiCtx}; use std::convert::TryInto; use std::io::{self, SeekFrom}; use std::ops::Deref; +use std::cmp::min; use tracing::{debug, trace}; use wiggle::{GuestPtr, GuestSliceMut}; @@ -304,15 +305,32 @@ impl<'a> WasiSnapshotPreview1 for WasiCtx { let name_raw = name.as_bytes(); let name_len = name_raw.len().try_into()?; let offset = dirent_len.checked_add(name_len).ok_or(Error::Overflow)?; - if (buf_len - bufused) < offset { - break; - } else { - buf.as_array(dirent_len).copy_from_slice(&dirent_raw)?; - buf = buf.add(dirent_len)?; - buf.as_array(name_len).copy_from_slice(name_raw)?; - buf = buf.add(name_len)?; - bufused += offset; + + // Copy as many bytes of the dirent as we can, up to the end of the buffer. + let dirent_copy_len = min(dirent_len, buf_len - bufused); + buf.as_array(dirent_copy_len).copy_from_slice(&dirent_raw[..dirent_copy_len as usize])?; + + // If the dirent struct wasn't copied entirely, return that we + // filled the buffer, which tells libc that we're not at EOF. + if dirent_copy_len < dirent_len { + return Ok(buf_len); } + + buf = buf.add(dirent_copy_len)?; + + // Copy as many bytes of the name as we can, up to the end of the buffer. + let name_copy_len = min(name_len, buf_len - bufused); + buf.as_array(name_copy_len).copy_from_slice(&name_raw[..name_copy_len as usize])?; + + // If the dirent struct wasn't copied entirely, return that we + // filled the buffer, which tells libc that we're not at EOF. + if name_copy_len < name_len { + return Ok(buf_len); + } + + buf = buf.add(name_copy_len)?; + + bufused += offset; } Ok(bufused) From 88a073eac3dbfa6793a33cd23904a6e8bbd74cf3 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 9 Dec 2020 17:05:09 -0800 Subject: [PATCH 57/63] rustfmt --- .../wasi-common/src/snapshots/wasi_snapshot_preview1.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs b/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs index 769fc3601b..3e0bd6d4fc 100644 --- a/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs +++ b/crates/wasi-common/src/snapshots/wasi_snapshot_preview1.rs @@ -4,10 +4,10 @@ use crate::sys::{clock, poll}; use crate::wasi::types; use crate::wasi::wasi_snapshot_preview1::WasiSnapshotPreview1; use crate::{path, sched, Error, Result, WasiCtx}; +use std::cmp::min; use std::convert::TryInto; use std::io::{self, SeekFrom}; use std::ops::Deref; -use std::cmp::min; use tracing::{debug, trace}; use wiggle::{GuestPtr, GuestSliceMut}; @@ -308,7 +308,8 @@ impl<'a> WasiSnapshotPreview1 for WasiCtx { // Copy as many bytes of the dirent as we can, up to the end of the buffer. let dirent_copy_len = min(dirent_len, buf_len - bufused); - buf.as_array(dirent_copy_len).copy_from_slice(&dirent_raw[..dirent_copy_len as usize])?; + buf.as_array(dirent_copy_len) + .copy_from_slice(&dirent_raw[..dirent_copy_len as usize])?; // If the dirent struct wasn't copied entirely, return that we // filled the buffer, which tells libc that we're not at EOF. @@ -320,7 +321,8 @@ impl<'a> WasiSnapshotPreview1 for WasiCtx { // Copy as many bytes of the name as we can, up to the end of the buffer. let name_copy_len = min(name_len, buf_len - bufused); - buf.as_array(name_copy_len).copy_from_slice(&name_raw[..name_copy_len as usize])?; + buf.as_array(name_copy_len) + .copy_from_slice(&name_raw[..name_copy_len as usize])?; // If the dirent struct wasn't copied entirely, return that we // filled the buffer, which tells libc that we're not at EOF. From 1d90c329b40303437f8a4b14a7f8844a0822af5c Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 9 Dec 2020 21:46:43 -0800 Subject: [PATCH 58/63] Remove an unused variable. --- crates/test-programs/wasi-tests/src/bin/fd_readdir.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs b/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs index 6e4345107a..903329b744 100644 --- a/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs +++ b/crates/test-programs/wasi-tests/src/bin/fd_readdir.rs @@ -149,8 +149,6 @@ unsafe fn test_fd_readdir(dir_fd: wasi::Fd) { } unsafe fn test_fd_readdir_lots(dir_fd: wasi::Fd) { - let stat = wasi::fd_filestat_get(dir_fd).expect("failed filestat"); - // Add a file and check the behavior for count in 0..1000 { let file_fd = wasi::path_open( From 8f7f8ee0b4c5007ace6de29b45505c360450b1bb Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 11 Dec 2020 17:17:18 +0100 Subject: [PATCH 59/63] Fix iconst.i8 0 miscompilation --- cranelift/codegen/meta/src/isa/x86/encodings.rs | 5 ++++- cranelift/codegen/meta/src/isa/x86/opcodes.rs | 3 --- .../isa/x86/optimized-zero-constants-32bit.clif | 8 ++++---- .../filetests/isa/x86/optimized-zero-constants.clif | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cranelift/codegen/meta/src/isa/x86/encodings.rs b/cranelift/codegen/meta/src/isa/x86/encodings.rs index 9ee12656c0..de48c57c5d 100644 --- a/cranelift/codegen/meta/src/isa/x86/encodings.rs +++ b/cranelift/codegen/meta/src/isa/x86/encodings.rs @@ -549,10 +549,13 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r: } e.enc64(bconst.bind(B64), rec_pu_id_bool.opcodes(&MOV_IMM).rex()); + // You may expect that i8 encodings would use 0x30 (XORB) to indicate that encodings should be + // on 8-bit operands (f.ex "xor %al, %al"). Cranelift currently does not know when it can + // safely drop the 0x66 prefix, so we explicitly select a wider but permissible opcode. let is_zero_int = InstructionPredicate::new_is_zero_int(&formats.unary_imm, "imm"); e.enc_both_instp( iconst.bind(I8), - rec_u_id_z.opcodes(&XORB), + rec_u_id_z.opcodes(&XOR), is_zero_int.clone(), ); diff --git a/cranelift/codegen/meta/src/isa/x86/opcodes.rs b/cranelift/codegen/meta/src/isa/x86/opcodes.rs index 09c07c458f..595d13ba2b 100644 --- a/cranelift/codegen/meta/src/isa/x86/opcodes.rs +++ b/cranelift/codegen/meta/src/isa/x86/opcodes.rs @@ -711,9 +711,6 @@ pub static XOR_IMM8_SIGN_EXTEND: [u8; 1] = [0x83]; /// r/m{16,32,64} XOR register of the same size. pub static XOR: [u8; 1] = [0x31]; -/// r/m8 XOR r8. -pub static XORB: [u8; 1] = [0x30]; - /// Bitwise logical XOR of packed double-precision floating-point values. pub static XORPD: [u8; 3] = [0x66, 0x0f, 0x57]; diff --git a/cranelift/filetests/filetests/isa/x86/optimized-zero-constants-32bit.clif b/cranelift/filetests/filetests/isa/x86/optimized-zero-constants-32bit.clif index 0f0f06e6f2..6ce39a5c38 100644 --- a/cranelift/filetests/filetests/isa/x86/optimized-zero-constants-32bit.clif +++ b/cranelift/filetests/filetests/isa/x86/optimized-zero-constants-32bit.clif @@ -44,9 +44,9 @@ block0: function %zero_byte() -> i8 fast { block0: - ; asm: xor %al, %al - [-,%rax] v0 = iconst.i8 0 ; bin: 30 c0 - ; asm: xor %dh, %dh - [-,%rdi] v1 = iconst.i8 0 ; bin: 30 ff + ; asm: xor %eax, %eax + [-,%rax] v0 = iconst.i8 0 ; bin: 31 c0 + ; asm: xor %edi, %edi + [-,%rdi] v1 = iconst.i8 0 ; bin: 31 ff return v0 } diff --git a/cranelift/filetests/filetests/isa/x86/optimized-zero-constants.clif b/cranelift/filetests/filetests/isa/x86/optimized-zero-constants.clif index 7f5890a1ae..4ff2865a21 100644 --- a/cranelift/filetests/filetests/isa/x86/optimized-zero-constants.clif +++ b/cranelift/filetests/filetests/isa/x86/optimized-zero-constants.clif @@ -62,11 +62,11 @@ block0: function %zero_byte() -> i8 fast { block0: - ; asm: xor %r8b, %r8b - [-,%r15] v0 = iconst.i8 0 ; bin: 45 30 ff - ; asm: xor %al, %al - [-,%rax] v1 = iconst.i8 0 ; bin: 30 c0 - ; asm: xor %dh, %dh - [-,%rdi] v2 = iconst.i8 0 ; bin: 30 ff + ; asm: xor %r8d, %r8d + [-,%r15] v0 = iconst.i8 0 ; bin: 45 31 ff + ; asm: xor %eax, eax + [-,%rax] v1 = iconst.i8 0 ; bin: 31 c0 + ; asm: xor %edi, %edi + [-,%rdi] v2 = iconst.i8 0 ; bin: 31 ff return v0 } From c83dee07b73f8c23e7054d27e86f7172e6fb3569 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 14 Dec 2020 07:57:47 -0800 Subject: [PATCH 60/63] Fix a memory reservation bug in `reserve_modules` This method attempted to reserve space in the `results` list of final modules. Unfortunately `results.reserve(nmodules)` isn't enough here because this can be called many times before a module is actually finished and pushed onto the vector. The attempted logic to work around this was buggy, however, and would simply trigger geometric growth on every single reservation because it erroneously assumed that a reservation would be exactly met. This is fixed by avoiding looking at the vector's capacity and instead keeping track of modules-to-be in a side field. This is the incremented and passed to `reserve` as it represents the number of modules that will eventually make their way into the result vector. --- crates/environ/src/module_environ.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 6d36b4e7a9..7b2cca6110 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -31,6 +31,10 @@ pub struct ModuleEnvironment<'data> { /// the module linking proposal. results: Vec>, + /// How many modules that have not yet made their way into `results` which + /// are coming at some point. + modules_to_be: usize, + /// Intern'd types for this entire translation, shared by all modules. types: TypeTables, @@ -138,6 +142,7 @@ impl<'data> ModuleEnvironment<'data> { Self { result: ModuleTranslation::default(), results: Vec::with_capacity(1), + modules_to_be: 1, cur: 0, types: Default::default(), target_config, @@ -741,8 +746,8 @@ and for re-adding support for interface types you can see this issue: fn reserve_modules(&mut self, amount: u32) { // Go ahead and reserve space in the final `results` array for `amount` // more modules. - let extra = self.results.capacity() + (amount as usize) - self.results.len(); - self.results.reserve(extra); + self.modules_to_be += amount as usize; + self.results.reserve(self.modules_to_be); // Then also reserve space in our own local module's metadata fields // we'll be adding to. @@ -796,6 +801,7 @@ and for re-adding support for interface types you can see this issue: self.cur = index; assert_eq!(index, self.results.len()); self.results.push(prev); + self.modules_to_be -= 1; } fn module_end(&mut self, index: usize) { From 42adeba65dd064243c9f7b0c4d4e4858e29f1e47 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 14 Dec 2020 08:03:45 -0800 Subject: [PATCH 61/63] Fix fuzzer expectation about valid modules Recent changes to fuzzers made expectations more strict about handling errors while fuzzing, but this erroneously changed a module compilation step to always assume that the input wasm is valid. Instead a flag is now passed through indicating whether the wasm blob is known valid or invalid, and only if compilation fails and it's known valid do we panic. --- crates/fuzzing/src/oracles.rs | 22 +++++++++++++++---- .../fuzz_targets/instantiate-maybe-invalid.rs | 1 + fuzz/fuzz_targets/instantiate-swarm.rs | 2 +- fuzz/fuzz_targets/instantiate-wasm-smith.rs | 2 +- fuzz/fuzz_targets/instantiate.rs | 2 +- tests/all/fuzzing.rs | 6 ++--- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 91e676c58d..2d11255f58 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -57,8 +57,13 @@ fn log_wat(wat: &str) { /// Performs initial validation, and returns early if the Wasm is invalid. /// /// You can control which compiler is used via passing a `Strategy`. -pub fn instantiate(wasm: &[u8], strategy: Strategy) { - instantiate_with_config(wasm, crate::fuzz_default_config(strategy).unwrap(), None); +pub fn instantiate(wasm: &[u8], known_valid: bool, strategy: Strategy) { + instantiate_with_config( + wasm, + known_valid, + crate::fuzz_default_config(strategy).unwrap(), + None, + ); } /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected @@ -67,7 +72,12 @@ pub fn instantiate(wasm: &[u8], strategy: Strategy) { /// The engine will be configured using provided config. /// /// See also `instantiate` functions. -pub fn instantiate_with_config(wasm: &[u8], mut config: Config, timeout: Option) { +pub fn instantiate_with_config( + wasm: &[u8], + known_valid: bool, + mut config: Config, + timeout: Option, +) { crate::init_fuzzing(); config.interruptable(timeout.is_some()); @@ -91,7 +101,11 @@ pub fn instantiate_with_config(wasm: &[u8], mut config: Config, timeout: Option< } log_wasm(wasm); - let module = Module::new(&engine, wasm).unwrap(); + let module = match Module::new(&engine, wasm) { + Ok(module) => module, + Err(_) if !known_valid => return, + Err(e) => panic!("failed to compile module: {:?}", e), + }; let imports = dummy_imports(&store, module.imports()); match Instance::new(&store, &module, &imports) { diff --git a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs b/fuzz/fuzz_targets/instantiate-maybe-invalid.rs index 5b9657b924..219986a25a 100644 --- a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs +++ b/fuzz/fuzz_targets/instantiate-maybe-invalid.rs @@ -9,6 +9,7 @@ use wasmtime_fuzzing::oracles; fuzz_target!(|module: MaybeInvalidModule| { oracles::instantiate_with_config( &module.to_bytes(), + false, wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(), Some(Duration::from_secs(20)), ); diff --git a/fuzz/fuzz_targets/instantiate-swarm.rs b/fuzz/fuzz_targets/instantiate-swarm.rs index a3049ad998..092a32e925 100644 --- a/fuzz/fuzz_targets/instantiate-swarm.rs +++ b/fuzz/fuzz_targets/instantiate-swarm.rs @@ -9,5 +9,5 @@ use wasmtime_fuzzing::oracles; fuzz_target!(|module: ConfiguredModule| { let mut cfg = wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(); cfg.wasm_multi_memory(true); - oracles::instantiate_with_config(&module.to_bytes(), cfg, Some(Duration::from_secs(20))); + oracles::instantiate_with_config(&module.to_bytes(), true, cfg, Some(Duration::from_secs(20))); }); diff --git a/fuzz/fuzz_targets/instantiate-wasm-smith.rs b/fuzz/fuzz_targets/instantiate-wasm-smith.rs index 9c081eff88..546a198ff0 100644 --- a/fuzz/fuzz_targets/instantiate-wasm-smith.rs +++ b/fuzz/fuzz_targets/instantiate-wasm-smith.rs @@ -9,5 +9,5 @@ fuzz_target!(|module: Module| { let mut module = module; module.ensure_termination(1000); let wasm_bytes = module.to_bytes(); - oracles::instantiate(&wasm_bytes, Strategy::Auto); + oracles::instantiate(&wasm_bytes, true, Strategy::Auto); }); diff --git a/fuzz/fuzz_targets/instantiate.rs b/fuzz/fuzz_targets/instantiate.rs index 61b22c1877..ec41c59105 100644 --- a/fuzz/fuzz_targets/instantiate.rs +++ b/fuzz/fuzz_targets/instantiate.rs @@ -5,5 +5,5 @@ use wasmtime::Strategy; use wasmtime_fuzzing::oracles; fuzz_target!(|data: &[u8]| { - oracles::instantiate(data, Strategy::Auto); + oracles::instantiate(data, false, Strategy::Auto); }); diff --git a/tests/all/fuzzing.rs b/tests/all/fuzzing.rs index 9e71add575..3e46af2d72 100644 --- a/tests/all/fuzzing.rs +++ b/tests/all/fuzzing.rs @@ -11,13 +11,13 @@ use wasmtime_fuzzing::oracles; #[test] fn instantiate_empty_module() { let data = wat::parse_str(include_str!("./fuzzing/empty.wat")).unwrap(); - oracles::instantiate(&data, Strategy::Auto); + oracles::instantiate(&data, true, Strategy::Auto); } #[test] fn instantiate_empty_module_with_memory() { let data = wat::parse_str(include_str!("./fuzzing/empty_with_memory.wat")).unwrap(); - oracles::instantiate(&data, Strategy::Auto); + oracles::instantiate(&data, true, Strategy::Auto); } #[test] @@ -26,5 +26,5 @@ fn instantiate_module_that_compiled_to_x64_has_register_32() { let mut config = Config::new(); config.debug_info(true); let data = wat::parse_str(include_str!("./fuzzing/issue694.wat")).unwrap(); - oracles::instantiate_with_config(&data, config, None); + oracles::instantiate_with_config(&data, true, config, None); } From 467a1af83ab633efb7af88f9e437397a280200f7 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 9 Dec 2020 16:23:10 +0100 Subject: [PATCH 62/63] Support explicit endianness in Cranelift IR MemFlags WebAssembly memory operations are by definition little-endian even on big-endian target platforms. However, other memory accesses will require native target endianness (e.g. to access parts of the VMContext that is also accessed by VM native code). This means on big-endian targets, the code generator will have to handle both little- and big-endian memory accesses. However, there is currently no way to encode that distinction into the Cranelift IR that describes memory accesses. This patch provides such a way by adding an (optional) explicit endianness marker to an instance of MemFlags. Since each Cranelift IR instruction that describes memory accesses already has an instance of MemFlags attached, this can now be used to provide endianness information. Note that by default, memory accesses will continue to use the native target ISA endianness. To override this to specify an explicit endianness, a MemFlags value that was built using the set_endianness routine must be used. This patch does so for accesses that implement WebAssembly memory operations. This patch addresses issue #2124. --- cranelift/codegen/src/ir/memflags.rs | 54 ++++++++++++++++++++++++-- cranelift/codegen/src/ir/mod.rs | 2 +- cranelift/codegen/src/isa/mod.rs | 8 ++++ cranelift/codegen/src/legalizer/mod.rs | 12 +++++- cranelift/wasm/src/code_translator.rs | 19 ++++++--- 5 files changed, 83 insertions(+), 12 deletions(-) diff --git a/cranelift/codegen/src/ir/memflags.rs b/cranelift/codegen/src/ir/memflags.rs index 87fd6bf3ab..3c9c8c98ba 100644 --- a/cranelift/codegen/src/ir/memflags.rs +++ b/cranelift/codegen/src/ir/memflags.rs @@ -6,15 +6,31 @@ enum FlagBit { Notrap, Aligned, Readonly, + LittleEndian, + BigEndian, } -const NAMES: [&str; 3] = ["notrap", "aligned", "readonly"]; +const NAMES: [&str; 5] = ["notrap", "aligned", "readonly", "little", "big"]; + +/// Endianness of a memory access. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum Endianness { + /// Little-endian + Little, + /// Big-endian + Big, +} /// Flags for memory operations like load/store. /// /// Each of these flags introduce a limited form of undefined behavior. The flags each enable /// certain optimizations that need to make additional assumptions. Generally, the semantics of a /// program does not change when a flag is removed, but adding a flag will. +/// +/// In addition, the flags determine the endianness of the memory access. By default, +/// any memory access uses the native endianness determined by the target ISA. This can +/// be overridden for individual accesses by explicitly specifying little- or big-endian +/// semantics via the flags. #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] pub struct MemFlags { bits: u8, @@ -48,16 +64,48 @@ impl MemFlags { /// Set a flag bit by name. /// /// Returns true if the flag was found and set, false for an unknown flag name. + /// Will also return false when trying to set inconsistent endianness flags. pub fn set_by_name(&mut self, name: &str) -> bool { match NAMES.iter().position(|&s| s == name) { Some(bit) => { - self.bits |= 1 << bit; - true + let bits = self.bits | 1 << bit; + if (bits & (1 << FlagBit::LittleEndian as usize)) != 0 + && (bits & (1 << FlagBit::BigEndian as usize)) != 0 + { + false + } else { + self.bits = bits; + true + } } None => false, } } + /// Return endianness of the memory access. This will return the endianness + /// explicitly specified by the flags if any, and will default to the native + /// endianness otherwise. The native endianness has to be provided by the + /// caller since it is not explicitly encoded in CLIF IR -- this allows a + /// front end to create IR without having to know the target endianness. + pub fn endianness(self, native_endianness: Endianness) -> Endianness { + if self.read(FlagBit::LittleEndian) { + Endianness::Little + } else if self.read(FlagBit::BigEndian) { + Endianness::Big + } else { + native_endianness + } + } + + /// Set endianness of the memory access. + pub fn set_endianness(&mut self, endianness: Endianness) { + match endianness { + Endianness::Little => self.set(FlagBit::LittleEndian), + Endianness::Big => self.set(FlagBit::BigEndian), + }; + assert!(!(self.read(FlagBit::LittleEndian) && self.read(FlagBit::BigEndian))); + } + /// Test if the `notrap` flag is set. /// /// Normally, trapping is part of the semantics of a load/store operation. If the platform diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index 4dbe90df34..c5e827db3d 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -50,7 +50,7 @@ pub use crate::ir::instructions::{ pub use crate::ir::jumptable::JumpTableData; pub use crate::ir::layout::Layout; pub use crate::ir::libcall::{get_probestack_funcref, LibCall}; -pub use crate::ir::memflags::MemFlags; +pub use crate::ir::memflags::{Endianness, MemFlags}; pub use crate::ir::progpoint::{ExpandedProgramPoint, ProgramOrder, ProgramPoint}; pub use crate::ir::sourceloc::SourceLoc; pub use crate::ir::stackslot::{StackLayoutInfo, StackSlotData, StackSlotKind, StackSlots}; diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index a1a4c3c397..1ce10a155e 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -235,6 +235,14 @@ pub trait TargetIsa: fmt::Display + Send + Sync { CallConv::triple_default(self.triple()) } + /// Get the endianness of this ISA. + fn endianness(&self) -> ir::Endianness { + match self.triple().endianness().unwrap() { + target_lexicon::Endianness::Little => ir::Endianness::Little, + target_lexicon::Endianness::Big => ir::Endianness::Big, + } + } + /// Get the pointer type of this ISA. fn pointer_type(&self) -> ir::Type { ir::Type::int(u16::from(self.pointer_bits())).unwrap() diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs index 1900b144ce..149a65b639 100644 --- a/cranelift/codegen/src/legalizer/mod.rs +++ b/cranelift/codegen/src/legalizer/mod.rs @@ -659,7 +659,7 @@ fn narrow_load( inst: ir::Inst, func: &mut ir::Function, _cfg: &mut ControlFlowGraph, - _isa: &dyn TargetIsa, + isa: &dyn TargetIsa, ) { let mut pos = FuncCursor::new(func).at_inst(inst); pos.use_srcloc(inst); @@ -684,6 +684,10 @@ fn narrow_load( ptr, offset.try_add_i64(8).expect("load offset overflow"), ); + let (al, ah) = match flags.endianness(isa.endianness()) { + ir::Endianness::Little => (al, ah), + ir::Endianness::Big => (ah, al), + }; pos.func.dfg.replace(inst).iconcat(al, ah); } @@ -692,7 +696,7 @@ fn narrow_store( inst: ir::Inst, func: &mut ir::Function, _cfg: &mut ControlFlowGraph, - _isa: &dyn TargetIsa, + isa: &dyn TargetIsa, ) { let mut pos = FuncCursor::new(func).at_inst(inst); pos.use_srcloc(inst); @@ -708,6 +712,10 @@ fn narrow_store( }; let (al, ah) = pos.ins().isplit(val); + let (al, ah) = match flags.endianness(isa.endianness()) { + ir::Endianness::Little => (al, ah), + ir::Endianness::Big => (ah, al), + }; pos.ins().store(flags, al, ptr, offset); pos.ins().store( flags, diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs index c522feb059..04c5163eba 100644 --- a/cranelift/wasm/src/code_translator.rs +++ b/cranelift/wasm/src/code_translator.rs @@ -2056,7 +2056,9 @@ fn prepare_load( // Note that we don't set `is_aligned` here, even if the load instruction's // alignment immediate says it's aligned, because WebAssembly's immediate // field is just a hint, while Cranelift's aligned flag needs a guarantee. - let flags = MemFlags::new(); + // WebAssembly memory accesses are always little-endian. + let mut flags = MemFlags::new(); + flags.set_endianness(ir::Endianness::Little); Ok((flags, base, offset.into())) } @@ -2103,7 +2105,8 @@ fn translate_store( builder, ); // See the comments in `prepare_load` about the flags. - let flags = MemFlags::new(); + let mut flags = MemFlags::new(); + flags.set_endianness(ir::Endianness::Little); builder .ins() .Store(opcode, val_ty, flags, offset.into(), val, base); @@ -2207,7 +2210,8 @@ fn translate_atomic_rmw( finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; // See the comments in `prepare_load` about the flags. - let flags = MemFlags::new(); + let mut flags = MemFlags::new(); + flags.set_endianness(ir::Endianness::Little); let mut res = builder .ins() .atomic_rmw(access_ty, flags, op, final_effective_address, arg2); @@ -2260,7 +2264,8 @@ fn translate_atomic_cas( finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; // See the comments in `prepare_load` about the flags. - let flags = MemFlags::new(); + let mut flags = MemFlags::new(); + flags.set_endianness(ir::Endianness::Little); let mut res = builder .ins() .atomic_cas(flags, final_effective_address, expected, replacement); @@ -2302,7 +2307,8 @@ fn translate_atomic_load( finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; // See the comments in `prepare_load` about the flags. - let flags = MemFlags::new(); + let mut flags = MemFlags::new(); + flags.set_endianness(ir::Endianness::Little); let mut res = builder .ins() .atomic_load(access_ty, flags, final_effective_address); @@ -2348,7 +2354,8 @@ fn translate_atomic_store( finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; // See the comments in `prepare_load` about the flags. - let flags = MemFlags::new(); + let mut flags = MemFlags::new(); + flags.set_endianness(ir::Endianness::Little); builder .ins() .atomic_store(flags, data, final_effective_address); From 2d7c2fba5f832ffd0832b8c16b8b5d11db6d0ac7 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 14 Dec 2020 13:39:38 -0600 Subject: [PATCH 63/63] Update wasm-smith (#2509) Brings in a few bug fixes for generating module-linking modules --- Cargo.lock | 4 ++-- crates/fuzzing/Cargo.toml | 2 +- crates/fuzzing/src/oracles.rs | 2 +- fuzz/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64237674ee..1de98625b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2365,9 +2365,9 @@ dependencies = [ [[package]] name = "wasm-smith" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd382e46cf44347f4d0c29122143cbab85cf248b9a8f680845c0239b6842a85" +checksum = "509904d9c4c4659ac238a3f27c3656dd6d3931697eddd4b0f32e335769c298d0" dependencies = [ "arbitrary", "leb128", diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index b0dd625c02..7120601b2e 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -16,7 +16,7 @@ wasmparser = "0.70" wasmprinter = "0.2.17" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } -wasm-smith = "0.2.0" +wasm-smith = "0.3.0" wasmi = "0.7.0" [dev-dependencies] diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 2d11255f58..9f04dc44df 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -460,7 +460,7 @@ pub fn table_ops(config: crate::generators::Config, ops: crate::generators::tabl /// Configuration options for wasm-smith such that generated modules always /// conform to certain specifications. -#[derive(Default, Debug, Arbitrary)] +#[derive(Default, Debug, Arbitrary, Clone)] pub struct DifferentialWasmiModuleConfig; impl wasm_smith::Config for DifferentialWasmiModuleConfig { diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 6220bdcb91..73bd003e1e 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -17,7 +17,7 @@ target-lexicon = "0.11" peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true } wasmtime = { path = "../crates/wasmtime" } wasmtime-fuzzing = { path = "../crates/fuzzing" } -wasm-smith = "0.2.0" +wasm-smith = "0.3.0" [features] experimental_x64 = ["wasmtime-fuzzing/experimental_x64"]