diff --git a/filetests/grow.wat b/filetests/grow.wat new file mode 100644 index 0000000000..f05105cfc7 --- /dev/null +++ b/filetests/grow.wat @@ -0,0 +1,28 @@ +(module + (memory 1) + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + (func $main (local i32) + (call $assert + (i32.eq + (grow_memory (i32.const 1)) + (i32.const 1) + ) + ) + (call $assert + (i32.eq + (current_memory) + (i32.const 2) + ) + ) + ) + (start $main) + (data (i32.const 0) "\04\03\02\01") +) + diff --git a/lib/environ/src/compilation.rs b/lib/environ/src/compilation.rs index bae21f753e..a2f71298da 100644 --- a/lib/environ/src/compilation.rs +++ b/lib/environ/src/compilation.rs @@ -46,16 +46,19 @@ impl binemit::RelocSink for RelocSink { name: &ExternalName, addend: binemit::Addend, ) { - // FIXME: Handle grow_memory/current_memory. - let func_index = if let ExternalName::User { namespace, index } = *name { + let reloc_target = if let ExternalName::User { namespace, index } = *name { debug_assert!(namespace == 0); - index + RelocationTarget::UserFunc(index as usize) + } else if *name == ExternalName::testcase("grow_memory") { + RelocationTarget::GrowMemory + } else if *name == ExternalName::testcase("current_memory") { + RelocationTarget::CurrentMemory } else { panic!("unrecognized external name") - } as usize; + }; self.func_relocs.push(Relocation { reloc, - func_index, + reloc_target, offset, addend, }); @@ -83,14 +86,25 @@ impl RelocSink { pub struct Relocation { /// The relocation code. pub reloc: binemit::Reloc, - /// The function index. - pub func_index: FunctionIndex, + /// Relocation target. + pub reloc_target: RelocationTarget, /// The offset where to apply the relocation. pub offset: binemit::CodeOffset, /// The addend to add to the relocation value. pub addend: binemit::Addend, } +/// Destination function. Can be either user function or some special one, like grow_memory. +#[derive(Debug)] +pub enum RelocationTarget { + /// The user function index. + UserFunc(FunctionIndex), + /// Function for growing the default memory by the specified amount of pages. + GrowMemory, + /// Function for query current size of the default linear memory. + CurrentMemory, +} + /// Relocations to apply to function bodies. pub type Relocations = Vec>; diff --git a/lib/environ/src/environ.rs b/lib/environ/src/environ.rs index d97db70d77..06980b3987 100644 --- a/lib/environ/src/environ.rs +++ b/lib/environ/src/environ.rs @@ -3,8 +3,7 @@ use cranelift_codegen::ir; use cranelift_codegen::ir::immediates::Offset32; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{ - AbiParam, ArgumentExtension, ArgumentLoc, ArgumentPurpose, ExtFuncData, ExternalName, FuncRef, - Function, InstBuilder, Signature, + AbiParam, ArgumentPurpose, ExtFuncData, ExternalName, FuncRef, Function, InstBuilder, Signature, }; use cranelift_codegen::isa; use cranelift_codegen::settings; @@ -132,12 +131,10 @@ impl<'data, 'module> cranelift_wasm::ModuleEnvironment<'data> fn declare_signature(&mut self, sig: &ir::Signature) { let mut sig = sig.clone(); - sig.params.push(AbiParam { - value_type: self.pointer_type(), - purpose: ArgumentPurpose::VMContext, - extension: ArgumentExtension::None, - location: ArgumentLoc::Unassigned, - }); + sig.params.push(AbiParam::special( + self.pointer_type(), + ArgumentPurpose::VMContext, + )); // TODO: Deduplicate signatures. self.module.signatures.push(sig); } @@ -377,7 +374,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m let sig_ref = pos.func.import_signature(Signature { call_conv: self.isa.flags().call_conv(), argument_bytes: None, - params: vec![AbiParam::new(I32)], + params: vec![ + AbiParam::new(I32), + AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), + ], returns: vec![AbiParam::new(I32)], }); // We currently allocate all code segments independently, so nothing @@ -391,7 +391,8 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m }) }); self.grow_memory_extfunc = Some(grow_mem_func); - let call_inst = pos.ins().call(grow_mem_func, &[val]); + let vmctx = pos.func.special_param(ArgumentPurpose::VMContext).unwrap(); + let call_inst = pos.ins().call(grow_mem_func, &[val, vmctx]); Ok(*pos.func.dfg.inst_results(call_inst).first().unwrap()) } @@ -406,7 +407,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m let sig_ref = pos.func.import_signature(Signature { call_conv: self.isa.flags().call_conv(), argument_bytes: None, - params: Vec::new(), + params: vec![AbiParam::special( + self.pointer_type(), + ArgumentPurpose::VMContext, + )], returns: vec![AbiParam::new(I32)], }); // We currently allocate all code segments independently, so nothing @@ -420,7 +424,8 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m }) }); self.current_memory_extfunc = Some(cur_mem_func); - let call_inst = pos.ins().call(cur_mem_func, &[]); + let vmctx = pos.func.special_param(ArgumentPurpose::VMContext).unwrap(); + let call_inst = pos.ins().call(cur_mem_func, &[vmctx]); Ok(*pos.func.dfg.inst_results(call_inst).first().unwrap()) } } diff --git a/lib/environ/src/lib.rs b/lib/environ/src/lib.rs index 482ac483d2..689e68db61 100644 --- a/lib/environ/src/lib.rs +++ b/lib/environ/src/lib.rs @@ -23,6 +23,6 @@ mod compilation; mod environ; mod module; -pub use compilation::{compile_module, Compilation, Relocation, Relocations}; +pub use compilation::{compile_module, Compilation, Relocation, RelocationTarget, Relocations}; pub use environ::{ModuleEnvironment, ModuleTranslation}; pub use module::{DataInitializer, Module, TableElements}; diff --git a/lib/execute/Cargo.toml b/lib/execute/Cargo.toml index edd7c9afb0..2613c17e9d 100644 --- a/lib/execute/Cargo.toml +++ b/lib/execute/Cargo.toml @@ -12,3 +12,4 @@ cranelift-codegen = "0.18.1" cranelift-wasm = "0.18.1" region = "0.3.0" wasmtime-environ = { path = "../environ" } +memmap = "0.6.2" diff --git a/lib/execute/src/execute.rs b/lib/execute/src/execute.rs index 434067ef08..84c1879261 100644 --- a/lib/execute/src/execute.rs +++ b/lib/execute/src/execute.rs @@ -1,11 +1,14 @@ use cranelift_codegen::binemit::Reloc; use cranelift_codegen::isa::TargetIsa; use instance::Instance; +use memory::LinearMemory; use region::protect; use region::Protection; use std::mem::transmute; use std::ptr::write_unaligned; -use wasmtime_environ::{compile_module, Compilation, Module, ModuleTranslation, Relocation}; +use wasmtime_environ::{ + compile_module, Compilation, Module, ModuleTranslation, Relocation, RelocationTarget, +}; /// Executes a module that has been translated with the `wasmtime-environ` environment /// implementation. @@ -33,7 +36,12 @@ fn relocate(compilation: &mut Compilation, relocations: &[Vec]) { // TODO: Support architectures other than x64, and other reloc kinds. for (i, function_relocs) in relocations.iter().enumerate() { for r in function_relocs { - let target_func_address: isize = compilation.functions[r.func_index].as_ptr() as isize; + let target_func_address: isize = match r.reloc_target { + RelocationTarget::UserFunc(index) => compilation.functions[index].as_ptr() as isize, + RelocationTarget::GrowMemory => grow_memory as isize, + RelocationTarget::CurrentMemory => current_memory as isize, + }; + let body = &mut compilation.functions[i]; match r.reloc { Reloc::Abs8 => unsafe { @@ -56,16 +64,30 @@ fn relocate(compilation: &mut Compilation, relocations: &[Vec]) { } } +extern "C" fn grow_memory(size: u32, vmctx: *mut *mut u8) -> u32 { + unsafe { + let instance = (*vmctx.offset(2)) as *mut Instance; + (*instance) + .memory_mut(0) + .grow(size) + .unwrap_or(u32::max_value()) + } +} + +extern "C" fn current_memory(vmctx: *mut *mut u8) -> u32 { + unsafe { + let instance = (*vmctx.offset(2)) as *mut Instance; + (*instance).memory_mut(0).current_size() + } +} + /// Create the VmCtx data structure for the JIT'd code to use. This must /// match the VmCtx layout in the environment. -fn make_vmctx(instance: &mut Instance) -> Vec<*mut u8> { - let mut memories = Vec::new(); +fn make_vmctx(instance: &mut Instance, mem_base_addrs: &mut [*mut u8]) -> Vec<*mut u8> { let mut vmctx = Vec::new(); vmctx.push(instance.globals.as_mut_ptr()); - for mem in &mut instance.memories { - memories.push(mem.as_mut_ptr()); - } - vmctx.push(memories.as_mut_ptr() as *mut u8); + vmctx.push(mem_base_addrs.as_mut_ptr() as *mut u8); + vmctx.push(instance as *mut Instance as *mut u8); vmctx } @@ -100,7 +122,13 @@ pub fn execute( let code_buf = &compilation.functions[start_index]; - let vmctx = make_vmctx(instance); + // Collect all memory base addresses and Vec. + let mut mem_base_addrs = instance + .memories + .iter_mut() + .map(LinearMemory::base_addr) + .collect::>(); + let vmctx = make_vmctx(instance, &mut mem_base_addrs); // Rather than writing inline assembly to jump to the code region, we use the fact that // the Rust ABI for calling a function with no arguments and no return matches the one of diff --git a/lib/execute/src/instance.rs b/lib/execute/src/instance.rs index 6ae0909a12..9096fdd2c8 100644 --- a/lib/execute/src/instance.rs +++ b/lib/execute/src/instance.rs @@ -3,10 +3,9 @@ use cranelift_codegen::ir; use cranelift_wasm::GlobalIndex; +use memory::LinearMemory; use wasmtime_environ::{DataInitializer, Module, TableElements}; -const PAGE_SIZE: usize = 65536; - /// An Instance of a WebAssemby module. #[derive(Debug)] pub struct Instance { @@ -14,7 +13,7 @@ pub struct Instance { pub tables: Vec>, /// WebAssembly linear memory data. - pub memories: Vec>, + pub memories: Vec, /// WebAssembly global variable data. pub globals: Vec, @@ -58,15 +57,13 @@ impl Instance { // Allocate the underlying memory and initialize it to all zeros. self.memories.reserve_exact(module.memories.len()); for memory in &module.memories { - let len = memory.pages_count * PAGE_SIZE; - let mut v = Vec::with_capacity(len); - v.resize(len, 0); + let v = LinearMemory::new(memory.pages_count as u32, memory.maximum.map(|m| m as u32)); self.memories.push(v); } for init in data_initializers { debug_assert!(init.base.is_none(), "globalvar base not supported yet"); - let to_init = - &mut self.memories[init.memory_index][init.offset..init.offset + init.data.len()]; + let mem_mut = self.memories[init.memory_index].as_mut(); + let to_init = &mut mem_mut[init.offset..init.offset + init.data.len()]; to_init.copy_from_slice(init.data); } } @@ -80,13 +77,20 @@ impl Instance { self.globals.resize(globals_data_size, 0); } + /// Returns a mutable reference to a linear memory under the specified index. + pub fn memory_mut(&mut self, memory_index: usize) -> &mut LinearMemory { + self.memories + .get_mut(memory_index) + .unwrap_or_else(|| panic!("no memory for index {}", memory_index)) + } + /// Returns a slice of the contents of allocated linear memory. pub fn inspect_memory(&self, memory_index: usize, address: usize, len: usize) -> &[u8] { &self .memories .get(memory_index) .unwrap_or_else(|| panic!("no memory for index {}", memory_index)) - [address..address + len] + .as_ref()[address..address + len] } /// Shows the value of a global variable. diff --git a/lib/execute/src/lib.rs b/lib/execute/src/lib.rs index 55a310cb54..82ec74e08c 100644 --- a/lib/execute/src/lib.rs +++ b/lib/execute/src/lib.rs @@ -14,11 +14,13 @@ extern crate cranelift_codegen; extern crate cranelift_wasm; +extern crate memmap; extern crate region; extern crate wasmtime_environ; mod execute; mod instance; +mod memory; pub use execute::{compile_and_link_module, execute}; pub use instance::Instance; diff --git a/lib/execute/src/memory.rs b/lib/execute/src/memory.rs new file mode 100644 index 0000000000..faa87ce7a5 --- /dev/null +++ b/lib/execute/src/memory.rs @@ -0,0 +1,90 @@ +use memmap; +use std::fmt; + +const PAGE_SIZE: u32 = 65536; +const MAX_PAGES: u32 = 65536; + +/// A linear memory instance. +/// +/// This linear memory has a stable base address and at the same time allows +/// for dynamical growing. +pub struct LinearMemory { + mmap: memmap::MmapMut, + current: u32, + maximum: u32, +} + +impl LinearMemory { + /// Create a new linear memory instance with specified initial and maximum number of pages. + /// + /// `maximum` cannot be set to more than `65536` pages. If `maximum` is `None` then it + /// will be treated as `65336`. + pub fn new(initial: u32, maximum: Option) -> Self { + let maximum = maximum.unwrap_or(MAX_PAGES); + + assert!(initial <= MAX_PAGES); + assert!(maximum <= MAX_PAGES); + + let len = maximum.saturating_mul(MAX_PAGES); + let mmap = memmap::MmapMut::map_anon(len as usize).unwrap(); + Self { + mmap, + current: initial, + maximum, + } + } + + /// Returns an base address of this linear memory. + pub fn base_addr(&mut self) -> *mut u8 { + self.mmap.as_mut_ptr() + } + + /// Returns a number of allocated wasm pages. + pub fn current_size(&self) -> u32 { + self.current + } + + /// Grow memory by the specified amount of pages. + /// + /// Returns `None` if memory can't be grown by the specified amount + /// of pages. + pub fn grow(&mut self, add_pages: u32) -> Option { + let new_pages = match self.current.checked_add(add_pages) { + Some(new_pages) => new_pages, + None => return None, + }; + + let prev_pages = self.current; + self.current = new_pages; + + // Ensure that newly allocated area is zeroed. + let new_start_offset = (prev_pages * PAGE_SIZE) as usize; + let new_end_offset = (new_pages * PAGE_SIZE) as usize; + for i in new_start_offset..new_end_offset - 1 { + self.mmap[i] = 0; + } + + Some(prev_pages) + } +} + +impl fmt::Debug for LinearMemory { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("LinearMemory") + .field("current", &self.current) + .field("maximum", &self.maximum) + .finish() + } +} + +impl AsRef<[u8]> for LinearMemory { + fn as_ref(&self) -> &[u8] { + &self.mmap + } +} + +impl AsMut<[u8]> for LinearMemory { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.mmap + } +}