T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wasmtime: Add support for func.ref and table.grow with funcrefs

Browse Source 
`funcref`s are implemented as `NonNull<VMCallerCheckedAnyfunc>`.

This should be more efficient than using a `VMExternRef` that points at a
`VMCallerCheckedAnyfunc` because it gets rid of an indirection, dynamic
allocation, and some reference counting.

Note that the null function reference is *NOT* a null pointer; it is a
`VMCallerCheckedAnyfunc` that has a null `func_ptr` member.

Part of #929
This commit is contained in:
Nick Fitzgerald
2020-06-18 11:04:40 -07:00
parent ddc2ce8080
commit 58bb5dd953
37 changed files with 603 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
crates/runtime/src/export.rs
View File

@@ -1,7 +1,7 @@
use crate::vmcontext::{
VMContext, VMFunctionBody, VMGlobalDefinition, VMMemoryDefinition, VMSharedSignatureIndex,
VMTableDefinition,
VMCallerCheckedAnyfunc, VMContext, VMGlobalDefinition, VMMemoryDefinition, VMTableDefinition,
};
use std::ptr::NonNull;
use wasmtime_environ::wasm::Global;
use wasmtime_environ::{MemoryPlan, TablePlan};
@@ -24,14 +24,11 @@ pub enum Export {
/// A function export value.
#[derive(Debug, Clone)]
pub struct ExportFunction {
/// The address of the native-code function.
pub address: *const VMFunctionBody,
/// Pointer to the containing `VMContext`.
pub vmctx: *mut VMContext,
/// The function signature declaration, used for compatibilty checking.
/// The `VMCallerCheckedAnyfunc` for this exported function.
///
/// Note that this indexes within the module associated with `vmctx`.
pub signature: VMSharedSignatureIndex,
/// Note that exported functions cannot be a null funcref, so this is a
/// non-null pointer.
pub anyfunc: NonNull<VMCallerCheckedAnyfunc>,
}
impl From<ExportFunction> for Export {

7
crates/runtime/src/externref.rs
View File

@@ -545,7 +545,10 @@ impl VMExternRefActivationsTable {
return Err(externref);
}
debug_assert!((*next.as_ref().get()).is_none());
debug_assert!(
(*next.as_ref().get()).is_none(),
"slots >= the `next` bump finger are always `None`"
);
ptr::write(next.as_ptr(), UnsafeCell::new(Some(externref)));
let next = NonNull::new_unchecked(next.as_ptr().add(1));
@@ -1121,6 +1124,7 @@ mod tests {
num_imported_tables: 0,
num_imported_memories: 0,
num_imported_globals: 0,
num_defined_functions: 0,
num_defined_tables: 0,
num_defined_memories: 0,
num_defined_globals: 0,
@@ -1147,6 +1151,7 @@ mod tests {
num_imported_tables: 0,
num_imported_memories: 0,
num_imported_globals: 0,
num_defined_functions: 0,
num_defined_tables: 0,
num_defined_memories: 0,
num_defined_globals: 0,

114
crates/runtime/src/instance.rs
View File

@@ -21,13 +21,15 @@ use std::any::Any;
use std::cell::RefCell;
use std::collections::HashMap;
use std::convert::TryFrom;
use std::ptr::NonNull;
use std::sync::Arc;
use std::{mem, ptr, slice};
use thiserror::Error;
use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap};
use wasmtime_environ::wasm::{
DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,
ElemIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, TableIndex,
ElemIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, TableElementType,
TableIndex,
};
use wasmtime_environ::{ir, DataInitializer, EntityIndex, Module, TableElements, VMOffsets};
@@ -54,7 +56,7 @@ pub(crate) struct Instance {
/// Passive elements in this instantiation. As `elem.drop`s happen, these
/// entries get removed. A missing entry is considered equivalent to an
/// empty slice.
passive_elements: RefCell<HashMap<ElemIndex, Box<[VMCallerCheckedAnyfunc]>>>,
passive_elements: RefCell<HashMap<ElemIndex, Box<[*mut VMCallerCheckedAnyfunc]>>>,
/// Passive data segments from our module. As `data.drop`s happen, entries
/// get removed. A missing entry is considered equivalent to an empty slice.
@@ -273,23 +275,10 @@ impl Instance {
pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export {
match export {
EntityIndex::Function(index) => {
let signature = self.signature_id(self.module.local.functions[*index]);
let (address, vmctx) =
if let Some(def_index) = self.module.local.defined_func_index(*index) {
(
self.finished_functions[def_index] as *const _,
self.vmctx_ptr(),
)
} else {
let import = self.imported_function(*index);
(import.body, import.vmctx)
};
ExportFunction {
address,
signature,
vmctx,
}
.into()
let anyfunc = self.get_caller_checked_anyfunc(*index).unwrap();
let anyfunc =
NonNull::new(anyfunc as *const VMCallerCheckedAnyfunc as *mut _).unwrap();
ExportFunction { anyfunc }.into()
}
EntityIndex::Table(index) => {
let (definition, vmctx) =
@@ -448,6 +437,11 @@ impl Instance {
foreign_instance.memory_size(foreign_index)
}
pub(crate) fn table_element_type(&self, table_index: TableIndex) -> TableElementType {
let table = self.get_table(table_index);
table.element_type()
}
/// Grow table by the specified amount of elements, filling them with
/// `init_value`.
///
@@ -513,30 +507,25 @@ impl Instance {
Layout::from_size_align(size, align).unwrap()
}
/// Get a `VMCallerCheckedAnyfunc` for the given `FuncIndex`.
fn get_caller_checked_anyfunc(&self, index: FuncIndex) -> VMCallerCheckedAnyfunc {
/// Get a `&VMCallerCheckedAnyfunc` for the given `FuncIndex`.
///
/// Returns `None` if the index is the reserved index value.
///
/// The returned reference is a stable reference that won't be moved and can
/// be passed into JIT code.
pub(crate) fn get_caller_checked_anyfunc(
&self,
index: FuncIndex,
) -> Option<&VMCallerCheckedAnyfunc> {
if index == FuncIndex::reserved_value() {
return VMCallerCheckedAnyfunc::default();
return None;
}
let sig = self.module.local.functions[index];
let type_index = self.signature_id(sig);
Some(unsafe { &*self.anyfunc_ptr(index) })
}
let (func_ptr, vmctx) = if let Some(def_index) = self.module.local.defined_func_index(index)
{
(
self.finished_functions[def_index] as *const _,
self.vmctx_ptr(),
)
} else {
let import = self.imported_function(index);
(import.body, import.vmctx)
};
VMCallerCheckedAnyfunc {
func_ptr,
type_index,
vmctx,
}
unsafe fn anyfunc_ptr(&self, index: FuncIndex) -> *mut VMCallerCheckedAnyfunc {
self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))
}
/// The `table.init` operation: initializes a portion of a table with a
@@ -574,7 +563,7 @@ impl Instance {
// TODO(#983): investigate replacing this get/set loop with a `memcpy`.
for (dst, src) in (dst..dst + len).zip(src..src + len) {
table
.set(dst, TableElement::FuncRef(elem[src as usize].clone()))
.set(dst, TableElement::FuncRef(elem[src as usize]))
.expect("should never panic because we already did the bounds check above");
}
@@ -932,6 +921,30 @@ impl InstanceHandle {
*instance.externref_activations_table() = externref_activations_table;
*instance.stack_map_registry() = stack_map_registry;
for (index, sig) in instance.module.local.functions.iter() {
let type_index = instance.signature_id(*sig);
let (func_ptr, vmctx) =
if let Some(def_index) = instance.module.local.defined_func_index(index) {
(
NonNull::new(instance.finished_functions[def_index] as *mut _).unwrap(),
instance.vmctx_ptr(),
)
} else {
let import = instance.imported_function(index);
(import.body, import.vmctx)
};
ptr::write(
instance.anyfunc_ptr(index),
VMCallerCheckedAnyfunc {
func_ptr,
type_index,
vmctx,
},
);
}
// Perform infallible initialization in this constructor, while fallible
// initialization is deferred to the `initialize` method.
initialize_passive_elements(instance);
@@ -1246,12 +1259,14 @@ fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> {
}
for (i, func_idx) in init.elements.iter().enumerate() {
let anyfunc = instance.get_caller_checked_anyfunc(*func_idx);
let anyfunc = instance.get_caller_checked_anyfunc(*func_idx).map_or(
ptr::null_mut(),
|f: &VMCallerCheckedAnyfunc| {
f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc
},
);
table
.set(
u32::try_from(start + i).unwrap(),
TableElement::FuncRef(anyfunc),
)
.set(u32::try_from(start + i).unwrap(), anyfunc.into())
.unwrap();
}
}
@@ -1280,7 +1295,14 @@ fn initialize_passive_elements(instance: &Instance) {
*idx,
segments
.iter()
.map(|s| instance.get_caller_checked_anyfunc(*s))
.map(|s| {
instance.get_caller_checked_anyfunc(*s).map_or(
ptr::null_mut(),
|f: &VMCallerCheckedAnyfunc| {
f as *const VMCallerCheckedAnyfunc as *mut _
},
)
})
.collect(),
)
}),

11
crates/runtime/src/lib.rs
View File

@@ -56,3 +56,14 @@ pub use crate::vmcontext::{
/// Version number of this crate.
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// The Cranelift IR type used for reference types for this target architecture.
pub fn ref_type() -> wasmtime_environ::ir::Type {
if cfg!(target_pointer_width = "32") {
wasmtime_environ::ir::types::R32
} else if cfg!(target_pointer_width = "64") {
wasmtime_environ::ir::types::R64
} else {
unreachable!()
}
}

44
crates/runtime/src/libcalls.rs
View File

@@ -57,10 +57,12 @@
//! ```
use crate::externref::VMExternRef;
use crate::table::{Table, TableElement};
use crate::table::Table;
use crate::traphandlers::raise_lib_trap;
use crate::vmcontext::VMContext;
use wasmtime_environ::wasm::{DataIndex, DefinedMemoryIndex, ElemIndex, MemoryIndex, TableIndex};
use crate::vmcontext::{VMCallerCheckedAnyfunc, VMContext};
use wasmtime_environ::wasm::{
DataIndex, DefinedMemoryIndex, ElemIndex, MemoryIndex, TableElementType, TableIndex,
};
/// Implementation of f32.ceil
pub extern "C" fn wasmtime_f32_ceil(x: f32) -> f32 {
@@ -226,24 +228,38 @@ pub unsafe extern "C" fn wasmtime_imported_memory32_size(
instance.imported_memory_size(memory_index)
}
/// Implementation of `table.grow` for `externref`s.
pub unsafe extern "C" fn wasmtime_table_grow_extern_ref(
/// Implementation of `table.grow`.
pub unsafe extern "C" fn wasmtime_table_grow(
vmctx: *mut VMContext,
table_index: u32,
delta: u32,
// NB: we don't know whether this is a pointer to a `VMCallerCheckedAnyfunc`
// or is a `VMExternRef` until we look at the table type.
init_value: *mut u8,
) -> u32 {
let init_value = if init_value.is_null() {
None
} else {
Some(VMExternRef::clone_from_raw(init_value))
};
let instance = (&mut *vmctx).instance();
let table_index = TableIndex::from_u32(table_index);
instance
.table_grow(table_index, delta, TableElement::ExternRef(init_value))
.unwrap_or(-1_i32 as u32)
match instance.table_element_type(table_index) {
TableElementType::Func => {
let func = init_value as *mut VMCallerCheckedAnyfunc;
instance
.table_grow(table_index, delta, func.into())
.unwrap_or(-1_i32 as u32)
}
TableElementType::Val(ty) => {
debug_assert_eq!(ty, crate::ref_type());
let init_value = if init_value.is_null() {
None
} else {
Some(VMExternRef::clone_from_raw(init_value))
};
instance
.table_grow(table_index, delta, init_value.into())
.unwrap_or(-1_i32 as u32)
}
}
}
/// Implementation of `table.copy`.

42
crates/runtime/src/table.rs
View File

@@ -6,6 +6,7 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition};
use crate::{Trap, VMExternRef};
use std::cell::RefCell;
use std::convert::{TryFrom, TryInto};
use std::ptr;
use wasmtime_environ::wasm::TableElementType;
use wasmtime_environ::{ir, TablePlan, TableStyle};
@@ -20,35 +21,28 @@ pub struct Table {
#[derive(Clone, Debug)]
pub enum TableElement {
/// A `funcref`.
FuncRef(VMCallerCheckedAnyfunc),
FuncRef(*mut VMCallerCheckedAnyfunc),
/// An `exrernref`.
ExternRef(Option<VMExternRef>),
}
#[derive(Debug)]
enum TableElements {
FuncRefs(Vec<VMCallerCheckedAnyfunc>),
FuncRefs(Vec<*mut VMCallerCheckedAnyfunc>),
ExternRefs(Vec<Option<VMExternRef>>),
}
impl Table {
/// Create a new table instance with specified minimum and maximum number of elements.
pub fn new(plan: &TablePlan) -> Self {
let elements =
RefCell::new(match plan.table.ty {
TableElementType::Func => TableElements::FuncRefs(vec![
VMCallerCheckedAnyfunc::default();
usize::try_from(plan.table.minimum).unwrap()
]),
TableElementType::Val(ty)
if (cfg!(target_pointer_width = "64") && ty == ir::types::R64)
|| (cfg!(target_pointer_width = "32") && ty == ir::types::R32) =>
{
let min = usize::try_from(plan.table.minimum).unwrap();
TableElements::ExternRefs(vec![None; min])
}
TableElementType::Val(ty) => unimplemented!("unsupported table type ({})", ty),
});
let min = usize::try_from(plan.table.minimum).unwrap();
let elements = RefCell::new(match plan.table.ty {
TableElementType::Func => TableElements::FuncRefs(vec![ptr::null_mut(); min]),
TableElementType::Val(ty) => {
debug_assert_eq!(ty, crate::ref_type());
TableElements::ExternRefs(vec![None; min])
}
});
match plan.style {
TableStyle::CallerChecksSignature => Self {
elements,
@@ -57,6 +51,14 @@ impl Table {
}
}
/// Returns the type of the elements in this table.
pub fn element_type(&self) -> TableElementType {
match &*self.elements.borrow() {
TableElements::FuncRefs(_) => TableElementType::Func,
TableElements::ExternRefs(_) => TableElementType::Val(crate::ref_type()),
}
}
/// Returns the number of allocated elements.
pub fn size(&self) -> u32 {
match &*self.elements.borrow() {
@@ -199,7 +201,7 @@ impl Table {
}
}
impl TryFrom<TableElement> for VMCallerCheckedAnyfunc {
impl TryFrom<TableElement> for *mut VMCallerCheckedAnyfunc {
type Error = TableElement;
fn try_from(e: TableElement) -> Result<Self, Self::Error> {
@@ -221,8 +223,8 @@ impl TryFrom<TableElement> for Option<VMExternRef> {
}
}
impl From<VMCallerCheckedAnyfunc> for TableElement {
fn from(f: VMCallerCheckedAnyfunc) -> TableElement {
impl From<*mut VMCallerCheckedAnyfunc> for TableElement {
fn from(f: *mut VMCallerCheckedAnyfunc) -> TableElement {
TableElement::FuncRef(f)
}
}

23
crates/runtime/src/vmcontext.rs
View File

@@ -3,8 +3,9 @@
use crate::instance::Instance;
use std::any::Any;
use std::ptr::NonNull;
use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
use std::{ptr, u32};
use std::u32;
use wasmtime_environ::BuiltinFunctionIndex;
/// An imported function.
@@ -12,7 +13,7 @@ use wasmtime_environ::BuiltinFunctionIndex;
#[repr(C)]
pub struct VMFunctionImport {
/// A pointer to the imported function body.
pub body: *const VMFunctionBody,
pub body: NonNull<VMFunctionBody>,
/// A pointer to the `VMContext` that owns the function.
pub vmctx: *mut VMContext,
@@ -475,7 +476,7 @@ impl Default for VMSharedSignatureIndex {
#[repr(C)]
pub struct VMCallerCheckedAnyfunc {
/// Function body.
pub func_ptr: *const VMFunctionBody,
pub func_ptr: NonNull<VMFunctionBody>,
/// Function signature id.
pub type_index: VMSharedSignatureIndex,
/// Function `VMContext`.
@@ -513,16 +514,6 @@ mod test_vmcaller_checked_anyfunc {
}
}
impl Default for VMCallerCheckedAnyfunc {
fn default() -> Self {
Self {
func_ptr: ptr::null_mut(),
type_index: Default::default(),
vmctx: ptr::null_mut(),
}
}
}
/// An array that stores addresses of builtin functions. We translate code
/// to use indirect calls. This way, we don't have to patch the code.
#[repr(C)]
@@ -549,8 +540,10 @@ impl VMBuiltinFunctionsArray {
ptrs[BuiltinFunctionIndex::imported_memory32_size().index() as usize] =
wasmtime_imported_memory32_size as usize;
ptrs[BuiltinFunctionIndex::table_copy().index() as usize] = wasmtime_table_copy as usize;
ptrs[BuiltinFunctionIndex::table_grow_extern_ref().index() as usize] =
wasmtime_table_grow_extern_ref as usize;
ptrs[BuiltinFunctionIndex::table_grow_funcref().index() as usize] =
wasmtime_table_grow as usize;
ptrs[BuiltinFunctionIndex::table_grow_externref().index() as usize] =
wasmtime_table_grow as usize;
ptrs[BuiltinFunctionIndex::table_init().index() as usize] = wasmtime_table_init as usize;
ptrs[BuiltinFunctionIndex::elem_drop().index() as usize] = wasmtime_elem_drop as usize;
ptrs[BuiltinFunctionIndex::defined_memory_copy().index() as usize] =