Make the storage of wasmtime_runtime::Table consistent.

This change makes the storage of `Table` more internally consistent.

Elements are stored as raw pointers for both static and dynamic table storage.

Explicitly storing elements as pointers removes assumptions being made by the
pooling allocator in terms of the size and default representation of the
elements.

However, care must be made to properly clone externrefs for table operations.

crates/runtime/src/externref.rs

@@ -351,8 +351,35 @@ impl VMExternRef {
         ptr
     }
 
+    /// Consume this `VMExternRef` into a raw, untyped pointer.
+    ///
+    /// # Safety
+    ///
+    /// This method forgets self, so it is possible to create a leak of the
+    /// underlying reference counted data if not used carefully.
+    ///
+    /// Use `from_raw` to recreate the `VMExternRef`.
+    pub unsafe fn into_raw(self) -> *mut u8 {
+        let ptr = self.0.cast::<u8>().as_ptr();
+        std::mem::forget(self);
+        ptr
+    }
+
     /// Recreate a `VMExternRef` from a pointer returned from a previous call to
-    /// `VMExternRef::as_raw`.
+    /// `as_raw`.
+    ///
+    /// # Safety
+    ///
+    /// Unlike `clone_from_raw`, this does not increment the reference count of the
+    /// underlying data.  It is not safe to continue to use the pointer passed to this
+    /// function.
+    pub unsafe fn from_raw(ptr: *mut u8) -> Self {
+        debug_assert!(!ptr.is_null());
+        VMExternRef(NonNull::new_unchecked(ptr).cast())
+    }
+
+    /// Recreate a `VMExternRef` from a pointer returned from a previous call to
+    /// `as_raw`.
     ///
     /// # Safety
     ///

crates/runtime/src/instance/allocator/pooling.rs

@@ -11,7 +11,7 @@ use super::{
     initialize_instance, initialize_vmcontext, FiberStackError, InstanceAllocationRequest,
     InstanceAllocator, InstanceHandle, InstantiationError,
 };
-use crate::{instance::Instance, table::max_table_element_size, Memory, Mmap, Table, VMContext};
+use crate::{instance::Instance, Memory, Mmap, Table, VMContext};
 use anyhow::{anyhow, bail, Context, Result};
 use rand::Rng;
 use std::cell::RefCell;
@@ -427,8 +427,12 @@ impl InstancePool {
         let instance = unsafe { &mut *handle.instance };
 
         // Decommit any linear memories that were used
-        for (mem, base) in instance.memories.values().zip(self.memories.get(index)) {
+        for (memory, base) in instance.memories.values_mut().zip(self.memories.get(index)) {
-            let size = (mem.size() * WASM_PAGE_SIZE) as usize;
+            let memory = mem::take(memory);
+            debug_assert!(memory.is_static());
+
+            let size = (memory.size() * WASM_PAGE_SIZE) as usize;
+            drop(memory);
             decommit_memory_pages(base, size).unwrap();
         }
 
@@ -436,13 +440,16 @@ impl InstancePool {
         instance.dropped_data.borrow_mut().clear();
 
         // Decommit any tables that were used
-        let table_element_size = max_table_element_size();
+        for (table, base) in instance.tables.values_mut().zip(self.tables.get(index)) {
-        for (table, base) in instance.tables.values().zip(self.tables.get(index)) {
+            let table = mem::take(table);
+            debug_assert!(table.is_static());
+
             let size = round_up_to_pow2(
-                table.size() as usize * table_element_size,
+                table.size() as usize * mem::size_of::<*mut u8>(),
                 self.tables.page_size,
             );
 
+            drop(table);
             decommit_table_pages(base, size).unwrap();
         }
 
@@ -503,12 +510,12 @@ impl InstancePool {
         for plan in (&module.table_plans.values().as_slice()[module.num_imported_tables..]).iter() {
             let base = tables.next().unwrap();
 
-            commit_table_pages(base, max_elements as usize * max_table_element_size())
+            commit_table_pages(base, max_elements as usize * mem::size_of::<*mut u8>())
                 .map_err(|e| InstantiationError::Resource(e.to_string()))?;
 
             instance
                 .tables
-                .push(Table::new_static(plan, base, max_elements));
+                .push(Table::new_static(plan, base as _, max_elements));
         }
 
         let mut dropped_elements = instance.dropped_elements.borrow_mut();
@@ -646,7 +653,7 @@ impl TablePool {
 
         let table_size = if module_limits.table_elements > 0 {
             round_up_to_pow2(
-                max_table_element_size()
+                mem::size_of::<*mut u8>()
                     .checked_mul(module_limits.table_elements as usize)
                     .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?,
                 page_size,

crates/runtime/src/memory.rs

@@ -9,6 +9,7 @@ use more_asserts::{assert_ge, assert_le};
 use std::cell::{Cell, RefCell};
 use std::cmp::min;
 use std::convert::TryFrom;
+use std::ptr;
 use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE};
 
 /// A memory allocator
@@ -184,16 +185,12 @@ enum MemoryStorage {
 }
 
 /// Represents an instantiation of a WebAssembly memory.
-pub struct Memory {
+pub struct Memory(MemoryStorage);
-    storage: MemoryStorage,
-}
 
 impl Memory {
     /// Create a new dynamic (movable) memory instance for the specified plan.
     pub fn new_dynamic(plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator) -> Result<Self> {
-        Ok(Self {
+        Ok(Self(MemoryStorage::Dynamic(creator.new_memory(plan)?)))
-            storage: MemoryStorage::Dynamic(creator.new_memory(plan)?),
-        })
     }
 
     /// Create a new static (immovable) memory instance for the specified plan.
@@ -207,30 +204,37 @@ impl Memory {
             make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize)?;
         }
 
-        Ok(Self {
+        Ok(Self(MemoryStorage::Static {
-            storage: MemoryStorage::Static {
+            base,
-                base,
+            size: Cell::new(plan.memory.minimum),
-                size: Cell::new(plan.memory.minimum),
+            maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum),
-                maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum),
+            make_accessible,
-                make_accessible,
+        }))
-            },
-        })
     }
 
     /// Returns the number of allocated wasm pages.
     pub fn size(&self) -> u32 {
-        match &self.storage {
+        match &self.0 {
             MemoryStorage::Static { size, .. } => size.get(),
             MemoryStorage::Dynamic(mem) => mem.size(),
         }
     }
 
+    /// Returns whether or not the underlying storage of the memory is "static".
+    pub(crate) fn is_static(&self) -> bool {
+        if let MemoryStorage::Static { .. } = &self.0 {
+            true
+        } else {
+            false
+        }
+    }
+
     /// Grow memory by the specified amount of wasm pages.
     ///
     /// Returns `None` if memory can't be grown by the specified amount
     /// of wasm pages.
     pub fn grow(&self, delta: u32) -> Option<u32> {
-        match &self.storage {
+        match &self.0 {
             MemoryStorage::Static {
                 base,
                 size,
@@ -264,7 +268,7 @@ impl Memory {
 
     /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code.
     pub fn vmmemory(&self) -> VMMemoryDefinition {
-        match &self.storage {
+        match &self.0 {
             MemoryStorage::Static { base, size, .. } => VMMemoryDefinition {
                 base: *base,
                 current_length: size.get() as usize * WASM_PAGE_SIZE as usize,
@@ -273,3 +277,19 @@ impl Memory {
         }
     }
 }
+
+// The default memory representation is an empty memory that cannot grow.
+impl Default for Memory {
+    fn default() -> Self {
+        fn make_accessible(_ptr: *mut u8, _len: usize) -> Result<()> {
+            unreachable!()
+        }
+
+        Self(MemoryStorage::Static {
+            base: ptr::null_mut(),
+            size: Cell::new(0),
+            maximum: 0,
+            make_accessible,
+        })
+    }
+}

crates/runtime/src/table.rs

@@ -6,12 +6,15 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition};
 use crate::{Trap, VMExternRef};
 use std::cell::{Cell, RefCell};
 use std::cmp::min;
-use std::convert::{TryFrom, TryInto};
+use std::convert::TryInto;
+use std::ops::Range;
 use std::ptr;
 use wasmtime_environ::wasm::TableElementType;
-use wasmtime_environ::{ir, TablePlan, TableStyle};
+use wasmtime_environ::{ir, TablePlan};
 
 /// An element going into or coming out of a table.
+///
+/// Table elements are stored as pointers and are default-initialized with `ptr::null_mut`.
 #[derive(Clone, Debug)]
 pub enum TableElement {
     /// A `funcref`.
@@ -20,24 +23,53 @@ pub enum TableElement {
     ExternRef(Option<VMExternRef>),
 }
 
-impl TryFrom<TableElement> for *mut VMCallerCheckedAnyfunc {
+impl TableElement {
-    type Error = ();
+    /// Consumes the given raw pointer into a table element.
-
+    ///
-    fn try_from(e: TableElement) -> Result<Self, Self::Error> {
+    /// # Safety
-        match e {
+    ///
-            TableElement::FuncRef(f) => Ok(f),
+    /// This is unsafe as it will *not* clone any externref, leaving the reference count unchanged.
-            _ => Err(()),
+    ///
+    /// This should only be used if the raw pointer is no longer in use.
+    unsafe fn from_raw(ty: TableElementType, ptr: *mut u8) -> Self {
+        match ty {
+            TableElementType::Func => Self::FuncRef(ptr as _),
+            TableElementType::Val(_) => Self::ExternRef(if ptr.is_null() {
+                None
+            } else {
+                Some(VMExternRef::from_raw(ptr))
+            }),
         }
     }
-}
 
-impl TryFrom<TableElement> for Option<VMExternRef> {
+    /// Clones a table element from the underlying raw pointer.
-    type Error = ();
+    ///
+    /// # Safety
+    ///
+    /// This is unsafe as it will clone any externref, incrementing the reference count.
+    unsafe fn clone_from_raw(ty: TableElementType, ptr: *mut u8) -> Self {
+        match ty {
+            TableElementType::Func => Self::FuncRef(ptr as _),
+            TableElementType::Val(_) => Self::ExternRef(if ptr.is_null() {
+                None
+            } else {
+                Some(VMExternRef::clone_from_raw(ptr))
+            }),
+        }
+    }
 
-    fn try_from(e: TableElement) -> Result<Self, Self::Error> {
+    /// Consumes a table element into a raw pointer.
-        match e {
+    ///
-            TableElement::ExternRef(x) => Ok(x),
+    /// # Safety
-            _ => Err(()),
+    ///
+    /// This is unsafe as it will consume any underlying externref into a raw pointer without modifying
+    /// the reference count.
+    ///
+    /// Use `from_raw` to properly drop any table elements stored as raw pointers.
+    unsafe fn into_raw(self) -> *mut u8 {
+        match self {
+            Self::FuncRef(e) => e as _,
+            Self::ExternRef(e) => e.map(|e| e.into_raw()).unwrap_or(ptr::null_mut()),
         }
     }
 }
@@ -60,107 +92,79 @@ impl From<VMExternRef> for TableElement {
     }
 }
 
-#[derive(Debug)]
-enum TableElements {
-    FuncRefs(Vec<*mut VMCallerCheckedAnyfunc>),
-    ExternRefs(Vec<Option<VMExternRef>>),
-}
-
-// Ideally this should be static assertion that table elements are pointer-sized
-#[inline(always)]
-pub(crate) fn max_table_element_size() -> usize {
-    debug_assert_eq!(
-        std::mem::size_of::<*mut VMCallerCheckedAnyfunc>(),
-        std::mem::size_of::<*const ()>()
-    );
-    debug_assert_eq!(
-        std::mem::size_of::<Option<VMExternRef>>(),
-        std::mem::size_of::<*const ()>()
-    );
-    std::mem::size_of::<*const ()>()
-}
-
 #[derive(Debug)]
 enum TableStorage {
     Static {
-        data: *mut u8,
+        data: *mut *mut u8,
         size: Cell<u32>,
         ty: TableElementType,
         maximum: u32,
     },
     Dynamic {
-        elements: RefCell<TableElements>,
+        elements: RefCell<Vec<*mut u8>>,
+        ty: TableElementType,
         maximum: Option<u32>,
     },
 }
 
 /// Represents an instance's table.
 #[derive(Debug)]
-pub struct Table {
+pub struct Table(TableStorage);
-    storage: TableStorage,
-}
 
 impl Table {
     /// Create a new dynamic (movable) table instance for the specified table plan.
     pub fn new_dynamic(plan: &TablePlan) -> Self {
-        let min = usize::try_from(plan.table.minimum).unwrap();
+        let elements = RefCell::new(vec![ptr::null_mut(); plan.table.minimum as usize]);
-        let elements = RefCell::new(match plan.table.ty {
+        let ty = plan.table.ty.clone();
-            TableElementType::Func => TableElements::FuncRefs(vec![ptr::null_mut(); min]),
+        let maximum = plan.table.maximum;
-            TableElementType::Val(ty) => {
+        Self(TableStorage::Dynamic {
-                debug_assert_eq!(ty, crate::ref_type());
+            elements,
-                TableElements::ExternRefs(vec![None; min])
+            ty,
-            }
+            maximum,
-        });
+        })
-
-        match plan.style {
-            TableStyle::CallerChecksSignature => Self {
-                storage: TableStorage::Dynamic {
-                    elements,
-                    maximum: plan.table.maximum,
-                },
-            },
-        }
     }
 
     /// Create a new static (immovable) table instance for the specified table plan.
-    pub fn new_static(plan: &TablePlan, data: *mut u8, maximum: u32) -> Self {
+    pub fn new_static(plan: &TablePlan, data: *mut *mut u8, maximum: u32) -> Self {
-        match plan.style {
+        let size = Cell::new(plan.table.minimum);
-            TableStyle::CallerChecksSignature => Self {
+        let ty = plan.table.ty.clone();
-                storage: TableStorage::Static {
+        let maximum = min(plan.table.maximum.unwrap_or(maximum), maximum);
-                    data,
+        Self(TableStorage::Static {
-                    size: Cell::new(plan.table.minimum),
+            data,
-                    ty: plan.table.ty.clone(),
+            size,
-                    maximum: min(plan.table.maximum.unwrap_or(maximum), maximum),
+            ty,
-                },
+            maximum,
-            },
+        })
-        }
     }
 
     /// Returns the type of the elements in this table.
     pub fn element_type(&self) -> TableElementType {
-        match &self.storage {
+        match &self.0 {
             TableStorage::Static { ty, .. } => *ty,
-            TableStorage::Dynamic { elements, .. } => match &*elements.borrow() {
+            TableStorage::Dynamic { ty, .. } => *ty,
-                TableElements::FuncRefs(_) => TableElementType::Func,
+        }
-                TableElements::ExternRefs(_) => TableElementType::Val(crate::ref_type()),
+    }
-            },
+
+    /// Returns whether or not the underlying storage of the table is "static".
+    pub(crate) fn is_static(&self) -> bool {
+        if let TableStorage::Static { .. } = &self.0 {
+            true
+        } else {
+            false
         }
     }
 
     /// Returns the number of allocated elements.
     pub fn size(&self) -> u32 {
-        match &self.storage {
+        match &self.0 {
             TableStorage::Static { size, .. } => size.get(),
-            TableStorage::Dynamic { elements, .. } => match &*elements.borrow() {
+            TableStorage::Dynamic { elements, .. } => elements.borrow().len().try_into().unwrap(),
-                TableElements::FuncRefs(x) => x.len().try_into().unwrap(),
-                TableElements::ExternRefs(x) => x.len().try_into().unwrap(),
-            },
         }
     }
 
     /// Returns the maximum number of elements.
     pub fn maximum(&self) -> Option<u32> {
-        match &self.storage {
+        match &self.0 {
             TableStorage::Static { maximum, .. } => Some(*maximum),
             TableStorage::Dynamic { maximum, .. } => maximum.clone(),
         }
@@ -170,31 +174,30 @@ impl Table {
     ///
     /// Returns a trap error on out-of-bounds accesses.
     pub fn fill(&self, dst: u32, val: TableElement, len: u32) -> Result<(), Trap> {
-        let start = dst;
+        let start = dst as usize;
         let end = start
-            .checked_add(len)
+            .checked_add(len as usize)
             .ok_or_else(|| Trap::wasm(ir::TrapCode::TableOutOfBounds))?;
 
-        if end > self.size() {
+        if end > self.size() as usize {
             return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds));
         }
 
-        match val {
+        debug_assert!(self.type_matches(&val));
-            TableElement::FuncRef(r) => unsafe {
-                self.with_funcrefs_mut(move |elements| {
-                    let elements = elements.unwrap();
-                    elements[start as usize..end as usize].fill(r);
-                });
-            },
-            TableElement::ExternRef(r) => unsafe {
-                self.with_externrefs_mut(move |elements| {
-                    let elements = elements.unwrap();
-                    elements[start as usize..end as usize].fill(r);
-                });
-            },
-        }
 
-        Ok(())
+        self.with_elements_mut(|elements| {
+            if let Some((last, elements)) = elements[start..end].split_last_mut() {
+                let ty = self.element_type();
+
+                for e in elements {
+                    Self::set_raw(ty, e, val.clone());
+                }
+
+                Self::set_raw(self.element_type(), last, val);
+            }
+
+            Ok(())
+        })
     }
 
     /// Grow table by the specified amount of elements.
@@ -223,41 +226,34 @@ impl Table {
             }
         }
 
-        match &self.storage {
+        debug_assert!(self.type_matches(&init_value));
+
+        // First resize the storage and then fill with the init value
+        match &self.0 {
             TableStorage::Static { size, .. } => {
                 size.set(new_size);
-                self.fill(old_size, init_value, delta)
-                    .ok()
-                    .map(|_| old_size)
             }
             TableStorage::Dynamic { elements, .. } => {
-                let new_len = usize::try_from(new_size).unwrap();
+                let mut elements = elements.borrow_mut();
-
+                elements.resize(new_size as usize, ptr::null_mut());
-                match &mut *elements.borrow_mut() {
-                    TableElements::FuncRefs(x) => x.resize(new_len, init_value.try_into().ok()?),
-                    TableElements::ExternRefs(x) => x.resize(new_len, init_value.try_into().ok()?),
-                }
-
-                Some(old_size)
             }
         }
+
+        self.fill(old_size, init_value, delta)
+            .expect("table should not be out of bounds");
+
+        Some(old_size)
     }
 
     /// Get reference to the specified element.
     ///
     /// Returns `None` if the index is out of bounds.
     pub fn get(&self, index: u32) -> Option<TableElement> {
-        unsafe {
+        self.with_elements(|elements| {
-            match self.element_type() {
+            elements
-                TableElementType::Func => self.with_funcrefs(|elements| {
+                .get(index as usize)
-                    elements.and_then(|e| e.get(index as usize).cloned().map(TableElement::FuncRef))
+                .map(|p| unsafe { TableElement::clone_from_raw(self.element_type(), *p) })
-                }),
+        })
-                TableElementType::Val(_) => self.with_externrefs(|elements| {
-                    elements
-                        .and_then(|e| e.get(index as usize).cloned().map(TableElement::ExternRef))
-                }),
-            }
-        }
     }
 
     /// Set reference to the specified element.
@@ -267,22 +263,15 @@ impl Table {
     /// Returns an error if `index` is out of bounds or if this table type does
     /// not match the element type.
     pub fn set(&self, index: u32, elem: TableElement) -> Result<(), ()> {
-        unsafe {
+        if !self.type_matches(&elem) {
-            match self.element_type() {
+            return Err(());
-                TableElementType::Func => self.with_funcrefs_mut(move |elements| {
-                    let elements = elements.ok_or(())?;
-                    let e = elements.get_mut(index as usize).ok_or(())?;
-                    *e = elem.try_into()?;
-                    Ok(())
-                }),
-                TableElementType::Val(_) => self.with_externrefs_mut(move |elements| {
-                    let elements = elements.ok_or(())?;
-                    let e = elements.get_mut(index as usize).ok_or(())?;
-                    *e = elem.try_into()?;
-                    Ok(())
-                }),
-            }
         }
+
+        self.with_elements_mut(|elements| {
+            let e = elements.get_mut(index as usize).ok_or(())?;
+            Self::set_raw(self.element_type(), e, elem);
+            Ok(())
+        })
     }
 
     /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`.
@@ -310,49 +299,19 @@ impl Table {
             return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds));
         }
 
-        // Check if the source and destination are the same table
+        debug_assert!(
-        // This ensures we don't `borrow` and `borrow_mut` the same underlying RefCell
+            dst_table.element_type() == src_table.element_type(),
-        let same_table = ptr::eq(dst_table, src_table);
+            "table element type mismatch"
+        );
 
         let src_range = src_index as usize..src_index as usize + len as usize;
         let dst_range = dst_index as usize..dst_index as usize + len as usize;
 
-        unsafe {
+        // Check if the tables are the same as we cannot mutably borrow and also borrow the same `RefCell`
-            match dst_table.element_type() {
+        if ptr::eq(dst_table, src_table) {
-                TableElementType::Func => dst_table.with_funcrefs_mut(|dst| {
+            Self::copy_elements_within(dst_table, dst_range, src_range);
-                    let dst = dst.unwrap();
+        } else {
-
+            Self::copy_elements(dst_table, src_table, dst_range, src_range);
-                    if same_table {
-                        dst.copy_within(src_range, dst_index as usize);
-                    } else {
-                        src_table.with_funcrefs(|src| {
-                            let src = src.unwrap();
-                            dst[dst_range].copy_from_slice(&src[src_range]);
-                        })
-                    }
-                }),
-                TableElementType::Val(_) => dst_table.with_externrefs_mut(|dst| {
-                    let dst = dst.unwrap();
-
-                    if same_table {
-                        // As there's no `slice::clone_within` because cloning can't be done with memmove, use a loop
-                        if dst_index <= src_index {
-                            for (s, d) in (src_range).zip(dst_range) {
-                                dst[d] = dst[s].clone();
-                            }
-                        } else {
-                            for (s, d) in src_range.rev().zip(dst_range.rev()) {
-                                dst[d] = dst[s].clone();
-                            }
-                        }
-                    } else {
-                        src_table.with_externrefs(|src| {
-                            let src = src.unwrap();
-                            dst[dst_range].clone_from_slice(&src[src_range]);
-                        })
-                    }
-                }),
-            }
         }
 
         Ok(())
@@ -360,97 +319,153 @@ impl Table {
 
     /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
     pub fn vmtable(&self) -> VMTableDefinition {
-        match &self.storage {
+        match &self.0 {
             TableStorage::Static { data, size, .. } => VMTableDefinition {
-                base: *data,
+                base: *data as _,
                 current_elements: size.get(),
             },
-            TableStorage::Dynamic { elements, .. } => match &*elements.borrow() {
+            TableStorage::Dynamic { elements, .. } => {
-                TableElements::FuncRefs(x) => VMTableDefinition {
+                let elements = elements.borrow();
-                    base: x.as_ptr() as *const u8 as _,
+                VMTableDefinition {
-                    current_elements: x.len().try_into().unwrap(),
+                    base: elements.as_ptr() as _,
-                },
+                    current_elements: elements.len().try_into().unwrap(),
-                TableElements::ExternRefs(x) => VMTableDefinition {
+                }
-                    base: x.as_ptr() as *const u8 as _,
+            }
-                    current_elements: x.len().try_into().unwrap(),
-                },
-            },
         }
     }
 
-    unsafe fn with_funcrefs<F, R>(&self, with: F) -> R
+    fn type_matches(&self, val: &TableElement) -> bool {
-    where
+        match (&val, self.element_type()) {
-        F: FnOnce(Option<&[*mut VMCallerCheckedAnyfunc]>) -> R,
+            (TableElement::FuncRef(_), TableElementType::Func) => true,
-    {
+            (TableElement::ExternRef(_), TableElementType::Val(_)) => true,
-        match &self.storage {
+            _ => false,
-            TableStorage::Static { data, size, ty, .. } => match ty {
-                TableElementType::Func => with(Some(std::slice::from_raw_parts(
-                    *data as *const _,
-                    size.get() as usize,
-                ))),
-                _ => with(None),
-            },
-            TableStorage::Dynamic { elements, .. } => match &*elements.borrow() {
-                TableElements::FuncRefs(x) => with(Some(x.as_slice())),
-                _ => with(None),
-            },
         }
     }
 
-    unsafe fn with_funcrefs_mut<F, R>(&self, with: F) -> R
+    fn with_elements<F, R>(&self, f: F) -> R
     where
-        F: FnOnce(Option<&mut [*mut VMCallerCheckedAnyfunc]>) -> R,
+        F: FnOnce(&[*mut u8]) -> R,
     {
-        match &self.storage {
+        match &self.0 {
-            TableStorage::Static { data, size, ty, .. } => match ty {
+            TableStorage::Static { data, size, .. } => unsafe {
-                TableElementType::Func => with(Some(std::slice::from_raw_parts_mut(
+                f(std::slice::from_raw_parts(*data, size.get() as usize))
-                    *data as *mut _,
-                    size.get() as usize,
-                ))),
-                _ => with(None),
-            },
-            TableStorage::Dynamic { elements, .. } => match &mut *elements.borrow_mut() {
-                TableElements::FuncRefs(x) => with(Some(x.as_mut_slice())),
-                _ => with(None),
             },
+            TableStorage::Dynamic { elements, .. } => {
+                let elements = elements.borrow();
+                f(elements.as_slice())
+            }
         }
     }
 
-    unsafe fn with_externrefs<F, R>(&self, with: F) -> R
+    fn with_elements_mut<F, R>(&self, f: F) -> R
     where
-        F: FnOnce(Option<&[Option<VMExternRef>]>) -> R,
+        F: FnOnce(&mut [*mut u8]) -> R,
     {
-        match &self.storage {
+        match &self.0 {
-            TableStorage::Static { data, size, ty, .. } => match ty {
+            TableStorage::Static { data, size, .. } => unsafe {
-                TableElementType::Val(_) => with(Some(std::slice::from_raw_parts(
+                f(std::slice::from_raw_parts_mut(*data, size.get() as usize))
-                    *data as *const _,
-                    size.get() as usize,
-                ))),
-                _ => with(None),
-            },
-            TableStorage::Dynamic { elements, .. } => match &*elements.borrow() {
-                TableElements::ExternRefs(x) => with(Some(x.as_slice())),
-                _ => with(None),
             },
+            TableStorage::Dynamic { elements, .. } => {
+                let mut elements = elements.borrow_mut();
+                f(elements.as_mut_slice())
+            }
         }
     }
 
-    unsafe fn with_externrefs_mut<F, R>(&self, with: F) -> R
+    fn set_raw(ty: TableElementType, e: &mut *mut u8, val: TableElement) {
-    where
+        unsafe {
-        F: FnOnce(Option<&mut [Option<VMExternRef>]>) -> R,
+            // Drop the existing element
-    {
+            let _ = TableElement::from_raw(ty, *e);
-        match &self.storage {
+            *e = val.into_raw();
-            TableStorage::Static { data, size, ty, .. } => match ty {
+        }
-                TableElementType::Val(_) => with(Some(std::slice::from_raw_parts_mut(
+    }
-                    *data as *mut _,
+
-                    size.get() as usize,
+    fn copy_elements(
-                ))),
+        dst_table: &Self,
-                _ => with(None),
+        src_table: &Self,
-            },
+        dst_range: Range<usize>,
-            TableStorage::Dynamic { elements, .. } => match &mut *elements.borrow_mut() {
+        src_range: Range<usize>,
-                TableElements::ExternRefs(x) => with(Some(x.as_mut_slice())),
+    ) {
-                _ => with(None),
+        // This can only be used when copying between different tables
-            },
+        debug_assert!(!ptr::eq(dst_table, src_table));
+
+        let ty = dst_table.element_type();
+
+        match ty {
+            TableElementType::Func => {
+                // `funcref` are `Copy`, so just do a mempcy
+                dst_table.with_elements_mut(|dst| {
+                    src_table.with_elements(|src| dst[dst_range].copy_from_slice(&src[src_range]))
+                });
+            }
+            TableElementType::Val(_) => {
+                // We need to clone each `externref`
+                dst_table.with_elements_mut(|dst| {
+                    src_table.with_elements(|src| {
+                        for (s, d) in src_range.zip(dst_range) {
+                            let elem = unsafe { TableElement::clone_from_raw(ty, src[s]) };
+                            Self::set_raw(ty, &mut dst[d], elem);
+                        }
+                    })
+                });
+            }
+        }
+    }
+
+    fn copy_elements_within(table: &Self, dst_range: Range<usize>, src_range: Range<usize>) {
+        let ty = table.element_type();
+
+        match ty {
+            TableElementType::Func => {
+                // `funcref` are `Copy`, so just do a memmove
+                table.with_elements_mut(|dst| dst.copy_within(src_range, dst_range.start));
+            }
+            TableElementType::Val(_) => {
+                // We need to clone each `externref` while handling overlapping ranges
+                table.with_elements_mut(|dst| {
+                    if dst_range.start <= src_range.start {
+                        for (s, d) in src_range.zip(dst_range) {
+                            let elem = unsafe { TableElement::clone_from_raw(ty, dst[s]) };
+                            Self::set_raw(ty, &mut dst[d], elem);
+                        }
+                    } else {
+                        for (s, d) in src_range.rev().zip(dst_range.rev()) {
+                            let elem = unsafe { TableElement::clone_from_raw(ty, dst[s]) };
+                            Self::set_raw(ty, &mut dst[d], elem);
+                        }
+                    }
+                });
+            }
         }
     }
 }
+
+impl Drop for Table {
+    fn drop(&mut self) {
+        let ty = self.element_type();
+
+        // funcref tables can skip this
+        if let TableElementType::Func = ty {
+            return;
+        }
+
+        // Properly drop any table elements stored in the table
+        self.with_elements(|elements| {
+            for element in elements.iter() {
+                let _ = unsafe { TableElement::from_raw(ty, *element) };
+            }
+        });
+    }
+}
+
+// The default table representation is an empty funcref table that cannot grow.
+impl Default for Table {
+    fn default() -> Self {
+        Self(TableStorage::Static {
+            data: std::ptr::null_mut(),
+            size: Cell::new(0),
+            ty: TableElementType::Func,
+            maximum: 0,
+        })
+    }
+}