externref: implement stack map-based garbage collection

For host VM code, we use plain reference counting, where cloning increments
the reference count, and dropping decrements it. We can avoid many of the
on-stack increment/decrement operations that typically plague the
performance of reference counting via Rust's ownership and borrowing system.
Moving a `VMExternRef` avoids mutating its reference count, and borrowing it
either avoids the reference count increment or delays it until if/when the
`VMExternRef` is cloned.

When passing a `VMExternRef` into compiled Wasm code, we don't want to do
reference count mutations for every compiled `local.{get,set}`, nor for
every function call. Therefore, we use a variation of **deferred reference
counting**, where we only mutate reference counts when storing
`VMExternRef`s somewhere that outlives the activation: into a global or
table. Simultaneously, we over-approximate the set of `VMExternRef`s that
are inside Wasm function activations. Periodically, we walk the stack at GC
safe points, and use stack map information to precisely identify the set of
`VMExternRef`s inside Wasm activations. Then we take the difference between
this precise set and our over-approximation, and decrement the reference
count for each of the `VMExternRef`s that are in our over-approximation but
not in the precise set. Finally, the over-approximation is replaced with the
precise set.

The `VMExternRefActivationsTable` implements the over-approximized set of
`VMExternRef`s referenced by Wasm activations. Calling a Wasm function and
passing it a `VMExternRef` moves the `VMExternRef` into the table, and the
compiled Wasm function logically "borrows" the `VMExternRef` from the
table. Similarly, `global.get` and `table.get` operations clone the gotten
`VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the
reference out of the table.

When a `VMExternRef` is returned to host code from a Wasm function, the host
increments the reference count (because the reference is logically
"borrowed" from the `VMExternRefActivationsTable` and the reference count
from the table will be dropped at the next GC).

For more general information on deferred reference counting, see *An
Examination of Deferred Reference Counting and Cycle Detection* by Quinane:
https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf

cc #929

Fixes #1804

crates/environ/src/cache.rs

@@ -1,5 +1,5 @@
 use crate::address_map::{ModuleAddressMap, ValueLabelsRanges};
-use crate::compilation::{Compilation, Relocations, Traps};
+use crate::compilation::{Compilation, Relocations, StackMaps, Traps};
 use cranelift_codegen::ir;
 use cranelift_entity::PrimaryMap;
 use cranelift_wasm::DefinedFuncIndex;
@@ -35,6 +35,7 @@ pub struct ModuleCacheData {
     value_ranges: ValueLabelsRanges,
     stack_slots: PrimaryMap<DefinedFuncIndex, ir::StackSlots>,
     traps: Traps,
+    stack_maps: StackMaps,
 }
 
 /// A type alias over the module cache data as a tuple.
@@ -45,6 +46,7 @@ pub type ModuleCacheDataTupleType = (
     ValueLabelsRanges,
     PrimaryMap<DefinedFuncIndex, ir::StackSlots>,
     Traps,
+    StackMaps,
 );
 
 struct Sha256Hasher(Sha256);
@@ -204,6 +206,7 @@ impl ModuleCacheData {
             value_ranges: data.3,
             stack_slots: data.4,
             traps: data.5,
+            stack_maps: data.6,
         }
     }
 
@@ -215,6 +218,7 @@ impl ModuleCacheData {
             self.value_ranges,
             self.stack_slots,
             self.traps,
+            self.stack_maps,
         )
     }
 }

crates/environ/src/compilation.rs

@@ -144,6 +144,22 @@ pub struct TrapInformation {
 /// Information about traps associated with the functions where the traps are placed.
 pub type Traps = PrimaryMap<DefinedFuncIndex, Vec<TrapInformation>>;
 
+/// The offset within a function of a GC safepoint, and its associated stack
+/// map.
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct StackMapInformation {
+    /// The offset of the GC safepoint within the function's native code. It is
+    /// relative to the beginning of the function.
+    pub code_offset: binemit::CodeOffset,
+
+    /// The stack map for identifying live GC refs at the GC safepoint.
+    pub stack_map: binemit::Stackmap,
+}
+
+/// Information about GC safepoints and their associated stack maps within each
+/// function.
+pub type StackMaps = PrimaryMap<DefinedFuncIndex, Vec<StackMapInformation>>;
+
 /// An error while compiling WebAssembly to machine code.
 #[derive(Error, Debug)]
 pub enum CompileError {

crates/environ/src/cranelift.rs

@@ -88,7 +88,8 @@
 use crate::address_map::{FunctionAddressMap, InstructionAddressMap};
 use crate::cache::{ModuleCacheDataTupleType, ModuleCacheEntry};
 use crate::compilation::{
-    Compilation, CompileError, CompiledFunction, Relocation, RelocationTarget, TrapInformation,
+    Compilation, CompileError, CompiledFunction, Relocation, RelocationTarget, StackMapInformation,
+    TrapInformation,
 };
 use crate::func_environ::{get_func_name, FuncEnvironment};
 use crate::{CacheConfig, FunctionBodyData, ModuleLocal, ModuleTranslation, Tunables};
@@ -204,6 +205,27 @@ impl binemit::TrapSink for TrapSink {
     }
 }
 
+#[derive(Default)]
+struct StackMapSink {
+    infos: Vec<StackMapInformation>,
+}
+
+impl binemit::StackmapSink for StackMapSink {
+    fn add_stackmap(&mut self, code_offset: binemit::CodeOffset, stack_map: binemit::Stackmap) {
+        self.infos.push(StackMapInformation {
+            code_offset,
+            stack_map,
+        });
+    }
+}
+
+impl StackMapSink {
+    fn finish(mut self) -> Vec<StackMapInformation> {
+        self.infos.sort_by_key(|info| info.code_offset);
+        self.infos
+    }
+}
+
 fn get_function_address_map<'data>(
     context: &Context,
     data: &FunctionBodyData<'data>,
@@ -294,6 +316,7 @@ fn compile(env: CompileEnv<'_>) -> Result<ModuleCacheDataTupleType, CompileError
     let mut value_ranges = PrimaryMap::with_capacity(env.function_body_inputs.len());
     let mut stack_slots = PrimaryMap::with_capacity(env.function_body_inputs.len());
     let mut traps = PrimaryMap::with_capacity(env.function_body_inputs.len());
+    let mut stack_maps = PrimaryMap::with_capacity(env.function_body_inputs.len());
 
     env.function_body_inputs
         .into_iter()
@@ -354,14 +377,14 @@ fn compile(env: CompileEnv<'_>) -> Result<ModuleCacheDataTupleType, CompileError
             let mut code_buf: Vec<u8> = Vec::new();
             let mut reloc_sink = RelocSink::new(func_index);
             let mut trap_sink = TrapSink::new();
-            let mut stackmap_sink = binemit::NullStackmapSink {};
+            let mut stack_map_sink = StackMapSink::default();
             context
                 .compile_and_emit(
                     isa,
                     &mut code_buf,
                     &mut reloc_sink,
                     &mut trap_sink,
-                    &mut stackmap_sink,
+                    &mut stack_map_sink,
                 )
                 .map_err(|error| {
                     CompileError::Codegen(pretty_error(&context.func, Some(isa), error))
@@ -391,6 +414,7 @@ fn compile(env: CompileEnv<'_>) -> Result<ModuleCacheDataTupleType, CompileError
                 context.func.stack_slots,
                 trap_sink.traps,
                 unwind_info,
+                stack_map_sink.finish(),
             ))
         })
         .collect::<Result<Vec<_>, CompileError>>()?
@@ -405,6 +429,7 @@ fn compile(env: CompileEnv<'_>) -> Result<ModuleCacheDataTupleType, CompileError
                 sss,
                 function_traps,
                 unwind_info,
+                stack_map,
             )| {
                 functions.push(CompiledFunction {
                     body: function,
@@ -416,6 +441,7 @@ fn compile(env: CompileEnv<'_>) -> Result<ModuleCacheDataTupleType, CompileError
                 value_ranges.push(ranges.unwrap_or_default());
                 stack_slots.push(sss);
                 traps.push(function_traps);
+                stack_maps.push(stack_map);
             },
         );
 
@@ -428,6 +454,7 @@ fn compile(env: CompileEnv<'_>) -> Result<ModuleCacheDataTupleType, CompileError
         value_ranges,
         stack_slots,
         traps,
+        stack_maps,
     ))
 }
 

crates/environ/src/data_structures.rs

@@ -1,6 +1,7 @@
 #![doc(hidden)]
 
 pub mod ir {
+    pub use cranelift_codegen::binemit::Stackmap;
     pub use cranelift_codegen::ir::{
         types, AbiParam, ArgumentPurpose, Signature, SourceLoc, StackSlots, TrapCode, Type,
         ValueLabel, ValueLoc,

crates/environ/src/func_environ.rs

@@ -658,13 +658,6 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm
     fn target_config(&self) -> TargetFrontendConfig {
         self.target_config
     }
-
-    fn reference_type(&self) -> ir::Type {
-        // For now, the only reference types we support are `externref`, which
-        // don't require tracing GC and stack maps. So we just use the target's
-        // pointer type. This will have to change once we move to tracing GC.
-        self.pointer_type()
-    }
 }
 
 impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'module_environment> {

crates/environ/src/lib.rs

@@ -47,7 +47,7 @@ pub use crate::cache::create_new_config as cache_create_new_config;
 pub use crate::cache::CacheConfig;
 pub use crate::compilation::{
     Compilation, CompileError, CompiledFunction, Compiler, Relocation, RelocationTarget,
-    Relocations, TrapInformation, Traps,
+    Relocations, StackMapInformation, StackMaps, TrapInformation, Traps,
 };
 pub use crate::cranelift::Cranelift;
 pub use crate::data_structures::*;

crates/environ/src/lightbeam.rs

@@ -34,6 +34,7 @@ impl crate::compilation::Compiler for Lightbeam {
         );
         let mut relocations = PrimaryMap::with_capacity(translation.function_body_inputs.len());
         let mut traps = PrimaryMap::with_capacity(translation.function_body_inputs.len());
+        let stack_maps = PrimaryMap::with_capacity(translation.function_body_inputs.len());
 
         let mut codegen_session: CodeGenSession<_> = CodeGenSession::new(
             translation.function_body_inputs.len() as u32,
@@ -81,6 +82,7 @@ impl crate::compilation::Compiler for Lightbeam {
             ValueLabelsRanges::new(),
             PrimaryMap::new(),
             traps,
+            stack_maps,
         ))
     }
 }

crates/environ/src/module_environ.rs

@@ -92,13 +92,6 @@ impl<'data> TargetEnvironment for ModuleEnvironment<'data> {
     fn target_config(&self) -> TargetFrontendConfig {
         self.result.target_config
     }
-
-    fn reference_type(&self) -> ir::Type {
-        // For now, the only reference types we support are `externref`, which
-        // don't require tracing GC and stack maps. So we just use the target's
-        // pointer type. This will have to change once we move to tracing GC.
-        self.pointer_type()
-    }
 }
 
 /// This trait is useful for `translate_module` because it tells how to translate

crates/environ/src/vmoffsets.rs

@@ -5,6 +5,8 @@
 //
 // struct VMContext {
 //      interrupts: *const VMInterrupts,
+//      externref_activations_table: *mut VMExternRefActivationsTable,
+//      stack_map_registry: *mut StackMapRegistry,
 //      signature_ids: [VMSharedSignatureIndex; module.num_signature_ids],
 //      imported_functions: [VMFunctionImport; module.num_imported_functions],
 //      imported_tables: [VMTableImport; module.num_imported_tables],
@@ -286,9 +288,23 @@ impl VMOffsets {
         0
     }
 
+    /// The offset of the `VMExternRefActivationsTable` member.
+    pub fn vmctx_externref_activations_table(&self) -> u32 {
+        self.vmctx_interrupts()
+            .checked_add(u32::from(self.pointer_size))
+            .unwrap()
+    }
+
+    /// The offset of the `*mut StackMapRegistry` member.
+    pub fn vmctx_stack_map_registry(&self) -> u32 {
+        self.vmctx_externref_activations_table()
+            .checked_add(u32::from(self.pointer_size))
+            .unwrap()
+    }
+
     /// The offset of the `signature_ids` array.
     pub fn vmctx_signature_ids_begin(&self) -> u32 {
-        self.vmctx_interrupts()
+        self.vmctx_stack_map_registry()
             .checked_add(u32::from(self.pointer_size))
             .unwrap()
     }
@@ -591,6 +607,19 @@ impl VMOffsets {
     }
 }
 
+/// Offsets for `VMExternRefActivationsTable`.
+impl VMOffsets {
+    /// Return the offset for `VMExternRefActivationsTable::next`.
+    pub fn vm_extern_ref_activation_table_next(&self) -> u32 {
+        0
+    }
+
+    /// Return the offset for `VMExternRefActivationsTable::end`.
+    pub fn vm_extern_ref_activation_table_end(&self) -> u32 {
+        self.pointer_size.into()
+    }
+}
+
 /// Target specific type for shared signature index.
 #[derive(Debug, Copy, Clone)]
 pub struct TargetSharedSignatureIndex(u32);

crates/jit/src/code_memory.rs

@@ -134,7 +134,7 @@ impl CodeMemory {
 
             if !m.is_empty() {
                 unsafe {
-                    region::protect(m.as_mut_ptr(), m.len(), region::Protection::READ_EXECUTE)
+                    region::protect(m.as_mut_ptr(), m.len(), region::Protection::ReadExecute)
                 }
                 .expect("unable to make memory readonly and executable");
             }

crates/jit/src/compiler.rs

@@ -15,7 +15,7 @@ use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex,
 use wasmtime_environ::{
     CacheConfig, CompileError, CompiledFunction, Compiler as _C, Module, ModuleAddressMap,
     ModuleMemoryOffset, ModuleTranslation, ModuleVmctxInfo, Relocation, RelocationTarget,
-    Relocations, Traps, Tunables, VMOffsets, ValueLabelsRanges,
+    Relocations, StackMaps, Traps, Tunables, VMOffsets, ValueLabelsRanges,
 };
 use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline};
 
@@ -138,6 +138,7 @@ pub struct Compilation {
     pub jt_offsets: PrimaryMap<DefinedFuncIndex, ir::JumpTableOffsets>,
     pub dwarf_sections: Vec<DwarfSection>,
     pub traps: Traps,
+    pub stack_maps: StackMaps,
     pub address_transform: ModuleAddressMap,
 }
 
@@ -165,27 +166,34 @@ impl Compiler {
     ) -> Result<Compilation, SetupError> {
         let mut code_memory = CodeMemory::new();
 
-        let (compilation, relocations, address_transform, value_ranges, stack_slots, traps) =
-            match self.strategy {
-                // For now, interpret `Auto` as `Cranelift` since that's the most stable
-                // implementation.
-                CompilationStrategy::Auto | CompilationStrategy::Cranelift => {
-                    wasmtime_environ::cranelift::Cranelift::compile_module(
-                        translation,
-                        &*self.isa,
-                        &self.cache_config,
-                    )
-                }
-                #[cfg(feature = "lightbeam")]
-                CompilationStrategy::Lightbeam => {
-                    wasmtime_environ::lightbeam::Lightbeam::compile_module(
-                        translation,
-                        &*self.isa,
-                        &self.cache_config,
-                    )
-                }
+        let (
+            compilation,
+            relocations,
+            address_transform,
+            value_ranges,
+            stack_slots,
+            traps,
+            stack_maps,
+        ) = match self.strategy {
+            // For now, interpret `Auto` as `Cranelift` since that's the most stable
+            // implementation.
+            CompilationStrategy::Auto | CompilationStrategy::Cranelift => {
+                wasmtime_environ::cranelift::Cranelift::compile_module(
+                    translation,
+                    &*self.isa,
+                    &self.cache_config,
+                )
             }
-            .map_err(SetupError::Compile)?;
+            #[cfg(feature = "lightbeam")]
+            CompilationStrategy::Lightbeam => {
+                wasmtime_environ::lightbeam::Lightbeam::compile_module(
+                    translation,
+                    &*self.isa,
+                    &self.cache_config,
+                )
+            }
+        }
+        .map_err(SetupError::Compile)?;
 
         let dwarf_sections = if debug_data.is_some() && !compilation.is_empty() {
             transform_dwarf_data(
@@ -239,6 +247,7 @@ impl Compiler {
             jt_offsets,
             dwarf_sections,
             traps,
+            stack_maps,
             address_transform,
         })
     }

crates/jit/src/instantiate.rs

@@ -10,6 +10,7 @@ use crate::link::link_module;
 use crate::resolver::Resolver;
 use std::any::Any;
 use std::collections::HashMap;
+use std::rc::Rc;
 use std::sync::Arc;
 use thiserror::Error;
 use wasmtime_debug::{read_debuginfo, write_debugsections_image, DwarfSection};
@@ -18,13 +19,13 @@ use wasmtime_environ::isa::TargetIsa;
 use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex};
 use wasmtime_environ::{
     CompileError, DataInitializer, DataInitializerLocation, Module, ModuleAddressMap,
-    ModuleEnvironment, ModuleTranslation, Traps,
+    ModuleEnvironment, ModuleTranslation, StackMaps, Traps,
 };
 use wasmtime_profiling::ProfilingAgent;
 use wasmtime_runtime::VMInterrupts;
 use wasmtime_runtime::{
     GdbJitImageRegistration, InstanceHandle, InstantiationError, RuntimeMemoryCreator,
-    SignatureRegistry, VMFunctionBody, VMTrampoline,
+    SignatureRegistry, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, VMTrampoline,
 };
 
 /// An error condition while setting up a wasm instance, be it validation,
@@ -69,6 +70,7 @@ pub struct CompiledModule {
     trampolines: PrimaryMap<SignatureIndex, VMTrampoline>,
     data_initializers: Box<[OwnedDataInitializer]>,
     traps: Traps,
+    stack_maps: StackMaps,
     address_transform: ModuleAddressMap,
 }
 
@@ -99,6 +101,7 @@ impl CompiledModule {
             jt_offsets,
             dwarf_sections,
             traps,
+            stack_maps,
             address_transform,
         } = compiler.compile(&translation, debug_data)?;
 
@@ -149,6 +152,7 @@ impl CompiledModule {
             trampolines,
             data_initializers,
             traps,
+            stack_maps,
             address_transform,
         })
     }
@@ -169,6 +173,8 @@ impl CompiledModule {
         mem_creator: Option<&dyn RuntimeMemoryCreator>,
         interrupts: Arc<VMInterrupts>,
         host_state: Box<dyn Any>,
+        externref_activations_table: Rc<VMExternRefActivationsTable>,
+        stack_map_registry: Arc<StackMapRegistry>,
     ) -> Result<InstanceHandle, InstantiationError> {
         // Compute indices into the shared signature table.
         let signatures = {
@@ -200,6 +206,8 @@ impl CompiledModule {
             signatures.into_boxed_slice(),
             host_state,
             interrupts,
+            externref_activations_table,
+            stack_map_registry,
         )
     }
 
@@ -229,11 +237,16 @@ impl CompiledModule {
         &self.finished_functions.0
     }
 
-    /// Returns the a map for all traps in this module.
+    /// Returns the map for all traps in this module.
     pub fn traps(&self) -> &Traps {
         &self.traps
     }
 
+    /// Returns the map for each of this module's stack maps.
+    pub fn stack_maps(&self) -> &StackMaps {
+        &self.stack_maps
+    }
+
     /// Returns a map of compiled addresses back to original bytecode offsets.
     pub fn address_transform(&self) -> &ModuleAddressMap {
         &self.address_transform

crates/runtime/Cargo.toml

@@ -15,6 +15,7 @@ edition = "2018"
 wasmtime-environ = { path = "../environ", version = "0.18.0" }
 region = "2.0.0"
 libc = { version = "0.2.70", default-features = false }
+log = "0.4.8"
 memoffset = "0.5.3"
 indexmap = "1.0.2"
 thiserror = "1.0.4"

crates/runtime/src/externref.rs

@@ -58,29 +58,59 @@
 //! need a ton of excess padding between the `VMExternData` and the value for
 //! values with large alignment.
 //!
-//! ## Reference Counting Protocol and Wasm Functions
+//! ## Reference Counting, Wasm Functions, and Garbage Collection
 //!
-//! Currently, `VMExternRef`s passed into compiled Wasm functions have move
-//! semantics: the host code gives up ownership and does not decrement the
-//! reference count. Similarly, `VMExternRef`s returned from compiled Wasm
-//! functions also have move semantics: host code takes ownership and the
-//! reference count is not incremented.
+//! For host VM code, we use plain reference counting, where cloning increments
+//! the reference count, and dropping decrements it. We can avoid many of the
+//! on-stack increment/decrement operations that typically plague the
+//! performance of reference counting via Rust's ownership and borrowing system.
+//! Moving a `VMExternRef` avoids mutating its reference count, and borrowing it
+//! either avoids the reference count increment or delays it until if/when the
+//! `VMExternRef` is cloned.
 //!
-//! This works well when a reference is passed into Wasm and then passed back
-//! out again. However, if a reference is passed into Wasm, but not passed back
-//! out, then the reference is leaked. This is only a temporary state, and
-//! follow up work will leverage stack maps to fix this issue. Follow
-//! https://github.com/bytecodealliance/wasmtime/issues/929 to keep an eye on
-//! this.
+//! When passing a `VMExternRef` into compiled Wasm code, we don't want to do
+//! reference count mutations for every compiled `local.{get,set}`, nor for
+//! every function call. Therefore, we use a variation of **deferred reference
+//! counting**, where we only mutate reference counts when storing
+//! `VMExternRef`s somewhere that outlives the activation: into a global or
+//! table. Simultaneously, we over-approximate the set of `VMExternRef`s that
+//! are inside Wasm function activations. Periodically, we walk the stack at GC
+//! safe points, and use stack map information to precisely identify the set of
+//! `VMExternRef`s inside Wasm activations. Then we take the difference between
+//! this precise set and our over-approximation, and decrement the reference
+//! count for each of the `VMExternRef`s that are in our over-approximation but
+//! not in the precise set. Finally, the over-approximation is replaced with the
+//! precise set.
+//!
+//! The `VMExternRefActivationsTable` implements the over-approximized set of
+//! `VMExternRef`s referenced by Wasm activations. Calling a Wasm function and
+//! passing it a `VMExternRef` moves the `VMExternRef` into the table, and the
+//! compiled Wasm function logically "borrows" the `VMExternRef` from the
+//! table. Similarly, `global.get` and `table.get` operations clone the gotten
+//! `VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the
+//! reference out of the table.
+//!
+//! When a `VMExternRef` is returned to host code from a Wasm function, the host
+//! increments the reference count (because the reference is logically
+//! "borrowed" from the `VMExternRefActivationsTable` and the reference count
+//! from the table will be dropped at the next GC).
+//!
+//! For more general information on deferred reference counting, see *An
+//! Examination of Deferred Reference Counting and Cycle Detection* by Quinane:
+//! https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf
 
 use std::alloc::Layout;
 use std::any::Any;
-use std::cell::UnsafeCell;
+use std::cell::{Cell, RefCell, UnsafeCell};
 use std::cmp::Ordering;
+use std::collections::BTreeMap;
+use std::collections::HashSet;
 use std::hash::Hasher;
 use std::mem;
 use std::ops::Deref;
 use std::ptr::{self, NonNull};
+use std::sync::{Arc, RwLock};
+use wasmtime_environ::{ir::Stackmap, StackMapInformation};
 
 /// An external reference to some opaque data.
 ///
@@ -307,35 +337,73 @@ impl VMExternRef {
         }
     }
 
+    // /// Turn this `VMExternRef` into a raw, untyped pointer.
+    // ///
+    // /// This forgets `self` and does *not* decrement the reference count on the
+    // /// pointed-to data.
+    // ///
+    // /// This `VMExternRef` may be recovered with `VMExternRef::from_raw`.
+    // pub fn into_raw(self) -> *mut u8 {
+    //     let ptr = self.as_raw();
+    //     mem::forget(self);
+    //     ptr
+    // }
+
+    // /// Recreate a `VMExternRef` from a pointer returned from a previous call to
+    // /// `VMExternRef::into_raw`.
+    // ///
+    // /// # Safety
+    // ///
+    // /// Wildly unsafe to use with anything other than the result of a previous
+    // /// `into_raw` call!
+    // ///
+    // /// This method does *not* increment the reference count on the pointed-to
+    // /// data, so `from_raw` must be called at most *once* on the result of a
+    // /// previous `into_raw` call. (Ideally, every `into_raw` is later followed
+    // /// by a `from_raw`, but it is technically memory safe to never call
+    // /// `from_raw` after `into_raw`: it will leak the pointed-to value, which is
+    // /// memory safe).
+    // pub unsafe fn from_raw(ptr: *mut u8) -> Self {
+    //     debug_assert!(!ptr.is_null());
+    //     VMExternRef(NonNull::new_unchecked(ptr).cast())
+    // }
+
     /// Turn this `VMExternRef` into a raw, untyped pointer.
     ///
-    /// This forgets `self` and does *not* decrement the reference count on the
-    /// pointed-to data.
+    /// Unlike `into_raw`, this does not consume and forget `self`. It is *not*
+    /// safe to use `from_raw` on pointers returned from this method; only use
+    /// `clone_from_raw`!
     ///
-    /// This `VMExternRef` may be recovered with `VMExternRef::from_raw`.
-    pub fn into_raw(self) -> *mut u8 {
+    ///  Nor does this method increment the reference count. You must ensure
+    ///  that `self` (or some other clone of `self`) stays alive until
+    ///  `clone_from_raw` is called.
+    pub fn as_raw(&self) -> *mut u8 {
         let ptr = self.0.cast::<u8>().as_ptr();
         mem::forget(self);
         ptr
     }
 
-    /// Create a `VMExternRef` from a pointer returned from a previous call to
-    /// `VMExternRef::into_raw`.
+    /// Recreate a `VMExternRef` from a pointer returned from a previous call to
+    /// `VMExternRef::as_raw`.
     ///
     /// # Safety
     ///
     /// Wildly unsafe to use with anything other than the result of a previous
-    /// `into_raw` call!
+    /// `as_raw` call!
     ///
-    /// This method does *not* increment the reference count on the pointed-to
-    /// data, so `from_raw` must be called at most *once* on the result of a
-    /// previous `into_raw` call. (Ideally, every `into_raw` is later followed
-    /// by a `from_raw`, but it is technically memory safe to never call
-    /// `from_raw` after `into_raw`: it will leak the pointed-to value, which is
-    /// memory safe).
-    pub unsafe fn from_raw(ptr: *mut u8) -> Self {
+    /// Additionally, it is your responsibility to ensure that this raw
+    /// `VMExternRef`'s reference count has not dropped to zero. Failure to do
+    /// so will result in use after free!
+    pub unsafe fn clone_from_raw(ptr: *mut u8) -> Self {
         debug_assert!(!ptr.is_null());
-        VMExternRef(NonNull::new_unchecked(ptr).cast())
+        let x = VMExternRef(NonNull::new_unchecked(ptr).cast());
+        x.extern_data().increment_ref_count();
+        x
+    }
+
+    /// Get the reference count for this `VMExternRef`.
+    pub fn get_reference_count(&self) -> usize {
+        self.extern_data().get_ref_count()
     }
 
     #[inline]
@@ -393,6 +461,556 @@ impl Deref for VMExternRef {
     }
 }
 
+type TableElem = UnsafeCell<Option<VMExternRef>>;
+
+/// A table that over-approximizes the set of `VMExternRef`s that any Wasm
+/// activation on this thread is currently using.
+///
+/// Under the covers, this is a simple bump allocator that allows duplicate
+/// entries. Deduplication happens at GC time.
+#[repr(C)]
+pub struct VMExternRefActivationsTable {
+    /// Bump-allocation finger within the current chunk.
+    ///
+    /// NB: this is an `UnsafeCell` because it is read from and written to by
+    /// compiled Wasm code.
+    next: UnsafeCell<NonNull<TableElem>>,
+
+    /// Pointer to just after the current chunk.
+    ///
+    /// This is *not* within the current chunk and therefore is not a valid
+    /// place to insert a reference!
+    ///
+    /// This is only updated from host code.
+    end: Cell<NonNull<TableElem>>,
+
+    /// The chunks within which we are bump allocating.
+    ///
+    /// This is only updated from host code.
+    chunks: RefCell<Vec<Box<[TableElem]>>>,
+
+    /// The precise set of on-stack, inside-Wasm GC roots that we discover via
+    /// walking the stack and interpreting stack maps.
+    ///
+    /// That is, this is the precise set that the bump allocation table is
+    /// over-approximating.
+    ///
+    /// This is *only* used inside the `gc` function, and is empty otherwise. It
+    /// is just part of this struct so that we can reuse the allocation, rather
+    /// than create a new hash set every GC.
+    precise_stack_roots: RefCell<HashSet<NonNull<VMExternData>>>,
+}
+
+impl VMExternRefActivationsTable {
+    const INITIAL_CHUNK_SIZE: usize = 4096 / mem::size_of::<usize>();
+
+    /// Create a new `VMExternRefActivationsTable`.
+    pub fn new() -> Self {
+        let chunk = Self::new_chunk(Self::INITIAL_CHUNK_SIZE);
+        let next = chunk.as_ptr() as *mut TableElem;
+        let end = unsafe { next.add(chunk.len()) };
+
+        VMExternRefActivationsTable {
+            next: UnsafeCell::new(NonNull::new(next).unwrap()),
+            end: Cell::new(NonNull::new(end).unwrap()),
+            chunks: RefCell::new(vec![chunk]),
+            precise_stack_roots: RefCell::new(HashSet::with_capacity(Self::INITIAL_CHUNK_SIZE)),
+        }
+    }
+
+    fn new_chunk(size: usize) -> Box<[UnsafeCell<Option<VMExternRef>>]> {
+        assert!(size >= Self::INITIAL_CHUNK_SIZE);
+        let mut chunk = Vec::with_capacity(size);
+        for _ in 0..size {
+            chunk.push(UnsafeCell::new(None));
+        }
+        chunk.into_boxed_slice()
+    }
+
+    /// Try and insert a `VMExternRef` into this table.
+    ///
+    /// This is a fast path that only succeeds when the current chunk has the
+    /// capacity for the requested insertion.
+    ///
+    /// If the insertion fails, then the `VMExternRef` is given back. Callers
+    /// may attempt a GC to free up space and try again, or may call
+    /// `insert_slow_path` to allocate a new bump chunk for this insertion.
+    #[inline]
+    pub fn try_insert(&self, externref: VMExternRef) -> Result<(), VMExternRef> {
+        unsafe {
+            let next = *self.next.get();
+            let end = self.end.get();
+            if next == end {
+                return Err(externref);
+            }
+
+            debug_assert!((*next.as_ref().get()).is_none());
+            ptr::write(next.as_ptr(), UnsafeCell::new(Some(externref)));
+            let next = NonNull::new_unchecked(next.as_ptr().add(1));
+            debug_assert!(next <= end);
+            *self.next.get() = next;
+            Ok(())
+        }
+    }
+
+    /// This is a slow path for inserting a reference into the table when the
+    /// current bump chunk is full.
+    ///
+    /// This method is infallible, and will allocate an additional bump chunk if
+    /// necessary.
+    #[inline(never)]
+    pub fn insert_slow_path(&self, externref: VMExternRef) {
+        let externref = match self.try_insert(externref) {
+            Ok(()) => return,
+            Err(x) => x,
+        };
+
+        {
+            let mut chunks = self.chunks.borrow_mut();
+
+            let new_size = chunks.last().unwrap().len() * 2;
+            let new_chunk = Self::new_chunk(new_size);
+
+            unsafe {
+                let next = new_chunk.as_ptr() as *mut TableElem;
+                debug_assert!(!next.is_null());
+                *self.next.get() = NonNull::new_unchecked(next);
+
+                let end = next.add(new_chunk.len());
+                debug_assert!(!end.is_null());
+                self.end.set(NonNull::new_unchecked(end));
+            }
+
+            chunks.push(new_chunk);
+        }
+
+        self.try_insert(externref)
+            .expect("insertion should always succeed after we allocate a new chunk");
+    }
+
+    fn num_filled_in_last_chunk(&self, chunks: &[Box<[TableElem]>]) -> usize {
+        let last_chunk = chunks.last().unwrap();
+        let next = unsafe { *self.next.get() };
+        let end = self.end.get();
+        let num_unused_in_last_chunk =
+            ((end.as_ptr() as usize) - (next.as_ptr() as usize)) / mem::size_of::<usize>();
+        last_chunk.len().saturating_sub(num_unused_in_last_chunk)
+    }
+
+    fn elements(&self, mut f: impl FnMut(&VMExternRef)) {
+        // Every chunk except the last one is full, so we can simply iterate
+        // over all of their elements.
+        let chunks = self.chunks.borrow();
+        for chunk in chunks.iter().take(chunks.len() - 1) {
+            for elem in chunk.iter() {
+                if let Some(elem) = unsafe { &*elem.get() } {
+                    f(elem);
+                }
+            }
+        }
+
+        // The last chunk is not all the way full, so we only iterate over its
+        // full parts.
+        let num_filled_in_last_chunk = self.num_filled_in_last_chunk(&chunks);
+        for elem in chunks.last().unwrap().iter().take(num_filled_in_last_chunk) {
+            if let Some(elem) = unsafe { &*elem.get() } {
+                f(elem);
+            }
+        }
+    }
+
+    fn insert_precise_stack_root(&self, root: NonNull<VMExternData>) {
+        let mut precise_stack_roots = self.precise_stack_roots.borrow_mut();
+        if precise_stack_roots.insert(root) {
+            // If this root was not already in the set, then we need to
+            // increment its reference count, so that it doesn't get freed in
+            // `reset` when we're overwriting old bump allocation table entries
+            // with new ones.
+            unsafe {
+                root.as_ref().increment_ref_count();
+            }
+        }
+    }
+
+    /// Refill the bump allocation table with our precise stack roots, and sweep
+    /// away everything else.
+    fn reset(&self) {
+        let mut chunks = self.chunks.borrow_mut();
+
+        let mut precise_roots = self.precise_stack_roots.borrow_mut();
+        if precise_roots.is_empty() {
+            // Get rid of all but our first bump chunk, and set our `next` and
+            // `end` bump allocation fingers into it.
+            unsafe {
+                let chunk = chunks.first().unwrap();
+
+                let next = chunk.as_ptr() as *mut TableElem;
+                debug_assert!(!next.is_null());
+                *self.next.get() = NonNull::new_unchecked(next);
+
+                let end = next.add(chunk.len());
+                debug_assert!(!end.is_null());
+                self.end.set(NonNull::new_unchecked(end));
+            }
+            chunks.truncate(1);
+        } else {
+            // Drain our precise stack roots into the bump allocation table.
+            //
+            // This overwrites old entries, which drops them and decrements their
+            // reference counts. Safety relies on the reference count increment in
+            // `insert_precise_stack_root` to avoid over-eagerly dropping references
+            // that are in `self.precise_stack_roots` but haven't been inserted into
+            // the bump allocation table yet.
+            let mut precise_roots = precise_roots.drain();
+            'outer: for (chunk_index, chunk) in chunks.iter().enumerate() {
+                for (slot_index, slot) in chunk.iter().enumerate() {
+                    if let Some(root) = precise_roots.next() {
+                        unsafe {
+                            // NB: there is no reference count increment here
+                            // because everything in `self.precise_stack_roots`
+                            // already had its reference count incremented for us,
+                            // and this is logically a move out from there, rather
+                            // than a clone.
+                            *slot.get() = Some(VMExternRef(root));
+                        }
+                    } else {
+                        // We've inserted all of our precise, on-stack roots back
+                        // into the bump allocation table. Update our `next` and
+                        // `end` bump pointer members for the new current chunk, and
+                        // free any excess chunks.
+                        let start = chunk.as_ptr() as *mut TableElem;
+                        unsafe {
+                            let next = start.add(slot_index + 1);
+                            debug_assert!(!next.is_null());
+                            *self.next.get() = NonNull::new_unchecked(next);
+
+                            let end = start.add(chunk.len());
+                            debug_assert!(!end.is_null());
+                            self.end.set(NonNull::new_unchecked(end));
+                        }
+                        chunks.truncate(chunk_index + 1);
+                        break 'outer;
+                    }
+                }
+            }
+
+            debug_assert!(
+                precise_roots.next().is_none(),
+                "should always have enough capacity in the bump allocations table \
+                 to hold all of our precise, on-stack roots"
+            );
+        }
+
+        // Finally, sweep away excess capacity within our new last/current
+        // chunk, so that old, no-longer-live roots get dropped.
+        let num_filled_in_last_chunk = self.num_filled_in_last_chunk(&chunks);
+        for slot in chunks.last().unwrap().iter().skip(num_filled_in_last_chunk) {
+            unsafe {
+                *slot.get() = None;
+            }
+        }
+    }
+}
+
+/// A registry of stack maps for currently active Wasm modules.
+#[derive(Default)]
+pub struct StackMapRegistry {
+    inner: RwLock<StackMapRegistryInner>,
+}
+
+#[derive(Default)]
+struct StackMapRegistryInner {
+    /// A map from the highest pc in a module, to its stack maps.
+    ///
+    /// For details, see the comment above `GlobalFrameInfo::ranges`.
+    ranges: BTreeMap<usize, ModuleStackMaps>,
+}
+
+#[derive(Debug)]
+struct ModuleStackMaps {
+    /// The range of PCs that this module covers. Different modules should
+    /// always have distinct ranges.
+    range: std::ops::Range<usize>,
+
+    /// A map from a PC in this module (that is a GC safepoint) to its
+    /// associated stack map.
+    pc_to_stack_map: Vec<(usize, Arc<Stackmap>)>,
+}
+
+impl StackMapRegistry {
+    /// Register the stack maps for a given module.
+    ///
+    /// The stack maps should be given as an iterator over a function's PC range
+    /// in memory (that is, where the JIT actually allocated and emitted the
+    /// function's code at), and the stack maps and code offsets within that
+    /// range for each of its GC safepoints.
+    ///
+    /// The return value is an RAII registration for the stack maps. The
+    /// registration should not be dropped until its associated module is
+    /// dropped. Dropping the registration will unregister its stack
+    /// maps.
+    ///
+    /// # Safety
+    ///
+    /// Dropping the returned registration before the module is dropped, or when
+    /// there are still active frames from the module on the stack, means we
+    /// will no longer be able to find GC roots for the module's frames anymore,
+    /// which could lead to freeing still-in-use objects and use-after-free!
+    pub unsafe fn register_stack_maps<'a>(
+        self: &Arc<Self>,
+        stack_maps: impl IntoIterator<Item = (std::ops::Range<usize>, &'a [StackMapInformation])>,
+    ) -> Option<StackMapRegistration> {
+        let mut min = usize::max_value();
+        let mut max = 0;
+        let mut pc_to_stack_map = vec![];
+
+        for (range, infos) in stack_maps {
+            let len = range.end - range.start;
+
+            min = std::cmp::min(min, range.start);
+            max = std::cmp::max(max, range.end);
+
+            for info in infos {
+                assert!((info.code_offset as usize) < len);
+                pc_to_stack_map.push((
+                    range.start + (info.code_offset as usize),
+                    Arc::new(info.stack_map.clone()),
+                ));
+            }
+        }
+
+        if pc_to_stack_map.is_empty() {
+            // Nothing to register.
+            return None;
+        }
+
+        let module_stack_maps = ModuleStackMaps {
+            range: min..max,
+            pc_to_stack_map,
+        };
+
+        let mut inner = self.inner.write().unwrap();
+
+        // Assert that this chunk of ranges doesn't collide with any other known
+        // chunks.
+        if let Some((_, prev)) = inner.ranges.range(max..).next() {
+            assert!(prev.range.start > max);
+        }
+        if let Some((prev_end, _)) = inner.ranges.range(..=min).next_back() {
+            assert!(*prev_end < min);
+        }
+
+        let old = inner.ranges.insert(max, module_stack_maps);
+        assert!(old.is_none());
+
+        Some(StackMapRegistration {
+            key: max,
+            registry: self.clone(),
+        })
+    }
+
+    /// Lookup the stack map for the given PC, if any.
+    pub fn lookup_stack_map(&self, pc: usize) -> Option<Arc<Stackmap>> {
+        let inner = self.inner.read().unwrap();
+        let stack_maps = inner.module_stack_maps(pc)?;
+
+        // Do a binary search to find the stack map for the given PC.
+        //
+        // Because GC safepoints are technically only associated with a single
+        // PC, we should ideally only care about `Ok(index)` values returned
+        // from the binary search. However, safepoints are inserted right before
+        // calls, and there are two things that can disturb the PC/offset
+        // associated with the safepoint versus the PC we actually use to query
+        // for the stack map:
+        //
+        // 1. The `backtrace` crate gives us the PC in a frame that will be
+        //    *returned to*, and where execution will continue from, rather than
+        //    the PC of the call we are currently at. So we would need to
+        //    disassemble one instruction backwards to query the actual PC for
+        //    the stack map.
+        //
+        //    TODO: One thing we *could* do to make this a little less error
+        //    prone, would be to assert/check that the nearest GC safepoint
+        //    found is within `max_encoded_size(any kind of call instruction)`
+        //    our queried PC for the target architecture.
+        //
+        // 2. Cranelift's stack maps only handle the stack, not
+        //    registers. However, some references that are arguments to a call
+        //    may need to be in registers. In these cases, what Cranelift will
+        //    do is:
+        //
+        //      a. spill all the live references,
+        //      b. insert a GC safepoint for those references,
+        //      c. reload the references into registers, and finally
+        //      d. make the call.
+        //
+        //    Step (c) adds drift between the GC safepoint and the location of
+        //    the call, which is where we actually walk the stack frame and
+        //    collect its live references.
+        //
+        //    Luckily, the spill stack slots for the live references are still
+        //    up to date, so we can still find all the on-stack roots.
+        //    Furthermore, we do not have a moving GC, so we don't need to worry
+        //    whether the following code will reuse the references in registers
+        //    (which would not have been updated to point to the moved objects)
+        //    or reload from the stack slots (which would have been updated to
+        //    point to the moved objects).
+        let index = match stack_maps
+            .pc_to_stack_map
+            .binary_search_by_key(&pc, |(pc, _stack_map)| *pc)
+        {
+            // Exact hit.
+            Ok(i) => i,
+
+            Err(n) => {
+                // `Err(0)` means that the associated stack map would have been
+                // the first element in the array if this pc had an associated
+                // stack map, but this pc does not have an associated stack
+                // map. That doesn't make sense since every call and trap inside
+                // Wasm is a GC safepoint and should have a stack map, and the
+                // only way to have Wasm frames under this native frame is if we
+                // are at a call or a trap.
+                debug_assert!(n != 0);
+
+                n - 1
+            }
+        };
+
+        let stack_map = stack_maps.pc_to_stack_map[index].1.clone();
+        Some(stack_map)
+    }
+}
+
+impl StackMapRegistryInner {
+    fn module_stack_maps(&self, pc: usize) -> Option<&ModuleStackMaps> {
+        let (end, stack_maps) = self.ranges.range(pc..).next()?;
+        if pc < stack_maps.range.start || *end < pc {
+            None
+        } else {
+            Some(stack_maps)
+        }
+    }
+}
+
+/// The registration for a module's stack maps.
+///
+/// Unsafe to drop earlier than its module is dropped. See
+/// `StackMapRegistry::register_stack_maps` for details.
+pub struct StackMapRegistration {
+    key: usize,
+    registry: Arc<StackMapRegistry>,
+}
+
+impl Drop for StackMapRegistration {
+    fn drop(&mut self) {
+        if let Ok(mut inner) = self.registry.inner.write() {
+            inner.ranges.remove(&self.key);
+        }
+    }
+}
+
+#[cfg(debug_assertions)]
+#[derive(Debug, Default)]
+struct DebugOnly<T> {
+    inner: T,
+}
+
+#[cfg(debug_assertions)]
+impl<T> std::ops::Deref for DebugOnly<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.inner
+    }
+}
+
+#[cfg(debug_assertions)]
+impl<T> std::ops::DerefMut for DebugOnly<T> {
+    fn deref_mut(&mut self) -> &mut T {
+        &mut self.inner
+    }
+}
+
+#[cfg(not(debug_assertions))]
+#[derive(Debug, Default)]
+struct DebugOnly<T> {
+    _phantom: PhantomData<T>,
+}
+
+#[cfg(not(debug_assertions))]
+impl<T> std::ops::Deref for DebugOnly<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        panic!("only deref `DebugOnly` inside `debug_assert!`s")
+    }
+}
+
+#[cfg(not(debug_assertions))]
+impl<T> std::ops::DerefMut for DebugOnly<T> {
+    fn deref_mut(&mut self) -> &mut T {
+        panic!("only deref `DebugOnly` inside `debug_assert!`s")
+    }
+}
+
+/// Perform garbage collection of `VMExternRef`s.
+pub fn gc(
+    stack_maps_registry: &StackMapRegistry,
+    externref_activations_table: &VMExternRefActivationsTable,
+) {
+    log::debug!("start GC");
+
+    debug_assert!({
+        // This set is only non-empty within this function. It is built up when
+        // walking the stack and interpreting stack maps, and then drained back
+        // into the activations table's bump-allocated space at the end.
+        let precise_stack_roots = externref_activations_table.precise_stack_roots.borrow();
+        precise_stack_roots.is_empty()
+    });
+
+    let mut activations_table_set: DebugOnly<HashSet<_>> = Default::default();
+    if cfg!(debug_assertions) {
+        externref_activations_table.elements(|elem| {
+            activations_table_set.insert(elem.as_raw() as *mut VMExternData);
+        });
+    }
+
+    backtrace::trace(|frame| {
+        let pc = frame.ip() as usize;
+
+        if let Some(stack_map) = stack_maps_registry.lookup_stack_map(pc) {
+            let ptr_to_frame = frame.sp() as usize;
+
+            for i in 0..(stack_map.mapped_words() as usize) {
+                if stack_map.get_bit(i) {
+                    // Stack maps have one bit per word in the frame, and the
+                    // zero^th bit is the *lowest* addressed word in the frame,
+                    // i.e. the closest to the SP. So to get the `i`^th word in
+                    // this frame, we add `i * sizeof(word)` to the
+                    // lowest-addressed word within this frame.
+                    let ptr_to_ref = ptr_to_frame + i * mem::size_of::<usize>();
+
+                    let r = unsafe { std::ptr::read(ptr_to_ref as *const *mut VMExternData) };
+                    debug_assert!(
+                        r.is_null() || activations_table_set.contains(&r),
+                        "every on-stack externref inside a Wasm frame should \
+                         have an entry in the VMExternRefActivationsTable"
+                    );
+                    if let Some(r) = NonNull::new(r) {
+                        externref_activations_table.insert_precise_stack_root(r);
+                    }
+                }
+            }
+        }
+
+        // Keep walking the stack.
+        true
+    });
+
+    externref_activations_table.reset();
+    log::debug!("end GC");
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -434,4 +1052,65 @@ mod tests {
             actual_offset.try_into().unwrap(),
         );
     }
+
+    #[test]
+    fn table_next_is_at_correct_offset() {
+        let table = VMExternRefActivationsTable::new();
+
+        let table_ptr = &table as *const _;
+        let next_ptr = &table.next as *const _;
+
+        let actual_offset = (next_ptr as usize) - (table_ptr as usize);
+
+        let offsets = wasmtime_environ::VMOffsets {
+            pointer_size: 8,
+            num_signature_ids: 0,
+            num_imported_functions: 0,
+            num_imported_tables: 0,
+            num_imported_memories: 0,
+            num_imported_globals: 0,
+            num_defined_tables: 0,
+            num_defined_memories: 0,
+            num_defined_globals: 0,
+        };
+        assert_eq!(offsets.vm_extern_ref_activation_table_next(), actual_offset);
+    }
+
+    #[test]
+    fn table_end_is_at_correct_offset() {
+        let table = VMExternRefActivationsTable::new();
+
+        let table_ptr = &table as *const _;
+        let end_ptr = &table.end as *const _;
+
+        let actual_offset = (end_ptr as usize) - (table_ptr as usize);
+
+        let offsets = wasmtime_environ::VMOffsets {
+            pointer_size: 8,
+            num_signature_ids: 0,
+            num_imported_functions: 0,
+            num_imported_tables: 0,
+            num_imported_memories: 0,
+            num_imported_globals: 0,
+            num_defined_tables: 0,
+            num_defined_memories: 0,
+            num_defined_globals: 0,
+        };
+        assert_eq!(offsets.vm_extern_ref_activation_table_end(), actual_offset);
+    }
+
+    fn assert_is_send<T: Send>() {}
+    fn assert_is_sync<T: Send>() {}
+
+    #[test]
+    fn stack_map_registry_is_send_sync() {
+        assert_is_send::<StackMapRegistry>();
+        assert_is_sync::<StackMapRegistry>();
+    }
+
+    #[test]
+    fn stack_map_registration_is_send_sync() {
+        assert_is_send::<StackMapRegistration>();
+        assert_is_sync::<StackMapRegistration>();
+    }
 }

crates/runtime/src/instance.rs

@@ -3,6 +3,7 @@
 //! `InstanceHandle` is a reference-counting handle for an `Instance`.
 
 use crate::export::Export;
+use crate::externref::{StackMapRegistry, VMExternRefActivationsTable};
 use crate::imports::Imports;
 use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory, RuntimeMemoryCreator};
 use crate::table::Table;
@@ -20,6 +21,7 @@ use std::any::Any;
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::convert::TryFrom;
+use std::rc::Rc;
 use std::sync::Arc;
 use std::{mem, ptr, slice};
 use thiserror::Error;
@@ -72,6 +74,19 @@ pub(crate) struct Instance {
     /// interrupted.
     pub(crate) interrupts: Arc<VMInterrupts>,
 
+    /// A handle to the (over-approximized) set of `externref`s that Wasm code
+    /// is using.
+    ///
+    /// The `vmctx` also holds a raw pointer to the table and relies on this
+    /// member to keep it alive.
+    pub(crate) externref_activations_table: Rc<VMExternRefActivationsTable>,
+
+    /// A handle to the stack map registry for this thread.
+    ///
+    /// The `vmctx` also holds a raw pointer to the registry and relies on this
+    /// member to keep it alive.
+    pub(crate) stack_map_registry: Arc<StackMapRegistry>,
+
     /// Additional context used by compiled wasm code. This field is last, and
     /// represents a dynamically-sized array that extends beyond the nominal
     /// end of the struct (similar to a flexible array member).
@@ -238,6 +253,16 @@ impl Instance {
         unsafe { self.vmctx_plus_offset(self.offsets.vmctx_interrupts()) }
     }
 
+    /// Return a pointer to the `VMExternRefActivationsTable`.
+    pub fn externref_activations_table(&self) -> *mut *mut VMExternRefActivationsTable {
+        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_externref_activations_table()) }
+    }
+
+    /// Return a pointer to the `StackMapRegistry`.
+    pub fn stack_map_registry(&self) -> *mut *mut StackMapRegistry {
+        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_stack_map_registry()) }
+    }
+
     /// Return a reference to the vmctx used by compiled wasm code.
     pub fn vmctx(&self) -> &VMContext {
         &self.vmctx
@@ -784,6 +809,8 @@ impl InstanceHandle {
         vmshared_signatures: BoxedSlice<SignatureIndex, VMSharedSignatureIndex>,
         host_state: Box<dyn Any>,
         interrupts: Arc<VMInterrupts>,
+        externref_activations_table: Rc<VMExternRefActivationsTable>,
+        stack_map_registry: Arc<StackMapRegistry>,
     ) -> Result<Self, InstantiationError> {
         let tables = create_tables(&module);
         let memories = create_memories(&module, mem_creator.unwrap_or(&DefaultMemoryCreator {}))?;
@@ -819,6 +846,8 @@ impl InstanceHandle {
                 trampolines,
                 host_state,
                 interrupts,
+                externref_activations_table,
+                stack_map_registry,
                 vmctx: VMContext {},
             };
             let layout = instance.alloc_layout();
@@ -878,6 +907,9 @@ impl InstanceHandle {
             VMBuiltinFunctionsArray::initialized(),
         );
         *instance.interrupts() = &*instance.interrupts;
+        *instance.externref_activations_table() =
+            &*instance.externref_activations_table as *const _ as *mut _;
+        *instance.stack_map_registry() = &*instance.stack_map_registry as *const _ as *mut _;
 
         // Perform infallible initialization in this constructor, while fallible
         // initialization is deferred to the `initialize` method.

crates/runtime/src/lib.rs

@@ -37,7 +37,7 @@ pub mod debug_builtins;
 pub mod libcalls;
 
 pub use crate::export::*;
-pub use crate::externref::VMExternRef;
+pub use crate::externref::*;
 pub use crate::imports::Imports;
 pub use crate::instance::{InstanceHandle, InstantiationError, LinkError};
 pub use crate::jit_int::GdbJitImageRegistration;

crates/runtime/src/mmap.rs

@@ -182,7 +182,7 @@ impl Mmap {
 
         // Commit the accessible size.
         let ptr = self.ptr as *const u8;
-        unsafe { region::protect(ptr.add(start), len, region::Protection::READ_WRITE) }
+        unsafe { region::protect(ptr.add(start), len, region::Protection::ReadWrite) }
             .map_err(|e| e.to_string())
     }
 

crates/wasmtime/src/func.rs

@@ -265,14 +265,14 @@ impl Func {
             // values produced are correct. There could be a bug in `func` that
             // produces the wrong number or wrong types of values, and we need
             // to catch that here.
-            for (i, (ret, ty)) in returns.iter_mut().zip(ty_clone.results()).enumerate() {
+            for (i, (ret, ty)) in returns.into_iter().zip(ty_clone.results()).enumerate() {
                 if ret.ty() != *ty {
                     return Err(Trap::new(
                         "function attempted to return an incompatible value",
                     ));
                 }
                 unsafe {
-                    ret.write_value_to(values_vec.add(i));
+                    ret.write_value_to(&store, values_vec.add(i));
                 }
             }
             Ok(())
@@ -535,7 +535,7 @@ impl Func {
 
         // Store the argument values into `values_vec`.
         let param_tys = my_ty.params().iter();
-        for ((arg, slot), ty) in params.iter().zip(&mut values_vec).zip(param_tys) {
+        for ((arg, slot), ty) in params.iter().cloned().zip(&mut values_vec).zip(param_tys) {
             if arg.ty() != *ty {
                 bail!(
                     "argument type mismatch: found {} but expected {}",
@@ -547,7 +547,7 @@ impl Func {
                 bail!("cross-`Store` values are not currently supported");
             }
             unsafe {
-                arg.write_value_to(slot);
+                arg.write_value_to(&self.instance.store, slot);
             }
         }
 

crates/wasmtime/src/instance.rs

@@ -3,9 +3,13 @@ use crate::{Engine, Export, Extern, Func, Global, Memory, Module, Store, Table,
 use anyhow::{bail, Error, Result};
 use std::any::Any;
 use std::mem;
+use std::rc::Rc;
+use std::sync::Arc;
 use wasmtime_environ::EntityIndex;
 use wasmtime_jit::{CompiledModule, Resolver};
-use wasmtime_runtime::{InstantiationError, VMContext, VMFunctionBody};
+use wasmtime_runtime::{
+    InstantiationError, StackMapRegistry, VMContext, VMExternRefActivationsTable, VMFunctionBody,
+};
 
 struct SimpleResolver<'a> {
     imports: &'a [Extern],
@@ -24,6 +28,8 @@ fn instantiate(
     compiled_module: &CompiledModule,
     imports: &[Extern],
     host: Box<dyn Any>,
+    externref_activations_table: Rc<VMExternRefActivationsTable>,
+    stack_map_registry: Arc<StackMapRegistry>,
 ) -> Result<StoreInstanceHandle, Error> {
     // For now we have a restriction that the `Store` that we're working
     // with is the same for everything involved here.
@@ -50,6 +56,8 @@ fn instantiate(
             config.memory_creator.as_ref().map(|a| a as _),
             store.interrupts().clone(),
             host,
+            externref_activations_table,
+            stack_map_registry,
         )?;
 
         // After we've created the `InstanceHandle` we still need to run
@@ -183,10 +191,27 @@ impl Instance {
             bail!("cross-`Engine` instantiation is not currently supported");
         }
 
-        let info = module.register_frame_info();
-        store.register_jit_code(module.compiled_module().jit_code_ranges());
+        let host_info = Box::new({
+            let frame_info_registration = module.register_frame_info();
+            store.register_jit_code(module.compiled_module().jit_code_ranges());
 
-        let handle = instantiate(store, module.compiled_module(), imports, Box::new(info))?;
+            // We need to make sure that we keep this alive as long as the instance
+            // is alive, or else we could miss GC roots, reclaim objects too early,
+            // and get user-after-frees.
+            let stack_map_registration =
+                unsafe { module.register_stack_maps(&*store.stack_map_registry()) };
+
+            (frame_info_registration, stack_map_registration)
+        });
+
+        let handle = instantiate(
+            store,
+            module.compiled_module(),
+            imports,
+            host_info,
+            store.externref_activations_table().clone(),
+            store.stack_map_registry().clone(),
+        )?;
 
         Ok(Instance {
             handle,

crates/wasmtime/src/module.rs

@@ -6,6 +6,7 @@ use std::path::Path;
 use std::sync::{Arc, Mutex};
 use wasmparser::validate;
 use wasmtime_jit::CompiledModule;
+use wasmtime_runtime::{StackMapRegistration, StackMapRegistry};
 
 /// A compiled WebAssembly module, ready to be instantiated.
 ///
@@ -80,6 +81,7 @@ pub struct Module {
     engine: Engine,
     compiled: Arc<CompiledModule>,
     frame_info_registration: Arc<Mutex<Option<Option<Arc<GlobalFrameInfoRegistration>>>>>,
+    stack_map_registration: Arc<Mutex<Option<Option<Arc<StackMapRegistration>>>>>,
 }
 
 impl Module {
@@ -307,6 +309,7 @@ impl Module {
             engine: engine.clone(),
             compiled: Arc::new(compiled),
             frame_info_registration: Arc::new(Mutex::new(None)),
+            stack_map_registration: Arc::new(Mutex::new(None)),
         })
     }
 
@@ -534,6 +537,41 @@ impl Module {
         *info = Some(ret.clone());
         return ret;
     }
+
+    /// Register this module's stack maps.
+    ///
+    /// # Safety
+    ///
+    /// The same as `wasmtime_runtime::StackMapRegistry::register_stack_maps`.
+    pub(crate) unsafe fn register_stack_maps(
+        &self,
+        registry: &Arc<StackMapRegistry>,
+    ) -> Option<Arc<StackMapRegistration>> {
+        let mut registration = self.stack_map_registration.lock().unwrap();
+        if let Some(registration) = &*registration {
+            return registration.clone();
+        }
+
+        let module = &self.compiled;
+        let ret = registry
+            .register_stack_maps(
+                module
+                    .finished_functions()
+                    .values()
+                    .zip(module.stack_maps().values())
+                    .map(|(func, stack_maps)| {
+                        let ptr = (**func).as_ptr();
+                        let len = (**func).len();
+                        let start = ptr as usize;
+                        let end = ptr as usize + len;
+                        let range = start..end;
+                        (range, &stack_maps[..])
+                    }),
+            )
+            .map(Arc::new);
+        *registration = Some(ret.clone());
+        ret
+    }
 }
 
 fn _assert_send_sync() {

crates/wasmtime/src/ref.rs

@@ -36,6 +36,11 @@ impl ExternRef {
         &*self.inner
     }
 
+    /// Get the reference count for this `ExternRef`.
+    pub fn get_reference_count(&self) -> usize {
+        self.inner.get_reference_count()
+    }
+
     /// Does this `ExternRef` point to the same inner value as `other`?0
     ///
     /// This is *only* pointer equality, and does *not* run any inner value's

crates/wasmtime/src/runtime.rs

@@ -19,7 +19,8 @@ use wasmtime_jit::{native, CompilationStrategy, Compiler};
 use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent};
 use wasmtime_runtime::{
     debug_builtins, InstanceHandle, RuntimeMemoryCreator, SignalHandler, SignatureRegistry,
-    VMExternRef, VMInterrupts, VMSharedSignatureIndex,
+    StackMapRegistry, VMExternRef, VMExternRefActivationsTable, VMInterrupts,
+    VMSharedSignatureIndex,
 };
 
 // Runtime Environment
@@ -194,10 +195,15 @@ impl Config {
         self.validating_config
             .operator_config
             .enable_reference_types = enable;
-        // The reference types proposal depends on the bulk memory proposal
+        self.flags
+            .set("enable_safepoints", if enable { "true" } else { "false" })
+            .unwrap();
+
+        // The reference types proposal depends on the bulk memory proposal.
         if enable {
             self.wasm_bulk_memory(true);
         }
+
         self
     }
 
@@ -724,6 +730,7 @@ pub struct Engine {
 struct EngineInner {
     config: Config,
     compiler: Compiler,
+    stack_map_registry: Arc<StackMapRegistry>,
 }
 
 impl Engine {
@@ -735,6 +742,7 @@ impl Engine {
             inner: Arc::new(EngineInner {
                 config: config.clone(),
                 compiler: config.build_compiler(),
+                stack_map_registry: Arc::new(StackMapRegistry::default()),
             }),
         }
     }
@@ -792,6 +800,8 @@ pub(crate) struct StoreInner {
     signal_handler: RefCell<Option<Box<SignalHandler<'static>>>>,
     jit_code_ranges: RefCell<Vec<(usize, usize)>>,
     host_info: RefCell<HashMap<HostInfoKey, Rc<RefCell<dyn Any>>>>,
+    externref_activations_table: Rc<VMExternRefActivationsTable>,
+    stack_map_registry: Arc<StackMapRegistry>,
 }
 
 struct HostInfoKey(VMExternRef);
@@ -832,6 +842,8 @@ impl Store {
                 signal_handler: RefCell::new(None),
                 jit_code_ranges: RefCell::new(Vec::new()),
                 host_info: RefCell::new(HashMap::new()),
+                externref_activations_table: Rc::new(VMExternRefActivationsTable::new()),
+                stack_map_registry: engine.inner.stack_map_registry.clone(),
             }),
         }
     }
@@ -1074,6 +1086,22 @@ impl Store {
             bail!("interrupts aren't enabled for this `Store`")
         }
     }
+
+    pub(crate) fn externref_activations_table(&self) -> &Rc<VMExternRefActivationsTable> {
+        &self.inner.externref_activations_table
+    }
+
+    pub(crate) fn stack_map_registry(&self) -> &Arc<StackMapRegistry> {
+        &self.inner.engine.inner.stack_map_registry
+    }
+
+    /// Perform garbage collection of `ExternRef`s.
+    pub fn gc(&self) {
+        wasmtime_runtime::gc(
+            &*self.inner.stack_map_registry,
+            &*self.inner.externref_activations_table,
+        );
+    }
 }
 
 impl Default for Store {

crates/wasmtime/src/trampoline/create_handle.rs

@@ -46,6 +46,8 @@ pub(crate) fn create_handle(
             signatures.into_boxed_slice(),
             state,
             store.interrupts().clone(),
+            store.externref_activations_table().clone(),
+            store.stack_map_registry().clone(),
         )?;
         Ok(store.add_instance(handle))
     }

crates/wasmtime/src/trampoline/func.rs

@@ -2,7 +2,7 @@
 
 use super::create_handle::create_handle;
 use crate::trampoline::StoreInstanceHandle;
-use crate::{FuncType, Store, Trap};
+use crate::{FuncType, Store, Trap, ValType};
 use anyhow::{bail, Result};
 use std::any::Any;
 use std::cmp;
@@ -11,7 +11,9 @@ use std::mem;
 use std::panic::{self, AssertUnwindSafe};
 use wasmtime_environ::entity::PrimaryMap;
 use wasmtime_environ::isa::TargetIsa;
-use wasmtime_environ::{ir, settings, CompiledFunction, EntityIndex, Module};
+use wasmtime_environ::{
+    ir, settings, settings::Configurable, CompiledFunction, EntityIndex, Module,
+};
 use wasmtime_jit::trampoline::ir::{
     ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind,
 };
@@ -210,7 +212,14 @@ pub fn create_handle_with_function(
 ) -> Result<(StoreInstanceHandle, VMTrampoline)> {
     let isa = {
         let isa_builder = native::builder();
-        let flag_builder = settings::builder();
+        let mut flag_builder = settings::builder();
+
+        if ft.params().iter().any(|p| *p == ValType::ExternRef)
+            || ft.results().iter().any(|r| *r == ValType::ExternRef)
+        {
+            flag_builder.set("enable_safepoints", "true").unwrap();
+        }
+
         isa_builder.finish(settings::Flags::new(flag_builder))
     };
 

crates/wasmtime/src/types.rs

@@ -106,6 +106,7 @@ impl ValType {
             ValType::F32 => Some(ir::types::F32),
             ValType::F64 => Some(ir::types::F64),
             ValType::V128 => Some(ir::types::I8X16),
+            ValType::ExternRef => Some(ir::types::R64),
             _ => None,
         }
     }
@@ -117,6 +118,7 @@ impl ValType {
             ir::types::F32 => Some(ValType::F32),
             ir::types::F64 => Some(ValType::F64),
             ir::types::I8X16 => Some(ValType::V128),
+            ir::types::R64 => Some(ValType::ExternRef),
             _ => None,
         }
     }

crates/wasmtime/src/values.rs

@@ -79,15 +79,22 @@ impl Val {
         }
     }
 
-    pub(crate) unsafe fn write_value_to(&self, p: *mut u128) {
+    pub(crate) unsafe fn write_value_to(self, store: &Store, p: *mut u128) {
         match self {
-            Val::I32(i) => ptr::write(p as *mut i32, *i),
-            Val::I64(i) => ptr::write(p as *mut i64, *i),
-            Val::F32(u) => ptr::write(p as *mut u32, *u),
-            Val::F64(u) => ptr::write(p as *mut u64, *u),
-            Val::V128(b) => ptr::write(p as *mut u128, *b),
+            Val::I32(i) => ptr::write(p as *mut i32, i),
+            Val::I64(i) => ptr::write(p as *mut i64, i),
+            Val::F32(u) => ptr::write(p as *mut u32, u),
+            Val::F64(u) => ptr::write(p as *mut u64, u),
+            Val::V128(b) => ptr::write(p as *mut u128, b),
             Val::ExternRef(None) => ptr::write(p, 0),
-            Val::ExternRef(Some(x)) => ptr::write(p as *mut *mut u8, x.inner.clone().into_raw()),
+            Val::ExternRef(Some(x)) => {
+                let externref_ptr = x.inner.as_raw();
+                if let Err(inner) = store.externref_activations_table().try_insert(x.inner) {
+                    store.gc();
+                    store.externref_activations_table().insert_slow_path(inner);
+                }
+                ptr::write(p as *mut *mut u8, externref_ptr)
+            }
             _ => unimplemented!("Val::write_value_to"),
         }
     }
@@ -105,7 +112,7 @@ impl Val {
                     Val::ExternRef(None)
                 } else {
                     Val::ExternRef(Some(ExternRef {
-                        inner: VMExternRef::from_raw(raw),
+                        inner: VMExternRef::clone_from_raw(raw),
                         store: store.weak(),
                     }))
                 }