externref: implement stack map-based garbage collection

For host VM code, we use plain reference counting, where cloning increments
the reference count, and dropping decrements it. We can avoid many of the
on-stack increment/decrement operations that typically plague the
performance of reference counting via Rust's ownership and borrowing system.
Moving a `VMExternRef` avoids mutating its reference count, and borrowing it
either avoids the reference count increment or delays it until if/when the
`VMExternRef` is cloned.

When passing a `VMExternRef` into compiled Wasm code, we don't want to do
reference count mutations for every compiled `local.{get,set}`, nor for
every function call. Therefore, we use a variation of **deferred reference
counting**, where we only mutate reference counts when storing
`VMExternRef`s somewhere that outlives the activation: into a global or
table. Simultaneously, we over-approximate the set of `VMExternRef`s that
are inside Wasm function activations. Periodically, we walk the stack at GC
safe points, and use stack map information to precisely identify the set of
`VMExternRef`s inside Wasm activations. Then we take the difference between
this precise set and our over-approximation, and decrement the reference
count for each of the `VMExternRef`s that are in our over-approximation but
not in the precise set. Finally, the over-approximation is replaced with the
precise set.

The `VMExternRefActivationsTable` implements the over-approximized set of
`VMExternRef`s referenced by Wasm activations. Calling a Wasm function and
passing it a `VMExternRef` moves the `VMExternRef` into the table, and the
compiled Wasm function logically "borrows" the `VMExternRef` from the
table. Similarly, `global.get` and `table.get` operations clone the gotten
`VMExternRef` into the `VMExternRefActivationsTable` and then "borrow" the
reference out of the table.

When a `VMExternRef` is returned to host code from a Wasm function, the host
increments the reference count (because the reference is logically
"borrowed" from the `VMExternRefActivationsTable` and the reference count
from the table will be dropped at the next GC).

For more general information on deferred reference counting, see *An
Examination of Deferred Reference Counting and Cycle Detection* by Quinane:
https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf

cc #929

Fixes #1804

crates/jit/src/code_memory.rs

@@ -134,7 +134,7 @@ impl CodeMemory {
 
             if !m.is_empty() {
                 unsafe {
-                    region::protect(m.as_mut_ptr(), m.len(), region::Protection::READ_EXECUTE)
+                    region::protect(m.as_mut_ptr(), m.len(), region::Protection::ReadExecute)
                 }
                 .expect("unable to make memory readonly and executable");
             }

crates/jit/src/compiler.rs

@@ -15,7 +15,7 @@ use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex,
 use wasmtime_environ::{
     CacheConfig, CompileError, CompiledFunction, Compiler as _C, Module, ModuleAddressMap,
     ModuleMemoryOffset, ModuleTranslation, ModuleVmctxInfo, Relocation, RelocationTarget,
-    Relocations, Traps, Tunables, VMOffsets, ValueLabelsRanges,
+    Relocations, StackMaps, Traps, Tunables, VMOffsets, ValueLabelsRanges,
 };
 use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline};
 
@@ -138,6 +138,7 @@ pub struct Compilation {
     pub jt_offsets: PrimaryMap<DefinedFuncIndex, ir::JumpTableOffsets>,
     pub dwarf_sections: Vec<DwarfSection>,
     pub traps: Traps,
+    pub stack_maps: StackMaps,
     pub address_transform: ModuleAddressMap,
 }
 
@@ -165,27 +166,34 @@ impl Compiler {
     ) -> Result<Compilation, SetupError> {
         let mut code_memory = CodeMemory::new();
 
-        let (compilation, relocations, address_transform, value_ranges, stack_slots, traps) =
-            match self.strategy {
-                // For now, interpret `Auto` as `Cranelift` since that's the most stable
-                // implementation.
-                CompilationStrategy::Auto | CompilationStrategy::Cranelift => {
-                    wasmtime_environ::cranelift::Cranelift::compile_module(
-                        translation,
-                        &*self.isa,
-                        &self.cache_config,
-                    )
-                }
-                #[cfg(feature = "lightbeam")]
-                CompilationStrategy::Lightbeam => {
-                    wasmtime_environ::lightbeam::Lightbeam::compile_module(
-                        translation,
-                        &*self.isa,
-                        &self.cache_config,
-                    )
-                }
+        let (
+            compilation,
+            relocations,
+            address_transform,
+            value_ranges,
+            stack_slots,
+            traps,
+            stack_maps,
+        ) = match self.strategy {
+            // For now, interpret `Auto` as `Cranelift` since that's the most stable
+            // implementation.
+            CompilationStrategy::Auto | CompilationStrategy::Cranelift => {
+                wasmtime_environ::cranelift::Cranelift::compile_module(
+                    translation,
+                    &*self.isa,
+                    &self.cache_config,
+                )
             }
-            .map_err(SetupError::Compile)?;
+            #[cfg(feature = "lightbeam")]
+            CompilationStrategy::Lightbeam => {
+                wasmtime_environ::lightbeam::Lightbeam::compile_module(
+                    translation,
+                    &*self.isa,
+                    &self.cache_config,
+                )
+            }
+        }
+        .map_err(SetupError::Compile)?;
 
         let dwarf_sections = if debug_data.is_some() && !compilation.is_empty() {
             transform_dwarf_data(
@@ -239,6 +247,7 @@ impl Compiler {
             jt_offsets,
             dwarf_sections,
             traps,
+            stack_maps,
             address_transform,
         })
     }

crates/jit/src/instantiate.rs

@@ -10,6 +10,7 @@ use crate::link::link_module;
 use crate::resolver::Resolver;
 use std::any::Any;
 use std::collections::HashMap;
+use std::rc::Rc;
 use std::sync::Arc;
 use thiserror::Error;
 use wasmtime_debug::{read_debuginfo, write_debugsections_image, DwarfSection};
@@ -18,13 +19,13 @@ use wasmtime_environ::isa::TargetIsa;
 use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex};
 use wasmtime_environ::{
     CompileError, DataInitializer, DataInitializerLocation, Module, ModuleAddressMap,
-    ModuleEnvironment, ModuleTranslation, Traps,
+    ModuleEnvironment, ModuleTranslation, StackMaps, Traps,
 };
 use wasmtime_profiling::ProfilingAgent;
 use wasmtime_runtime::VMInterrupts;
 use wasmtime_runtime::{
     GdbJitImageRegistration, InstanceHandle, InstantiationError, RuntimeMemoryCreator,
-    SignatureRegistry, VMFunctionBody, VMTrampoline,
+    SignatureRegistry, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, VMTrampoline,
 };
 
 /// An error condition while setting up a wasm instance, be it validation,
@@ -69,6 +70,7 @@ pub struct CompiledModule {
     trampolines: PrimaryMap<SignatureIndex, VMTrampoline>,
     data_initializers: Box<[OwnedDataInitializer]>,
     traps: Traps,
+    stack_maps: StackMaps,
     address_transform: ModuleAddressMap,
 }
 
@@ -99,6 +101,7 @@ impl CompiledModule {
             jt_offsets,
             dwarf_sections,
             traps,
+            stack_maps,
             address_transform,
         } = compiler.compile(&translation, debug_data)?;
 
@@ -149,6 +152,7 @@ impl CompiledModule {
             trampolines,
             data_initializers,
             traps,
+            stack_maps,
             address_transform,
         })
     }
@@ -169,6 +173,8 @@ impl CompiledModule {
         mem_creator: Option<&dyn RuntimeMemoryCreator>,
         interrupts: Arc<VMInterrupts>,
         host_state: Box<dyn Any>,
+        externref_activations_table: Rc<VMExternRefActivationsTable>,
+        stack_map_registry: Arc<StackMapRegistry>,
     ) -> Result<InstanceHandle, InstantiationError> {
         // Compute indices into the shared signature table.
         let signatures = {
@@ -200,6 +206,8 @@ impl CompiledModule {
             signatures.into_boxed_slice(),
             host_state,
             interrupts,
+            externref_activations_table,
+            stack_map_registry,
         )
     }
 
@@ -229,11 +237,16 @@ impl CompiledModule {
         &self.finished_functions.0
     }
 
-    /// Returns the a map for all traps in this module.
+    /// Returns the map for all traps in this module.
     pub fn traps(&self) -> &Traps {
         &self.traps
     }
 
+    /// Returns the map for each of this module's stack maps.
+    pub fn stack_maps(&self) -> &StackMaps {
+        &self.stack_maps
+    }
+
     /// Returns a map of compiled addresses back to original bytecode offsets.
     pub fn address_transform(&self) -> &ModuleAddressMap {
         &self.address_transform