Implement user fault handling with userfaultfd on Linux.

This commit implements the `uffd` feature which turns on support for utilizing
the `userfaultfd` system call on Linux for the pooling instance allocator.

By handling page faults in userland, we are able to detect guard page accesses
without having to constantly change memory page protections.

This should help reduce the number of syscalls as well as kernel lock
contentions when many threads are allocating and deallocating instances.

Additionally, the user fault handler can lazy initialize linear
memories of an instance (implementation to come).

crates/runtime/src/instance.rs

@@ -67,6 +67,11 @@ pub(crate) struct Instance {
     /// Hosts can store arbitrary per-instance information here.
     host_state: Box<dyn Any>,
 
+    /// Stores guard page faults in memory relating to the instance.
+    /// This is used for the pooling allocator with uffd enabled on Linux.
+    #[cfg(all(feature = "uffd", target_os = "linux"))]
+    guard_page_faults: RefCell<Vec<(*mut u8, usize, unsafe fn(*mut u8, usize) -> bool)>>,
+
     /// Additional context used by compiled wasm code. This field is last, and
     /// represents a dynamically-sized array that extends beyond the nominal
     /// end of the struct (similar to a flexible array member).
@@ -376,6 +381,10 @@ impl Instance {
     /// Returns `None` if memory can't be grown by the specified amount
     /// of pages.
     pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option<u32> {
+        // Reset all guard pages before growing any memory
+        #[cfg(all(feature = "uffd", target_os = "linux"))]
+        self.reset_guard_pages().ok()?;
+
         let result = self
             .memories
             .get(memory_index)
@@ -803,6 +812,40 @@ impl Instance {
             (foreign_table_index, foreign_instance)
         }
     }
+
+    /// Records a faulted guard page.
+    ///
+    /// This is used to track faulted guard pages that need to be reset.
+    #[cfg(all(feature = "uffd", target_os = "linux"))]
+    pub(crate) fn record_guard_page_fault(
+        &self,
+        page_addr: *mut u8,
+        size: usize,
+        reset: unsafe fn(*mut u8, usize) -> bool,
+    ) {
+        self.guard_page_faults
+            .borrow_mut()
+            .push((page_addr, size, reset));
+    }
+
+    /// Resets previously faulted guard pages.
+    ///
+    /// This is used to reset the protection of any guard pages that were previously faulted.
+    ///
+    /// Resetting the guard pages is required before growing memory.
+    #[cfg(all(feature = "uffd", target_os = "linux"))]
+    pub(crate) fn reset_guard_pages(&self) -> Result<(), String> {
+        let mut faults = self.guard_page_faults.borrow_mut();
+        for (addr, len, reset) in faults.drain(..) {
+            unsafe {
+                if !reset(addr, len) {
+                    return Err("failed to reset previously faulted memory guard page".into());
+                }
+            }
+        }
+
+        Ok(())
+    }
 }
 
 /// A handle holding an `Instance` of a WebAssembly module.