memfd: make "dense image" heuristic limit configurable. (#3831)

In #3820 we see an issue with the new heuristics that control use of
memfd: it's entirely possible for a reasonable Wasm module produced by a
snapshotting system to have a relatively sparse heap (less than 50%
filled). A system that avoids memfd because of this would have an
undesirable performance reduction on such modules.

Ultimately we should try to implement a hybrid scheme where we support
outlier/leftover initializers, but for now this PR makes the "always
allow dense" limit configurable. This way, embedders that want to ensure
that memfd is used can do so, if they have other knowledge about the
maximum heap size allowed in their system.

(Partially addresses #3820 but let's leave it open to track the hybrid
idea)

crates/wasmtime/src/config.rs

@@ -105,6 +105,7 @@ pub struct Config {
     pub(crate) parallel_compilation: bool,
     pub(crate) paged_memory_initialization: bool,
     pub(crate) memfd: bool,
+    pub(crate) memfd_guaranteed_dense_image_size: u64,
 }
 
 impl Config {
@@ -131,6 +132,7 @@ impl Config {
             // Default to paged memory initialization when using uffd on linux
             paged_memory_initialization: cfg!(all(target_os = "linux", feature = "uffd")),
             memfd: false,
+            memfd_guaranteed_dense_image_size: 16 << 20,
         };
         #[cfg(compiler)]
         {
@@ -1199,6 +1201,47 @@ impl Config {
         self
     }
 
+    /// Configures the "guaranteed dense image size" for memfd.
+    ///
+    /// When using the memfd feature to initialize memory efficiently,
+    /// compiled modules contain an image of the module's initial
+    /// heap. If the module has a fairly sparse initial heap, with
+    /// just a few data segments at very different offsets, this could
+    /// result in a large region of zero bytes in the image. In other
+    /// words, it's not very memory-efficient.
+    ///
+    /// We normally use a heuristic to avoid this: if less than half
+    /// of the initialized range (first non-zero to last non-zero
+    /// byte) of any memory in the module has pages with nonzero
+    /// bytes, then we avoid memfd for the entire module.
+    ///
+    /// However, if the embedder always needs the instantiation-time
+    /// efficiency of memfd, and is otherwise carefully controlling
+    /// parameters of the modules (for example, by limiting the
+    /// maximum heap size of the modules), then it may be desirable to
+    /// ensure memfd is used even if this could go against the
+    /// heuristic above. Thus, we add another condition: there is a
+    /// size of initialized data region up to which we *always* allow
+    /// memfd. The embedder can set this to a known maximum heap size
+    /// if they desire to always get the benefits of memfd.
+    ///
+    /// In the future we may implement a "best of both worlds"
+    /// solution where we have a dense image up to some limit, and
+    /// then support a sparse list of initializers beyond that; this
+    /// would get most of the benefit of memfd and pay the incremental
+    /// cost of eager initialization only for those bits of memory
+    /// that are out-of-bounds. However, for now, an embedder desiring
+    /// fast instantiation should ensure that this setting is as large
+    /// as the maximum module initial memory content size.
+    ///
+    /// By default this value is 16 MiB.
+    #[cfg(feature = "memfd")]
+    #[cfg_attr(nightlydoc, doc(cfg(feature = "memfd")))]
+    pub fn memfd_guaranteed_dense_image_size(&mut self, size_in_bytes: u64) -> &mut Self {
+        self.memfd_guaranteed_dense_image_size = size_in_bytes;
+        self
+    }
+
     pub(crate) fn build_allocator(&self) -> Result<Box<dyn InstanceAllocator>> {
         #[cfg(feature = "async")]
         let stack_size = self.async_stack_size;
@@ -1269,6 +1312,7 @@ impl Clone for Config {
             parallel_compilation: self.parallel_compilation,
             paged_memory_initialization: self.paged_memory_initialization,
             memfd: self.memfd,
+            memfd_guaranteed_dense_image_size: self.memfd_guaranteed_dense_image_size,
         }
     }
 }

crates/wasmtime/src/module.rs

@@ -432,7 +432,8 @@ impl Module {
             // such as mmap'ing from a file to get copy-on-write.
             if engine.config().memfd {
                 let align = engine.compiler().page_size_align();
-                translation.try_static_init(align);
+                let max_always_allowed = engine.config().memfd_guaranteed_dense_image_size;
+                translation.try_static_init(align, max_always_allowed);
             }
 
             // Attempt to convert table initializer segments to