fuzz: Implement finer memory limits per-store (#3149)

* fuzz: Implement finer memory limits per-store

This commit implements a custom resource limiter for fuzzing. Locally I
was seeing a lot of ooms while fuzzing and I believe it was generally
caused from not actually having any runtime limits for wasm modules. I'm
actually surprised that this hasn't come up more on oss-fuzz more in
reality, but with a custom store limiter I think this'll get the job
done where we have an easier knob to turn for controlling the memory
usage of fuzz-generated modules.

For now I figure a 2gb limit should be good enough for limiting fuzzer
execution. Additionally the "out of resources" check if instantiation
fails now looks for the `oom` flag to be set instead of pattern matching
on some error messages about resources.

* Fix tests

crates/fuzzing/src/oracles.rs

@@ -45,18 +45,53 @@ fn log_wasm(wasm: &[u8]) {
 fn create_store(engine: &Engine) -> Store<StoreLimits> {
     let mut store = Store::new(
         &engine,
-        StoreLimitsBuilder::new()
+        StoreLimits {
-            // The limits here are chosen based on the default "maximum type size"
+            // Limits tables/memories within a store to at most 2gb for now to
-            // configured in wasm-smith, which is 1000. This means that instances
+            // exercise some larger address but not overflow various limits.
-            // are allowed to, for example, export up to 1000 memories. We bump that
+            remaining_memory: 2 << 30,
-            // a little bit here to give us some slop.
+            oom: false,
-            .instances(1100)
+        },
-            .tables(1100)
-            .memories(1100)
-            .build(),
     );
     store.limiter(|s| s as &mut dyn ResourceLimiter);
-    store
+    return store;
+}
+
+struct StoreLimits {
+    /// Remaining memory, in bytes, left to allocate
+    remaining_memory: usize,
+    /// Whether or not an allocation request has been denied
+    oom: bool,
+}
+
+impl StoreLimits {
+    fn alloc(&mut self, amt: usize) -> bool {
+        match self.remaining_memory.checked_sub(amt) {
+            Some(mem) => {
+                self.remaining_memory = mem;
+                true
+            }
+            None => {
+                self.oom = true;
+                false
+            }
+        }
+    }
+}
+
+impl ResourceLimiter for StoreLimits {
+    fn memory_growing(&mut self, current: u32, desired: u32, _maximum: Option<u32>) -> bool {
+        // Units provided are in wasm pages, so adjust them to bytes to see if
+        // we are ok to allocate this much.
+        self.alloc((desired - current) as usize * 16 * 1024)
+    }
+
+    fn table_growing(&mut self, current: u32, desired: u32, _maximum: Option<u32>) -> bool {
+        // Units provided are in table elements, and for now we allocate one
+        // pointer per table element, so use that size for an adjustment into
+        // bytes.
+        let delta = (desired - current) as usize * std::mem::size_of::<usize>();
+        self.alloc(delta)
+    }
 }
 
 /// Methods of timing out execution of a WebAssembly module
@@ -159,22 +194,26 @@ pub fn instantiate_with_config(
     match linker.instantiate(&mut store, &module) {
         Ok(_) => {}
         Err(e) => {
-            let string = e.to_string();
+            // If the instantiation hit OOM for some reason then that's ok, it's
+            // expected that fuzz-generated programs try to allocate lots of
+            // stuff.
+            if store.data().oom {
+                return;
+            }
+
             // Allow traps which can happen normally with `unreachable` or a
-            // timeout
+            // timeout or such
-            if e.downcast_ref::<Trap>().is_some()
+            if e.downcast_ref::<Trap>().is_some() {
-                // Allow resource exhaustion since this is something that
+                return;
-                // our wasm-smith generator doesn't guarantee is forbidden.
+            }
-                || string.contains("resource limit exceeded")
+
-                // Also allow errors related to fuel consumption
+            let string = e.to_string();
-                || string.contains("all fuel consumed")
+            // Also allow errors related to fuel consumption
+            if string.contains("all fuel consumed")
                 // Currently we instantiate with a `Linker` which can't instantiate
                 // every single module under the sun due to using name-based resolution
                 // rather than positional-based resolution
                 || string.contains("incompatible import type")
-                // If we ran out of resources instantiating this wasm module that's
-                // ok, no need to consider that a fatal error.
-                || string.contains("Insufficient resources")
             {
                 return;
             }

crates/fuzzing/src/oracles/dummy.rs

@@ -41,7 +41,7 @@ pub fn dummy_extern<T>(store: &mut Store<T>, ty: ExternType) -> Result<Extern> {
         ExternType::Global(global_ty) => Extern::Global(dummy_global(store, global_ty)),
         ExternType::Table(table_ty) => Extern::Table(dummy_table(store, table_ty)),
         ExternType::Memory(mem_ty) => Extern::Memory(dummy_memory(store, mem_ty)?),
-        ExternType::Instance(instance_ty) => Extern::Instance(dummy_instance(store, instance_ty)),
+        ExternType::Instance(instance_ty) => Extern::Instance(dummy_instance(store, instance_ty)?),
         ExternType::Module(module_ty) => Extern::Module(dummy_module(store.engine(), module_ty)),
     })
 }
@@ -95,13 +95,13 @@ pub fn dummy_memory<T>(store: &mut Store<T>, ty: MemoryType) -> Result<Memory> {
 ///
 /// This is done by using the expected type to generate a module on-the-fly
 /// which we the instantiate.
-pub fn dummy_instance<T>(store: &mut Store<T>, ty: InstanceType) -> Instance {
+pub fn dummy_instance<T>(store: &mut Store<T>, ty: InstanceType) -> Result<Instance> {
     let mut wat = WatGenerator::new();
     for ty in ty.exports() {
         wat.export(&ty);
     }
     let module = Module::new(store.engine(), &wat.finish()).unwrap();
-    Instance::new(store, &module, &[]).unwrap()
+    Instance::new(store, &module, &[])
 }
 
 /// Construct a dummy module for the given module type.
@@ -469,7 +469,7 @@ mod tests {
         instance_ty.add_named_export("instance0", InstanceType::new().into());
         instance_ty.add_named_export("instance1", InstanceType::new().into());
 
-        let instance = dummy_instance(&mut store, instance_ty.clone());
+        let instance = dummy_instance(&mut store, instance_ty.clone()).unwrap();
 
         let mut expected_exports = vec![
             "func0",