T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve stability for fuzz targets. (#3804)

Browse Source 
This commit improves the stability of the fuzz targets by ensuring the
generated configs and modules are congruent, especially when the pooling
allocator is being used.

For the `differential` target, this means both configurations must use the same
allocation strategy for now as one side generates the module that might not be
compatible with another arbitrary config now that we fuzz the pooling
allocator.

These changes also ensure that constraints put on the config are more
consistently applied, especially when using a fuel-based timeout.
This commit is contained in:
Peter Huene
2022-02-15 12:59:04 -08:00
committed by GitHub
parent 0b4263333b
commit 6ffcd4ead9
9 changed files with 198 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
crates/fuzzing/src/generators.rs
View File

@@ -185,7 +185,7 @@ impl InstanceAllocationStrategy {
/// This configuration guides what modules are generated, how wasmtime
/// configuration is generated, and is typically itself generated through a call
/// to `Arbitrary` which allows for a form of "swarm testing".
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct Config {
/// Configuration related to the `wasmtime::Config`.
pub wasmtime: WasmtimeConfig,
@@ -210,9 +210,17 @@ impl<'a> Arbitrary<'a> for Config {
// Force the use of a normal memory config when using the pooling allocator and
// limit the static memory maximum to be the same as the pooling allocator's memory
// page limit.
let mut memory_config: NormalMemoryConfig = u.arbitrary()?;
memory_config.static_memory_maximum_size = Some(limits.memory_pages * 0x10000);
config.wasmtime.memory_config = MemoryConfig::Normal(memory_config);
config.wasmtime.memory_config = match config.wasmtime.memory_config {
MemoryConfig::Normal(mut config) => {
config.static_memory_maximum_size = Some(limits.memory_pages * 0x10000);
MemoryConfig::Normal(config)
}
MemoryConfig::CustomUnaligned => {
let mut config: NormalMemoryConfig = u.arbitrary()?;
config.static_memory_maximum_size = Some(limits.memory_pages * 0x10000);
MemoryConfig::Normal(config)
}
};
let cfg = &mut config.module_config.config;
cfg.max_imports = limits.imported_functions.min(
@@ -245,7 +253,8 @@ pub struct WasmtimeConfig {
canonicalize_nans: bool,
interruptable: bool,
pub(crate) consume_fuel: bool,
memory_config: MemoryConfig,
/// The Wasmtime memory configuration to use.
pub memory_config: MemoryConfig,
force_jump_veneers: bool,
memfd: bool,
use_precompiled_cwasm: bool,
@@ -254,8 +263,9 @@ pub struct WasmtimeConfig {
codegen: CodegenSettings,
}
/// Configuration for linear memories in Wasmtime.
#[derive(Arbitrary, Clone, Debug, Eq, Hash, PartialEq)]
enum MemoryConfig {
pub enum MemoryConfig {
/// Configuration for linear memories which correspond to normal
/// configuration settings in `wasmtime` itself. This will tweak various
/// parameters about static/dynamic memories.
@@ -267,8 +277,10 @@ enum MemoryConfig {
CustomUnaligned,
}
/// Represents a normal memory configuration for Wasmtime with the given
/// static and dynamic memory sizes.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct NormalMemoryConfig {
pub struct NormalMemoryConfig {
static_memory_maximum_size: Option<u64>,
static_memory_guard_size: Option<u64>,
dynamic_memory_guard_size: Option<u64>,
@@ -289,6 +301,117 @@ impl<'a> Arbitrary<'a> for NormalMemoryConfig {
}
impl Config {
/// Indicates that this configuration is being used for differential
/// execution so only a single function should be generated since that's all
/// that's going to be exercised.
pub fn set_differential_config(&mut self) {
let config = &mut self.module_config.config;
config.allow_start_export = false;
// Make sure there's a type available for the function.
config.min_types = 1;
config.max_types = 1;
// Generate one and only one function
config.min_funcs = 1;
config.max_funcs = 1;
// Give the function a memory, but keep it small
config.min_memories = 1;
config.max_memories = 1;
config.max_memory_pages = 1;
config.memory_max_size_required = true;
// Don't allow any imports
config.max_imports = 0;
// Try to get the function and the memory exported
config.min_exports = 2;
config.max_exports = 4;
// NaN is canonicalized at the wasm level for differential fuzzing so we
// can paper over NaN differences between engines.
config.canonicalize_nans = true;
// When diffing against a non-wasmtime engine then disable wasm
// features to get selectively re-enabled against each differential
// engine.
config.bulk_memory_enabled = false;
config.reference_types_enabled = false;
config.simd_enabled = false;
config.memory64_enabled = false;
// If using the pooling allocator, update the module limits too
if let InstanceAllocationStrategy::Pooling {
module_limits: limits,
..
} = &mut self.wasmtime.strategy
{
// No imports
limits.imported_functions = 0;
limits.imported_tables = 0;
limits.imported_memories = 0;
limits.imported_globals = 0;
// One type, one function, and one single-page memory
limits.types = 1;
limits.functions = 1;
limits.memories = 1;
limits.memory_pages = 1;
match &mut self.wasmtime.memory_config {
MemoryConfig::Normal(config) => {
config.static_memory_maximum_size = Some(limits.memory_pages * 0x10000);
}
MemoryConfig::CustomUnaligned => unreachable!(), // Arbitrary impl for `Config` should have prevented this
}
}
}
/// Uses this configuration and the supplied source of data to generate
/// a wasm module.
///
/// If a `default_fuel` is provided, the resulting module will be configured
/// to ensure termination; as doing so will add an additional global to the module,
/// the pooling allocator, if configured, will also have its globals limit updated.
pub fn generate(
&mut self,
input: &mut Unstructured<'_>,
default_fuel: Option<u32>,
) -> arbitrary::Result<wasm_smith::Module> {
let mut module = wasm_smith::Module::new(self.module_config.config.clone(), input)?;
if let Some(default_fuel) = default_fuel {
module.ensure_termination(default_fuel);
// Bump the allowed global count by 1
if let InstanceAllocationStrategy::Pooling { module_limits, .. } =
&mut self.wasmtime.strategy
{
module_limits.globals += 1;
}
}
Ok(module)
}
/// Indicates that this configuration should be spec-test-compliant,
/// disabling various features the spec tests assert are disabled.
pub fn set_spectest_compliant(&mut self) {
let config = &mut self.module_config.config;
config.memory64_enabled = false;
config.simd_enabled = false;
config.bulk_memory_enabled = true;
config.reference_types_enabled = true;
config.max_memories = 1;
if let InstanceAllocationStrategy::Pooling { module_limits, .. } =
&mut self.wasmtime.strategy
{
module_limits.memories = 1;
}
}
/// Converts this to a `wasmtime::Config` object
pub fn to_wasmtime(&self) -> wasmtime::Config {
crate::init_fuzzing();
@@ -496,63 +619,6 @@ pub struct ModuleConfig {
pub config: SwarmConfig,
}
impl ModuleConfig {
/// Uses this configuration and the supplied source of data to generate
/// a wasm module.
pub fn generate(&self, input: &mut Unstructured<'_>) -> arbitrary::Result<wasm_smith::Module> {
wasm_smith::Module::new(self.config.clone(), input)
}
/// Indicates that this configuration should be spec-test-compliant,
/// disabling various features the spec tests assert are disabled.
pub fn set_spectest_compliant(&mut self) {
self.config.memory64_enabled = false;
self.config.simd_enabled = false;
self.config.bulk_memory_enabled = true;
self.config.reference_types_enabled = true;
self.config.max_memories = 1;
}
/// Indicates that this configuration is being used for differential
/// execution so only a single function should be generated since that's all
/// that's going to be exercised.
pub fn set_differential_config(&mut self) {
self.config.allow_start_export = false;
// Make sure there's a type available for the function.
self.config.min_types = 1;
self.config.max_types = 1;
// Generate one and only one function
self.config.min_funcs = 1;
self.config.max_funcs = 1;
// Give the function a memory, but keep it small
self.config.min_memories = 1;
self.config.max_memories = 1;
self.config.max_memory_pages = 1;
self.config.memory_max_size_required = true;
// Don't allow any imports
self.config.max_imports = 0;
// Try to get the function and the memory exported
self.config.min_exports = 2;
self.config.max_exports = 4;
// NaN is canonicalized at the wasm level for differential fuzzing so we
// can paper over NaN differences between engines.
self.config.canonicalize_nans = true;
// When diffing against a non-wasmtime engine then disable wasm
// features to get selectively re-enabled against each differential
// engine.
self.config.bulk_memory_enabled = false;
self.config.reference_types_enabled = false;
self.config.simd_enabled = false;
self.config.memory64_enabled = false;
}
}
impl<'a> Arbitrary<'a> for ModuleConfig {
fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<ModuleConfig> {
let mut config = SwarmConfig::arbitrary(u)?;

12
crates/fuzzing/src/generators/api.rs
View File

@@ -14,7 +14,7 @@
//!
//! [swarm testing]: https://www.cs.utah.edu/~regehr/papers/swarm12.pdf
use crate::generators::{Config, ModuleConfig};
use crate::generators::Config;
use arbitrary::{Arbitrary, Unstructured};
use std::collections::BTreeSet;
@@ -44,7 +44,7 @@ struct Scope {
id_counter: usize,
modules: BTreeSet<usize>,
instances: BTreeSet<usize>,
module_config: ModuleConfig,
config: Config,
}
impl Scope {
@@ -70,14 +70,13 @@ impl<'a> Arbitrary<'a> for ApiCalls {
let mut calls = vec![];
let config = Config::arbitrary(input)?;
let module_config = config.module_config.clone();
calls.push(StoreNew(config));
calls.push(StoreNew(config.clone()));
let mut scope = Scope {
id_counter: 0,
modules: BTreeSet::default(),
instances: BTreeSet::default(),
module_config,
config,
};
// Total limit on number of API calls we'll generate. This exists to
@@ -94,8 +93,7 @@ impl<'a> Arbitrary<'a> for ApiCalls {
if swarm.module_new {
choices.push(|input, scope| {
let id = scope.next_id();
let mut wasm = scope.module_config.generate(input)?;
wasm.ensure_termination(1000);
let wasm = scope.config.generate(input, Some(1000))?;
scope.modules.insert(id);
Ok(ModuleNew {
id,

40
crates/fuzzing/src/oracles.rs
View File

@@ -295,11 +295,13 @@ fn instantiate_with_dummy(store: &mut Store<StoreLimits>, module: &Module) -> Op
/// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are
/// or aren't enabled for different configs, we should get the same results when
/// we call the exported functions for all of our different configs.
///
/// Returns `None` if a fuzz configuration was rejected (should happen rarely).
pub fn differential_execution(
wasm: &[u8],
module_config: &generators::ModuleConfig,
configs: &[generators::WasmtimeConfig],
) {
) -> Option<()> {
use std::collections::{HashMap, HashSet};
// We need at least two configs.
@@ -307,7 +309,7 @@ pub fn differential_execution(
// And all the configs should be unique.
|| configs.iter().collect::<HashSet<_>>().len() != configs.len()
{
return;
return None;
}
let mut export_func_results: HashMap<String, Result<Box<[Val]>, Trap>> = Default::default();
@@ -321,7 +323,7 @@ pub fn differential_execution(
log::debug!("fuzz config: {:?}", fuzz_config);
let mut store = fuzz_config.to_store();
let module = fuzz_config.compile(store.engine(), &wasm).unwrap();
let module = compile_module(store.engine(), &wasm, true, &fuzz_config)?;
// TODO: we should implement tracing versions of these dummy imports
// that record a trace of the order that imported functions were called
@@ -357,6 +359,8 @@ pub fn differential_execution(
}
}
return Some(());
fn assert_same_export_func_result(
lhs: &Result<Box<[Val]>, Trap>,
rhs: &Result<Box<[Val]>, Trap>,
@@ -505,7 +509,7 @@ pub fn make_api_calls(api: generators::api::ApiCalls) {
/// Ensures that spec tests pass regardless of the `Config`.
pub fn spectest(mut fuzz_config: generators::Config, test: generators::SpecTest) {
crate::init_fuzzing();
fuzz_config.module_config.set_spectest_compliant();
fuzz_config.set_spectest_compliant();
log::debug!("running {:?} with {:?}", test.file, fuzz_config);
let mut wast_context = WastContext::new(fuzz_config.to_store());
wast_context.register_spectest().unwrap();
@@ -531,9 +535,9 @@ pub fn table_ops(mut fuzz_config: generators::Config, ops: generators::table_ops
let wasm = ops.to_wasm_binary();
log_wasm(&wasm);
let module = match fuzz_config.compile(store.engine(), &wasm) {
Ok(m) => m,
Err(_) => return,
let module = match compile_module(store.engine(), &wasm, false, &fuzz_config) {
Some(m) => m,
None => return,
};
let mut linker = Linker::new(store.engine());
@@ -677,6 +681,7 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &generators::Config) ->
// If wasmi succeeded then we assert that wasmtime will also succeed.
let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config);
let wasmtime_module = wasmtime_module?;
let wasmtime_instance = Instance::new(&mut wasmtime_store, &wasmtime_module, &[])
.expect("Wasmtime can instantiate module");
@@ -790,7 +795,7 @@ pub fn differential_spec_execution(wasm: &[u8], config: &generators::Config) ->
match (&spec_vals, &wasmtime_vals) {
// Compare the returned values, failing if they do not match.
(Ok(spec_vals), Ok(wasmtime_vals)) => {
(Ok(spec_vals), Ok(Some(wasmtime_vals))) => {
let all_match = spec_vals
.iter()
.zip(wasmtime_vals)
@@ -802,6 +807,10 @@ pub fn differential_spec_execution(wasm: &[u8], config: &generators::Config) ->
);
}
}
(_, Ok(None)) => {
// `run_in_wasmtime` rejected the config
return None;
}
// If both sides fail, skip this fuzz execution.
(Err(spec_error), Err(wasmtime_error)) => {
// The `None` value returned here indicates that both sides
@@ -833,11 +842,9 @@ pub fn differential_spec_execution(wasm: &[u8], config: &generators::Config) ->
fn differential_store(
wasm: &[u8],
fuzz_config: &generators::Config,
) -> (Module, Store<StoreLimits>) {
) -> (Option<Module>, Store<StoreLimits>) {
let store = fuzz_config.to_store();
let module = fuzz_config
.compile(store.engine(), wasm)
.expect("Wasmtime can compile module");
let module = compile_module(store.engine(), wasm, true, fuzz_config);
(module, store)
}
@@ -847,9 +854,14 @@ fn run_in_wasmtime(
wasm: &[u8],
config: &generators::Config,
params: &[Val],
) -> anyhow::Result<Vec<Val>> {
) -> anyhow::Result<Option<Vec<Val>>> {
// Instantiate wasmtime module and instance.
let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config);
let wasmtime_module = match wasmtime_module {
Some(m) => m,
None => return Ok(None),
};
let wasmtime_instance = Instance::new(&mut wasmtime_store, &wasmtime_module, &[])
.context("Wasmtime cannot instantiate module")?;
@@ -864,7 +876,7 @@ fn run_in_wasmtime(
let mut results = vec![Val::I32(0); ty.results().len()];
wasmtime_main
.call(&mut wasmtime_store, params, &mut results)
.map(|()| results)
.map(|()| Some(results))
}
// Introspect wasmtime module to find the name of the first exported function.

1
crates/fuzzing/src/oracles/v8.rs
View File

@@ -17,6 +17,7 @@ pub fn differential_v8_execution(wasm: &[u8], config: &crate::generators::Config
// Wasmtime setup
log_wasm(wasm);
let (wasmtime_module, mut wasmtime_store) = super::differential_store(wasm, config);
let wasmtime_module = wasmtime_module?;
log::trace!("compiled module with wasmtime");
// V8 setup

29
fuzz/fuzz_targets/differential.rs
View File

@@ -2,6 +2,7 @@
use libfuzzer_sys::arbitrary::{Result, Unstructured};
use libfuzzer_sys::fuzz_target;
use wasmtime_fuzzing::generators::InstanceAllocationStrategy;
use wasmtime_fuzzing::{generators, oracles};
fuzz_target!(|data: &[u8]| {
@@ -13,12 +14,28 @@ fuzz_target!(|data: &[u8]| {
fn run(data: &[u8]) -> Result<()> {
let mut u = Unstructured::new(data);
let lhs: generators::WasmtimeConfig = u.arbitrary()?;
let rhs: generators::WasmtimeConfig = u.arbitrary()?;
let config: generators::ModuleConfig = u.arbitrary()?;
let mut module = config.generate(&mut u)?;
module.ensure_termination(1000);
let mut config: generators::Config = u.arbitrary()?;
let module = config.generate(&mut u, Some(1000))?;
oracles::differential_execution(&module.to_bytes(), &config, &[lhs, rhs]);
let lhs = config.wasmtime;
let mut rhs: generators::WasmtimeConfig = u.arbitrary()?;
// Use the same allocation strategy between the two configs.
//
// Ideally this wouldn't be necessary, but if the lhs is using ondemand
// and the rhs is using the pooling allocator (or vice versa), then
// the module may have been generated in such a way that is incompatible
// with the other allocation strategy.
//
// We can remove this in the future when it's possible to access the
// fields of `wasm_smith::Module` to constrain the pooling allocator
// based on what was actually generated.
rhs.strategy = lhs.strategy.clone();
if let InstanceAllocationStrategy::Pooling { .. } = &rhs.strategy {
// Also use the same memory configuration when using the pooling allocator
rhs.memory_config = lhs.memory_config.clone();
}
oracles::differential_execution(&module.to_bytes(), &config.module_config, &[lhs, rhs]);
Ok(())
}

5
fuzz/fuzz_targets/differential_v8.rs
View File

@@ -13,15 +13,14 @@ fuzz_target!(|data: &[u8]| {
fn run(data: &[u8]) -> Result<()> {
let mut u = Unstructured::new(data);
let mut config: generators::Config = u.arbitrary()?;
config.module_config.set_differential_config();
config.set_differential_config();
// Enable features that v8 has implemented
config.module_config.config.simd_enabled = true;
config.module_config.config.bulk_memory_enabled = true;
config.module_config.config.reference_types_enabled = true;
let mut module = config.module_config.generate(&mut u)?;
module.ensure_termination(1000);
let module = config.generate(&mut u, Some(1000))?;
oracles::differential_v8_execution(&module.to_bytes(), &config);
Ok(())
}

5
fuzz/fuzz_targets/differential_wasmi.rs
View File

@@ -13,9 +13,8 @@ fuzz_target!(|data: &[u8]| {
fn run(data: &[u8]) -> Result<()> {
let mut u = Unstructured::new(data);
let mut config: generators::Config = u.arbitrary()?;
config.module_config.set_differential_config();
let mut module = config.module_config.generate(&mut u)?;
module.ensure_termination(1000);
config.set_differential_config();
let module = config.generate(&mut u, Some(1000))?;
oracles::differential_wasmi_execution(&module.to_bytes(), &config);
Ok(())
}

2
fuzz/fuzz_targets/instantiate-many.rs
View File

@@ -26,7 +26,7 @@ fn run(data: &[u8]) -> Result<()> {
// Create the modules to instantiate
let modules = (0..u.int_in_range(1..=MAX_MODULES)?)
.map(|_| Ok(config.module_config.generate(&mut u)?.to_bytes()))
.map(|_| Ok(config.generate(&mut u, None)?.to_bytes()))
.collect::<Result<Vec<_>>>()?;
let max_instances = match &config.wasmtime.strategy {

23
fuzz/fuzz_targets/instantiate.rs
View File

@@ -2,7 +2,6 @@
use libfuzzer_sys::arbitrary::{Result, Unstructured};
use libfuzzer_sys::fuzz_target;
use wasmtime_fuzzing::generators::InstanceAllocationStrategy;
use wasmtime_fuzzing::oracles::Timeout;
use wasmtime_fuzzing::{generators, oracles};
@@ -28,21 +27,15 @@ fn run(data: &[u8]) -> Result<()> {
// Enable module linking for this fuzz target specifically
config.module_config.config.module_linking_enabled = u.arbitrary()?;
// When using the pooling allocator without a timeout, we must
// allow at least 1 more global because the `ensure_termination` call below
// will define one.
if let Timeout::None = timeout {
if let InstanceAllocationStrategy::Pooling { module_limits, .. } =
&mut config.wasmtime.strategy
{
module_limits.globals += 1;
}
}
let module = config.generate(
&mut u,
if let Timeout::None = timeout {
Some(1000)
} else {
None
},
)?;
let mut module = config.module_config.generate(&mut u)?;
if let Timeout::None = timeout {
module.ensure_termination(1000);
}
oracles::instantiate(&module.to_bytes(), true, &config, timeout);
Ok(())
}