From 1b8efa7bbdb9f86f0444661c06e28d9c208a9946 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Tue, 11 May 2021 01:00:34 -0700 Subject: [PATCH] Implement simple benchmarks for instantiation. This adds benchmarks around module instantiation using criterion. Both the default (i.e. on-demand) and pooling allocators are tested sequentially and in parallel using a thread pool. Instantiation is tested with an empty module, a module with a single page linear memory, a larger linear memory with a data initializer, and a "hello world" Rust WASI program. --- .github/workflows/main.yml | 13 ++ Cargo.lock | 2 + Cargo.toml | 8 +- benches/.gitignore | 1 + benches/instantiation.rs | 158 ++++++++++++++++++ benches/instantiation/data_segments.wat | 5 + benches/instantiation/empty.wat | 3 + benches/instantiation/small_memory.wat | 4 + crates/jit/src/instantiate.rs | 4 +- .../runtime/src/instance/allocator/pooling.rs | 4 +- 10 files changed, 197 insertions(+), 5 deletions(-) create mode 100644 benches/.gitignore create mode 100644 benches/instantiation.rs create mode 100644 benches/instantiation/data_segments.wat create mode 100644 benches/instantiation/empty.wat create mode 100644 benches/instantiation/small_memory.wat diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71dc0a3b3c..c0c3a26987 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -363,6 +363,19 @@ jobs: env: RUST_BACKTRACE: 1 + bench: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - run: rustup target add wasm32-wasi + - name: Install Rust + run: rustup update stable && rustup default stable + - run: cargo bench + - run: cargo bench --features uffd + # Verify that cranelift's code generation is deterministic meta_determinist_check: name: Meta deterministic check diff --git a/Cargo.lock b/Cargo.lock index aad4a2bc11..d4c86e863b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3599,6 +3599,7 @@ name = "wasmtime-cli" version = "0.26.0" dependencies = [ "anyhow", + "criterion", "env_logger 0.8.3", "file-per-thread-logger", "filecheck", @@ -3607,6 +3608,7 @@ dependencies = [ "libc", "log", "more-asserts", + "num_cpus", "object", "pretty_env_logger", "rayon", diff --git a/Cargo.toml b/Cargo.toml index bc121c2082..8691e7399e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ file-per-thread-logger = "0.1.1" wat = "1.0.37" libc = "0.2.60" log = "0.4.8" -rayon = "1.2.1" +rayon = "1.5.0" humantime = "2.0.0" wasmparser = "0.77.0" lazy_static = "1.4.0" @@ -57,6 +57,8 @@ wasmtime-runtime = { path = "crates/runtime" } tokio = { version = "1.5.0", features = ["rt", "time", "macros", "rt-multi-thread"] } tracing-subscriber = "0.2.16" wast = "35.0.0" +criterion = "0.3.4" +num_cpus = "1.13.0" [build-dependencies] anyhow = "1.0.19" @@ -116,3 +118,7 @@ required-features = ["wasmtime-wasi/tokio"] [profile.dev.package.backtrace] debug = false # FIXME(#1813) + +[[bench]] +name = "instantiation" +harness = false diff --git a/benches/.gitignore b/benches/.gitignore new file mode 100644 index 0000000000..3021dd2576 --- /dev/null +++ b/benches/.gitignore @@ -0,0 +1 @@ +instantiation/wasi.wasm diff --git a/benches/instantiation.rs b/benches/instantiation.rs new file mode 100644 index 0000000000..67109d43a0 --- /dev/null +++ b/benches/instantiation.rs @@ -0,0 +1,158 @@ +use anyhow::Result; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use rayon::{prelude::*, ThreadPoolBuilder}; +use std::{path::PathBuf, process::Command}; +use wasmtime::*; +use wasmtime_wasi::{sync::WasiCtxBuilder, Wasi}; + +fn instantiate(module: &Module) -> Result { + let store = Store::new(&module.engine()); + + // As we don't actually invoke Wasm code in this benchmark, we still add + // the WASI context to the store as it is considered part of getting a + // module that depends on WASI "ready to run". + Wasi::set_context(&store, WasiCtxBuilder::new().build()?) + .map_err(|_| anyhow::anyhow!("wasi set_context failed"))?; + + let linker = Linker::new(&store); + let instance = linker.instantiate(module)?; + + Ok(instance) +} + +fn benchmark_name<'a>(strategy: &InstanceAllocationStrategy) -> &'static str { + match strategy { + InstanceAllocationStrategy::OnDemand => "default", + #[cfg(any(not(feature = "uffd"), not(target_os = "linux")))] + InstanceAllocationStrategy::Pooling { .. } => "pooling", + #[cfg(all(feature = "uffd", target_os = "linux"))] + InstanceAllocationStrategy::Pooling { .. } => "uffd", + } +} + +fn bench_sequential(c: &mut Criterion, modules: &[&str]) { + let mut group = c.benchmark_group("sequential"); + + for strategy in &[ + // Skip the on-demand allocator when uffd is enabled + #[cfg(any(not(feature = "uffd"), not(target_os = "linux")))] + InstanceAllocationStrategy::OnDemand, + InstanceAllocationStrategy::pooling(), + ] { + for file_name in modules { + let mut path = PathBuf::new(); + path.push("benches"); + path.push("instantiation"); + path.push(file_name); + + let mut config = Config::default(); + Wasi::add_to_config(&mut config); + config.allocation_strategy(strategy.clone()); + + let engine = Engine::new(&config).expect("failed to create engine"); + let module = Module::from_file(&engine, &path) + .expect(&format!("failed to load benchmark `{}`", path.display())); + + group.bench_function(BenchmarkId::new(benchmark_name(strategy), file_name), |b| { + b.iter(|| instantiate(&module).expect("failed to instantiate module")); + }); + } + } + + group.finish(); +} + +fn bench_parallel(c: &mut Criterion) { + const PARALLEL_INSTANCES: usize = 1000; + + let mut group = c.benchmark_group("parallel"); + + for strategy in &[ + // Skip the on-demand allocator when uffd is enabled + #[cfg(any(not(feature = "uffd"), not(target_os = "linux")))] + InstanceAllocationStrategy::OnDemand, + InstanceAllocationStrategy::pooling(), + ] { + let mut config = Config::default(); + Wasi::add_to_config(&mut config); + config.allocation_strategy(strategy.clone()); + + let engine = Engine::new(&config).expect("failed to create engine"); + let module = Module::from_file(&engine, "benches/instantiation/wasi.wasm") + .expect("failed to load WASI example module"); + + for threads in 1..=num_cpus::get_physical() { + let pool = ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .unwrap(); + + group.bench_function( + BenchmarkId::new( + benchmark_name(strategy), + format!( + "{} instances with {} thread{}", + PARALLEL_INSTANCES, + threads, + if threads == 1 { "" } else { "s" } + ), + ), + |b| { + b.iter(|| { + pool.install(|| { + (0..PARALLEL_INSTANCES).into_par_iter().for_each(|_| { + instantiate(&module).expect("failed to instantiate module"); + }) + }) + }); + }, + ); + } + } + + group.finish(); +} + +fn build_wasi_example() { + println!("Building WASI example module..."); + if !Command::new("cargo") + .args(&[ + "build", + "--release", + "-p", + "example-wasi-wasm", + "--target", + "wasm32-wasi", + ]) + .spawn() + .expect("failed to run cargo to build WASI example") + .wait() + .expect("failed to wait for cargo to build") + .success() + { + panic!("failed to build WASI example for target `wasm32-wasi`"); + } + + std::fs::copy( + "target/wasm32-wasi/release/wasi.wasm", + "benches/instantiation/wasi.wasm", + ) + .expect("failed to copy WASI example module"); +} + +fn bench_instantiation(c: &mut Criterion) { + build_wasi_example(); + bench_sequential( + c, + &[ + "empty.wat", + "small_memory.wat", + "data_segments.wat", + "wasi.wasm", + ], + ); + bench_parallel(c); +} + +criterion_group!(benches, bench_instantiation); +criterion_main!(benches); diff --git a/benches/instantiation/data_segments.wat b/benches/instantiation/data_segments.wat new file mode 100644 index 0000000000..876d891291 --- /dev/null +++ b/benches/instantiation/data_segments.wat @@ -0,0 +1,5 @@ +(module + (memory 17) + (func (export "_start")) + (data (i32.const 0) "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789") +) diff --git a/benches/instantiation/empty.wat b/benches/instantiation/empty.wat new file mode 100644 index 0000000000..6e9f4f5ffd --- /dev/null +++ b/benches/instantiation/empty.wat @@ -0,0 +1,3 @@ +(module + (func (export "_start")) +) diff --git a/benches/instantiation/small_memory.wat b/benches/instantiation/small_memory.wat new file mode 100644 index 0000000000..b3410b71d9 --- /dev/null +++ b/benches/instantiation/small_memory.wat @@ -0,0 +1,4 @@ +(module + (memory 1) + (func (export "_start")) +) diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index dd3f3577c7..70894bdfeb 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -35,7 +35,7 @@ pub enum SetupError { #[error("Validation error: {0}")] Validate(String), - /// A wasm translation error occured. + /// A wasm translation error occurred. #[error("WebAssembly failed to compile")] Compile(#[from] CompileError), @@ -44,7 +44,7 @@ pub enum SetupError { #[error("Instantiation failed during setup")] Instantiate(#[from] InstantiationError), - /// Debug information generation error occured. + /// Debug information generation error occurred. #[error("Debug information error")] DebugInfo(#[from] anyhow::Error), } diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 7a17b2b143..a0d1e85672 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -1610,7 +1610,7 @@ mod test { } #[test] - fn test_pooling_allocator_with_memory_pages_exeeded() { + fn test_pooling_allocator_with_memory_pages_exceeded() { assert_eq!( PoolingInstanceAllocator::new( PoolingAllocationStrategy::Random, @@ -1631,7 +1631,7 @@ mod test { } #[test] - fn test_pooling_allocator_with_reservation_size_exeeded() { + fn test_pooling_allocator_with_reservation_size_exceeded() { assert_eq!( PoolingInstanceAllocator::new( PoolingAllocationStrategy::Random,