Merge remote-tracking branch 'origin/main' into pch/wasi_common_cap_std

This commit is contained in:
Pat Hickey
2021-01-04 11:19:02 -08:00
41 changed files with 1005 additions and 380 deletions

View File

@@ -0,0 +1,25 @@
[package]
name = "wasmtime-bench-api"
version = "0.19.0"
authors = ["The Wasmtime Project Developers"]
description = "Exposes a benchmarking API for the Wasmtime runtime"
license = "Apache-2.0 WITH LLVM-exception"
repository = "https://github.com/bytecodealliance/wasmtime"
readme = "README.md"
edition = "2018"
publish = false
[lib]
name = "wasmtime_bench_api"
crate-type = ["rlib", "cdylib"]
# The rlib is only included here so that `cargo test` will run.
[dependencies]
anyhow = "1.0"
wasmtime = { path = "../wasmtime", default-features = false }
wasmtime-wasi = { path = "../wasi" }
wasi-common = { path = "../wasi-common" }
[dev-dependencies]
wat = "1.0"

248
crates/bench-api/src/lib.rs Normal file
View File

@@ -0,0 +1,248 @@
//! A C API for benchmarking Wasmtime's WebAssembly compilation, instantiation,
//! and execution.
//!
//! The API expects sequential calls to:
//!
//! - `wasm_bench_create`
//! - `wasm_bench_compile`
//! - `wasm_bench_instantiate`
//! - `wasm_bench_execute`
//! - `wasm_bench_free`
//!
//! You may repeat this sequence of calls multiple times to take multiple
//! measurements of compilation, instantiation, and execution time within a
//! single process.
//!
//! All API calls must happen on the same thread.
//!
//! Functions which return pointers use null as an error value. Function which
//! return `int` use `0` as OK and non-zero as an error value.
//!
//! # Example
//!
//! ```
//! use wasmtime_bench_api::*;
//!
//! let engine = unsafe { wasm_bench_create() };
//! assert!(!engine.is_null());
//!
//! let wasm = wat::parse_bytes(br#"
//! (module
//! (func $bench_start (import "bench" "start"))
//! (func $bench_end (import "bench" "end"))
//! (func $start (export "_start")
//! call $bench_start
//! i32.const 1
//! i32.const 2
//! i32.add
//! drop
//! call $bench_end
//! )
//! )
//! "#).unwrap();
//!
//! // Start your compilation timer here.
//! let code = unsafe { wasm_bench_compile(engine, wasm.as_ptr(), wasm.len()) };
//! // End your compilation timer here.
//! assert_eq!(code, OK);
//!
//! // The Wasm benchmark will expect us to provide functions to start ("bench"
//! // "start") and stop ("bench" "stop") the measurement counters/timers during
//! // execution.
//! extern "C" fn bench_start() {
//! // Start your execution timer here.
//! }
//! extern "C" fn bench_stop() {
//! // End your execution timer here.
//! }
//!
//! // Start your instantiation timer here.
//! let code = unsafe { wasm_bench_instantiate(engine, bench_start, bench_stop) };
//! // End your instantiation timer here.
//! assert_eq!(code, OK);
//!
//! // No need to start timers for the execution since, by convention, the timer
//! // functions we passed during instantiation will be called by the benchmark
//! // at the appropriate time (before and after the benchmarked section).
//! let code = unsafe { wasm_bench_execute(engine) };
//! assert_eq!(code, OK);
//!
//! unsafe {
//! wasm_bench_free(engine);
//! }
//! ```
use anyhow::{anyhow, Context, Result};
use std::os::raw::{c_int, c_void};
use std::slice;
use wasi_common::WasiCtxBuilder;
use wasmtime::{Config, Engine, Instance, Linker, Module, Store};
use wasmtime_wasi::Wasi;
pub type ExitCode = c_int;
pub const OK: ExitCode = 0;
pub const ERR: ExitCode = -1;
/// Exposes a C-compatible way of creating the engine from the bytes of a single
/// Wasm module.
///
/// This function returns a pointer to a structure that contains the engine's
/// initialized state.
#[no_mangle]
pub extern "C" fn wasm_bench_create() -> *mut c_void {
let state = Box::new(BenchState::new());
Box::into_raw(state) as _
}
/// Free the engine state allocated by this library.
#[no_mangle]
pub extern "C" fn wasm_bench_free(state: *mut c_void) {
assert!(!state.is_null());
unsafe {
Box::from_raw(state as *mut BenchState);
}
}
/// Compile the Wasm benchmark module.
#[no_mangle]
pub extern "C" fn wasm_bench_compile(
state: *mut c_void,
wasm_bytes: *const u8,
wasm_bytes_length: usize,
) -> ExitCode {
let state = unsafe { (state as *mut BenchState).as_mut().unwrap() };
let wasm_bytes = unsafe { slice::from_raw_parts(wasm_bytes, wasm_bytes_length) };
let result = state.compile(wasm_bytes).context("failed to compile");
to_exit_code(result)
}
/// Instantiate the Wasm benchmark module.
#[no_mangle]
pub extern "C" fn wasm_bench_instantiate(
state: *mut c_void,
bench_start: extern "C" fn(),
bench_end: extern "C" fn(),
) -> ExitCode {
let state = unsafe { (state as *mut BenchState).as_mut().unwrap() };
let result = state
.instantiate(bench_start, bench_end)
.context("failed to instantiate");
to_exit_code(result)
}
/// Execute the Wasm benchmark module.
#[no_mangle]
pub extern "C" fn wasm_bench_execute(state: *mut c_void) -> ExitCode {
let state = unsafe { (state as *mut BenchState).as_mut().unwrap() };
let result = state.execute().context("failed to execute");
to_exit_code(result)
}
/// Helper function for converting a Rust result to a C error code.
///
/// This will print an error indicating some information regarding the failure.
fn to_exit_code<T>(result: impl Into<Result<T>>) -> ExitCode {
match result.into() {
Ok(_) => OK,
Err(error) => {
eprintln!("{:?}", error);
ERR
}
}
}
/// This structure contains the actual Rust implementation of the state required
/// to manage the Wasmtime engine between calls.
struct BenchState {
engine: Engine,
store: Store,
module: Option<Module>,
instance: Option<Instance>,
did_execute: bool,
}
impl BenchState {
fn new() -> Self {
let mut config = Config::new();
config.wasm_simd(true);
// NB: do not configure a code cache.
let engine = Engine::new(&config);
let store = Store::new(&engine);
Self {
engine,
store,
module: None,
instance: None,
did_execute: false,
}
}
fn compile(&mut self, bytes: &[u8]) -> Result<()> {
assert!(
self.module.is_none(),
"create a new engine to repeat compilation"
);
self.module = Some(Module::from_binary(&self.engine, bytes)?);
Ok(())
}
fn instantiate(
&mut self,
bench_start: extern "C" fn(),
bench_end: extern "C" fn(),
) -> Result<()> {
assert!(
self.instance.is_none(),
"create a new engine to repeat instantiation"
);
let module = self
.module
.as_mut()
.expect("compile the module before instantiating it");
let mut linker = Linker::new(&self.store);
// Import a very restricted WASI environment.
let mut cx = WasiCtxBuilder::new();
cx.inherit_stdio();
let cx = cx.build()?;
let wasi = Wasi::new(linker.store(), cx);
wasi.add_to_linker(&mut linker)?;
// Import the specialized benchmarking functions.
linker.func("bench", "start", move || bench_start())?;
linker.func("bench", "end", move || bench_end())?;
self.instance = Some(linker.instantiate(&module)?);
Ok(())
}
fn execute(&mut self) -> Result<()> {
assert!(!self.did_execute, "create a new engine to repeat execution");
self.did_execute = true;
let instance = self
.instance
.as_ref()
.expect("instantiate the module before executing it");
let start_func = instance.get_func("_start").expect("a _start function");
let runnable_func = start_func.get0::<()>()?;
match runnable_func() {
Ok(_) => Ok(()),
Err(trap) => {
// Since _start will likely return by using the system `exit` call, we must
// check the trap code to see if it actually represents a successful exit.
match trap.i32_exit_status() {
Some(0) => Ok(()),
Some(n) => Err(anyhow!("_start exited with a non-zero code: {}", n)),
None => Err(anyhow!(
"executing the benchmark resulted in a trap: {}",
trap
)),
}
}
}
}
}

View File

@@ -13,8 +13,8 @@ edition = "2018"
[dependencies]
gimli = "0.23.0"
wasmparser = "0.70"
object = { version = "0.22.0", default-features = false, features = ["read", "write"] }
wasmparser = "0.71"
object = { version = "0.22.0", default-features = false, features = ["read_core", "elf", "write"] }
wasmtime-environ = { path = "../environ", version = "0.21.0" }
target-lexicon = { version = "0.11.0", default-features = false }
anyhow = "1.0"

View File

@@ -16,7 +16,7 @@ anyhow = "1.0"
cranelift-codegen = { path = "../../cranelift/codegen", version = "0.68.0", features = ["enable-serde"] }
cranelift-entity = { path = "../../cranelift/entity", version = "0.68.0", features = ["enable-serde"] }
cranelift-wasm = { path = "../../cranelift/wasm", version = "0.68.0", features = ["enable-serde"] }
wasmparser = "0.70"
wasmparser = "0.71"
indexmap = { version = "1.0.2", features = ["serde-1"] }
thiserror = "1.0.4"
serde = { version = "1.0.94", features = ["derive"] }

View File

@@ -5,6 +5,7 @@ edition = "2018"
name = "wasmtime-fuzzing"
publish = false
version = "0.19.0"
license = "Apache-2.0 WITH LLVM-exception"
[dependencies]
anyhow = "1.0.22"
@@ -12,10 +13,11 @@ arbitrary = { version = "0.4.1", features = ["derive"] }
env_logger = "0.8.1"
log = "0.4.8"
rayon = "1.2.1"
wasmparser = "0.70"
wasmparser = "0.71"
wasmprinter = "0.2.17"
wasmtime = { path = "../wasmtime" }
wasmtime-wast = { path = "../wast" }
wasm-encoder = "0.2"
wasm-smith = "0.3.0"
wasmi = "0.7.0"

View File

@@ -1,8 +1,11 @@
//! Generating series of `table.get` and `table.set` operations.
use arbitrary::Arbitrary;
use std::fmt::Write;
use std::ops::Range;
use wasm_encoder::{
CodeSection, EntityType, Export, ExportSection, Function, FunctionSection, ImportSection,
Instruction, Limits, Module, TableSection, TableType, TypeSection, ValType,
};
/// A description of a Wasm module that makes a series of `externref` table
/// operations.
@@ -32,7 +35,7 @@ impl TableOps {
table_size
}
/// Convert this into a WAT string.
/// Serialize this module into a Wasm binary.
///
/// The module requires a single import: `(import "" "gc" (func))`. This
/// should be a function to trigger GC.
@@ -43,32 +46,62 @@ impl TableOps {
/// The "run" function is guaranteed to terminate (no loops or recursive
/// calls), but is not guaranteed to avoid traps (might access out-of-bounds
/// of the table).
pub fn to_wat_string(&self) -> String {
let mut wat = "(module\n".to_string();
pub fn to_wasm_binary(&self) -> Vec<u8> {
let mut module = Module::new();
// Import the GC function.
wat.push_str(" (import \"\" \"gc\" (func))\n");
let mut imports = ImportSection::new();
imports.import("", Some("gc"), EntityType::Function(0));
// Define our table.
wat.push_str(" (table $table ");
write!(&mut wat, "{}", self.table_size()).unwrap();
wat.push_str(" externref)\n");
let mut tables = TableSection::new();
tables.table(TableType {
element_type: ValType::ExternRef,
limits: Limits {
min: self.table_size(),
max: None,
},
});
// Encode the types for all functions that we are using.
let mut types = TypeSection::new();
types.function(vec![], vec![]); // 0: "gc"
let mut params: Vec<ValType> = Vec::with_capacity(self.num_params() as usize);
for _i in 0..self.num_params() {
params.push(ValType::ExternRef);
}
let results = vec![];
types.function(params, results); // 1: "run"
// Define the "run" function export.
wat.push_str(r#" (func (export "run") (param"#);
for _ in 0..self.num_params() {
wat.push_str(" externref");
}
wat.push_str(")\n");
for op in self.ops.iter().take(MAX_OPS) {
wat.push_str(" ");
op.to_wat_string(&mut wat);
wat.push('\n');
}
wat.push_str(" )\n");
let mut functions = FunctionSection::new();
functions.function(1);
wat.push_str(")\n");
wat
let mut exports = ExportSection::new();
exports.export("run", Export::Function(1));
let mut params: Vec<(u32, ValType)> = Vec::with_capacity(self.num_params() as usize);
for _i in 0..self.num_params() {
params.push((0, ValType::ExternRef));
}
let mut func = Function::new(params);
for op in self.ops.iter().take(MAX_OPS) {
op.insert(&mut func);
}
let mut code = CodeSection::new();
code.function(&func);
module
.section(&types)
.section(&imports)
.section(&functions)
.section(&tables)
.section(&exports)
.section(&code);
module.finish()
}
}
@@ -77,37 +110,34 @@ pub(crate) enum TableOp {
// `(call 0)`
Gc,
// `(drop (table.get x))`
Get(u32),
Get(i32),
// `(table.set x (local.get y))`
SetFromParam(u32, u8),
SetFromParam(i32, u32),
// `(table.set x (table.get y))`
SetFromGet(u32, u32),
SetFromGet(i32, i32),
}
impl TableOp {
fn to_wat_string(&self, wat: &mut String) {
fn insert(&self, func: &mut Function) {
match self {
Self::Gc => {
wat.push_str("(call 0)");
func.instruction(Instruction::Call(0));
}
Self::Get(x) => {
wat.push_str("(drop (table.get $table (i32.const ");
write!(wat, "{}", x).unwrap();
wat.push_str(")))");
func.instruction(Instruction::I32Const(*x));
func.instruction(Instruction::TableGet { table: 0 });
func.instruction(Instruction::Drop);
}
Self::SetFromParam(x, y) => {
wat.push_str("(table.set $table (i32.const ");
write!(wat, "{}", x).unwrap();
wat.push_str(") (local.get ");
write!(wat, "{}", y).unwrap();
wat.push_str("))");
func.instruction(Instruction::I32Const(*x));
func.instruction(Instruction::LocalGet(*y));
func.instruction(Instruction::TableSet { table: 0 });
}
Self::SetFromGet(x, y) => {
wat.push_str("(table.set $table (i32.const ");
write!(wat, "{}", x).unwrap();
wat.push_str(") (table.get $table (i32.const ");
write!(wat, "{}", y).unwrap();
wat.push_str(")))");
func.instruction(Instruction::I32Const(*x));
func.instruction(Instruction::I32Const(*y));
func.instruction(Instruction::TableGet { table: 0 });
func.instruction(Instruction::TableSet { table: 0 });
}
}
}
@@ -132,17 +162,26 @@ mod tests {
let expected = r#"
(module
(import "" "gc" (func))
(table $table 10 externref)
(func (export "run") (param externref externref)
(call 0)
(drop (table.get $table (i32.const 0)))
(table.set $table (i32.const 1) (local.get 2))
(table.set $table (i32.const 3) (table.get $table (i32.const 4)))
)
)
(type (;0;) (func))
(type (;1;) (func (param externref externref)))
(import "" "gc" (func (;0;) (type 0)))
(func (;1;) (type 1) (param externref externref)
call 0
i32.const 0
table.get 0
drop
i32.const 1
local.get 2
table.set 0
i32.const 3
i32.const 4
table.get 0
table.set 0)
(table (;0;) 10 externref)
(export "run" (func 1)))
"#;
let actual = ops.to_wat_string();
let actual = ops.to_wasm_binary();
let actual = wasmprinter::print_bytes(&actual).unwrap();
assert_eq!(actual.trim(), expected.trim());
}
}

View File

@@ -40,17 +40,6 @@ fn log_wasm(wasm: &[u8]) {
}
}
fn log_wat(wat: &str) {
if !log::log_enabled!(log::Level::Debug) {
return;
}
let i = CNT.fetch_add(1, SeqCst);
let name = format!("testcase{}.wat", i);
log::debug!("wrote wat file to `{}`", name);
std::fs::write(&name, wat).expect("failed to write wat file");
}
/// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected
/// panic or segfault or anything else that can be detected "passively".
///
@@ -418,9 +407,9 @@ pub fn table_ops(config: crate::generators::Config, ops: crate::generators::tabl
let engine = Engine::new(&config);
let store = Store::new(&engine);
let wat = ops.to_wat_string();
log_wat(&wat);
let module = match Module::new(&engine, &wat) {
let wasm = ops.to_wasm_binary();
log_wasm(&wasm);
let module = match Module::new(&engine, &wasm) {
Ok(m) => m,
Err(_) => return,
};

View File

@@ -28,7 +28,7 @@ rayon = { version = "1.0", optional = true }
region = "2.1.0"
thiserror = "1.0.4"
target-lexicon = { version = "0.11.0", default-features = false }
wasmparser = "0.70"
wasmparser = "0.71"
more-asserts = "0.2.1"
anyhow = "1.0"
cfg-if = "1.0"

View File

@@ -24,7 +24,7 @@ more-asserts = "0.2.1"
smallvec = "1.0.0"
thiserror = "1.0.9"
typemap = "0.3"
wasmparser = "0.70"
wasmparser = "0.71"
[dev-dependencies]
lazy_static = "1.2"

View File

@@ -13,6 +13,6 @@ edition = "2018"
[dependencies]
lightbeam = { path = "..", version = "0.21.0" }
wasmparser = "0.70"
wasmparser = "0.71"
cranelift-codegen = { path = "../../../cranelift/codegen", version = "0.68.0" }
wasmtime-environ = { path = "../../environ", version = "0.21.0" }

View File

@@ -249,11 +249,7 @@ impl State {
let header = FileHeader {
timestamp: self.get_time_stamp(),
e_machine: self.get_e_machine(),
magic: if cfg!(target_endian = "little") {
0x4A695444
} else {
0x4454694a
},
magic: 0x4A695444,
version: 1,
size: mem::size_of::<FileHeader>() as u32,
pad1: 0,

View File

@@ -5,6 +5,7 @@ authors = ["The Wasmtime Project Developers"]
readme = "README.md"
edition = "2018"
publish = false
license = "Apache-2.0 WITH LLVM-exception"
[build-dependencies]
cfg-if = "1.0"

View File

@@ -17,7 +17,7 @@
// TODO it might worth re-investigating the suitability of this type on Windows.
use super::{fd, AsFile};
use crate::handle::{Fdflags, Filetype, Handle, HandleRights, Rights, RightsExt, Size};
use crate::handle::{Fdflags, Filestat, Filetype, Handle, HandleRights, Rights, RightsExt, Size};
use crate::sandboxed_tty_writer::SandboxedTTYWriter;
use crate::{Error, Result};
use std::any::Any;
@@ -65,6 +65,9 @@ impl Handle for Stdin {
}
Ok(())
}
fn filestat_get(&self) -> Result<Filestat> {
fd::filestat_get(&*self.as_file()?)
}
fn read_vectored(&self, iovs: &mut [io::IoSliceMut]) -> Result<usize> {
let nread = io::stdin().read_vectored(iovs)?;
Ok(nread)
@@ -111,6 +114,9 @@ impl Handle for Stdout {
}
Ok(())
}
fn filestat_get(&self) -> Result<Filestat> {
fd::filestat_get(&*self.as_file()?)
}
fn write_vectored(&self, iovs: &[io::IoSlice]) -> Result<usize> {
// lock for the duration of the scope
let stdout = io::stdout();
@@ -165,6 +171,9 @@ impl Handle for Stderr {
}
Ok(())
}
fn filestat_get(&self) -> Result<Filestat> {
fd::filestat_get(&*self.as_file()?)
}
fn write_vectored(&self, iovs: &[io::IoSlice]) -> Result<usize> {
// Always sanitize stderr, even if it's not directly connected to a tty,
// because stderr is meant for diagnostics rather than binary output,

View File

@@ -9,7 +9,8 @@ pub(crate) mod poll;
pub(crate) mod stdio;
cfg_if::cfg_if! {
if #[cfg(target_os = "linux")] {
if #[cfg(any(target_os = "linux",
target_os = "android"))] {
mod linux;
use linux as sys_impl;
} else if #[cfg(target_os = "emscripten")] {

View File

@@ -36,9 +36,15 @@ impl Dir {
}
/// Set the position of the directory stream, see `seekdir(3)`.
#[cfg(not(target_os = "android"))]
pub fn seek(&mut self, loc: SeekLoc) {
unsafe { libc::seekdir(self.0.as_ptr(), loc.0) }
// https://github.com/rust-lang/libc/pull/1996
#[cfg(not(target_os = "android"))]
use libc::seekdir;
#[cfg(target_os = "android")]
extern "C" {
fn seekdir(dirp: *mut libc::DIR, loc: libc::c_long);
}
unsafe { seekdir(self.0.as_ptr(), loc.0) }
}
/// Reset directory stream, see `rewinddir(3)`.
@@ -50,10 +56,16 @@ impl Dir {
///
/// If this location is given to `Dir::seek`, the entries up to the previously returned
/// will be omitted and the iteration will start from the currently pending directory entry.
#[cfg(not(target_os = "android"))]
#[allow(dead_code)]
pub fn tell(&self) -> SeekLoc {
let loc = unsafe { libc::telldir(self.0.as_ptr()) };
#[cfg(not(target_os = "android"))]
use libc::telldir;
#[cfg(target_os = "android")]
extern "C" {
fn telldir(dirp: *mut libc::DIR) -> libc::c_long;
}
// https://github.com/rust-lang/libc/pull/1996
let loc = unsafe { telldir(self.0.as_ptr()) };
SeekLoc(loc)
}
@@ -93,11 +105,9 @@ impl Entry {
}
}
#[cfg(not(target_os = "android"))]
#[derive(Clone, Copy, Debug)]
pub struct SeekLoc(pub(crate) libc::c_long);
#[cfg(not(target_os = "android"))]
impl SeekLoc {
pub fn to_raw(&self) -> i64 {
self.0.into()

View File

@@ -90,11 +90,26 @@ bitflags! {
const WRONLY = libc::O_WRONLY;
const RDWR = libc::O_RDWR;
#[cfg(any(target_os = "linux",
target_os = "android",
target_os = "netbsd",
target_os = "openbsd",
target_os = "wasi",
target_os = "emscripten"))]
const RSYNC = libc::O_RSYNC;
const RSYNC = {
// Have to use cfg_if: https://github.com/bitflags/bitflags/issues/137
cfg_if! {
if #[cfg(any(target_os = "linux",
target_os = "netbsd",
target_os = "openbsd",
target_os = "wasi",
target_os = "emscripten"))] {
libc::O_RSYNC
} else if #[cfg(target_os = "android")] {
// Android defines O_RSYNC as O_SYNC
libc::O_SYNC
}
}
};
const SYNC = libc::O_SYNC;
const TRUNC = libc::O_TRUNC;
#[cfg(any(target_os = "linux",

View File

@@ -53,5 +53,8 @@ pub fn utimensat(
return Err(err);
}
super::utimesat::utimesat(dirfd, path, atime, mtime, symlink_nofollow)
#[cfg(not(target_os = "android"))]
return super::utimesat::utimesat(dirfd, path, atime, mtime, symlink_nofollow);
#[cfg(target_os = "android")]
unreachable!();
}

View File

@@ -2,4 +2,5 @@ pub(crate) mod dir;
pub(crate) mod fadvise;
pub(crate) mod file;
pub(crate) mod filetime;
#[cfg(not(target_os = "android"))]
pub(crate) mod utimesat;

View File

@@ -16,7 +16,7 @@ wasmtime-jit = { path = "../jit", version = "0.21.0" }
wasmtime-cache = { path = "../cache", version = "0.21.0", optional = true }
wasmtime-profiling = { path = "../profiling", version = "0.21.0" }
target-lexicon = { version = "0.11.0", default-features = false }
wasmparser = "0.70"
wasmparser = "0.71"
anyhow = "1.0.19"
region = "2.2.0"
libc = "0.2"