T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into pch/wiggle_sync_shimming

Browse Source
This commit is contained in:
Pat Hickey
2021-05-06 17:54:03 -07:00
parent e9f410d1db 002b0744eb
commit b7593cb8fe
142 changed files with 4443 additions and 1771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
Cargo.lock generated
View File

@@ -189,30 +189,6 @@ dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.55.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75b13ce559e6433d360c26305643803cb52cfbabbc2b9c47ce04a58493dfb443"
dependencies = [
"bitflags",
"cexpr",
"cfg-if 0.1.10",
"clang-sys",
"clap",
"env_logger 0.7.1",
"lazy_static",
"lazycell",
"log",
"peeking_take_while",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"which",
]
[[package]]
name = "bindgen"
version = "0.57.0"
@@ -293,9 +269,9 @@ checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
[[package]]
name = "cap-fs-ext"
version = "0.13.7"
version = "0.13.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee87a3a916d6f051fc6809c39c4627f0c3a73b2a803bcfbb5fdf2bdfa1da0cb"
checksum = "c4a7f9bdd49f3979c659811723041929803c986b3ed5381af726a4082e806c27"
dependencies = [
"cap-primitives",
"cap-std",
@@ -306,9 +282,9 @@ dependencies = [
[[package]]
name = "cap-primitives"
version = "0.13.7"
version = "0.13.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c3e3ea29994a34f3bc67b5396a43c87597d302d9e2e5e3b3d5ba952d86c7b41"
checksum = "0e206e688244a8f8158f91d77d3d8ed5891eb27aef2542bb26ceb98b970d5597"
dependencies = [
"errno",
"fs-set-times",
@@ -326,18 +302,18 @@ dependencies = [
[[package]]
name = "cap-rand"
version = "0.13.7"
version = "0.13.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0418058b38db7efc6021c5ce012e3a39c57e1a4d7bf2ddcd3789771de505d2f"
checksum = "0842ea6c9a2e078ab901b3b805608af65e11fcbfb87024d37fb129b2b74cb1ac"
dependencies = [
"rand 0.8.3",
]
[[package]]
name = "cap-std"
version = "0.13.7"
version = "0.13.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5f20cbb3055e9c72b16ba45913fe9f92836d2aa7a880e1ffacb8d244f454319"
checksum = "36d8cb4b761a2fb1ccf0e92d795c717bba9dcb1e54636ee0579fca628abfa305"
dependencies = [
"cap-primitives",
"posish",
@@ -358,9 +334,9 @@ dependencies = [
[[package]]
name = "cap-time-ext"
version = "0.13.7"
version = "0.13.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b684f9db089b0558520076b4eeda2b719a5c4c06f329be96c9497f2b48c3944"
checksum = "bc5c279d213dfe8f71c02c4cd740bd130d1dce39f45fdcdb745ae3de8a777fbd"
dependencies = [
"cap-primitives",
"once_cell",
@@ -561,7 +537,6 @@ name = "cranelift-codegen"
version = "0.73.0"
dependencies = [
"bincode",
"byteorder",
"cranelift-bforest",
"cranelift-codegen-meta",
"cranelift-codegen-shared",
@@ -577,7 +552,6 @@ dependencies = [
"smallvec",
"souper-ir",
"target-lexicon",
"thiserror",
"wast 35.0.2",
]
@@ -608,7 +582,6 @@ name = "cranelift-filetests"
version = "0.73.0"
dependencies = [
"anyhow",
"byteorder",
"cranelift-codegen",
"cranelift-frontend",
"cranelift-interpreter",
@@ -678,7 +651,6 @@ dependencies = [
"cranelift-entity",
"hashbrown",
"log",
"thiserror",
]
[[package]]
@@ -718,7 +690,6 @@ dependencies = [
"cranelift-codegen",
"smallvec",
"target-lexicon",
"thiserror",
]
[[package]]
@@ -1389,9 +1360,9 @@ checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
[[package]]
name = "iter-enum"
version = "0.2.7"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cad34f24d3b48ceffdff38af2df5ce1b7d1d9cc113e503d8e86fe8cdb889c871"
checksum = "4f947f0d9df7e69c4df60a950c0a83741455bb9ebd8fd9b5a87994dda4dbb005"
dependencies = [
"derive_utils",
"quote",
@@ -1784,22 +1755,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openvino"
version = "0.1.8"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43eeb44285b7ce8e2012b92bec32968622e1dad452e812e6edea9e001e5e9410"
checksum = "0cb74b3d8c653f7a9928bda494d329e6363ea0b428d3a3e5805b45ebb74ace76"
dependencies = [
"openvino-sys",
"thiserror",
]
[[package]]
name = "openvino-sys"
version = "0.1.8"
name = "openvino-finder"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fb64bef270a1ff665b0b2e28ebfa213e6205a007ce88223d020730225d6008f"
checksum = "426587a131841eb1e1111b0fea96cbd4fd0fd5d7b6526fb9c41400587d1c525c"
[[package]]
name = "openvino-sys"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83d5e5d5e913f4e9aa42b2a7ae9c8719aedb4bc0eb443bf92f07d9ee9a05e7b1"
dependencies = [
"bindgen 0.55.1",
"cmake",
"lazy_static",
"libloading",
"openvino-finder",
]
[[package]]
@@ -3066,7 +3045,7 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ada4f4ae167325015f52cc65f9fb6c251b868d8fb3b6dd0ce2d60e497c4870a"
dependencies = [
"bindgen 0.57.0",
"bindgen",
"cc",
"cfg-if 0.1.10",
]
@@ -3449,7 +3428,6 @@ dependencies = [
name = "wasmtime-environ"
version = "0.26.0"
dependencies = [
"anyhow",
"cfg-if 1.0.0",
"cranelift-codegen",
"cranelift-entity",
@@ -3458,7 +3436,6 @@ dependencies = [
"indexmap",
"log",
"more-asserts",
"region",
"serde",
"thiserror",
"wasmparser",
@@ -3723,15 +3700,6 @@ dependencies = [
"wast 35.0.2",
]
[[package]]
name = "which"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
dependencies = [
"libc",
]
[[package]]
name = "wiggle"
version = "0.26.0"

2
Cargo.toml
View File

@@ -87,7 +87,7 @@ members = [
]
[features]
default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation"]
default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation", "wasi-nn"]
lightbeam = ["wasmtime/lightbeam"]
jitdump = ["wasmtime/jitdump"]
vtune = ["wasmtime/vtune"]

13
RELEASES.md
View File

@@ -4,6 +4,19 @@
## Unreleased
### Added
* Added `Store::with_limits`, `StoreLimits`, and `ResourceLimiter` to the
Wasmtime API to help with enforcing resource limits at runtime. The
`ResourceLimiter` trait can be implemented by custom resource limiters to
decide if linear memories or tables can be grown.
### Changed
* Breaking: `Memory::new` has been changed to return `Result` as creating a
host memory object is now a fallible operation when the initial size of
the memory exceeds the store limits.
## 0.26.0
Released 2021-04-05.

4
build.rs
View File

@@ -220,10 +220,6 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
},
"Cranelift" => match (testsuite, testname) {
("simd", _) if cfg!(feature = "old-x86-backend") => return true, // skip all SIMD tests on old backend.
// These are only implemented on x64.
("simd", "simd_i64x2_arith2") | ("simd", "simd_boolean") => {
return !platform_is_x64() || cfg!(feature = "old-x86-backend")
}
// These are new instructions that are not really implemented in any backend.
("simd", "simd_i8x16_arith2")
| ("simd", "simd_conversions")

2
ci/run-wasi-crypto-example.sh
View File

@@ -7,4 +7,4 @@ pushd "$RUST_BINDINGS"
cargo build --release --target=wasm32-wasi
popd
cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm"
cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm" --wasi-modules=experimental-wasi-crypto

2
ci/run-wasi-nn-example.sh
View File

@@ -37,7 +37,7 @@ cp target/wasm32-wasi/release/wasi-nn-example.wasm $TMP_DIR
popd
# Run the example in Wasmtime (note that the example uses `fixture` as the expected location of the model/tensor files).
OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo run --features wasi-nn -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm
cargo run -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm --wasi-modules=experimental-wasi-nn
# Clean up the temporary directory only if it was not specified (users may want to keep the directory around).
if [[ $REMOVE_TMP_DIR -eq 1 ]]; then

2
cranelift/codegen/Cargo.toml
View File

@@ -23,8 +23,6 @@ serde = { version = "1.0.94", features = ["derive"], optional = true }
bincode = { version = "1.2.1", optional = true }
gimli = { version = "0.23.0", default-features = false, features = ["write"], optional = true }
smallvec = { version = "1.6.1" }
thiserror = "1.0.4"
byteorder = { version = "1.3.2", default-features = false }
peepmatic = { path = "../peepmatic", optional = true, version = "0.73.0" }
peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.73.0" }
peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.73.0" }

3
cranelift/codegen/src/binemit/mod.rs
View File

@@ -60,6 +60,8 @@ pub enum Reloc {
Arm64Call,
/// RISC-V call target
RiscvCall,
/// s390x PC-relative 4-byte offset
S390xPCRel32Dbl,
/// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
ElfX86_64TlsGd,
@@ -75,6 +77,7 @@ impl fmt::Display for Reloc {
match *self {
Self::Abs4 => write!(f, "Abs4"),
Self::Abs8 => write!(f, "Abs8"),
Self::S390xPCRel32Dbl => write!(f, "PCRel32Dbl"),
Self::X86PCRel4 => write!(f, "PCRel4"),
Self::X86PCRelRodata4 => write!(f, "PCRelRodata4"),
Self::X86CallPCRel4 => write!(f, "CallPCRel4"),

30
cranelift/codegen/src/data_value.rs
View File

@@ -5,7 +5,6 @@ use crate::ir::{types, ConstantData, Type};
use core::convert::TryInto;
use core::fmt::{self, Display, Formatter};
use core::ptr;
use thiserror::Error;
/// Represent a data value. Where [Value] is an SSA reference, [DataValue] is the type + value
/// that would be referred to by a [Value].
@@ -97,15 +96,38 @@ impl DataValue {
}
/// Record failures to cast [DataValue].
#[derive(Error, Debug, PartialEq)]
#[derive(Debug, PartialEq)]
#[allow(missing_docs)]
pub enum DataValueCastFailure {
#[error("unable to cast data value of type {0} to type {1}")]
TryInto(Type, Type),
#[error("unable to cast i64({0}) to a data value of type {1}")]
FromInteger(i64, Type),
}
// This is manually implementing Error and Display instead of using thiserror to reduce the amount
// of dependencies used by Cranelift.
impl std::error::Error for DataValueCastFailure {}
impl Display for DataValueCastFailure {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
DataValueCastFailure::TryInto(from, to) => {
write!(
f,
"unable to cast data value of type {} to type {}",
from, to
)
}
DataValueCastFailure::FromInteger(val, to) => {
write!(
f,
"unable to cast i64({}) to a data value of type {}",
val, to
)
}
}
}
}
/// Helper for creating conversion implementations for [DataValue].
macro_rules! build_conversion_impl {
( $rust_ty:ty, $data_value_ty:ident, $cranelift_ty:ident ) => {

6
cranelift/codegen/src/ir/entities.rs
View File

@@ -146,7 +146,7 @@ impl StackSlot {
/// [`VmContext`](super::GlobalValueData::VMContext) using
/// [`FuncEnvironment::make_global`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_global).
/// - When compiling to native code, you can use it for objects in static memory with
/// [`Module::declare_data_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html#method.declare_data_in_func).
/// [`Module::declare_data_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html#method.declare_data_in_func).
/// - For any compilation target, it can be registered with
/// [`FunctionBuilder::create_global_value`](https://docs.rs/cranelift-frontend/*/cranelift_frontend/struct.FunctionBuilder.html#method.create_global_value).
///
@@ -264,9 +264,9 @@ impl JumpTable {
///
/// - [`FunctionBuilder::import_function`](https://docs.rs/cranelift-frontend/*/cranelift_frontend/struct.FunctionBuilder.html#method.import_function)
/// for external functions
/// - [`Module::declare_func_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html#method.declare_func_in_func)
/// - [`Module::declare_func_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html#method.declare_func_in_func)
/// for functions declared elsewhere in the same native
/// [`Module`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html)
/// [`Module`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html)
/// - [`FuncEnvironment::make_direct_func`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_direct_func)
/// for functions declared in the same WebAssembly
/// [`FuncEnvironment`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_direct_func)

2
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -603,6 +603,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> (u64, SmallVec<[Inst; 16]>) {
let mut insts = SmallVec::new();
let (clobbered_int, clobbered_vec) = get_regs_saved_in_prologue(call_conv, clobbers);
@@ -819,6 +820,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();
let (clobbered_int, clobbered_vec) = get_regs_saved_in_prologue(call_conv, clobbers);

17
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -427,6 +427,15 @@ fn enc_vec_rr_misc(qu: u32, size: u32, bits_12_16: u32, rd: Writable<Reg>, rn: R
| machreg_to_vec(rd.to_reg())
}
fn enc_vec_rr_pair(bits_12_16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
debug_assert_eq!(bits_12_16 & 0b11111, bits_12_16);
0b010_11110_11_11000_11011_10_00000_00000
| bits_12_16 << 12
| machreg_to_vec(rn) << 5
| machreg_to_vec(rd.to_reg())
}
fn enc_vec_lanes(q: u32, u: u32, size: u32, opcode: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
debug_assert_eq!(q & 0b1, q);
debug_assert_eq!(u & 0b1, u);
@@ -1628,6 +1637,7 @@ impl MachInstEmit for Inst {
debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16);
(0b0, 0b00101, enc_size)
}
VecMisc2::Cmeq0 => (0b0, 0b01001, enc_size),
};
sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn));
}
@@ -2054,6 +2064,13 @@ impl MachInstEmit for Inst {
| machreg_to_vec(rd.to_reg()),
);
}
&Inst::VecRRPair { op, rd, rn } => {
let bits_12_16 = match op {
VecPairOp::Addp => 0b11011,
};
sink.put4(enc_vec_rr_pair(bits_12_16, rd, rn));
}
&Inst::VecRRR {
rd,
rn,

21
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -2311,6 +2311,16 @@ fn test_aarch64_binemit() {
"sqxtun v16.8b, v23.8h",
));
insns.push((
Inst::VecRRPair {
op: VecPairOp::Addp,
rd: writable_vreg(0),
rn: vreg(30),
},
"C0BBF15E",
"addp d0, v30.2d",
));
insns.push((
Inst::VecRRR {
alu_op: VecALUOp::Sqadd,
@@ -3803,6 +3813,17 @@ fn test_aarch64_binemit() {
"cnt v23.8b, v5.8b",
));
insns.push((
Inst::VecMisc {
op: VecMisc2::Cmeq0,
rd: writable_vreg(12),
rn: vreg(27),
size: VectorSize::Size16x8,
},
"6C9B604E",
"cmeq v12.8h, v27.8h, #0",
));
insns.push((
Inst::VecLanes {
op: VecLanesOp::Uminv,

100
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -334,6 +334,8 @@ pub enum VecMisc2 {
Frintp,
/// Population count per byte
Cnt,
/// Compare bitwise equal to 0
Cmeq0,
}
/// A Vector narrowing operation with two registers.
@@ -347,6 +349,13 @@ pub enum VecMiscNarrowOp {
Sqxtun,
}
/// A vector operation on a pair of elements with one register.
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum VecPairOp {
/// Add pair of elements
Addp,
}
/// An operation across the lanes of vectors.
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum VecLanesOp {
@@ -1011,6 +1020,13 @@ pub enum Inst {
high_half: bool,
},
/// 1-operand vector instruction that operates on a pair of elements.
VecRRPair {
op: VecPairOp,
rd: Writable<Reg>,
rn: Reg,
},
/// A vector ALU op.
VecRRR {
alu_op: VecALUOp,
@@ -2028,6 +2044,10 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_def(rd);
}
}
&Inst::VecRRPair { rd, rn, .. } => {
collector.add_def(rd);
collector.add_use(rn);
}
&Inst::VecRRR {
alu_op, rd, rn, rm, ..
} => {
@@ -2816,6 +2836,14 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
map_def(mapper, rd);
}
}
&mut Inst::VecRRPair {
ref mut rd,
ref mut rn,
..
} => {
map_def(mapper, rd);
map_use(mapper, rn);
}
&mut Inst::VecRRR {
alu_op,
ref mut rd,
@@ -3856,6 +3884,15 @@ impl Inst {
};
format!("{} {}, {}", op, rd, rn)
}
&Inst::VecRRPair { op, rd, rn } => {
let op = match op {
VecPairOp::Addp => "addp",
};
let rd = show_vreg_scalar(rd.to_reg(), mb_rru, ScalarSize::Size64);
let rn = show_vreg_vector(rn, mb_rru, VectorSize::Size64x2);
format!("{} {}, {}", op, rd, rn)
}
&Inst::VecRRR {
rd,
rn,
@@ -3919,43 +3956,44 @@ impl Inst {
format!("{} {}, {}, {}", op, rd, rn, rm)
}
&Inst::VecMisc { op, rd, rn, size } => {
let is_shll = op == VecMisc2::Shll;
let suffix = match (is_shll, size) {
(true, VectorSize::Size8x8) => ", #8",
(true, VectorSize::Size16x4) => ", #16",
(true, VectorSize::Size32x2) => ", #32",
_ => "",
};
let (op, size) = match op {
VecMisc2::Not => (
"mvn",
if size.is_128bits() {
let (op, rd_size, size, suffix) = match op {
VecMisc2::Not => {
let size = if size.is_128bits() {
VectorSize::Size8x16
} else {
VectorSize::Size8x8
};
("mvn", size, size, "")
}
VecMisc2::Neg => ("neg", size, size, ""),
VecMisc2::Abs => ("abs", size, size, ""),
VecMisc2::Fabs => ("fabs", size, size, ""),
VecMisc2::Fneg => ("fneg", size, size, ""),
VecMisc2::Fsqrt => ("fsqrt", size, size, ""),
VecMisc2::Rev64 => ("rev64", size, size, ""),
VecMisc2::Shll => (
"shll",
size.widen(),
size,
match size {
VectorSize::Size8x8 => ", #8",
VectorSize::Size16x4 => ", #16",
VectorSize::Size32x2 => ", #32",
_ => panic!("Unexpected vector size: {:?}", size),
},
),
VecMisc2::Neg => ("neg", size),
VecMisc2::Abs => ("abs", size),
VecMisc2::Fabs => ("fabs", size),
VecMisc2::Fneg => ("fneg", size),
VecMisc2::Fsqrt => ("fsqrt", size),
VecMisc2::Rev64 => ("rev64", size),
VecMisc2::Shll => ("shll", size),
VecMisc2::Fcvtzs => ("fcvtzs", size),
VecMisc2::Fcvtzu => ("fcvtzu", size),
VecMisc2::Scvtf => ("scvtf", size),
VecMisc2::Ucvtf => ("ucvtf", size),
VecMisc2::Frintn => ("frintn", size),
VecMisc2::Frintz => ("frintz", size),
VecMisc2::Frintm => ("frintm", size),
VecMisc2::Frintp => ("frintp", size),
VecMisc2::Cnt => ("cnt", size),
VecMisc2::Fcvtzs => ("fcvtzs", size, size, ""),
VecMisc2::Fcvtzu => ("fcvtzu", size, size, ""),
VecMisc2::Scvtf => ("scvtf", size, size, ""),
VecMisc2::Ucvtf => ("ucvtf", size, size, ""),
VecMisc2::Frintn => ("frintn", size, size, ""),
VecMisc2::Frintz => ("frintz", size, size, ""),
VecMisc2::Frintm => ("frintm", size, size, ""),
VecMisc2::Frintp => ("frintp", size, size, ""),
VecMisc2::Cnt => ("cnt", size, size, ""),
VecMisc2::Cmeq0 => ("cmeq", size, size, ", #0"),
};
let rd_size = if is_shll { size.widen() } else { size };
let rd = show_vreg_vector(rd.to_reg(), mb_rru, rd_size);
let rn = show_vreg_vector(rn, mb_rru, size);
format!("{} {}, {}{}", op, rd, rn, suffix)

4
cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs
View File

@@ -56,8 +56,8 @@ impl crate::isa::unwind::systemv::RegisterMapper<Reg> for RegisterMapper {
fn sp(&self) -> u16 {
regs::stack_reg().get_hw_encoding().into()
}
fn fp(&self) -> u16 {
regs::fp_reg().get_hw_encoding().into()
fn fp(&self) -> Option<u16> {
Some(regs::fp_reg().get_hw_encoding().into())
}
fn lr(&self) -> Option<u16> {
Some(regs::link_reg().get_hw_encoding().into())

75
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -1950,6 +1950,40 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}
}
Opcode::VallTrue if ctx.input_ty(insn, 0) == I64X2 => {
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let tmp = ctx.alloc_tmp(I64X2).only_reg().unwrap();
// cmeq vtmp.2d, vm.2d, #0
// addp dtmp, vtmp.2d
// fcmp dtmp, dtmp
// cset xd, eq
//
// Note that after the ADDP the value of the temporary register will
// be either 0 when all input elements are true, i.e. non-zero, or a
// NaN otherwise (either -1 or -2 when represented as an integer);
// NaNs are the only floating-point numbers that compare unequal to
// themselves.
ctx.emit(Inst::VecMisc {
op: VecMisc2::Cmeq0,
rd: tmp,
rn: rm,
size: VectorSize::Size64x2,
});
ctx.emit(Inst::VecRRPair {
op: VecPairOp::Addp,
rd: tmp,
rn: tmp.to_reg(),
});
ctx.emit(Inst::FpuCmp64 {
rn: tmp.to_reg(),
rm: tmp.to_reg(),
});
materialize_bool_result(ctx, insn, rd, Cond::Eq);
}
Opcode::VanyTrue | Opcode::VallTrue => {
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
@@ -2180,6 +2214,47 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
size: VectorSize::Size32x4,
});
}
I64X2 => {
// mov dst_r, src_v.d[0]
// mov tmp_r0, src_v.d[1]
// lsr dst_r, dst_r, #63
// lsr tmp_r0, tmp_r0, #63
// add dst_r, dst_r, tmp_r0, lsl #1
ctx.emit(Inst::MovFromVec {
rd: dst_r,
rn: src_v,
idx: 0,
size: VectorSize::Size64x2,
});
ctx.emit(Inst::MovFromVec {
rd: tmp_r0,
rn: src_v,
idx: 1,
size: VectorSize::Size64x2,
});
ctx.emit(Inst::AluRRImmShift {
alu_op: ALUOp::Lsr64,
rd: dst_r,
rn: dst_r.to_reg(),
immshift: ImmShift::maybe_from_u64(63).unwrap(),
});
ctx.emit(Inst::AluRRImmShift {
alu_op: ALUOp::Lsr64,
rd: tmp_r0,
rn: tmp_r0.to_reg(),
immshift: ImmShift::maybe_from_u64(63).unwrap(),
});
ctx.emit(Inst::AluRRRShift {
alu_op: ALUOp::Add32,
rd: dst_r,
rn: dst_r.to_reg(),
rm: tmp_r0.to_reg(),
shiftop: ShiftOpAndAmt::new(
ShiftOp::LSL,
ShiftOpShiftImm::maybe_from_shift(1).unwrap(),
),
});
}
_ => panic!("arm64 isel: VhighBits unhandled, ty = {:?}", ty),
}
}

2
cranelift/codegen/src/isa/arm32/abi.rs
View File

@@ -319,6 +319,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
_flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> (u64, SmallVec<[Inst; 16]>) {
let mut insts = SmallVec::new();
if fixed_frame_storage_size > 0 {
@@ -348,6 +349,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
_flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
_fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();
let clobbered_vec = get_callee_saves(clobbers);

20
cranelift/codegen/src/isa/mod.rs
View File

@@ -69,7 +69,6 @@ use core::fmt;
use core::fmt::{Debug, Formatter};
use core::hash::Hasher;
use target_lexicon::{triple, Architecture, OperatingSystem, PointerWidth, Triple};
use thiserror::Error;
#[cfg(feature = "riscv")]
mod riscv;
@@ -178,17 +177,30 @@ pub fn lookup_by_name(name: &str) -> Result<Builder, LookupError> {
}
/// Describes reason for target lookup failure
#[derive(Error, PartialEq, Eq, Copy, Clone, Debug)]
#[derive(PartialEq, Eq, Copy, Clone, Debug)]
pub enum LookupError {
/// Support for this target was disabled in the current build.
#[error("Support for this target is disabled")]
SupportDisabled,
/// Support for this target has not yet been implemented.
#[error("Support for this target has not been implemented yet")]
Unsupported,
}
// This is manually implementing Error and Display instead of using thiserror to reduce the amount
// of dependencies used by Cranelift.
impl std::error::Error for LookupError {}
impl fmt::Display for LookupError {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
LookupError::SupportDisabled => write!(f, "Support for this target is disabled"),
LookupError::Unsupported => {
write!(f, "Support for this target has not been implemented yet")
}
}
}
}
/// Builder for a `TargetIsa`.
/// Modify the ISA-specific settings before creating the `TargetIsa` trait object with `finish`.
#[derive(Clone)]

5
cranelift/codegen/src/isa/unwind.rs
View File

@@ -225,6 +225,11 @@ pub enum UnwindInst {
/// the clobber area.
offset_downward_to_clobbers: u32,
},
/// The stack pointer was adjusted to allocate the stack.
StackAlloc {
/// Size to allocate.
size: u32,
},
/// The stack slot at the given offset from the clobber-area base has been
/// used to save the given register.
///

64
cranelift/codegen/src/isa/unwind/systemv.rs
View File

@@ -6,7 +6,6 @@ use crate::isa::unwind::UnwindInst;
use crate::result::{CodegenError, CodegenResult};
use alloc::vec::Vec;
use gimli::write::{Address, FrameDescriptionEntry};
use thiserror::Error;
#[cfg(feature = "enable-serde")]
use serde::{Deserialize, Serialize};
@@ -15,16 +14,32 @@ type Register = u16;
/// Enumerate the errors possible in mapping Cranelift registers to their DWARF equivalent.
#[allow(missing_docs)]
#[derive(Error, Debug, PartialEq, Eq)]
#[derive(Debug, PartialEq, Eq)]
pub enum RegisterMappingError {
#[error("unable to find bank for register info")]
MissingBank,
#[error("register mapping is currently only implemented for x86_64")]
UnsupportedArchitecture,
#[error("unsupported register bank: {0}")]
UnsupportedRegisterBank(&'static str),
}
// This is manually implementing Error and Display instead of using thiserror to reduce the amount
// of dependencies used by Cranelift.
impl std::error::Error for RegisterMappingError {}
impl std::fmt::Display for RegisterMappingError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
RegisterMappingError::MissingBank => write!(f, "unable to find bank for register info"),
RegisterMappingError::UnsupportedArchitecture => write!(
f,
"register mapping is currently only implemented for x86_64"
),
RegisterMappingError::UnsupportedRegisterBank(bank) => {
write!(f, "unsupported register bank: {}", bank)
}
}
}
}
// This mirrors gimli's CallFrameInstruction, but is serializable
// This excludes CfaExpression, Expression, ValExpression due to
// https://github.com/gimli-rs/gimli/issues/513.
@@ -122,8 +137,10 @@ pub(crate) trait RegisterMapper<Reg> {
fn map(&self, reg: Reg) -> Result<Register, RegisterMappingError>;
/// Gets stack pointer register.
fn sp(&self) -> Register;
/// Gets the frame pointer register.
fn fp(&self) -> Register;
/// Gets the frame pointer register, if any.
fn fp(&self) -> Option<Register> {
None
}
/// Gets the link register, if any.
fn lr(&self) -> Option<Register> {
None
@@ -151,6 +168,7 @@ pub(crate) fn create_unwind_info_from_insts<MR: RegisterMapper<regalloc::Reg>>(
) -> CodegenResult<UnwindInfo> {
let mut instructions = vec![];
let mut cfa_offset = 0;
let mut clobber_offset_to_cfa = 0;
for &(instruction_offset, ref inst) in insts {
match inst {
@@ -163,10 +181,14 @@ pub(crate) fn create_unwind_info_from_insts<MR: RegisterMapper<regalloc::Reg>>(
instruction_offset,
CallFrameInstruction::CfaOffset(offset_upward_to_caller_sp as i32),
));
// Note that we saved the old FP value on the stack.
// Note that we saved the old FP value on the stack. Use of this
// operation implies that the target defines a FP register.
instructions.push((
instruction_offset,
CallFrameInstruction::Offset(mr.fp(), -(offset_upward_to_caller_sp as i32)),
CallFrameInstruction::Offset(
mr.fp().unwrap(),
-(offset_upward_to_caller_sp as i32),
),
));
// If there is a link register on this architecture, note that
// we saved it as well.
@@ -188,15 +210,29 @@ pub(crate) fn create_unwind_info_from_insts<MR: RegisterMapper<regalloc::Reg>>(
// Define CFA in terms of FP. Note that we assume it was already
// defined correctly in terms of the current SP, and FP has just
// been set to the current SP, so we do not need to change the
// offset, only the register.
instructions.push((
instruction_offset,
CallFrameInstruction::CfaRegister(mr.fp()),
));
// offset, only the register. (This is done only if the target
// defines a frame pointer register.)
if let Some(fp) = mr.fp() {
instructions.push((instruction_offset, CallFrameInstruction::CfaRegister(fp)));
}
// Record initial CFA offset. This will be used with later
// StackAlloc calls if we do not have a frame pointer.
cfa_offset = offset_upward_to_caller_sp;
// Record distance from CFA downward to clobber area so we can
// express clobber offsets later in terms of CFA.
clobber_offset_to_cfa = offset_upward_to_caller_sp + offset_downward_to_clobbers;
}
&UnwindInst::StackAlloc { size } => {
// If we do not use a frame pointer, we need to update the
// CFA offset whenever the stack pointer changes.
if mr.fp().is_none() {
cfa_offset += size;
instructions.push((
instruction_offset,
CallFrameInstruction::CfaOffset(cfa_offset as i32),
));
}
}
&UnwindInst::SaveReg {
clobber_offset,
reg,

27
cranelift/codegen/src/isa/unwind/winx64.rs
View File

@@ -3,7 +3,6 @@
use crate::isa::unwind::input;
use crate::result::{CodegenError, CodegenResult};
use alloc::vec::Vec;
use byteorder::{ByteOrder, LittleEndian};
use log::warn;
#[cfg(feature = "enable-serde")]
use serde::{Deserialize, Serialize};
@@ -31,13 +30,13 @@ impl<'a> Writer<'a> {
self.offset += 1;
}
fn write_u16<T: ByteOrder>(&mut self, v: u16) {
T::write_u16(&mut self.buf[self.offset..(self.offset + 2)], v);
fn write_u16_le(&mut self, v: u16) {
self.buf[self.offset..(self.offset + 2)].copy_from_slice(&v.to_le_bytes());
self.offset += 2;
}
fn write_u32<T: ByteOrder>(&mut self, v: u32) {
T::write_u32(&mut self.buf[self.offset..(self.offset + 4)], v);
fn write_u32_le(&mut self, v: u32) {
self.buf[self.offset..(self.offset + 4)].copy_from_slice(&v.to_le_bytes());
self.offset += 4;
}
}
@@ -121,11 +120,11 @@ impl UnwindCode {
let scaled_stack_offset = stack_offset / 16;
if scaled_stack_offset <= core::u16::MAX as u32 {
writer.write_u8((*reg << 4) | (op_small as u8));
writer.write_u16::<LittleEndian>(scaled_stack_offset as u16);
writer.write_u16_le(scaled_stack_offset as u16);
} else {
writer.write_u8((*reg << 4) | (op_large as u8));
writer.write_u16::<LittleEndian>(*stack_offset as u16);
writer.write_u16::<LittleEndian>((stack_offset >> 16) as u16);
writer.write_u16_le(*stack_offset as u16);
writer.write_u16_le((stack_offset >> 16) as u16);
}
}
Self::StackAlloc {
@@ -143,10 +142,10 @@ impl UnwindCode {
);
} else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE {
writer.write_u8(UnwindOperation::LargeStackAlloc as u8);
writer.write_u16::<LittleEndian>((*size / 8) as u16);
writer.write_u16_le((*size / 8) as u16);
} else {
writer.write_u8((1 << 4) | (UnwindOperation::LargeStackAlloc as u8));
writer.write_u32::<LittleEndian>(*size);
writer.write_u32_le(*size);
}
}
Self::SetFPReg { instruction_offset } => {
@@ -248,7 +247,7 @@ impl UnwindInfo {
// To keep a 32-bit alignment, emit 2 bytes of padding if there's an odd number of 16-bit nodes
if (node_count & 1) == 1 {
writer.write_u16::<LittleEndian>(0);
writer.write_u16_le(0);
}
// Ensure the correct number of bytes was emitted
@@ -356,6 +355,12 @@ pub(crate) fn create_unwind_info_from_insts<MR: RegisterMapper<regalloc::Reg>>(
frame_register_offset = ensure_unwind_offset(offset_downward_to_clobbers)?;
unwind_codes.push(UnwindCode::SetFPReg { instruction_offset });
}
&UnwindInst::StackAlloc { size } => {
unwind_codes.push(UnwindCode::StackAlloc {
instruction_offset,
size,
});
}
&UnwindInst::SaveReg {
clobber_offset,
reg,

2
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -500,6 +500,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> (u64, SmallVec<[Self::I; 16]>) {
let mut insts = SmallVec::new();
// Find all clobbered registers that are callee-save.
@@ -574,6 +575,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> SmallVec<[Self::I; 16]> {
let mut insts = SmallVec::new();

4
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -460,9 +460,7 @@ pub(crate) enum InstructionSet {
BMI1,
#[allow(dead_code)] // never constructed (yet).
BMI2,
#[allow(dead_code)]
AVX512F,
#[allow(dead_code)]
AVX512VL,
}
@@ -995,13 +993,11 @@ impl fmt::Display for SseOpcode {
#[derive(Clone)]
pub enum Avx512Opcode {
#[allow(dead_code)]
Vpabsq,
}
impl Avx512Opcode {
/// Which `InstructionSet`s support the opcode?
#[allow(dead_code)]
pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
match self {
Avx512Opcode::Vpabsq => smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL],

22
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -6,9 +6,11 @@ use crate::isa::x64::inst::args::*;
use crate::isa::x64::inst::*;
use crate::machinst::{inst_common, MachBuffer, MachInstEmit, MachLabel};
use core::convert::TryInto;
use encoding::evex::{EvexInstruction, EvexVectorLength};
use encoding::rex::{
emit_simm, emit_std_enc_enc, emit_std_enc_mem, emit_std_reg_mem, emit_std_reg_reg, int_reg_enc,
low8_will_sign_extend_to_32, low8_will_sign_extend_to_64, reg_enc, LegacyPrefixes, RexFlags,
low8_will_sign_extend_to_32, low8_will_sign_extend_to_64, reg_enc, LegacyPrefixes, OpcodeMap,
RexFlags,
};
use log::debug;
use regalloc::{Reg, Writable};
@@ -1404,6 +1406,24 @@ pub(crate) fn emit(
};
}
Inst::XmmUnaryRmREvex { op, src, dst } => {
let opcode = match op {
Avx512Opcode::Vpabsq => 0x1f,
};
match src {
RegMem::Reg { reg: src } => EvexInstruction::new()
.length(EvexVectorLength::V128)
.prefix(LegacyPrefixes::_66)
.map(OpcodeMap::_0F38)
.w(true)
.opcode(opcode)
.reg(dst.to_reg().get_hw_encoding())
.rm(src.get_hw_encoding())
.encode(sink),
_ => todo!(),
};
}
Inst::XmmRmR {
op,
src: src_e,

7
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

@@ -3865,6 +3865,12 @@ fn test_x64_emit() {
"cvtdq2pd %xmm2, %xmm8",
));
insns.push((
Inst::xmm_unary_rm_r_evex(Avx512Opcode::Vpabsq, RegMem::reg(xmm2), w_xmm8),
"6272FD081FC2",
"vpabsq %xmm2, %xmm8",
));
// Xmm to int conversions, and conversely.
insns.push((
@@ -4276,6 +4282,7 @@ fn test_x64_emit() {
let mut isa_flag_builder = x64::settings::builder();
isa_flag_builder.enable("has_ssse3").unwrap();
isa_flag_builder.enable("has_sse41").unwrap();
isa_flag_builder.enable("has_avx512f").unwrap();
let isa_flags = x64::settings::Flags::new(&flags, isa_flag_builder);
let rru = regs::create_reg_universe_systemv(&flags);

396
cranelift/codegen/src/isa/x64/inst/encoding/evex.rs Normal file
View File

@@ -0,0 +1,396 @@
//! Encodes EVEX instructions. These instructions are those added by the AVX-512 extensions. The
//! EVEX encoding requires a 4-byte prefix:
//!
//! Byte 0: 0x62
//! ┌───┬───┬───┬───┬───┬───┬───┬───┐
//! Byte 1: │ R │ X │ B │ R'│ 0 │ 0 │ m │ m │
//! ├───┼───┼───┼───┼───┼───┼───┼───┤
//! Byte 2: │ W │ v │ v │ v │ v │ 1 │ p │ p │
//! ├───┼───┼───┼───┼───┼───┼───┼───┤
//! Byte 3: │ z │ L'│ L │ b │ V'│ a │ a │ a │
//! └───┴───┴───┴───┴───┴───┴───┴───┘
//!
//! The prefix is then followeded by the opcode byte, the ModR/M byte, and other optional suffixes
//! (e.g. SIB byte, displacements, immediates) based on the instruction (see section 2.6, Intel
//! Software Development Manual, volume 2A).
use super::rex::{encode_modrm, LegacyPrefixes, OpcodeMap};
use super::ByteSink;
use core::ops::RangeInclusive;
/// Constructs an EVEX-encoded instruction using a builder pattern. This approach makes it visually
/// easier to transform something the manual's syntax, `EVEX.256.66.0F38.W1 1F /r` to code:
/// `EvexInstruction::new().length(...).prefix(...).map(...).w(true).opcode(0x1F).reg(...).rm(...)`.
pub struct EvexInstruction {
bits: u32,
opcode: u8,
reg: Register,
rm: Register,
}
/// Because some of the bit flags in the EVEX prefix are reversed and users of `EvexInstruction` may
/// choose to skip setting fields, here we set some sane defaults. Note that:
/// - the first byte is always `0x62` but you will notice it at the end of the default `bits` value
/// implemented--remember the little-endian order
/// - some bits are always set to certain values: bits 10-11 to 0, bit 18 to 1
/// - the other bits set correspond to reversed bits: R, X, B, R' (byte 1), vvvv (byte 2), V' (byte
/// 3).
///
/// See the `default_emission` test for what these defaults are equivalent to (e.g. using RAX,
/// unsetting the W bit, etc.)
impl Default for EvexInstruction {
fn default() -> Self {
Self {
bits: 0x08_7C_F0_62,
opcode: 0,
reg: Register::default(),
rm: Register::default(),
}
}
}
#[allow(non_upper_case_globals)] // This makes it easier to match the bit range names to the manual's names.
impl EvexInstruction {
/// Construct a default EVEX instruction.
pub fn new() -> Self {
Self::default()
}
/// Set the length of the instruction . Note that there are sets of instructions (i.e. rounding,
/// memory broadcast) that modify the same underlying bits--at some point (TODO) we can add a
/// way to set those context bits and verify that both are not used (e.g. rounding AND length).
/// For now, this method is very convenient.
#[inline(always)]
pub fn length(mut self, length: EvexVectorLength) -> Self {
self.write(Self::LL, EvexContext::Other { length }.bits() as u32);
self
}
/// Set the legacy prefix byte of the instruction: None | 66 | F0 | F2 | F3. EVEX instructions
/// pack these into the prefix, not as separate bytes.
#[inline(always)]
pub fn prefix(mut self, prefix: LegacyPrefixes) -> Self {
self.write(Self::pp, prefix.bits() as u32);
self
}
/// Set the opcode map byte of the instruction: None | 0F | 0F38 | 0F3A. EVEX instructions pack
/// these into the prefix, not as separate bytes.
#[inline(always)]
pub fn map(mut self, map: OpcodeMap) -> Self {
self.write(Self::mm, map.bits() as u32);
self
}
/// Set the W bit, typically used to indicate an instruction using 64 bits of an operand (e.g.
/// 64 bit lanes). EVEX packs this bit in the EVEX prefix; previous encodings used the REX
/// prefix.
#[inline(always)]
pub fn w(mut self, w: bool) -> Self {
self.write(Self::W, w as u32);
self
}
/// Set the instruction opcode byte.
#[inline(always)]
pub fn opcode(mut self, opcode: u8) -> Self {
self.opcode = opcode;
self
}
/// Set the register to use for the `reg` bits; many instructions use this as the write operand.
/// Setting this affects both the ModRM byte (`reg` section) and the EVEX prefix (the extension
/// bits for register encodings > 8).
#[inline(always)]
pub fn reg(mut self, reg: impl Into<Register>) -> Self {
self.reg = reg.into();
let r = !(self.reg.0 >> 3) & 1;
let r_ = !(self.reg.0 >> 4) & 1;
self.write(Self::R, r as u32);
self.write(Self::R_, r_ as u32);
self
}
/// Set the mask to use. See section 2.6 in the Intel Software Developer's Manual, volume 2A for
/// more details.
#[allow(dead_code)]
#[inline(always)]
pub fn mask(mut self, mask: EvexMasking) -> Self {
self.write(Self::aaa, mask.aaa_bits() as u32);
self.write(Self::z, mask.z_bit() as u32);
self
}
/// Set the `vvvvv` register; some instructions allow using this as a second, non-destructive
/// source register in 3-operand instructions (e.g. 2 read, 1 write).
#[allow(dead_code)]
#[inline(always)]
pub fn vvvvv(mut self, reg: impl Into<Register>) -> Self {
let reg = reg.into();
self.write(Self::vvvv, !(reg.0 as u32) & 0b1111);
self.write(Self::V_, !(reg.0 as u32 >> 4) & 0b1);
self
}
/// Set the register to use for the `rm` bits; many instructions use this as the "read from
/// register/memory" operand. Currently this does not support memory addressing (TODO).Setting
/// this affects both the ModRM byte (`rm` section) and the EVEX prefix (the extension bits for
/// register encodings > 8).
#[inline(always)]
pub fn rm(mut self, reg: impl Into<Register>) -> Self {
self.rm = reg.into();
let b = !(self.rm.0 >> 3) & 1;
let x = !(self.rm.0 >> 4) & 1;
self.write(Self::X, x as u32);
self.write(Self::B, b as u32);
self
}
/// Emit the EVEX-encoded instruction to the code sink:
/// - first, the 4-byte EVEX prefix;
/// - then, the opcode byte;
/// - finally, the ModR/M byte.
///
/// Eventually this method should support encodings of more than just the reg-reg addressing mode (TODO).
pub fn encode<CS: ByteSink + ?Sized>(&self, sink: &mut CS) {
sink.put4(self.bits);
sink.put1(self.opcode);
sink.put1(encode_modrm(3, self.reg.0 & 7, self.rm.0 & 7));
}
// In order to simplify the encoding of the various bit ranges in the prefix, we specify those
// ranges according to the table below (extracted from the Intel Software Development Manual,
// volume 2A). Remember that, because we pack the 4-byte prefix into a little-endian `u32`, this
// chart should be read from right-to-left, top-to-bottom. Note also that we start ranges at bit
// 8, leaving bits 0-7 for the mandatory `0x62`.
// ┌───┬───┬───┬───┬───┬───┬───┬───┐
// Byte 1: │ R │ X │ B │ R'│ 0 │ 0 │ m │ m │
// ├───┼───┼───┼───┼───┼───┼───┼───┤
// Byte 2: │ W │ v │ v │ v │ v │ 1 │ p │ p │
// ├───┼───┼───┼───┼───┼───┼───┼───┤
// Byte 3: │ z │ L'│ L │ b │ V'│ a │ a │ a │
// └───┴───┴───┴───┴───┴───┴───┴───┘
// Byte 1:
const mm: RangeInclusive<u8> = 8..=9;
const R_: RangeInclusive<u8> = 12..=12;
const B: RangeInclusive<u8> = 13..=13;
const X: RangeInclusive<u8> = 14..=14;
const R: RangeInclusive<u8> = 15..=15;
// Byte 2:
const pp: RangeInclusive<u8> = 16..=17;
const vvvv: RangeInclusive<u8> = 19..=22;
const W: RangeInclusive<u8> = 23..=23;
// Byte 3:
const aaa: RangeInclusive<u8> = 24..=26;
const V_: RangeInclusive<u8> = 27..=27;
#[allow(dead_code)] // Will be used once broadcast and rounding controls are exposed.
const b: RangeInclusive<u8> = 28..=28;
const LL: RangeInclusive<u8> = 29..=30;
const z: RangeInclusive<u8> = 31..=31;
// A convenience method for writing the `value` bits to the given range in `self.bits`.
#[inline]
fn write(&mut self, range: RangeInclusive<u8>, value: u32) {
assert!(ExactSizeIterator::len(&range) > 0);
let size = range.end() - range.start() + 1; // Calculate the number of bits in the range.
let mask: u32 = (1 << size) - 1; // Generate a bit mask.
debug_assert!(
value <= mask,
"The written value should have fewer than {} bits.",
size
);
let mask_complement = !(mask << *range.start()); // Create the bitwise complement for the clear mask.
self.bits &= mask_complement; // Clear the bits in `range`; otherwise the OR below may allow previously-set bits to slip through.
let value = value << *range.start(); // Place the value in the correct location (assumes `value <= mask`).
self.bits |= value; // Modify the bits in `range`.
}
}
#[derive(Copy, Clone, Default)]
pub struct Register(u8);
impl From<u8> for Register {
fn from(reg: u8) -> Self {
debug_assert!(reg < 16);
Self(reg)
}
}
/// Defines the EVEX context for the `L'`, `L`, and `b` bits (bits 6:4 of EVEX P2 byte). Table 2-36 in
/// section 2.6.10 (Intel Software Development Manual, volume 2A) describes how these bits can be
/// used together for certain classes of instructions; i.e., special care should be taken to ensure
/// that instructions use an applicable correct `EvexContext`. Table 2-39 contains cases where
/// opcodes can result in an #UD.
#[allow(dead_code)] // Rounding and broadcast modes are not yet used.
pub enum EvexContext {
RoundingRegToRegFP {
rc: EvexRoundingControl,
},
NoRoundingFP {
sae: bool,
length: EvexVectorLength,
},
MemoryOp {
broadcast: bool,
length: EvexVectorLength,
},
Other {
length: EvexVectorLength,
},
}
impl Default for EvexContext {
fn default() -> Self {
Self::Other {
length: EvexVectorLength::default(),
}
}
}
impl EvexContext {
/// Encode the `L'`, `L`, and `b` bits (bits 6:4 of EVEX P2 byte) for merging with the P2 byte.
fn bits(&self) -> u8 {
match self {
Self::RoundingRegToRegFP { rc } => 0b001 | rc.bits() << 1,
Self::NoRoundingFP { sae, length } => (*sae as u8) | length.bits() << 1,
Self::MemoryOp { broadcast, length } => (*broadcast as u8) | length.bits() << 1,
Self::Other { length } => length.bits() << 1,
}
}
}
/// The EVEX format allows choosing a vector length in the `L'` and `L` bits; see `EvexContext`.
#[allow(dead_code)] // Wider-length vectors are not yet used.
pub enum EvexVectorLength {
V128,
V256,
V512,
}
impl EvexVectorLength {
/// Encode the `L'` and `L` bits for merging with the P2 byte.
fn bits(&self) -> u8 {
match self {
Self::V128 => 0b00,
Self::V256 => 0b01,
Self::V512 => 0b10,
// 0b11 is reserved (#UD).
}
}
}
impl Default for EvexVectorLength {
fn default() -> Self {
Self::V128
}
}
/// The EVEX format allows defining rounding control in the `L'` and `L` bits; see `EvexContext`.
#[allow(dead_code)] // Rounding controls are not yet used.
pub enum EvexRoundingControl {
RNE,
RD,
RU,
RZ,
}
impl EvexRoundingControl {
/// Encode the `L'` and `L` bits for merging with the P2 byte.
fn bits(&self) -> u8 {
match self {
Self::RNE => 0b00,
Self::RD => 0b01,
Self::RU => 0b10,
Self::RZ => 0b11,
}
}
}
/// Defines the EVEX masking behavior; masking support is described in section 2.6.4 of the Intel
/// Software Development Manual, volume 2A.
#[allow(dead_code)] // Masking is not yet used.
pub enum EvexMasking {
None,
Merging { k: u8 },
Zeroing { k: u8 },
}
impl Default for EvexMasking {
fn default() -> Self {
EvexMasking::None
}
}
impl EvexMasking {
/// Encode the `z` bit for merging with the P2 byte.
fn z_bit(&self) -> u8 {
match self {
Self::None | Self::Merging { .. } => 0,
Self::Zeroing { .. } => 1,
}
}
/// Encode the `aaa` bits for merging with the P2 byte.
fn aaa_bits(&self) -> u8 {
match self {
Self::None => 0b000,
Self::Merging { k } | Self::Zeroing { k } => {
debug_assert!(*k <= 7);
*k
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::isa::x64::inst::regs;
use std::vec::Vec;
// As a sanity test, we verify that the output of `xed-asmparse-main 'vpabsq xmm0{k0},
// xmm1'` matches this EVEX encoding machinery.
#[test]
fn vpabsq() {
let dst = regs::xmm0();
let src = regs::xmm1();
let mut sink0 = Vec::new();
EvexInstruction::new()
.prefix(LegacyPrefixes::_66)
.map(OpcodeMap::_0F38)
.w(true)
.opcode(0x1F)
.reg(dst.get_hw_encoding())
.rm(src.get_hw_encoding())
.length(EvexVectorLength::V128)
.encode(&mut sink0);
assert_eq!(sink0, vec![0x62, 0xf2, 0xfd, 0x08, 0x1f, 0xc1]);
}
/// Verify that the defaults are equivalent to an instruction with a `0x00` opcode using the
/// "0" register (i.e. `rax`), with sane defaults for the various configurable parameters. This
/// test is more interesting than it may appear because some of the parameters have flipped-bit
/// representations (e.g. `vvvvv`) so emitting 0s as a default will not work.
#[test]
fn default_emission() {
let mut sink0 = Vec::new();
EvexInstruction::new().encode(&mut sink0);
let mut sink1 = Vec::new();
EvexInstruction::new()
.length(EvexVectorLength::V128)
.prefix(LegacyPrefixes::None)
.map(OpcodeMap::None)
.w(false)
.opcode(0x00)
.reg(regs::rax().get_hw_encoding())
.rm(regs::rax().get_hw_encoding())
.mask(EvexMasking::None)
.encode(&mut sink1);
assert_eq!(sink0, sink1);
}
}

56
cranelift/codegen/src/isa/x64/inst/encoding/mod.rs
View File

@@ -1 +1,57 @@
use crate::{isa::x64, machinst::MachBuffer};
use std::vec::Vec;
pub mod evex;
pub mod rex;
pub mod vex;
pub trait ByteSink {
/// Add 1 byte to the code section.
fn put1(&mut self, _: u8);
/// Add 2 bytes to the code section.
fn put2(&mut self, _: u16);
/// Add 4 bytes to the code section.
fn put4(&mut self, _: u32);
/// Add 8 bytes to the code section.
fn put8(&mut self, _: u64);
}
impl ByteSink for MachBuffer<x64::inst::Inst> {
fn put1(&mut self, value: u8) {
self.put1(value)
}
fn put2(&mut self, value: u16) {
self.put2(value)
}
fn put4(&mut self, value: u32) {
self.put4(value)
}
fn put8(&mut self, value: u64) {
self.put8(value)
}
}
/// Provide a convenient implementation for testing.
impl ByteSink for Vec<u8> {
fn put1(&mut self, v: u8) {
self.extend_from_slice(&[v])
}
fn put2(&mut self, v: u16) {
self.extend_from_slice(&v.to_le_bytes())
}
fn put4(&mut self, v: u32) {
self.extend_from_slice(&v.to_le_bytes())
}
fn put8(&mut self, v: u64) {
self.extend_from_slice(&v.to_le_bytes())
}
}

65
cranelift/codegen/src/isa/x64/inst/encoding/rex.rs
View File

@@ -153,9 +153,37 @@ impl From<(OperandSize, Reg)> for RexFlags {
}
}
/// Allows using the same opcode byte in different "opcode maps" to allow for more instruction
/// encodings. See appendix A in the Intel Software Developer's Manual, volume 2A, for more details.
pub enum OpcodeMap {
None,
_0F,
_0F38,
_0F3A,
}
impl OpcodeMap {
/// Normally the opcode map is specified as bytes in the instruction, but some x64 encoding
/// formats pack this information as bits in a prefix (e.g. EVEX).
pub(crate) fn bits(&self) -> u8 {
match self {
OpcodeMap::None => 0b00,
OpcodeMap::_0F => 0b01,
OpcodeMap::_0F38 => 0b10,
OpcodeMap::_0F3A => 0b11,
}
}
}
impl Default for OpcodeMap {
fn default() -> Self {
Self::None
}
}
/// We may need to include one or more legacy prefix bytes before the REX prefix. This enum
/// covers only the small set of possibilities that we actually need.
pub(crate) enum LegacyPrefixes {
pub enum LegacyPrefixes {
/// No prefix bytes.
None,
/// Operand Size Override -- here, denoting "16-bit operation".
@@ -173,26 +201,47 @@ pub(crate) enum LegacyPrefixes {
}
impl LegacyPrefixes {
/// Emit the legacy prefix as bytes (e.g. in REX instructions).
#[inline(always)]
pub(crate) fn emit(&self, sink: &mut MachBuffer<Inst>) {
match self {
LegacyPrefixes::_66 => sink.put1(0x66),
LegacyPrefixes::_F0 => sink.put1(0xF0),
LegacyPrefixes::_66F0 => {
Self::_66 => sink.put1(0x66),
Self::_F0 => sink.put1(0xF0),
Self::_66F0 => {
// I don't think the order matters, but in any case, this is the same order that
// the GNU assembler uses.
sink.put1(0x66);
sink.put1(0xF0);
}
LegacyPrefixes::_F2 => sink.put1(0xF2),
LegacyPrefixes::_F3 => sink.put1(0xF3),
LegacyPrefixes::_66F3 => {
Self::_F2 => sink.put1(0xF2),
Self::_F3 => sink.put1(0xF3),
Self::_66F3 => {
sink.put1(0x66);
sink.put1(0xF3);
}
LegacyPrefixes::None => (),
Self::None => (),
}
}
/// Emit the legacy prefix as bits (e.g. for EVEX instructions).
#[inline(always)]
pub(crate) fn bits(&self) -> u8 {
match self {
Self::None => 0b00,
Self::_66 => 0b01,
Self::_F3 => 0b10,
Self::_F2 => 0b11,
_ => panic!(
"VEX and EVEX bits can only be extracted from single prefixes: None, 66, F3, F2"
),
}
}
}
impl Default for LegacyPrefixes {
fn default() -> Self {
Self::None
}
}
/// This is the core 'emit' function for instructions that reference memory.

2
cranelift/codegen/src/isa/x64/inst/encoding/vex.rs Normal file
View File

@@ -0,0 +1,2 @@
//! Encodes VEX instructions. These instructions are those added by the Advanced Vector Extensions
//! (AVX).

30
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -225,6 +225,12 @@ pub enum Inst {
dst: Writable<Reg>,
},
XmmUnaryRmREvex {
op: Avx512Opcode,
src: RegMem,
dst: Writable<Reg>,
},
/// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq
XmmMovRM {
op: SseOpcode,
@@ -571,6 +577,8 @@ impl Inst {
| Inst::XmmRmRImm { op, .. }
| Inst::XmmToGpr { op, .. }
| Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()],
Inst::XmmUnaryRmREvex { op, .. } => op.available_from(),
}
}
}
@@ -705,6 +713,12 @@ impl Inst {
Inst::XmmUnaryRmR { op, src, dst }
}
pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmUnaryRmREvex { op, src, dst }
}
pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
@@ -1391,6 +1405,13 @@ impl PrettyPrint for Inst {
show_ireg_sized(dst.to_reg(), mb_rru, 8),
),
Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!(
"{} {}, {}",
ljustify(op.to_string()),
src.show_rru_sized(mb_rru, 8),
show_ireg_sized(dst.to_reg(), mb_rru, 8),
),
Inst::XmmMovRM { op, src, dst, .. } => format!(
"{} {}, {}",
ljustify(op.to_string()),
@@ -1863,7 +1884,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_def(Writable::from_reg(regs::rdx()));
}
},
Inst::UnaryRmR { src, dst, .. } | Inst::XmmUnaryRmR { src, dst, .. } => {
Inst::UnaryRmR { src, dst, .. }
| Inst::XmmUnaryRmR { src, dst, .. }
| Inst::XmmUnaryRmREvex { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_def(*dst);
}
@@ -2210,6 +2233,11 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
ref mut dst,
..
}
| Inst::XmmUnaryRmREvex {
ref mut src,
ref mut dst,
..
}
| Inst::UnaryRmR {
ref mut src,
ref mut dst,

4
cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs
View File

@@ -89,8 +89,8 @@ impl crate::isa::unwind::systemv::RegisterMapper<Reg> for RegisterMapper {
fn sp(&self) -> u16 {
X86_64::RSP.0
}
fn fp(&self) -> u16 {
X86_64::RBP.0
fn fp(&self) -> Option<u16> {
Some(X86_64::RBP.0)
}
}

40
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -1855,25 +1855,29 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let ty = ty.unwrap();
if ty == types::I64X2 {
// This lowering could be a single instruction with AVX512F/VL's VPABSQ instruction.
// Instead, we use a separate register, `tmp`, to contain the results of `0 - src`
// and then blend in those results with `BLENDVPD` if the MSB of `tmp` was set to 1
// (i.e. if `tmp` was negative or, conversely, if `src` was originally positive).
if isa_flags.use_avx512f_simd() || isa_flags.use_avx512vl_simd() {
ctx.emit(Inst::xmm_unary_rm_r_evex(Avx512Opcode::Vpabsq, src, dst));
} else {
// If `VPABSQ` from AVX512 is unavailable, we use a separate register, `tmp`, to
// contain the results of `0 - src` and then blend in those results with
// `BLENDVPD` if the MSB of `tmp` was set to 1 (i.e. if `tmp` was negative or,
// conversely, if `src` was originally positive).
// Emit all 0s into the `tmp` register.
let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp));
// Subtract the lanes from 0 and set up `dst`.
ctx.emit(Inst::xmm_rm_r(SseOpcode::Psubq, src.clone(), tmp));
ctx.emit(Inst::gen_move(dst, tmp.to_reg(), ty));
// Choose the subtracted lanes when `tmp` has an MSB of 1. BLENDVPD's semantics
// require the "choice" mask to be in XMM0.
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::xmm0()),
tmp.to_reg(),
ty,
));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Blendvpd, src, dst));
// Emit all 0s into the `tmp` register.
let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp));
// Subtract the lanes from 0 and set up `dst`.
ctx.emit(Inst::xmm_rm_r(SseOpcode::Psubq, src.clone(), tmp));
ctx.emit(Inst::gen_move(dst, tmp.to_reg(), ty));
// Choose the subtracted lanes when `tmp` has an MSB of 1. BLENDVPD's semantics
// require the "choice" mask to be in XMM0.
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::xmm0()),
tmp.to_reg(),
ty,
));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Blendvpd, src, dst));
}
} else if ty.is_vector() {
let opcode = match ty {
types::I8X16 => SseOpcode::Pabsb,

4
cranelift/codegen/src/isa/x86/unwind/systemv.rs
View File

@@ -121,8 +121,8 @@ pub(crate) fn create_unwind_info(
fn sp(&self) -> u16 {
X86_64::RSP.0
}
fn fp(&self) -> u16 {
X86_64::RBP.0
fn fp(&self) -> Option<u16> {
Some(X86_64::RBP.0)
}
}
let map = RegisterMapper(isa);

4
cranelift/codegen/src/isa/x86/unwind/winx64.rs
View File

@@ -2,7 +2,7 @@
use crate::ir::Function;
use crate::isa::x86::registers::{FPR, GPR};
use crate::isa::{unwind::winx64::UnwindInfo, CallConv, RegUnit, TargetIsa};
use crate::isa::{unwind::winx64::UnwindInfo, RegUnit, TargetIsa};
use crate::result::CodegenResult;
pub(crate) fn create_unwind_info(
@@ -10,7 +10,7 @@ pub(crate) fn create_unwind_info(
isa: &dyn TargetIsa,
) -> CodegenResult<Option<UnwindInfo>> {
// Only Windows fastcall is supported for unwind information
if func.signature.call_conv != CallConv::WindowsFastcall || func.prologue_end.is_none() {
if !func.signature.call_conv.extends_windows_fastcall() || func.prologue_end.is_none() {
return Ok(None);
}

1
cranelift/codegen/src/lib.rs
View File

@@ -97,6 +97,7 @@ mod inst_predicates;
mod iterators;
mod legalizer;
mod licm;
mod log;
mod nan_canonicalization;
mod partition_slice;
mod postopt;

39
cranelift/codegen/src/log.rs Normal file
View File

@@ -0,0 +1,39 @@
//! This module implements deferred display helpers.
//!
//! These are particularly useful in logging contexts, where the maximum logging level filter might
//! be enabled, but we don't want the arguments to be evaluated early:
//!
//! ```
//! log::set_max_level(log::LevelFilter::max());
//! fn expensive_calculation() -> String {
//! "a string that is very slow to generate".into()
//! }
//! log::debug!("{}", expensive_calculation());
//! ```
//!
//! If the associated log implementation filters out log debug entries, the expensive calculation
//! would have been spurious. In this case, we can wrap the expensive computation within an
//! `DeferredDisplay`, so that the computation only happens when the actual `fmt` function is
//! called.
use core::fmt;
pub(crate) struct DeferredDisplay<F>(F);
impl<F: Fn() -> T, T: fmt::Display> DeferredDisplay<F> {
pub(crate) fn new(f: F) -> Self {
Self(f)
}
}
impl<F: Fn() -> T, T: fmt::Display> fmt::Display for DeferredDisplay<F> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0().fmt(f)
}
}
impl<F: Fn() -> T, T: fmt::Debug> fmt::Debug for DeferredDisplay<F> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0().fmt(f)
}
}

13
cranelift/codegen/src/machinst/abi.rs
View File

@@ -30,6 +30,12 @@ pub trait ABICallee {
/// Access the (possibly legalized) signature.
fn signature(&self) -> &Signature;
/// Accumulate outgoing arguments. This ensures that at least SIZE bytes
/// are allocated in the prologue to be available for use in function calls
/// to hold arguments and/or return values. If this function is called
/// multiple times, the maximum of all SIZE values will be available.
fn accumulate_outgoing_args_size(&mut self, size: u32);
/// Get the settings controlling this function's compilation.
fn flags(&self) -> &settings::Flags;
@@ -242,6 +248,13 @@ pub trait ABICaller {
/// Emit code to post-adjust the satck, after call return and return-value copies.
fn emit_stack_post_adjust<C: LowerCtx<I = Self::I>>(&self, ctx: &mut C);
/// Accumulate outgoing arguments. This ensures that the caller (as
/// identified via the CTX argument) allocates enough space in the
/// prologue to hold all arguments and return values for this call.
/// There is no code emitted at the call site, everything is done
/// in the caller's function prologue.
fn accumulate_outgoing_args_size<C: LowerCtx<I = Self::I>>(&self, ctx: &mut C);
/// Emit the call itself.
///
/// The returned instruction should have proper use- and def-sets according

44
cranelift/codegen/src/machinst/abi_impl.rs
View File

@@ -444,6 +444,7 @@ pub trait ABIMachineSpec {
flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
fixed_frame_storage_size: u32,
outgoing_args_size: u32,
) -> (u64, SmallVec<[Self::I; 16]>);
/// Generate a clobber-restore sequence. This sequence should perform the
@@ -455,6 +456,7 @@ pub trait ABIMachineSpec {
flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
fixed_frame_storage_size: u32,
outgoing_args_size: u32,
) -> SmallVec<[Self::I; 16]>;
/// Generate a call instruction/sequence. This method is provided one
@@ -576,6 +578,8 @@ pub struct ABICalleeImpl<M: ABIMachineSpec> {
stackslots: PrimaryMap<StackSlot, u32>,
/// Total stack size of all stackslots.
stackslots_size: u32,
/// Stack size to be reserved for outgoing arguments.
outgoing_args_size: u32,
/// Clobbered registers, from regalloc.
clobbered: Set<Writable<RealReg>>,
/// Total number of spillslots, from regalloc.
@@ -691,6 +695,7 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
sig,
stackslots,
stackslots_size: stack_offset,
outgoing_args_size: 0,
clobbered: Set::empty(),
spillslots: None,
fixed_frame_storage_size: 0,
@@ -917,6 +922,12 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
}
}
fn accumulate_outgoing_args_size(&mut self, size: u32) {
if size > self.outgoing_args_size {
self.outgoing_args_size = size;
}
}
fn flags(&self) -> &settings::Flags {
&self.flags
}
@@ -1198,6 +1209,15 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
let spill_off = islot * M::word_bytes() as i64;
let sp_off = self.stackslots_size as i64 + spill_off;
trace!("load_spillslot: slot {:?} -> sp_off {}", slot, sp_off);
// Integer types smaller than word size have been spilled as words below,
// and therefore must be reloaded in the same type.
let ty = if ty.is_int() && ty.bytes() < M::word_bytes() {
M::word_type()
} else {
ty
};
gen_load_stack_multi::<M>(StackAMode::NominalSPOffset(sp_off, ty), into_regs, ty)
}
@@ -1213,6 +1233,19 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
let spill_off = islot * M::word_bytes() as i64;
let sp_off = self.stackslots_size as i64 + spill_off;
trace!("store_spillslot: slot {:?} -> sp_off {}", slot, sp_off);
// When reloading from a spill slot, we might have lost information about real integer
// types. For instance, on the x64 backend, a zero-extension can become spurious and
// optimized into a move, causing vregs of types I32 and I64 to share the same coalescing
// equivalency class. As a matter of fact, such a value can be spilled as an I32 and later
// reloaded as an I64; to make sure the high bits are always defined, do a word-sized store
// all the time, in this case.
let ty = if ty.is_int() && ty.bytes() < M::word_bytes() {
M::word_type()
} else {
ty
};
gen_store_stack_multi::<M>(StackAMode::NominalSPOffset(sp_off, ty), from_regs, ty)
}
@@ -1285,11 +1318,12 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
}
// Save clobbered registers.
let (_, clobber_insts) = M::gen_clobber_save(
let (clobber_size, clobber_insts) = M::gen_clobber_save(
self.call_conv,
&self.flags,
&self.clobbered,
self.fixed_frame_storage_size,
self.outgoing_args_size,
);
insts.extend(clobber_insts);
@@ -1304,7 +1338,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
// [crate::machinst::abi_impl](this module) for more details
// on stackframe layout and nominal SP maintenance.
self.total_frame_size = Some(total_stacksize);
self.total_frame_size = Some(total_stacksize + clobber_size as u32);
insts
}
@@ -1317,6 +1351,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
&self.flags,
&self.clobbered,
self.fixed_frame_storage_size,
self.outgoing_args_size,
));
// N.B.: we do *not* emit a nominal SP adjustment here, because (i) there will be no
@@ -1519,6 +1554,11 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
}
}
fn accumulate_outgoing_args_size<C: LowerCtx<I = Self::I>>(&self, ctx: &mut C) {
let off = self.sig.stack_arg_space + self.sig.stack_ret_space;
ctx.abi().accumulate_outgoing_args_size(off as u32);
}
fn emit_stack_pre_adjust<C: LowerCtx<I = Self::I>>(&self, ctx: &mut C) {
let off = self.sig.stack_arg_space + self.sig.stack_ret_space;
adjust_stack_and_nominal_sp::<M, C>(ctx, off as i32, /* is_sub = */ true)

28
cranelift/codegen/src/machinst/compile.rs
View File

@@ -1,11 +1,12 @@
//! Compilation backend pipeline: optimized IR to VCode / binemit.
use crate::ir::Function;
use crate::log::DeferredDisplay;
use crate::machinst::*;
use crate::settings;
use crate::timing;
use log::{debug, log_enabled, Level};
use log::debug;
use regalloc::{allocate_registers_with_opts, Algorithm, Options, PrettyPrint};
/// Compile the given function down to VCode with allocated registers, ready
@@ -29,15 +30,12 @@ where
lower.lower(b)?
};
// Creating the vcode string representation may be costly for large functions, so don't do it
// if the Debug level hasn't been statically (through features) or dynamically (through
// RUST_LOG) enabled.
if log_enabled!(Level::Debug) {
debug!(
"vcode from lowering: \n{}",
vcode.show_rru(Some(b.reg_universe()))
);
}
// Creating the vcode string representation may be costly for large functions, so defer its
// rendering.
debug!(
"vcode from lowering: \n{}",
DeferredDisplay::new(|| vcode.show_rru(Some(b.reg_universe())))
);
// Perform register allocation.
let (run_checker, algorithm) = match vcode.flags().regalloc() {
@@ -106,12 +104,10 @@ where
vcode.replace_insns_from_regalloc(result);
}
if log_enabled!(Level::Debug) {
debug!(
"vcode after regalloc: final version:\n{}",
vcode.show_rru(Some(b.reg_universe()))
);
}
debug!(
"vcode after regalloc: final version:\n{}",
DeferredDisplay::new(|| vcode.show_rru(Some(b.reg_universe())))
);
Ok(vcode)
}

44
cranelift/codegen/src/result.rs
View File

@@ -2,19 +2,17 @@
use crate::verifier::VerifierErrors;
use std::string::String;
use thiserror::Error;
/// A compilation error.
///
/// When Cranelift fails to compile a function, it will return one of these error codes.
#[derive(Error, Debug, PartialEq, Eq)]
#[derive(Debug, PartialEq, Eq)]
pub enum CodegenError {
/// A list of IR verifier errors.
///
/// This always represents a bug, either in the code that generated IR for Cranelift, or a bug
/// in Cranelift itself.
#[error("Verifier errors")]
Verifier(#[from] VerifierErrors),
Verifier(VerifierErrors),
/// An implementation limit was exceeded.
///
@@ -22,27 +20,57 @@ pub enum CodegenError {
/// limits][limits] that cause compilation to fail when they are exceeded.
///
/// [limits]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/docs/ir.md#implementation-limits
#[error("Implementation limit exceeded")]
ImplLimitExceeded,
/// The code size for the function is too large.
///
/// Different target ISAs may impose a limit on the size of a compiled function. If that limit
/// is exceeded, compilation fails.
#[error("Code for function is too large")]
CodeTooLarge,
/// Something is not supported by the code generator. This might be an indication that a
/// feature is used without explicitly enabling it, or that something is temporarily
/// unsupported by a given target backend.
#[error("Unsupported feature: {0}")]
Unsupported(String),
/// A failure to map Cranelift register representation to a DWARF register representation.
#[cfg(feature = "unwind")]
#[error("Register mapping error")]
RegisterMappingError(crate::isa::unwind::systemv::RegisterMappingError),
}
/// A convenient alias for a `Result` that uses `CodegenError` as the error type.
pub type CodegenResult<T> = Result<T, CodegenError>;
// This is manually implementing Error and Display instead of using thiserror to reduce the amount
// of dependencies used by Cranelift.
impl std::error::Error for CodegenError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
CodegenError::Verifier(source) => Some(source),
CodegenError::ImplLimitExceeded { .. }
| CodegenError::CodeTooLarge { .. }
| CodegenError::Unsupported { .. } => None,
#[cfg(feature = "unwind")]
CodegenError::RegisterMappingError { .. } => None,
}
}
}
impl std::fmt::Display for CodegenError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
CodegenError::Verifier(_) => write!(f, "Verifier errors"),
CodegenError::ImplLimitExceeded => write!(f, "Implementation limit exceeded"),
CodegenError::CodeTooLarge => write!(f, "Code for function is too large"),
CodegenError::Unsupported(feature) => write!(f, "Unsupported feature: {}", feature),
#[cfg(feature = "unwind")]
CodegenError::RegisterMappingError(_0) => write!(f, "Register mapping error"),
}
}
}
impl From<VerifierErrors> for CodegenError {
fn from(source: VerifierErrors) -> Self {
CodegenError::Verifier { 0: source }
}
}

22
cranelift/codegen/src/settings.rs
View File

@@ -26,7 +26,6 @@ use alloc::boxed::Box;
use alloc::string::{String, ToString};
use core::fmt;
use core::str;
use thiserror::Error;
/// A string-based configurator for settings groups.
///
@@ -261,21 +260,34 @@ impl Configurable for Builder {
}
/// An error produced when changing a setting.
#[derive(Error, Debug, PartialEq, Eq)]
#[derive(Debug, PartialEq, Eq)]
pub enum SetError {
/// No setting by this name exists.
#[error("No existing setting named '{0}'")]
BadName(String),
/// Type mismatch for setting (e.g., setting an enum setting as a bool).
#[error("Trying to set a setting with the wrong type")]
BadType,
/// This is not a valid value for this setting.
#[error("Unexpected value for a setting, expected {0}")]
BadValue(String),
}
impl std::error::Error for SetError {}
impl fmt::Display for SetError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
SetError::BadName(name) => write!(f, "No existing setting named '{}'", name),
SetError::BadType => {
write!(f, "Trying to set a setting with the wrong type")
}
SetError::BadValue(value) => {
write!(f, "Unexpected value for a setting, expected {}", value)
}
}
}
}
/// A result returned when changing a setting.
pub type SetResult<T> = Result<T, SetError>;

25
cranelift/codegen/src/verifier/mod.rs
View File

@@ -80,7 +80,6 @@ use alloc::vec::Vec;
use core::cmp::Ordering;
use core::fmt::{self, Display, Formatter, Write};
use log::debug;
use thiserror::Error;
pub use self::cssa::verify_cssa;
pub use self::liveness::verify_liveness;
@@ -92,8 +91,7 @@ mod liveness;
mod locations;
/// A verifier error.
#[derive(Error, Debug, PartialEq, Eq, Clone)]
#[error("{}{}: {}", .location, format_context(.context), .message)]
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct VerifierError {
/// The entity causing the verifier error.
pub location: AnyEntity,
@@ -104,11 +102,16 @@ pub struct VerifierError {
pub message: String,
}
/// Helper for formatting Verifier::Error context.
fn format_context(context: &Option<String>) -> String {
match context {
None => "".to_string(),
Some(c) => format!(" ({})", c),
// This is manually implementing Error and Display instead of using thiserror to reduce the amount
// of dependencies used by Cranelift.
impl std::error::Error for VerifierError {}
impl Display for VerifierError {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match &self.context {
None => write!(f, "{}: {}", self.location, self.message),
Some(context) => write!(f, "{} ({}): {}", self.location, context, self.message),
}
}
}
@@ -175,9 +178,13 @@ pub type VerifierStepResult<T> = Result<T, ()>;
pub type VerifierResult<T> = Result<T, VerifierErrors>;
/// List of verifier errors.
#[derive(Error, Debug, Default, PartialEq, Eq, Clone)]
#[derive(Debug, Default, PartialEq, Eq, Clone)]
pub struct VerifierErrors(pub Vec<VerifierError>);
// This is manually implementing Error and Display instead of using thiserror to reduce the amount
// of dependencies used by Cranelift.
impl std::error::Error for VerifierErrors {}
impl VerifierErrors {
/// Return a new `VerifierErrors` struct.
#[inline]

1
cranelift/filetests/Cargo.toml
View File

@@ -16,7 +16,6 @@ cranelift-interpreter = { path = "../interpreter", version = "0.73.0" }
cranelift-native = { path = "../native", version = "0.73.0" }
cranelift-reader = { path = "../reader", version = "0.73.0" }
cranelift-preopt = { path = "../preopt", version = "0.73.0" }
byteorder = { version = "1.3.2", default-features = false }
file-per-thread-logger = "0.1.2"
filecheck = "0.5.0"
gimli = { version = "0.23.0", default-features = false, features = ["read"] }

127
cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif Normal file
View File

@@ -0,0 +1,127 @@
test compile
target x86_64 machinst
;; The goal of this test is to ensure that stack spills of an integer value,
;; which width is less than the machine word's size, cause the full word to be
;; stored, and not only the lower bits.
;; Because of unsigned extensions which can be transformed into simple moves,
;; the source vreg of the extension operation can be coalesced with its
;; destination vreg, and if it happens to be spill, then the reload may use a
;; reload of a different, larger size.
function %f0(i32, i32, i32) -> i64 {
fn0 = %g(i32) -> i64
; check: pushq %rbp
; nextln: movq %rsp, %rbp
; nextln: subq $$64, %rsp
;; Stash all the callee-saved registers.
; nextln: movq %r12, 16(%rsp)
; nextln: movq %r13, 24(%rsp)
; nextln: movq %r14, 32(%rsp)
; nextln: movq %rbx, 40(%rsp)
; nextln: movq %r15, 48(%rsp)
block0(v0: i32, v1: i32, v2: i32):
;; First, create enough virtual registers so that the call instructions
;; causes at least one of them to be spilled onto the stack.
v3 = iadd.i32 v0, v1
v4 = iadd.i32 v1, v2
v5 = iadd.i32 v0, v2
v6 = iadd.i32 v3, v0
v7 = iadd.i32 v4, v0
v8 = iadd.i32 v5, v0
; nextln: movq %rdi, %r12
; nextln: addl %esi, %r12d
; nextln: movq %rsi, %r13
; nextln: addl %edx, %r13d
; nextln: movq %rdi, %r14
; nextln: addl %edx, %r14d
; nextln: movq %r12, %rbx
; nextln: addl %edi, %ebx
; nextln: movq %r13, %r15
; nextln: addl %edi, %r15d
; nextln: movq %r14, %rsi
;; This should be movq below, not movl.
; nextln: movq %rsi, rsp(0 + virtual offset)
; nextln: movq rsp(0 + virtual offset), %rsi
; nextln: addl %edi, %esi
;; Put an effectful instruction so that the live-ranges of the adds and
;; uextends are split here, and to prevent the uextend to be emitted
;; before the call. This will effectively causing the above i32 to be
;; spilled as an i32, and not a full i64.
v300 = call fn0(v0)
;; This should be movq below, not movl.
; nextln: movq %rsi, rsp(0 + virtual offset)
; nextln: load_ext_name %g+0, %rsi
; nextln: call *%rsi
v31 = uextend.i64 v3
v41 = uextend.i64 v4
v51 = uextend.i64 v5
v61 = uextend.i64 v6
v71 = uextend.i64 v7
v81 = uextend.i64 v8
;; None of the uextends are generated here yet.
;; At this point, I'd expect that this second call below would be not
;; necessary, but if it is removed, the uextend is applied before the call,
;; and the i64 is spilled (then reloaded), causing the bug to not appear. So
;; an additional call it is!
v100 = call fn0(v3)
; nextln: movq %r12, %rsi
; nextln: movq %rsi, rsp(8 + virtual offset)
; nextln: nop len=0
; nextln: movq %r12, %rdi
; nextln: load_ext_name %g+0, %rsi
; nextln: call *%rsi
;; Cause reloads of all the values. Most are in registers, but one of them
;; is on the stack. Make sure they're all used in the final computation.
v101 = iadd.i64 v100, v31
v102 = iadd.i64 v101, v41
v103 = iadd.i64 v102, v51
v104 = iadd.i64 v103, v61
v105 = iadd.i64 v104, v71
v200 = iadd.i64 v105, v81
; nextln: movq %rax, %rsi
; nextln: movq rsp(8 + virtual offset), %rdi
; nextln: addq %rdi, %rsi
; nextln: addq %r13, %rsi
; nextln: addq %r14, %rsi
; nextln: addq %rbx, %rsi
; nextln: addq %r15, %rsi
;; The reload operates on a full word, so uses movq.
; nextln: movq rsp(0 + virtual offset), %rdi
; nextln: addq %rdi, %rsi
; nextln: movq %rsi, %rax
; nextln: movq 16(%rsp), %r12
; nextln: movq 24(%rsp), %r13
; nextln: movq 32(%rsp), %r14
; nextln: movq 40(%rsp), %rbx
; nextln: movq 48(%rsp), %r15
; nextln: addq $$64, %rsp
return v200
; nextln: movq %rbp, %rsp
; nextln: popq %rbp
; nextln: ret
}

32
cranelift/filetests/src/test_unwind.rs
View File

@@ -72,7 +72,6 @@ impl SubTest for TestUnwind {
}
mod windowsx64 {
use byteorder::{ByteOrder, LittleEndian};
use std::fmt::Write;
pub fn dump<W: Write>(text: &mut W, mem: &[u8]) {
@@ -165,23 +164,24 @@ mod windowsx64 {
let op_and_info = mem[1];
let op = UnwindOperation::from(op_and_info & 0xF);
let info = (op_and_info & 0xF0) >> 4;
let unwind_le_bytes = |bytes| match (bytes, &mem[2..]) {
(2, &[b0, b1, ..]) => UnwindValue::U16(u16::from_le_bytes([b0, b1])),
(4, &[b0, b1, b2, b3, ..]) => {
UnwindValue::U32(u32::from_le_bytes([b0, b1, b2, b3]))
}
(_, _) => panic!("not enough bytes to unwind value"),
};
let value = match op {
UnwindOperation::LargeStackAlloc => match info {
0 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])),
1 => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])),
_ => panic!("unexpected stack alloc info value"),
},
UnwindOperation::SaveNonvolatileRegister => {
UnwindValue::U16(LittleEndian::read_u16(&mem[2..]))
}
UnwindOperation::SaveNonvolatileRegisterFar => {
UnwindValue::U32(LittleEndian::read_u32(&mem[2..]))
}
UnwindOperation::SaveXmm128 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])),
UnwindOperation::SaveXmm128Far => {
UnwindValue::U32(LittleEndian::read_u32(&mem[2..]))
let value = match (&op, info) {
(UnwindOperation::LargeStackAlloc, 0) => unwind_le_bytes(2),
(UnwindOperation::LargeStackAlloc, 1) => unwind_le_bytes(4),
(UnwindOperation::LargeStackAlloc, _) => {
panic!("unexpected stack alloc info value")
}
(UnwindOperation::SaveNonvolatileRegister, _) => unwind_le_bytes(2),
(UnwindOperation::SaveNonvolatileRegisterFar, _) => unwind_le_bytes(4),
(UnwindOperation::SaveXmm128, _) => unwind_le_bytes(2),
(UnwindOperation::SaveXmm128Far, _) => unwind_le_bytes(4),
_ => UnwindValue::None,
};

34
cranelift/frontend/src/frontend.rs
View File

@@ -640,6 +640,7 @@ impl<'a> FunctionBuilder<'a> {
dest_align: u8,
src_align: u8,
non_overlapping: bool,
mut flags: MemFlags,
) {
// Currently the result of guess work, not actual profiling.
const THRESHOLD: u64 = 4;
@@ -676,7 +677,6 @@ impl<'a> FunctionBuilder<'a> {
return;
}
let mut flags = MemFlags::new();
flags.set_aligned();
// Load all of the memory first. This is necessary in case `dest` overlaps.
@@ -732,6 +732,7 @@ impl<'a> FunctionBuilder<'a> {
ch: u8,
size: u64,
buffer_align: u8,
mut flags: MemFlags,
) {
// Currently the result of guess work, not actual profiling.
const THRESHOLD: u64 = 4;
@@ -763,7 +764,6 @@ impl<'a> FunctionBuilder<'a> {
let size = self.ins().iconst(config.pointer_type(), size as i64);
self.call_memset(config, buffer, ch, size);
} else {
let mut flags = MemFlags::new();
flags.set_aligned();
let ch = u64::from(ch);
@@ -851,7 +851,9 @@ mod tests {
use alloc::string::ToString;
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::types::*;
use cranelift_codegen::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature};
use cranelift_codegen::ir::{
AbiParam, ExternalName, Function, InstBuilder, MemFlags, Signature,
};
use cranelift_codegen::isa::CallConv;
use cranelift_codegen::settings;
use cranelift_codegen::verifier::verify_function;
@@ -1063,7 +1065,16 @@ block0:
let src = builder.use_var(x);
let dest = builder.use_var(y);
let size = 8;
builder.emit_small_memory_copy(target.frontend_config(), dest, src, size, 8, 8, true);
builder.emit_small_memory_copy(
target.frontend_config(),
dest,
src,
size,
8,
8,
true,
MemFlags::new(),
);
builder.ins().return_(&[dest]);
builder.seal_all_blocks();
@@ -1121,7 +1132,16 @@ block0:
let src = builder.use_var(x);
let dest = builder.use_var(y);
let size = 8192;
builder.emit_small_memory_copy(target.frontend_config(), dest, src, size, 8, 8, true);
builder.emit_small_memory_copy(
target.frontend_config(),
dest,
src,
size,
8,
8,
true,
MemFlags::new(),
);
builder.ins().return_(&[dest]);
builder.seal_all_blocks();
@@ -1179,7 +1199,7 @@ block0:
let dest = builder.use_var(y);
let size = 8;
builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8);
builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8, MemFlags::new());
builder.ins().return_(&[dest]);
builder.seal_all_blocks();
@@ -1232,7 +1252,7 @@ block0:
let dest = builder.use_var(y);
let size = 8192;
builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8);
builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8, MemFlags::new());
builder.ins().return_(&[dest]);
builder.seal_all_blocks();

2
cranelift/interpreter/Cargo.toml
View File

@@ -13,13 +13,13 @@ edition = "2018"
[dependencies]
cranelift-codegen = { path = "../codegen", version = "0.73.0", features = ["all-arch"] }
cranelift-entity = { path = "../entity", version = "0.73.0" }
cranelift-reader = { path = "../reader", version = "0.73.0" }
log = { version = "0.4.8", default-features = false }
smallvec = "1.6.1"
thiserror = "1.0.15"
[dev-dependencies]
cranelift-frontend = { path = "../frontend", version = "0.73.0" }
cranelift-reader = { path = "../reader", version = "0.73.0" }
[badges]
maintenance = { status = "experimental" }

7
cranelift/jit/src/backend.rs
View File

@@ -285,6 +285,13 @@ impl JITModule {
}
}
/// Returns the given function's entry in the Global Offset Table.
///
/// Panics if there's no entry in the table for the given function.
pub fn read_got_entry(&self, func_id: FuncId) -> *const u8 {
unsafe { *self.function_got_entries[func_id].unwrap().as_ptr() }
}
fn get_got_address(&self, name: &ir::ExternalName) -> *const u8 {
match *name {
ir::ExternalName::User { .. } => {

9
cranelift/jit/src/compiled_blob.rs
View File

@@ -72,6 +72,15 @@ impl CompiledBlob {
write_unaligned(at as *mut i32, pcrel)
};
}
Reloc::S390xPCRel32Dbl => {
let base = get_address(name);
let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
let pcrel = i32::try_from(((what as isize) - (at as isize)) >> 1).unwrap();
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut i32, pcrel)
};
}
_ => unimplemented!(),
}
}

1
cranelift/module/Cargo.toml
View File

@@ -15,7 +15,6 @@ cranelift-codegen = { path = "../codegen", version = "0.73.0", default-features
cranelift-entity = { path = "../entity", version = "0.73.0" }
hashbrown = { version = "0.9.1", optional = true }
log = { version = "0.4.6", default-features = false }
thiserror = "1.0.4"
anyhow = "1.0"
[features]

76
cranelift/module/src/module.rs
View File

@@ -12,7 +12,6 @@ use cranelift_codegen::entity::{entity_impl, PrimaryMap};
use cranelift_codegen::{ir, isa, CodegenError, Context};
use std::borrow::ToOwned;
use std::string::String;
use thiserror::Error;
/// A function identifier for use in the `Module` interface.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
@@ -168,30 +167,85 @@ impl FunctionDeclaration {
}
/// Error messages for all `Module` methods
#[derive(Error, Debug)]
#[derive(Debug)]
pub enum ModuleError {
/// Indicates an identifier was used before it was declared
#[error("Undeclared identifier: {0}")]
Undeclared(String),
/// Indicates an identifier was used as data/function first, but then used as the other
#[error("Incompatible declaration of identifier: {0}")]
IncompatibleDeclaration(String),
/// Indicates a function identifier was declared with a
/// different signature than declared previously
#[error("Function {0} signature {2:?} is incompatible with previous declaration {1:?}")]
IncompatibleSignature(String, ir::Signature, ir::Signature),
/// Indicates an identifier was defined more than once
#[error("Duplicate definition of identifier: {0}")]
DuplicateDefinition(String),
/// Indicates an identifier was defined, but was declared as an import
#[error("Invalid to define identifier declared as an import: {0}")]
InvalidImportDefinition(String),
/// Wraps a `cranelift-codegen` error
#[error("Compilation error: {0}")]
Compilation(#[from] CodegenError),
Compilation(CodegenError),
/// Wraps a generic error from a backend
#[error("Backend error: {0}")]
Backend(#[source] anyhow::Error),
Backend(anyhow::Error),
}
// This is manually implementing Error and Display instead of using thiserror to reduce the amount
// of dependencies used by Cranelift.
impl std::error::Error for ModuleError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Undeclared { .. }
| Self::IncompatibleDeclaration { .. }
| Self::IncompatibleSignature { .. }
| Self::DuplicateDefinition { .. }
| Self::InvalidImportDefinition { .. } => None,
Self::Compilation(source) => Some(source),
Self::Backend(source) => Some(&**source),
}
}
}
impl std::fmt::Display for ModuleError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Self::Undeclared(name) => {
write!(f, "Undeclared identifier: {}", name)
}
Self::IncompatibleDeclaration(name) => {
write!(f, "Incompatible declaration of identifier: {}", name,)
}
Self::IncompatibleSignature(name, prev_sig, new_sig) => {
write!(
f,
"Function {} signature {:?} is incompatible with previous declaration {:?}",
name, new_sig, prev_sig,
)
}
Self::DuplicateDefinition(name) => {
write!(f, "Duplicate definition of identifier: {}", name)
}
Self::InvalidImportDefinition(name) => {
write!(
f,
"Invalid to define identifier declared as an import: {}",
name,
)
}
Self::Compilation(err) => {
write!(f, "Compilation error: {}", err)
}
Self::Backend(err) => write!(f, "Backend error: {}", err),
}
}
}
impl std::convert::From<CodegenError> for ModuleError {
fn from(source: CodegenError) -> Self {
Self::Compilation { 0: source }
}
}
/// A convenient alias for a `Result` that uses `ModuleError` as the error type.

4
cranelift/object/src/backend.rs
View File

@@ -639,8 +639,8 @@ fn translate_linkage(linkage: Linkage) -> (SymbolScope, bool) {
(scope, weak)
}
/// This is the output of `Module`'s
/// [`finish`](../cranelift_module/struct.Module.html#method.finish) function.
/// This is the output of `ObjectModule`'s
/// [`finish`](../struct.ObjectModule.html#method.finish) function.
/// It contains the generated `Object` and other information produced during
/// compilation.
pub struct ObjectProduct {

1
cranelift/reader/Cargo.toml
View File

@@ -13,7 +13,6 @@ edition = "2018"
cranelift-codegen = { path = "../codegen", version = "0.73.0" }
smallvec = "1.6.1"
target-lexicon = "0.12"
thiserror = "1.0.15"
[badges]
maintenance = { status = "experimental" }

5
cranelift/src/disasm.rs
View File

@@ -153,6 +153,11 @@ cfg_if! {
cs.set_skipdata(true).map_err(map_caperr)?;
cs
}
Architecture::S390x {..} => Capstone::new()
.sysz()
.mode(arch::sysz::ArchMode::Default)
.build()
.map_err(map_caperr)?,
_ => anyhow::bail!("Unknown ISA"),
};

4
cranelift/wasm/src/environ/dummy.rs
View File

@@ -747,7 +747,7 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment {
&mut self,
_table_index: TableIndex,
_base: Option<GlobalIndex>,
_offset: usize,
_offset: u32,
_elements: Box<[FuncIndex]>,
) -> WasmResult<()> {
// We do nothing
@@ -792,7 +792,7 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment {
&mut self,
_memory_index: MemoryIndex,
_base: Option<GlobalIndex>,
_offset: usize,
_offset: u32,
_data: &'data [u8],
) -> WasmResult<()> {
// We do nothing

11
cranelift/wasm/src/environ/spec.rs
View File

@@ -937,7 +937,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
&mut self,
table_index: TableIndex,
base: Option<GlobalIndex>,
offset: usize,
offset: u32,
elements: Box<[FuncIndex]>,
) -> WasmResult<()>;
@@ -948,6 +948,13 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
elements: Box<[FuncIndex]>,
) -> WasmResult<()>;
/// Indicates that a declarative element segment was seen in the wasm
/// module.
fn declare_elements(&mut self, elements: Box<[FuncIndex]>) -> WasmResult<()> {
drop(elements);
Ok(())
}
/// Provides the number of passive data segments up front.
///
/// By default this does nothing, but implementations may use this to
@@ -984,7 +991,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
&mut self,
memory_index: MemoryIndex,
base: Option<GlobalIndex>,
offset: usize,
offset: u32,
data: &'data [u8],
) -> WasmResult<()>;

18
cranelift/wasm/src/sections_translator.rs
View File

@@ -377,7 +377,7 @@ pub fn parse_element_section<'data>(
} => {
let mut init_expr_reader = init_expr.get_binary_reader();
let (base, offset) = match init_expr_reader.read_operator()? {
Operator::I32Const { value } => (None, value as u32 as usize),
Operator::I32Const { value } => (None, value as u32),
Operator::GlobalGet { global_index } => {
(Some(GlobalIndex::from_u32(global_index)), 0)
}
@@ -388,12 +388,6 @@ pub fn parse_element_section<'data>(
));
}
};
// Check for offset + len overflow
if offset.checked_add(segments.len()).is_none() {
return Err(wasm_unsupported!(
"element segment offset and length overflows"
));
}
environ.declare_table_elements(
TableIndex::from_u32(table_index),
base,
@@ -406,7 +400,7 @@ pub fn parse_element_section<'data>(
environ.declare_passive_element(index, segments)?;
}
ElementKind::Declared => {
// Nothing to do here.
environ.declare_elements(segments)?;
}
}
}
@@ -429,7 +423,7 @@ pub fn parse_data_section<'data>(
} => {
let mut init_expr_reader = init_expr.get_binary_reader();
let (base, offset) = match init_expr_reader.read_operator()? {
Operator::I32Const { value } => (None, value as u32 as usize),
Operator::I32Const { value } => (None, value as u32),
Operator::GlobalGet { global_index } => {
(Some(GlobalIndex::from_u32(global_index)), 0)
}
@@ -440,12 +434,6 @@ pub fn parse_data_section<'data>(
))
}
};
// Check for offset + len overflow
if offset.checked_add(data.len()).is_none() {
return Err(wasm_unsupported!(
"data segment offset and length overflows"
));
}
environ.declare_data_initialization(
MemoryIndex::from_u32(memory_index),
base,

14
crates/c-api/include/wasmtime.h
View File

@@ -283,14 +283,6 @@ WASMTIME_CONFIG_PROP(void, static_memory_guard_size, uint64_t)
*/
WASMTIME_CONFIG_PROP(void, dynamic_memory_guard_size, uint64_t)
/**
* \brief Configures the maximum number of instances that can be created.
*
* For more information see the Rust documentation at
* https://bytecodealliance.github.io/wasmtime/api/wasmtime/struct.Config.html#method.max_instances.
*/
WASMTIME_CONFIG_PROP(void, max_instances, size_t)
/**
* \brief Enables Wasmtime's cache and loads configuration from the specified
* path.
@@ -1000,9 +992,13 @@ WASM_API_EXTERN own wasmtime_error_t* wasmtime_module_serialize(
/**
* \brief Build a module from serialized data.
* *
*
* This function does not take ownership of any of its arguments, but the
* returned error and module are owned by the caller.
*
* This function is not safe to receive arbitrary user input. See the Rust
* documentation for more information on what inputs are safe to pass in here
* (e.g. only that of #wasmtime_module_serialize)
*/
WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize(
wasm_engine_t *engine,

5
crates/c-api/src/config.rs
View File

@@ -176,8 +176,3 @@ pub extern "C" fn wasmtime_config_static_memory_guard_size_set(c: &mut wasm_conf
pub extern "C" fn wasmtime_config_dynamic_memory_guard_size_set(c: &mut wasm_config_t, size: u64) {
c.config.dynamic_memory_guard_size(size);
}
#[no_mangle]
pub extern "C" fn wasmtime_config_max_instances_set(c: &mut wasm_config_t, limit: usize) {
c.config.max_instances(limit);
}

8
crates/c-api/src/memory.rs
View File

@@ -31,13 +31,13 @@ impl wasm_memory_t {
pub extern "C" fn wasm_memory_new(
store: &wasm_store_t,
mt: &wasm_memorytype_t,
) -> Box<wasm_memory_t> {
let memory = Memory::new(&store.store, mt.ty().ty.clone());
Box::new(wasm_memory_t {
) -> Option<Box<wasm_memory_t>> {
let memory = Memory::new(&store.store, mt.ty().ty.clone()).ok()?;
Some(Box::new(wasm_memory_t {
ext: wasm_extern_t {
which: memory.into(),
},
})
}))
}
#[no_mangle]

11
crates/c-api/src/module.rs
View File

@@ -185,10 +185,13 @@ pub extern "C" fn wasmtime_module_deserialize(
binary: &wasm_byte_vec_t,
ret: &mut *mut wasm_module_t,
) -> Option<Box<wasmtime_error_t>> {
handle_result(Module::new(&engine.engine, binary.as_slice()), |module| {
let module = Box::new(wasm_module_t::new(module));
*ret = Box::into_raw(module);
})
handle_result(
unsafe { Module::deserialize(&engine.engine, binary.as_slice()) },
|module| {
let module = Box::new(wasm_module_t::new(module));
*ret = Box::into_raw(module);
},
)
}
#[no_mangle]

78
crates/debug/src/lib.rs
View File

@@ -3,6 +3,7 @@
#![allow(clippy::cast_ptr_alignment)]
use anyhow::{bail, ensure, Error};
use object::endian::{BigEndian, Endian, Endianness, LittleEndian};
use object::{RelocationEncoding, RelocationKind};
use std::collections::HashMap;
@@ -18,13 +19,20 @@ pub fn create_gdbjit_image(
defined_funcs_offset: usize,
funcs: &[*const u8],
) -> Result<Vec<u8>, Error> {
ensure_supported_elf_format(&mut bytes)?;
let e = ensure_supported_elf_format(&mut bytes)?;
// patch relocs
relocate_dwarf_sections(&mut bytes, defined_funcs_offset, funcs)?;
// elf is still missing details...
convert_object_elf_to_loadable_file(&mut bytes, code_region);
match e {
Endianness::Little => {
convert_object_elf_to_loadable_file::<LittleEndian>(&mut bytes, code_region)
}
Endianness::Big => {
convert_object_elf_to_loadable_file::<BigEndian>(&mut bytes, code_region)
}
}
// let mut file = ::std::fs::File::create(::std::path::Path::new("test.o")).expect("file");
// ::std::io::Write::write_all(&mut file, &bytes).expect("write");
@@ -83,20 +91,36 @@ fn relocate_dwarf_sections(
Ok(())
}
fn ensure_supported_elf_format(bytes: &mut Vec<u8>) -> Result<(), Error> {
fn ensure_supported_elf_format(bytes: &mut Vec<u8>) -> Result<Endianness, Error> {
use object::elf::*;
use object::endian::LittleEndian;
use object::read::elf::*;
use object::Bytes;
use std::mem::size_of;
let e = LittleEndian;
let header: &FileHeader64<LittleEndian> =
unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) };
ensure!(
header.e_ident.class == ELFCLASS64 && header.e_ident.data == ELFDATA2LSB,
"bits and endianess in .ELF",
);
let kind = match object::FileKind::parse(bytes) {
Ok(file) => file,
Err(err) => {
bail!("Failed to parse file: {}", err);
}
};
let header = match kind {
object::FileKind::Elf64 => {
match object::elf::FileHeader64::<Endianness>::parse(Bytes(bytes)) {
Ok(header) => header,
Err(err) => {
bail!("Unsupported ELF file: {}", err);
}
}
}
_ => {
bail!("only 64-bit ELF files currently supported")
}
};
let e = header.endian().unwrap();
match header.e_machine.get(e) {
EM_X86_64 => (),
EM_S390 => (),
machine => {
bail!("Unsupported ELF target machine: {:x}", machine);
}
@@ -106,23 +130,25 @@ fn ensure_supported_elf_format(bytes: &mut Vec<u8>) -> Result<(), Error> {
"program header table is empty"
);
let e_shentsize = header.e_shentsize.get(e);
ensure!(
e_shentsize as usize == size_of::<SectionHeader64<LittleEndian>>(),
"size of sh"
);
Ok(())
let req_shentsize = match e {
Endianness::Little => size_of::<SectionHeader64<LittleEndian>>(),
Endianness::Big => size_of::<SectionHeader64<BigEndian>>(),
};
ensure!(e_shentsize as usize == req_shentsize, "size of sh");
Ok(e)
}
fn convert_object_elf_to_loadable_file(bytes: &mut Vec<u8>, code_region: (*const u8, usize)) {
fn convert_object_elf_to_loadable_file<E: Endian>(
bytes: &mut Vec<u8>,
code_region: (*const u8, usize),
) {
use object::elf::*;
use object::endian::LittleEndian;
use std::ffi::CStr;
use std::mem::size_of;
use std::os::raw::c_char;
let e = LittleEndian;
let header: &FileHeader64<LittleEndian> =
unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) };
let e = E::default();
let header: &FileHeader64<E> = unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) };
let e_shentsize = header.e_shentsize.get(e);
let e_shoff = header.e_shoff.get(e);
@@ -130,7 +156,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec<u8>, code_region: (*const
let mut shstrtab_off = 0;
for i in 0..e_shnum {
let off = e_shoff as isize + i as isize * e_shentsize as isize;
let section: &SectionHeader64<LittleEndian> =
let section: &SectionHeader64<E> =
unsafe { &*(bytes.as_ptr().offset(off) as *const SectionHeader64<_>) };
if section.sh_type.get(e) != SHT_STRTAB {
continue;
@@ -140,7 +166,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec<u8>, code_region: (*const
let mut segment: Option<_> = None;
for i in 0..e_shnum {
let off = e_shoff as isize + i as isize * e_shentsize as isize;
let section: &mut SectionHeader64<LittleEndian> =
let section: &mut SectionHeader64<E> =
unsafe { &mut *(bytes.as_mut_ptr().offset(off) as *mut SectionHeader64<_>) };
if section.sh_type.get(e) != SHT_PROGBITS {
continue;
@@ -171,12 +197,12 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec<u8>, code_region: (*const
// LLDB wants segment with virtual address set, placing them at the end of ELF.
let ph_off = bytes.len();
let e_phentsize = size_of::<ProgramHeader64<LittleEndian>>();
let e_phentsize = size_of::<ProgramHeader64<E>>();
let e_phnum = 1;
bytes.resize(ph_off + e_phentsize * e_phnum, 0);
if let Some((sh_offset, sh_size)) = segment {
let (v_offset, size) = code_region;
let program: &mut ProgramHeader64<LittleEndian> =
let program: &mut ProgramHeader64<E> =
unsafe { &mut *(bytes.as_ptr().add(ph_off) as *mut ProgramHeader64<_>) };
program.p_type.set(e, PT_LOAD);
program.p_offset.set(e, sh_offset);
@@ -189,7 +215,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec<u8>, code_region: (*const
}
// It is somewhat loadable ELF file at this moment.
let header: &mut FileHeader64<LittleEndian> =
let header: &mut FileHeader64<E> =
unsafe { &mut *(bytes.as_mut_ptr() as *mut FileHeader64<_>) };
header.e_type.set(e, ET_DYN);
header.e_phoff.set(e, ph_off as u64);

28
crates/debug/src/transform/expression.rs
View File

@@ -512,24 +512,28 @@ where
}
};
}
// Find all landing pads by scanning bytes, do not care about
// false location at this moment.
// Looks hacky but it is fast; does not need to be really exact.
for i in 0..buf.len() - 2 {
let op = buf[i];
if op == gimli::constants::DW_OP_bra.0 || op == gimli::constants::DW_OP_skip.0 {
// TODO fix for big-endian
let offset = i16::from_le_bytes([buf[i + 1], buf[i + 2]]);
let origin = i + 3;
// Discarding out-of-bounds jumps (also some of falsely detected ops)
if (offset >= 0 && offset as usize + origin <= buf.len())
|| (offset < 0 && -offset as usize <= origin)
{
let target = buf.len() as isize - origin as isize - offset as isize;
jump_targets.insert(target as u64, JumpTargetMarker::new());
if buf.len() > 2 {
for i in 0..buf.len() - 2 {
let op = buf[i];
if op == gimli::constants::DW_OP_bra.0 || op == gimli::constants::DW_OP_skip.0 {
// TODO fix for big-endian
let offset = i16::from_le_bytes([buf[i + 1], buf[i + 2]]);
let origin = i + 3;
// Discarding out-of-bounds jumps (also some of falsely detected ops)
if (offset >= 0 && offset as usize + origin <= buf.len())
|| (offset < 0 && -offset as usize <= origin)
{
let target = buf.len() as isize - origin as isize - offset as isize;
jump_targets.insert(target as u64, JumpTargetMarker::new());
}
}
}
}
while !pc.is_empty() {
let unread_bytes = pc.len().into_u64();
if let Some(marker) = jump_targets.get(&unread_bytes) {

14
crates/debug/src/transform/unit.rs
View File

@@ -10,6 +10,7 @@ use anyhow::{Context, Error};
use gimli::write;
use gimli::{AttributeValue, DebuggingInformationEntry, Unit};
use std::collections::HashSet;
use wasmtime_environ::ir::Endianness;
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::wasm::DefinedFuncIndex;
use wasmtime_environ::{CompiledFunctions, ModuleMemoryOffset};
@@ -463,6 +464,19 @@ where
isa,
)?;
// Data in WebAssembly memory always uses little-endian byte order.
// If the native architecture is big-endian, we need to mark all
// base types used to refer to WebAssembly memory as little-endian
// using the DW_AT_endianity attribute, so that the debugger will
// be able to correctly access them.
if entry.tag() == gimli::DW_TAG_base_type && isa.endianness() == Endianness::Big {
let current_scope = comp_unit.get_mut(die_id);
current_scope.set(
gimli::DW_AT_endianity,
write::AttributeValue::Endianity(gimli::DW_END_little),
);
}
if entry.tag() == gimli::DW_TAG_subprogram && !current_scope_ranges.is_empty() {
append_vmctx_info(
comp_unit,

23
crates/debug/src/write_debuginfo.rs
View File

@@ -2,6 +2,7 @@ pub use crate::transform::transform_dwarf;
use gimli::write::{Address, Dwarf, EndianVec, FrameTable, Result, Sections, Writer};
use gimli::{RunTimeEndian, SectionId};
use wasmtime_environ::entity::EntityRef;
use wasmtime_environ::ir::Endianness;
use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa};
use wasmtime_environ::{CompiledFunctions, DebugInfoData, ModuleMemoryOffset};
@@ -26,10 +27,19 @@ pub struct DwarfSection {
}
fn emit_dwarf_sections(
isa: &dyn TargetIsa,
mut dwarf: Dwarf,
frames: Option<FrameTable>,
) -> anyhow::Result<Vec<DwarfSection>> {
let mut sections = Sections::new(WriterRelocate::default());
let endian = match isa.endianness() {
Endianness::Little => RunTimeEndian::Little,
Endianness::Big => RunTimeEndian::Big,
};
let writer = WriterRelocate {
relocs: Vec::new(),
writer: EndianVec::new(endian),
};
let mut sections = Sections::new(writer);
dwarf.write(&mut sections)?;
if let Some(frames) = frames {
frames.write_debug_frame(&mut sections.debug_frame)?;
@@ -54,15 +64,6 @@ pub struct WriterRelocate {
writer: EndianVec<RunTimeEndian>,
}
impl Default for WriterRelocate {
fn default() -> Self {
WriterRelocate {
relocs: Vec::new(),
writer: EndianVec::new(RunTimeEndian::Little),
}
}
}
impl Writer for WriterRelocate {
type Endian = RunTimeEndian;
@@ -156,6 +157,6 @@ pub fn emit_dwarf<'a>(
) -> anyhow::Result<Vec<DwarfSection>> {
let dwarf = transform_dwarf(isa, debuginfo_data, funcs, memory_offset)?;
let frame_table = create_frame_table(isa, funcs);
let sections = emit_dwarf_sections(dwarf, frame_table)?;
let sections = emit_dwarf_sections(isa, dwarf, frame_table)?;
Ok(sections)
}

2
crates/environ/Cargo.toml
View File

@@ -12,8 +12,6 @@ readme = "README.md"
edition = "2018"
[dependencies]
anyhow = "1.0"
region = "2.2.0"
cranelift-codegen = { path = "../../cranelift/codegen", version = "0.73.0", features = ["enable-serde"] }
cranelift-entity = { path = "../../cranelift/entity", version = "0.73.0", features = ["enable-serde"] }
cranelift-wasm = { path = "../../cranelift/wasm", version = "0.73.0", features = ["enable-serde"] }

4
crates/environ/src/data_structures.rs
View File

@@ -3,8 +3,8 @@
pub mod ir {
pub use cranelift_codegen::binemit::{Reloc, StackMap};
pub use cranelift_codegen::ir::{
types, AbiParam, ArgumentPurpose, JumpTableOffsets, LabelValueLoc, LibCall, Signature,
SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc,
types, AbiParam, ArgumentPurpose, Endianness, JumpTableOffsets, LabelValueLoc, LibCall,
Signature, SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc,
};
pub use cranelift_codegen::{ValueLabelsRanges, ValueLocRange};
}

19
crates/environ/src/module.rs
View File

@@ -7,6 +7,7 @@ use cranelift_wasm::*;
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::convert::TryFrom;
use std::sync::Arc;
/// Implemenation styles for WebAssembly linear memory.
@@ -86,7 +87,7 @@ pub struct MemoryInitializer {
/// Optionally, a global variable giving a base index.
pub base: Option<GlobalIndex>,
/// The offset to add to the base.
pub offset: usize,
pub offset: u32,
/// The data to write into the linear memory.
pub data: Box<[u8]>,
}
@@ -168,7 +169,15 @@ impl MemoryInitialization {
// Perform a bounds check on the segment
// As this segment is referencing a defined memory without a global base, the last byte
// written to by the segment cannot exceed the memory's initial minimum size
if (initializer.offset + initializer.data.len())
let offset = usize::try_from(initializer.offset).unwrap();
let end = match offset.checked_add(initializer.data.len()) {
Some(end) => end,
None => {
out_of_bounds = true;
continue;
}
};
if end
> ((module.memory_plans[initializer.memory_index].memory.minimum
as usize)
* WASM_PAGE_SIZE)
@@ -178,8 +187,8 @@ impl MemoryInitialization {
}
let pages = &mut map[index];
let mut page_index = initializer.offset / WASM_PAGE_SIZE;
let mut page_offset = initializer.offset % WASM_PAGE_SIZE;
let mut page_index = offset / WASM_PAGE_SIZE;
let mut page_offset = offset % WASM_PAGE_SIZE;
let mut data_offset = 0;
let mut data_remaining = initializer.data.len();
@@ -268,7 +277,7 @@ pub struct TableInitializer {
/// Optionally, a global variable giving a base index.
pub base: Option<GlobalIndex>,
/// The offset to add to the base.
pub offset: usize,
pub offset: u32,
/// The values to write into the table elements.
pub elements: Box<[FuncIndex]>,
}

11
crates/environ/src/module_environ.rs
View File

@@ -705,7 +705,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
&mut self,
table_index: TableIndex,
base: Option<GlobalIndex>,
offset: usize,
offset: u32,
elements: Box<[FuncIndex]>,
) -> WasmResult<()> {
for element in elements.iter() {
@@ -746,6 +746,13 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
Ok(())
}
fn declare_elements(&mut self, segments: Box<[FuncIndex]>) -> WasmResult<()> {
for element in segments.iter() {
self.flag_func_possibly_exported(*element);
}
Ok(())
}
fn reserve_function_bodies(&mut self, _count: u32, offset: u64) {
self.result.debuginfo.wasm_file.code_section_offset = offset;
}
@@ -794,7 +801,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
&mut self,
memory_index: MemoryIndex,
base: Option<GlobalIndex>,
offset: usize,
offset: u32,
data: &'data [u8],
) -> WasmResult<()> {
match &mut self.result.module.memory_initialization {

18
crates/environ/src/vmoffsets.rs
View File

@@ -6,7 +6,7 @@
// struct VMContext {
// interrupts: *const VMInterrupts,
// externref_activations_table: *mut VMExternRefActivationsTable,
// stack_map_registry: *mut StackMapRegistry,
// module_info_lookup: *const dyn ModuleInfoLookup,
// signature_ids: [VMSharedSignatureIndex; module.num_signature_ids],
// imported_functions: [VMFunctionImport; module.num_imported_functions],
// imported_tables: [VMTableImport; module.num_imported_tables],
@@ -77,7 +77,7 @@ pub struct VMOffsets {
// precalculated offsets of various member fields
interrupts: u32,
externref_activations_table: u32,
stack_map_registry: u32,
module_info_lookup: u32,
signature_ids: u32,
imported_functions: u32,
imported_tables: u32,
@@ -149,7 +149,7 @@ impl From<VMOffsetsFields> for VMOffsets {
num_defined_globals: fields.num_defined_globals,
interrupts: 0,
externref_activations_table: 0,
stack_map_registry: 0,
module_info_lookup: 0,
signature_ids: 0,
imported_functions: 0,
imported_tables: 0,
@@ -168,13 +168,13 @@ impl From<VMOffsetsFields> for VMOffsets {
.interrupts
.checked_add(u32::from(fields.pointer_size))
.unwrap();
ret.stack_map_registry = ret
ret.module_info_lookup = ret
.externref_activations_table
.checked_add(u32::from(fields.pointer_size))
.unwrap();
ret.signature_ids = ret
.stack_map_registry
.checked_add(u32::from(fields.pointer_size))
.module_info_lookup
.checked_add(u32::from(fields.pointer_size * 2))
.unwrap();
ret.imported_functions = ret
.signature_ids
@@ -507,10 +507,10 @@ impl VMOffsets {
self.externref_activations_table
}
/// The offset of the `*mut StackMapRegistry` member.
/// The offset of the `*const dyn ModuleInfoLookup` member.
#[inline]
pub fn vmctx_stack_map_registry(&self) -> u32 {
self.stack_map_registry
pub fn vmctx_module_info_lookup(&self) -> u32 {
self.module_info_lookup
}
/// The offset of the `signature_ids` array.

112
crates/fiber/src/arch/s390x.S Normal file
View File

@@ -0,0 +1,112 @@
// A WORD OF CAUTION
//
// This entire file basically needs to be kept in sync with itself. It's not
// really possible to modify just one bit of this file without understanding
// all the other bits. Documentation tries to reference various bits here and
// there but try to make sure to read over everything before tweaking things!
//
// Also at this time this file is heavily based off the x86_64 file, so you'll
// probably want to read that one as well.
#include "header.h"
// fn(top_of_stack(%x0): *mut u8)
HIDDEN(wasmtime_fiber_switch)
GLOBL(wasmtime_fiber_switch)
.p2align 2
TYPE(wasmtime_fiber_switch)
FUNCTION(wasmtime_fiber_switch):
// Save all callee-saved registers on the stack since we're assuming
// they're clobbered as a result of the stack switch.
stmg %r6, %r15, 48(%r15)
aghi %r15, -64
std %f8, 0(%r15)
std %f9, 8(%r15)
std %f10, 16(%r15)
std %f11, 24(%r15)
std %f12, 32(%r15)
std %f13, 40(%r15)
std %f14, 48(%r15)
std %f15, 56(%r15)
// Load our previously saved stack pointer to resume to, and save off our
// current stack pointer on where to come back to eventually.
lg %r1, -16(%r2)
stg %r15, -16(%r2)
// Switch to the new stack and restore all our callee-saved registers after
// the switch and return to our new stack.
ld %f8, 0(%r1)
ld %f9, 8(%r1)
ld %f10, 16(%r1)
ld %f11, 24(%r1)
ld %f12, 32(%r1)
ld %f13, 40(%r1)
ld %f14, 48(%r1)
ld %f15, 56(%r1)
lmg %r6, %r15, 112(%r1)
br %r14
SIZE(wasmtime_fiber_switch)
// fn(
// top_of_stack(%x0): *mut u8,
// entry_point(%x1): extern fn(*mut u8, *mut u8),
// entry_arg0(%x2): *mut u8,
// )
HIDDEN(wasmtime_fiber_init)
GLOBL(wasmtime_fiber_init)
.p2align 2
TYPE(wasmtime_fiber_init)
FUNCTION(wasmtime_fiber_init):
larl %r1, FUNCTION(wasmtime_fiber_start)
stg %r1, -48(%r2) // wasmtime_fiber_start - restored into %r14
stg %r2, -112(%r2) // top_of_stack - restored into %r6
stg %r3, -104(%r2) // entry_point - restored into %r7
stg %r4, -96(%r2) // entry_arg0 - restored into %r8
aghi %r2, -160 // 160 bytes register save area
stg %r2, 120(%r2) // bottom of register save area - restored into %r15
// `wasmtime_fiber_switch` has a 64 byte stack.
aghi %r2, -64
stg %r2, 208(%r2)
br %r14
SIZE(wasmtime_fiber_init)
.p2align 2
TYPE(wasmtime_fiber_start)
FUNCTION(wasmtime_fiber_start):
.cfi_startproc simple
// See the x86_64 file for more commentary on what these CFI directives are
// doing. Like over there note that the relative offsets to registers here
// match the frame layout in `wasmtime_fiber_switch`.
.cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \
7, /* the byte length of this expression */ \
0x7f, 0x90, 0x1, /* DW_OP_breg15 0x90 */ \
0x06, /* DW_OP_deref */ \
0x23, 0xe0, 0x1 /* DW_OP_plus_uconst 0xe0 */
.cfi_rel_offset 6, -112
.cfi_rel_offset 7, -104
.cfi_rel_offset 8, -96
.cfi_rel_offset 9, -88
.cfi_rel_offset 10, -80
.cfi_rel_offset 11, -72
.cfi_rel_offset 12, -64
.cfi_rel_offset 13, -56
.cfi_rel_offset 14, -48
.cfi_rel_offset 15, -40
// Load our two arguments prepared by `wasmtime_fiber_init`.
lgr %r2, %r8 // entry_arg0
lgr %r3, %r6 // top_of_stack
// ... and then we call the function! Note that this is a function call so
// our frame stays on the stack to backtrace through.
basr %r14, %r7 // entry_point
// .. technically we shouldn't get here, so just trap.
.word 0x0000
.cfi_endproc
SIZE(wasmtime_fiber_start)
FOOTER

7
crates/fuzzing/src/lib.rs
View File

@@ -39,13 +39,6 @@ pub fn fuzz_default_config(strategy: wasmtime::Strategy) -> anyhow::Result<wasmt
.wasm_bulk_memory(true)
.wasm_reference_types(true)
.wasm_module_linking(true)
// The limits here are chosen based on the default "maximum type size"
// configured in wasm-smith, which is 1000. This means that instances
// are allowed to, for example, export up to 1000 memories. We bump that
// a little bit here to give us some slop.
.max_instances(1100)
.max_tables(1100)
.max_memories(1100)
.strategy(strategy)?;
Ok(config)
}

27
crates/fuzzing/src/oracles.rs
View File

@@ -44,6 +44,21 @@ fn log_wasm(wasm: &[u8]) {
}
}
fn create_store(engine: &Engine) -> Store {
Store::new_with_limits(
&engine,
StoreLimitsBuilder::new()
// The limits here are chosen based on the default "maximum type size"
// configured in wasm-smith, which is 1000. This means that instances
// are allowed to, for example, export up to 1000 memories. We bump that
// a little bit here to give us some slop.
.instances(1100)
.tables(1100)
.memories(1100)
.build(),
)
}
/// Methods of timing out execution of a WebAssembly module
#[derive(Debug)]
pub enum Timeout {
@@ -95,7 +110,7 @@ pub fn instantiate_with_config(
_ => false,
});
let engine = Engine::new(&config).unwrap();
let store = Store::new(&engine);
let store = create_store(&engine);
let mut timeout_state = SignalOnDrop::default();
match timeout {
@@ -203,7 +218,7 @@ pub fn differential_execution(
config.wasm_module_linking(false);
let engine = Engine::new(&config).unwrap();
let store = Store::new(&engine);
let store = create_store(&engine);
let module = Module::new(&engine, &wasm).unwrap();
@@ -348,7 +363,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) {
ApiCall::StoreNew => {
log::trace!("creating store");
assert!(store.is_none());
store = Some(Store::new(engine.as_ref().unwrap()));
store = Some(create_store(engine.as_ref().unwrap()));
}
ApiCall::ModuleNew { id, wasm } => {
@@ -439,7 +454,7 @@ pub fn spectest(fuzz_config: crate::generators::Config, test: crate::generators:
config.wasm_reference_types(false);
config.wasm_bulk_memory(false);
config.wasm_module_linking(false);
let store = Store::new(&Engine::new(&config).unwrap());
let store = create_store(&Engine::new(&config).unwrap());
if fuzz_config.consume_fuel {
store.add_fuel(u64::max_value()).unwrap();
}
@@ -463,7 +478,7 @@ pub fn table_ops(
let mut config = fuzz_config.to_wasmtime();
config.wasm_reference_types(true);
let engine = Engine::new(&config).unwrap();
let store = Store::new(&engine);
let store = create_store(&engine);
if fuzz_config.consume_fuel {
store.add_fuel(u64::max_value()).unwrap();
}
@@ -578,7 +593,7 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Con
let mut wasmtime_config = config.to_wasmtime();
wasmtime_config.cranelift_nan_canonicalization(true);
let wasmtime_engine = Engine::new(&wasmtime_config).unwrap();
let wasmtime_store = Store::new(&wasmtime_engine);
let wasmtime_store = create_store(&wasmtime_engine);
if config.consume_fuel {
wasmtime_store.add_fuel(u64::max_value()).unwrap();
}

2
crates/fuzzing/src/oracles/dummy.rs
View File

@@ -87,7 +87,7 @@ pub fn dummy_table(store: &Store, ty: TableType) -> Table {
/// Construct a dummy memory for the given memory type.
pub fn dummy_memory(store: &Store, ty: MemoryType) -> Memory {
Memory::new(store, ty)
Memory::new(store, ty).unwrap()
}
/// Construct a dummy instance for the given instance type.

13
crates/jit/src/instantiate.rs
View File

@@ -176,11 +176,13 @@ struct FinishedFunctions(PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>);
unsafe impl Send for FinishedFunctions {}
unsafe impl Sync for FinishedFunctions {}
/// Information about a function, such as trap information, address map,
/// and stack maps.
#[derive(Serialize, Deserialize, Clone)]
struct FunctionInfo {
traps: Vec<TrapInformation>,
address_map: FunctionAddressMap,
stack_maps: Vec<StackMapInformation>,
pub struct FunctionInfo {
pub traps: Vec<TrapInformation>,
pub address_map: FunctionAddressMap,
pub stack_maps: Vec<StackMapInformation>,
}
/// This is intended to mirror the type tables in `wasmtime_environ`, except that
@@ -362,11 +364,10 @@ impl CompiledModule {
}
/// Gets the function information for a given function index.
pub fn func_info(&self, index: DefinedFuncIndex) -> (&FunctionAddressMap, &[TrapInformation]) {
pub fn func_info(&self, index: DefinedFuncIndex) -> &FunctionInfo {
self.artifacts
.funcs
.get(index)
.map(|f| (&f.address_map, f.traps.as_ref()))
.expect("defined function should be present")
}

13
crates/jit/src/link.rs
View File

@@ -111,6 +111,19 @@ fn apply_reloc(
);
write_unaligned(reloc_address as *mut u32, reloc_delta_u64 as u32);
},
#[cfg(target_pointer_width = "64")]
(RelocationKind::Relative, RelocationEncoding::S390xDbl, 32) => unsafe {
let reloc_address = body.add(offset as usize) as usize;
let reloc_addend = r.addend() as isize;
let reloc_delta_u64 = (target_func_address as u64)
.wrapping_sub(reloc_address as u64)
.wrapping_add(reloc_addend as u64);
assert!(
(reloc_delta_u64 as isize) >> 1 <= i32::max_value() as isize,
"relocation too large to fit in i32"
);
write_unaligned(reloc_address as *mut u32, (reloc_delta_u64 >> 1) as u32);
},
(RelocationKind::Elf(elf::R_AARCH64_CALL26), RelocationEncoding::Generic, 32) => unsafe {
let reloc_address = body.add(offset as usize) as usize;
let reloc_addend = r.addend() as isize;

2
crates/lightbeam/Cargo.toml
View File

@@ -17,7 +17,7 @@ cranelift-codegen = { path = "../../cranelift/codegen", version = "0.73.0" }
derive_more = "0.99"
dynasm = "1.0.0"
dynasmrt = "1.0.0"
iter-enum = "0.2"
iter-enum = "1"
itertools = "0.10.0"
memoffset = "0.6.0"
more-asserts = "0.2.1"

2
crates/obj/src/builder.rs
View File

@@ -80,6 +80,7 @@ fn to_object_relocations<'a>(
RelocationEncoding::Generic,
32,
),
Reloc::S390xPCRel32Dbl => (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32),
other => unimplemented!("Unimplemented relocation {:?}", other),
};
Some(ObjectRelocation {
@@ -102,6 +103,7 @@ fn to_object_architecture(
X86_64 => Architecture::X86_64,
Arm(_) => Architecture::Arm,
Aarch64(_) => Architecture::Aarch64,
S390x => Architecture::S390x,
architecture => {
anyhow::bail!("target architecture {:?} is unsupported", architecture,);
}

1
crates/profiling/src/jitdump_linux.rs
View File

@@ -241,6 +241,7 @@ impl State {
Architecture::X86_32(_) => elf::EM_386 as u32,
Architecture::Arm(_) => elf::EM_ARM as u32,
Architecture::Aarch64(_) => elf::EM_AARCH64 as u32,
Architecture::S390x => elf::EM_S390 as u32,
_ => unimplemented!("unrecognized architecture"),
}
}

241
crates/runtime/src/externref.rs
View File

@@ -99,18 +99,16 @@
//! Examination of Deferred Reference Counting and Cycle Detection* by Quinane:
//! <https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf>
use std::alloc::Layout;
use std::any::Any;
use std::cell::{Cell, RefCell, UnsafeCell};
use std::cmp::Ordering;
use std::collections::BTreeMap;
use std::collections::HashSet;
use std::hash::{Hash, Hasher};
use std::mem;
use std::ops::Deref;
use std::ptr::{self, NonNull};
use std::rc::Rc;
use wasmtime_environ::{ir::StackMap, StackMapInformation};
use std::{alloc::Layout, sync::Arc};
use wasmtime_environ::ir::StackMap;
/// An external reference to some opaque data.
///
@@ -596,10 +594,10 @@ impl VMExternRefActivationsTable {
pub unsafe fn insert_with_gc(
&self,
externref: VMExternRef,
stack_maps_registry: &StackMapRegistry,
module_info_lookup: &dyn ModuleInfoLookup,
) {
if let Err(externref) = self.try_insert(externref) {
self.gc_and_insert_slow(externref, stack_maps_registry);
self.gc_and_insert_slow(externref, module_info_lookup);
}
}
@@ -607,9 +605,9 @@ impl VMExternRefActivationsTable {
unsafe fn gc_and_insert_slow(
&self,
externref: VMExternRef,
stack_maps_registry: &StackMapRegistry,
module_info_lookup: &dyn ModuleInfoLookup,
) {
gc(stack_maps_registry, self);
gc(module_info_lookup, self);
// Might as well insert right into the hash set, rather than the bump
// chunk, since we are already on a slow path and we get de-duplication
@@ -743,182 +741,28 @@ impl VMExternRefActivationsTable {
}
}
/// A registry of stack maps for currently active Wasm modules.
#[derive(Default)]
pub struct StackMapRegistry {
inner: RefCell<StackMapRegistryInner>,
/// Used by the runtime to lookup information about a module given a
/// program counter value.
pub trait ModuleInfoLookup: 'static {
/// Lookup the module information from a program counter value.
fn lookup(&self, pc: usize) -> Option<Arc<dyn ModuleInfo>>;
}
#[derive(Default)]
struct StackMapRegistryInner {
/// A map from the highest pc in a module, to its stack maps.
///
/// For details, see the comment above `GlobalFrameInfo::ranges`.
ranges: BTreeMap<usize, ModuleStackMaps>,
/// Used by the runtime to query module information.
pub trait ModuleInfo {
/// Lookup the stack map at a program counter value.
fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap>;
}
#[derive(Debug)]
struct ModuleStackMaps {
/// The range of PCs that this module covers. Different modules must always
/// have distinct ranges.
range: std::ops::Range<usize>,
pub(crate) struct EmptyModuleInfoLookup;
/// A map from a PC in this module (that is a GC safepoint) to its
/// associated stack map. If `None` then it means that the PC is the start
/// of a range which has no stack map.
pc_to_stack_map: Vec<(usize, Option<Rc<StackMap>>)>,
}
impl StackMapRegistry {
/// Register the stack maps for a given module.
///
/// The stack maps should be given as an iterator over a function's PC range
/// in memory (that is, where the JIT actually allocated and emitted the
/// function's code at), and the stack maps and code offsets within that
/// range for each of its GC safepoints.
pub fn register_stack_maps<'a>(
&self,
stack_maps: impl IntoIterator<Item = (std::ops::Range<usize>, &'a [StackMapInformation])>,
) {
let mut min = usize::max_value();
let mut max = 0;
let mut pc_to_stack_map = vec![];
let mut last_is_none_marker = true;
for (range, infos) in stack_maps {
let len = range.end - range.start;
min = std::cmp::min(min, range.start);
max = std::cmp::max(max, range.end);
// Add a marker between functions indicating that this function's pc
// starts with no stack map so when our binary search later on finds
// a pc between the start of the function and the function's first
// stack map it doesn't think the previous stack map is our stack
// map.
//
// We skip this if the previous entry pushed was also a `None`
// marker, in which case the starting pc already has no stack map.
// This is also skipped if the first `code_offset` is zero since
// what we'll push applies for the first pc anyway.
if !last_is_none_marker && (infos.is_empty() || infos[0].code_offset > 0) {
pc_to_stack_map.push((range.start, None));
last_is_none_marker = true;
}
for info in infos {
assert!((info.code_offset as usize) < len);
pc_to_stack_map.push((
range.start + (info.code_offset as usize),
Some(Rc::new(info.stack_map.clone())),
));
last_is_none_marker = false;
}
}
if pc_to_stack_map.is_empty() {
// Nothing to register.
return;
}
let module_stack_maps = ModuleStackMaps {
range: min..max,
pc_to_stack_map,
};
let mut inner = self.inner.borrow_mut();
// Assert that this chunk of ranges doesn't collide with any other known
// chunks.
if let Some((_, prev)) = inner.ranges.range(max..).next() {
assert!(prev.range.start > max);
}
if let Some((prev_end, _)) = inner.ranges.range(..=min).next_back() {
assert!(*prev_end < min);
}
let old = inner.ranges.insert(max, module_stack_maps);
assert!(old.is_none());
}
/// Lookup the stack map for the given PC, if any.
pub fn lookup_stack_map(&self, pc: usize) -> Option<Rc<StackMap>> {
let inner = self.inner.borrow();
let stack_maps = inner.module_stack_maps(pc)?;
// Do a binary search to find the stack map for the given PC.
//
// Because GC safepoints are technically only associated with a single
// PC, we should ideally only care about `Ok(index)` values returned
// from the binary search. However, safepoints are inserted right before
// calls, and there are two things that can disturb the PC/offset
// associated with the safepoint versus the PC we actually use to query
// for the stack map:
//
// 1. The `backtrace` crate gives us the PC in a frame that will be
// *returned to*, and where execution will continue from, rather than
// the PC of the call we are currently at. So we would need to
// disassemble one instruction backwards to query the actual PC for
// the stack map.
//
// TODO: One thing we *could* do to make this a little less error
// prone, would be to assert/check that the nearest GC safepoint
// found is within `max_encoded_size(any kind of call instruction)`
// our queried PC for the target architecture.
//
// 2. Cranelift's stack maps only handle the stack, not
// registers. However, some references that are arguments to a call
// may need to be in registers. In these cases, what Cranelift will
// do is:
//
// a. spill all the live references,
// b. insert a GC safepoint for those references,
// c. reload the references into registers, and finally
// d. make the call.
//
// Step (c) adds drift between the GC safepoint and the location of
// the call, which is where we actually walk the stack frame and
// collect its live references.
//
// Luckily, the spill stack slots for the live references are still
// up to date, so we can still find all the on-stack roots.
// Furthermore, we do not have a moving GC, so we don't need to worry
// whether the following code will reuse the references in registers
// (which would not have been updated to point to the moved objects)
// or reload from the stack slots (which would have been updated to
// point to the moved objects).
let index = match stack_maps
.pc_to_stack_map
.binary_search_by_key(&pc, |(pc, _stack_map)| *pc)
{
// Exact hit.
Ok(i) => i,
// `Err(0)` means that the associated stack map would have been the
// first element in the array if this pc had an associated stack
// map, but this pc does not have an associated stack map. This can
// only happen inside a Wasm frame if there are no live refs at this
// pc.
Err(0) => return None,
Err(n) => n - 1,
};
let stack_map = stack_maps.pc_to_stack_map[index].1.as_ref()?.clone();
Some(stack_map)
impl ModuleInfoLookup for EmptyModuleInfoLookup {
fn lookup(&self, _pc: usize) -> Option<Arc<dyn ModuleInfo>> {
None
}
}
impl StackMapRegistryInner {
fn module_stack_maps(&self, pc: usize) -> Option<&ModuleStackMaps> {
let (end, stack_maps) = self.ranges.range(pc..).next()?;
if pc < stack_maps.range.start || *end < pc {
None
} else {
Some(stack_maps)
}
}
}
pub(crate) const EMPTY_MODULE_LOOKUP: EmptyModuleInfoLookup = EmptyModuleInfoLookup;
#[derive(Debug, Default)]
struct DebugOnly<T> {
@@ -965,7 +809,7 @@ impl<T> std::ops::DerefMut for DebugOnly<T> {
/// Additionally, you must have registered the stack maps for every Wasm module
/// that has frames on the stack with the given `stack_maps_registry`.
pub unsafe fn gc(
stack_maps_registry: &StackMapRegistry,
module_info_lookup: &dyn ModuleInfoLookup,
externref_activations_table: &VMExternRefActivationsTable,
) {
// We borrow the precise stack roots `RefCell` for the whole duration of
@@ -1003,8 +847,7 @@ pub unsafe fn gc(
if cfg!(debug_assertions) {
// Assert that there aren't any Wasm frames on the stack.
backtrace::trace(|frame| {
let stack_map = stack_maps_registry.lookup_stack_map(frame.ip() as usize);
assert!(stack_map.is_none());
assert!(module_info_lookup.lookup(frame.ip() as usize).is_none());
true
});
}
@@ -1048,28 +891,30 @@ pub unsafe fn gc(
let pc = frame.ip() as usize;
let sp = frame.sp() as usize;
if let Some(stack_map) = stack_maps_registry.lookup_stack_map(pc) {
debug_assert!(sp != 0, "we should always get a valid SP for Wasm frames");
if let Some(module_info) = module_info_lookup.lookup(pc) {
if let Some(stack_map) = module_info.lookup_stack_map(pc) {
debug_assert!(sp != 0, "we should always get a valid SP for Wasm frames");
for i in 0..(stack_map.mapped_words() as usize) {
if stack_map.get_bit(i) {
// Stack maps have one bit per word in the frame, and the
// zero^th bit is the *lowest* addressed word in the frame,
// i.e. the closest to the SP. So to get the `i`^th word in
// this frame, we add `i * sizeof(word)` to the SP.
let ptr_to_ref = sp + i * mem::size_of::<usize>();
for i in 0..(stack_map.mapped_words() as usize) {
if stack_map.get_bit(i) {
// Stack maps have one bit per word in the frame, and the
// zero^th bit is the *lowest* addressed word in the frame,
// i.e. the closest to the SP. So to get the `i`^th word in
// this frame, we add `i * sizeof(word)` to the SP.
let ptr_to_ref = sp + i * mem::size_of::<usize>();
let r = std::ptr::read(ptr_to_ref as *const *mut VMExternData);
debug_assert!(
r.is_null() || activations_table_set.contains(&r),
"every on-stack externref inside a Wasm frame should \
have an entry in the VMExternRefActivationsTable"
);
if let Some(r) = NonNull::new(r) {
VMExternRefActivationsTable::insert_precise_stack_root(
&mut precise_stack_roots,
r,
let r = std::ptr::read(ptr_to_ref as *const *mut VMExternData);
debug_assert!(
r.is_null() || activations_table_set.contains(&r),
"every on-stack externref inside a Wasm frame should \
have an entry in the VMExternRefActivationsTable"
);
if let Some(r) = NonNull::new(r) {
VMExternRefActivationsTable::insert_precise_stack_root(
&mut precise_stack_roots,
r,
);
}
}
}
}

174
crates/runtime/src/instance.rs
View File

@@ -3,7 +3,7 @@
//! `InstanceHandle` is a reference-counting handle for an `Instance`.
use crate::export::Export;
use crate::externref::{StackMapRegistry, VMExternRefActivationsTable};
use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable};
use crate::memory::{Memory, RuntimeMemoryCreator};
use crate::table::{Table, TableElement};
use crate::traphandlers::Trap;
@@ -37,6 +37,52 @@ mod allocator;
pub use allocator::*;
/// Used by hosts to limit resource consumption of instances.
///
/// An instance can be created with a resource limiter so that hosts can take into account
/// non-WebAssembly resource usage to determine if a linear memory or table should grow.
pub trait ResourceLimiter {
/// Notifies the resource limiter that an instance's linear memory has been requested to grow.
///
/// * `current` is the current size of the linear memory in WebAssembly page units.
/// * `desired` is the desired size of the linear memory in WebAssembly page units.
/// * `maximum` is either the linear memory's maximum or a maximum from an instance allocator,
/// also in WebAssembly page units. A value of `None` indicates that the linear memory is
/// unbounded.
///
/// This function should return `true` to indicate that the growing operation is permitted or
/// `false` if not permitted. Returning `true` when a maximum has been exceeded will have no
/// effect as the linear memory will not grow.
fn memory_growing(&self, current: u32, desired: u32, maximum: Option<u32>) -> bool;
/// Notifies the resource limiter that an instance's table has been requested to grow.
///
/// * `current` is the current number of elements in the table.
/// * `desired` is the desired number of elements in the table.
/// * `maximum` is either the table's maximum or a maximum from an instance allocator.
/// A value of `None` indicates that the table is unbounded.
///
/// This function should return `true` to indicate that the growing operation is permitted or
/// `false` if not permitted. Returning `true` when a maximum has been exceeded will have no
/// effect as the table will not grow.
fn table_growing(&self, current: u32, desired: u32, maximum: Option<u32>) -> bool;
/// The maximum number of instances that can be created for a `Store`.
///
/// Module instantiation will fail if this limit is exceeded.
fn instances(&self) -> usize;
/// The maximum number of tables that can be created for a `Store`.
///
/// Module instantiation will fail if this limit is exceeded.
fn tables(&self) -> usize;
/// The maximum number of tables that can be created for a `Store`.
///
/// Module instantiation will fail if this limit is exceeded.
fn memories(&self) -> usize;
}
/// Runtime representation of an instance value, which erases all `Instance`
/// information since instances are just a collection of values.
pub type RuntimeInstance = Rc<IndexMap<String, Export>>;
@@ -249,9 +295,9 @@ impl Instance {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_externref_activations_table()) }
}
/// Return a pointer to the `StackMapRegistry`.
pub fn stack_map_registry(&self) -> *mut *mut StackMapRegistry {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_stack_map_registry()) }
/// Return a pointer to the `ModuleInfoLookup`.
pub fn module_info_lookup(&self) -> *mut *const dyn ModuleInfoLookup {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_module_info_lookup()) }
}
/// Return a reference to the vmctx used by compiled wasm code.
@@ -378,11 +424,12 @@ impl Instance {
/// Returns `None` if memory can't be grown by the specified amount
/// of pages.
pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option<u32> {
let result = self
let memory = self
.memories
.get(memory_index)
.unwrap_or_else(|| panic!("no memory for index {}", memory_index.index()))
.grow(delta);
.unwrap_or_else(|| panic!("no memory for index {}", memory_index.index()));
let result = unsafe { memory.grow(delta) };
// Keep current the VMContext pointers used by compiled wasm code.
self.set_memory(memory_index, self.memories[memory_index].vmmemory());
@@ -460,19 +507,18 @@ impl Instance {
delta: u32,
init_value: TableElement,
) -> Option<u32> {
unsafe {
let orig_size = self
.tables
.get(table_index)
.unwrap_or_else(|| panic!("no table for index {}", table_index.index()))
.grow(delta, init_value)?;
let table = self
.tables
.get(table_index)
.unwrap_or_else(|| panic!("no table for index {}", table_index.index()));
// Keep the `VMContext` pointers used by compiled Wasm code up to
// date.
self.set_table(table_index, self.tables[table_index].vmtable());
let result = unsafe { table.grow(delta, init_value) };
Some(orig_size)
}
// Keep the `VMContext` pointers used by compiled Wasm code up to
// date.
self.set_table(table_index, self.tables[table_index].vmtable());
result
}
pub(crate) fn defined_table_fill(
@@ -527,11 +573,11 @@ impl Instance {
return None;
}
Some(unsafe { &*self.anyfunc_ptr(index) })
unsafe { Some(&*self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))) }
}
unsafe fn anyfunc_ptr(&self, index: FuncIndex) -> *mut VMCallerCheckedAnyfunc {
self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))
unsafe fn anyfunc_base(&self) -> *mut VMCallerCheckedAnyfunc {
self.vmctx_plus_offset(self.offsets.vmctx_anyfuncs_begin())
}
fn find_passive_segment<'a, I, D, T>(
@@ -565,38 +611,56 @@ impl Instance {
src: u32,
len: u32,
) -> Result<(), Trap> {
// https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init
let table = self.get_table(table_index);
let elements = Self::find_passive_segment(
elem_index,
&self.module.passive_elements_map,
&self.module.passive_elements,
&self.dropped_elements,
);
self.table_init_segment(table_index, elements, dst, src, len)
}
if src
.checked_add(len)
.map_or(true, |n| n as usize > elements.len())
|| dst.checked_add(len).map_or(true, |m| m > table.size())
pub(crate) fn table_init_segment(
&self,
table_index: TableIndex,
elements: &[FuncIndex],
dst: u32,
src: u32,
len: u32,
) -> Result<(), Trap> {
// https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init
let table = self.get_table(table_index);
let elements = match elements
.get(usize::try_from(src).unwrap()..)
.and_then(|s| s.get(..usize::try_from(len).unwrap()))
{
return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds));
Some(elements) => elements,
None => return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)),
};
match table.element_type() {
TableElementType::Func => unsafe {
let base = self.anyfunc_base();
table.init_funcs(
dst,
elements.iter().map(|idx| {
if *idx == FuncIndex::reserved_value() {
ptr::null_mut()
} else {
debug_assert!(idx.as_u32() < self.offsets.num_defined_functions);
base.add(usize::try_from(idx.as_u32()).unwrap())
}
}),
)?;
},
TableElementType::Val(_) => {
debug_assert!(elements.iter().all(|e| *e == FuncIndex::reserved_value()));
table.fill(dst, TableElement::ExternRef(None), len)?;
}
}
// TODO(#983): investigate replacing this get/set loop with a `memcpy`.
for (dst, src) in (dst..dst + len).zip(src..src + len) {
let elem = self
.get_caller_checked_anyfunc(elements[src as usize])
.map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| {
f as *const VMCallerCheckedAnyfunc as *mut _
});
table
.set(dst, TableElement::FuncRef(elem))
.expect("should never panic because we already did the bounds check above");
}
Ok(())
}
@@ -727,16 +791,26 @@ impl Instance {
src: u32,
len: u32,
) -> Result<(), Trap> {
// https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init
let memory = self.get_memory(memory_index);
let data = Self::find_passive_segment(
data_index,
&self.module.passive_data_map,
&self.module.passive_data,
&self.dropped_data,
);
self.memory_init_segment(memory_index, &data, dst, src, len)
}
pub(crate) fn memory_init_segment(
&self,
memory_index: MemoryIndex,
data: &[u8],
dst: u32,
src: u32,
len: u32,
) -> Result<(), Trap> {
// https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init
let memory = self.get_memory(memory_index);
if src
.checked_add(len)
@@ -818,10 +892,6 @@ pub struct InstanceHandle {
}
impl InstanceHandle {
pub(crate) unsafe fn new(instance: *mut Instance) -> Self {
Self { instance }
}
/// Create a new `InstanceHandle` pointing at the instance
/// pointed to by the given `VMContext` pointer.
///

138
crates/runtime/src/instance/allocator.rs
View File

@@ -1,8 +1,8 @@
use crate::externref::{StackMapRegistry, VMExternRefActivationsTable};
use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable, EMPTY_MODULE_LOOKUP};
use crate::imports::Imports;
use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator};
use crate::instance::{Instance, InstanceHandle, ResourceLimiter, RuntimeMemoryCreator};
use crate::memory::{DefaultMemoryCreator, Memory};
use crate::table::{Table, TableElement};
use crate::table::Table;
use crate::traphandlers::Trap;
use crate::vmcontext::{
VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport,
@@ -15,13 +15,13 @@ use std::any::Any;
use std::cell::RefCell;
use std::convert::TryFrom;
use std::ptr::{self, NonNull};
use std::rc::Rc;
use std::slice;
use std::sync::Arc;
use thiserror::Error;
use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap};
use wasmtime_environ::entity::{EntityRef, EntitySet, PrimaryMap};
use wasmtime_environ::wasm::{
DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex,
TableElementType, WasmType,
DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, GlobalInit, SignatureIndex, WasmType,
};
use wasmtime_environ::{
ir, MemoryInitialization, MemoryInitializer, Module, ModuleType, TableInitializer, VMOffsets,
@@ -57,8 +57,11 @@ pub struct InstanceAllocationRequest<'a> {
/// The pointer to the reference activations table to use for the instance.
pub externref_activations_table: *mut VMExternRefActivationsTable,
/// The pointer to the stack map registry to use for the instance.
pub stack_map_registry: *mut StackMapRegistry,
/// The pointer to the module info lookup to use for the instance.
pub module_info_lookup: Option<*const dyn ModuleInfoLookup>,
/// The resource limiter to use for the instance.
pub limiter: Option<&'a Rc<dyn ResourceLimiter>>,
}
/// An link error while instantiating a module.
@@ -208,7 +211,7 @@ impl<'a> From<&'a PrimaryMap<SignatureIndex, VMSharedSignatureIndex>> for Shared
fn get_table_init_start(
init: &TableInitializer,
instance: &Instance,
) -> Result<usize, InstantiationError> {
) -> Result<u32, InstantiationError> {
match init.base {
Some(base) => {
let val = unsafe {
@@ -219,7 +222,7 @@ fn get_table_init_start(
}
};
init.offset.checked_add(val as usize).ok_or_else(|| {
init.offset.checked_add(val).ok_or_else(|| {
InstantiationError::Link(LinkError(
"element segment global base overflows".to_owned(),
))
@@ -233,6 +236,7 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError
for init in &instance.module.table_initializers {
let table = instance.get_table(init.table_index);
let start = get_table_init_start(init, instance)?;
let start = usize::try_from(start).unwrap();
let end = start.checked_add(init.elements.len());
match end {
@@ -252,34 +256,15 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError
fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> {
for init in &instance.module.table_initializers {
let table = instance.get_table(init.table_index);
let start = get_table_init_start(init, instance)?;
let end = start.checked_add(init.elements.len());
match end {
Some(end) if end <= table.size() as usize => {
for (i, func_idx) in init.elements.iter().enumerate() {
let item = match table.element_type() {
TableElementType::Func => instance
.get_caller_checked_anyfunc(*func_idx)
.map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| {
f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc
})
.into(),
TableElementType::Val(_) => {
assert!(*func_idx == FuncIndex::reserved_value());
TableElement::ExternRef(None)
}
};
table.set(u32::try_from(start + i).unwrap(), item).unwrap();
}
}
_ => {
return Err(InstantiationError::Trap(Trap::wasm(
ir::TrapCode::TableOutOfBounds,
)))
}
}
instance
.table_init_segment(
init.table_index,
&init.elements,
get_table_init_start(init, instance)?,
0,
init.elements.len() as u32,
)
.map_err(InstantiationError::Trap)?;
}
Ok(())
@@ -288,7 +273,7 @@ fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> {
fn get_memory_init_start(
init: &MemoryInitializer,
instance: &Instance,
) -> Result<usize, InstantiationError> {
) -> Result<u32, InstantiationError> {
match init.base {
Some(base) => {
let val = unsafe {
@@ -299,7 +284,7 @@ fn get_memory_init_start(
}
};
init.offset.checked_add(val as usize).ok_or_else(|| {
init.offset.checked_add(val).ok_or_else(|| {
InstantiationError::Link(LinkError("data segment global base overflows".to_owned()))
})
}
@@ -307,24 +292,6 @@ fn get_memory_init_start(
}
}
unsafe fn get_memory_slice<'instance>(
init: &MemoryInitializer,
instance: &'instance Instance,
) -> &'instance mut [u8] {
let memory = if let Some(defined_memory_index) =
instance.module.defined_memory_index(init.memory_index)
{
instance.memory(defined_memory_index)
} else {
let import = instance.imported_memory(init.memory_index);
let foreign_instance = (&mut *(import).vmctx).instance();
let foreign_memory = &mut *(import).from;
let foreign_index = foreign_instance.memory_index(foreign_memory);
foreign_instance.memory(foreign_index)
};
&mut *ptr::slice_from_raw_parts_mut(memory.base, memory.current_length)
}
fn check_memory_init_bounds(
instance: &Instance,
initializers: &[MemoryInitializer],
@@ -332,6 +299,7 @@ fn check_memory_init_bounds(
for init in initializers {
let memory = instance.get_memory(init.memory_index);
let start = get_memory_init_start(init, instance)?;
let start = usize::try_from(start).unwrap();
let end = start.checked_add(init.data.len());
match end {
@@ -354,21 +322,15 @@ fn initialize_memories(
initializers: &[MemoryInitializer],
) -> Result<(), InstantiationError> {
for init in initializers {
let memory = instance.get_memory(init.memory_index);
let start = get_memory_init_start(init, instance)?;
let end = start.checked_add(init.data.len());
match end {
Some(end) if end <= memory.current_length => {
let mem_slice = unsafe { get_memory_slice(init, instance) };
mem_slice[start..end].copy_from_slice(&init.data);
}
_ => {
return Err(InstantiationError::Trap(Trap::wasm(
ir::TrapCode::HeapOutOfBounds,
)))
}
}
instance
.memory_init_segment(
init.memory_index,
&init.data,
get_memory_init_start(init, instance)?,
0,
init.data.len() as u32,
)
.map_err(InstantiationError::Trap)?;
}
Ok(())
@@ -447,7 +409,7 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque
*instance.interrupts() = req.interrupts;
*instance.externref_activations_table() = req.externref_activations_table;
*instance.stack_map_registry() = req.stack_map_registry;
*instance.module_info_lookup() = req.module_info_lookup.unwrap_or(&EMPTY_MODULE_LOOKUP);
// Initialize shared signatures
let mut ptr = instance.signature_ids_ptr();
@@ -492,6 +454,7 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque
);
// Initialize the functions
let mut base = instance.anyfunc_base();
for (index, sig) in instance.module.functions.iter() {
let type_index = req.shared_signatures.lookup(*sig);
@@ -506,13 +469,14 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque
};
ptr::write(
instance.anyfunc_ptr(index),
base,
VMCallerCheckedAnyfunc {
func_ptr,
type_index,
vmctx,
},
);
base = base.add(1);
}
// Initialize the defined tables
@@ -590,19 +554,23 @@ impl OnDemandInstanceAllocator {
}
}
fn create_tables(module: &Module) -> PrimaryMap<DefinedTableIndex, Table> {
fn create_tables(
module: &Module,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<PrimaryMap<DefinedTableIndex, Table>, InstantiationError> {
let num_imports = module.num_imported_tables;
let mut tables: PrimaryMap<DefinedTableIndex, _> =
PrimaryMap::with_capacity(module.table_plans.len() - num_imports);
for table in &module.table_plans.values().as_slice()[num_imports..] {
tables.push(Table::new_dynamic(table));
tables.push(Table::new_dynamic(table, limiter).map_err(InstantiationError::Resource)?);
}
tables
Ok(tables)
}
fn create_memories(
&self,
module: &Module,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<PrimaryMap<DefinedMemoryIndex, Memory>, InstantiationError> {
let creator = self
.mem_creator
@@ -612,8 +580,10 @@ impl OnDemandInstanceAllocator {
let mut memories: PrimaryMap<DefinedMemoryIndex, _> =
PrimaryMap::with_capacity(module.memory_plans.len() - num_imports);
for plan in &module.memory_plans.values().as_slice()[num_imports..] {
memories
.push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?);
memories.push(
Memory::new_dynamic(plan, creator, limiter)
.map_err(InstantiationError::Resource)?,
);
}
Ok(memories)
}
@@ -633,8 +603,8 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator {
&self,
mut req: InstanceAllocationRequest,
) -> Result<InstanceHandle, InstantiationError> {
let memories = self.create_memories(&req.module)?;
let tables = Self::create_tables(&req.module);
let memories = self.create_memories(&req.module, req.limiter)?;
let tables = Self::create_tables(&req.module, req.limiter)?;
let host_state = std::mem::replace(&mut req.host_state, Box::new(()));
@@ -657,7 +627,9 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator {
alloc::handle_alloc_error(layout);
}
ptr::write(instance_ptr, instance);
InstanceHandle::new(instance_ptr)
InstanceHandle {
instance: instance_ptr,
}
};
initialize_vmcontext(handle.instance(), req);

85
crates/runtime/src/instance/allocator/pooling.rs
View File

@@ -9,7 +9,7 @@
use super::{
initialize_instance, initialize_vmcontext, InstanceAllocationRequest, InstanceAllocator,
InstanceHandle, InstantiationError,
InstanceHandle, InstantiationError, ResourceLimiter,
};
use crate::{instance::Instance, Memory, Mmap, Table, VMContext};
use anyhow::{anyhow, bail, Context, Result};
@@ -18,6 +18,7 @@ use std::cell::RefCell;
use std::cmp::min;
use std::convert::TryFrom;
use std::mem;
use std::rc::Rc;
use std::sync::{Arc, Mutex};
use wasmtime_environ::{
entity::{EntitySet, PrimaryMap},
@@ -376,10 +377,45 @@ impl InstancePool {
}
}
unsafe fn setup_instance(
&self,
index: usize,
mut req: InstanceAllocationRequest,
) -> Result<InstanceHandle, InstantiationError> {
let instance = self.instance(index);
instance.module = req.module.clone();
instance.offsets = VMOffsets::new(
std::mem::size_of::<*const u8>() as u8,
instance.module.as_ref(),
);
instance.host_state = std::mem::replace(&mut req.host_state, Box::new(()));
Self::set_instance_memories(
instance,
self.memories.get(index),
self.memories.max_wasm_pages,
req.limiter,
)?;
Self::set_instance_tables(
instance,
self.tables.get(index),
self.tables.max_elements,
req.limiter,
)?;
initialize_vmcontext(instance, req);
Ok(InstanceHandle {
instance: instance as _,
})
}
fn allocate(
&self,
strategy: PoolingAllocationStrategy,
mut req: InstanceAllocationRequest,
req: InstanceAllocationRequest,
) -> Result<InstanceHandle, InstantiationError> {
let index = {
let mut free_list = self.free_list.lock().unwrap();
@@ -390,28 +426,15 @@ impl InstancePool {
free_list.swap_remove(free_index)
};
let host_state = std::mem::replace(&mut req.host_state, Box::new(()));
unsafe {
let instance = self.instance(index);
instance.module = req.module.clone();
instance.offsets = VMOffsets::new(
std::mem::size_of::<*const u8>() as u8,
instance.module.as_ref(),
);
instance.host_state = host_state;
Self::set_instance_memories(
instance,
self.memories.get(index),
self.memories.max_wasm_pages,
)?;
Self::set_instance_tables(instance, self.tables.get(index), self.tables.max_elements)?;
initialize_vmcontext(instance, req);
Ok(InstanceHandle::new(instance as _))
self.setup_instance(index, req).or_else(|e| {
// Deallocate the allocated instance on error
let instance = self.instance(index);
self.deallocate(&InstanceHandle {
instance: instance as _,
});
Err(e)
})
}
}
@@ -473,6 +496,7 @@ impl InstancePool {
instance: &mut Instance,
mut memories: impl Iterator<Item = *mut u8>,
max_pages: u32,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<(), InstantiationError> {
let module = instance.module.as_ref();
@@ -487,6 +511,7 @@ impl InstancePool {
memories.next().unwrap(),
max_pages,
commit_memory_pages,
limiter,
)
.map_err(InstantiationError::Resource)?,
);
@@ -503,6 +528,7 @@ impl InstancePool {
instance: &mut Instance,
mut tables: impl Iterator<Item = *mut u8>,
max_elements: u32,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<(), InstantiationError> {
let module = instance.module.as_ref();
@@ -514,9 +540,10 @@ impl InstancePool {
commit_table_pages(base, max_elements as usize * mem::size_of::<*mut u8>())
.map_err(InstantiationError::Resource)?;
instance
.tables
.push(Table::new_static(plan, base as _, max_elements));
instance.tables.push(
Table::new_static(plan, base as _, max_elements, limiter)
.map_err(InstantiationError::Resource)?,
);
}
let mut dropped_elements = instance.dropped_elements.borrow_mut();
@@ -1370,7 +1397,8 @@ mod test {
host_state: Box::new(()),
interrupts: std::ptr::null(),
externref_activations_table: std::ptr::null_mut(),
stack_map_registry: std::ptr::null_mut(),
module_info_lookup: None,
limiter: None,
},
)
.expect("allocation should succeed"),
@@ -1394,7 +1422,8 @@ mod test {
host_state: Box::new(()),
interrupts: std::ptr::null(),
externref_activations_table: std::ptr::null_mut(),
stack_map_registry: std::ptr::null_mut(),
module_info_lookup: None,
limiter: None,
},
) {
Err(InstantiationError::Limit(3)) => {}

3
crates/runtime/src/instance/allocator/pooling/uffd.rs
View File

@@ -523,7 +523,8 @@ mod test {
host_state: Box::new(()),
interrupts: ptr::null(),
externref_activations_table: ptr::null_mut(),
stack_map_registry: ptr::null_mut(),
module_info_lookup: None,
limiter: None,
},
)
.expect("instance should allocate"),

2
crates/runtime/src/lib.rs
View File

@@ -40,7 +40,7 @@ pub use crate::imports::Imports;
pub use crate::instance::{
InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstanceLimits,
InstantiationError, LinkError, ModuleLimits, OnDemandInstanceAllocator,
PoolingAllocationStrategy, PoolingInstanceAllocator, RuntimeInstance,
PoolingAllocationStrategy, PoolingInstanceAllocator, ResourceLimiter, RuntimeInstance,
};
pub use crate::jit_int::GdbJitImageRegistration;
pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator};

8
crates/runtime/src/libcalls.rs
View File

@@ -449,8 +449,8 @@ pub unsafe extern "C" fn wasmtime_activations_table_insert_with_gc(
let externref = VMExternRef::clone_from_raw(externref);
let instance = (&mut *vmctx).instance();
let activations_table = &**instance.externref_activations_table();
let registry = &**instance.stack_map_registry();
activations_table.insert_with_gc(externref, registry);
let module_info_lookup = &**instance.module_info_lookup();
activations_table.insert_with_gc(externref, module_info_lookup);
}
/// Perform a Wasm `global.get` for `externref` globals.
@@ -466,8 +466,8 @@ pub unsafe extern "C" fn wasmtime_externref_global_get(
Some(externref) => {
let raw = externref.as_raw();
let activations_table = &**instance.externref_activations_table();
let registry = &**instance.stack_map_registry();
activations_table.insert_with_gc(externref, registry);
let module_info_lookup = &**instance.module_info_lookup();
activations_table.insert_with_gc(externref, module_info_lookup);
raw
}
}

153
crates/runtime/src/memory.rs
View File

@@ -4,12 +4,14 @@
use crate::mmap::Mmap;
use crate::vmcontext::VMMemoryDefinition;
use anyhow::Result;
use crate::ResourceLimiter;
use anyhow::{bail, Result};
use more_asserts::{assert_ge, assert_le};
use std::cell::{Cell, RefCell};
use std::cmp::min;
use std::convert::TryFrom;
use std::ptr;
use std::rc::Rc;
use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE};
/// A memory allocator
@@ -33,6 +35,10 @@ pub trait RuntimeLinearMemory {
/// Returns the number of allocated wasm pages.
fn size(&self) -> u32;
/// Returns the maximum number of pages the memory can grow to.
/// Returns `None` if the memory is unbounded.
fn maximum(&self) -> Option<u32>;
/// Grow memory by the specified amount of wasm pages.
///
/// Returns `None` if memory can't be grown by the specified amount
@@ -105,6 +111,12 @@ impl RuntimeLinearMemory for MmapMemory {
self.mmap.borrow().size
}
/// Returns the maximum number of pages the memory can grow to.
/// Returns `None` if the memory is unbounded.
fn maximum(&self) -> Option<u32> {
self.maximum
}
/// Grow memory by the specified amount of wasm pages.
///
/// Returns `None` if memory can't be grown by the specified amount
@@ -189,12 +201,23 @@ enum MemoryStorage {
}
/// Represents an instantiation of a WebAssembly memory.
pub struct Memory(MemoryStorage);
pub struct Memory {
storage: MemoryStorage,
limiter: Option<Rc<dyn ResourceLimiter>>,
}
impl Memory {
/// Create a new dynamic (movable) memory instance for the specified plan.
pub fn new_dynamic(plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator) -> Result<Self> {
Ok(Self(MemoryStorage::Dynamic(creator.new_memory(plan)?)))
pub fn new_dynamic(
plan: &MemoryPlan,
creator: &dyn RuntimeMemoryCreator,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<Self> {
Self::new(
plan,
MemoryStorage::Dynamic(creator.new_memory(plan)?),
limiter,
)
}
/// Create a new static (immovable) memory instance for the specified plan.
@@ -203,32 +226,78 @@ impl Memory {
base: *mut u8,
maximum: u32,
make_accessible: fn(*mut u8, usize) -> Result<()>,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<Self> {
if plan.memory.minimum > 0 {
make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize)?;
}
Ok(Self(MemoryStorage::Static {
let storage = MemoryStorage::Static {
base,
size: Cell::new(plan.memory.minimum),
maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum),
make_accessible,
#[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: RefCell::new(Vec::new()),
}))
};
Self::new(plan, storage, limiter)
}
fn new(
plan: &MemoryPlan,
storage: MemoryStorage,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<Self> {
if let Some(limiter) = limiter {
if !limiter.memory_growing(0, plan.memory.minimum, plan.memory.maximum) {
bail!(
"memory minimum size of {} pages exceeds memory limits",
plan.memory.minimum
);
}
}
if let MemoryStorage::Static {
base,
make_accessible,
..
} = &storage
{
if plan.memory.minimum > 0 {
make_accessible(
*base,
plan.memory.minimum as usize * WASM_PAGE_SIZE as usize,
)?;
}
}
Ok(Self {
storage,
limiter: limiter.cloned(),
})
}
/// Returns the number of allocated wasm pages.
pub fn size(&self) -> u32 {
match &self.0 {
match &self.storage {
MemoryStorage::Static { size, .. } => size.get(),
MemoryStorage::Dynamic(mem) => mem.size(),
}
}
/// Returns the maximum number of pages the memory can grow to at runtime.
///
/// Returns `None` if the memory is unbounded.
///
/// The runtime maximum may not be equal to the maximum from the linear memory's
/// Wasm type when it is being constrained by an instance allocator.
pub fn maximum(&self) -> Option<u32> {
match &self.storage {
MemoryStorage::Static { maximum, .. } => Some(*maximum),
MemoryStorage::Dynamic(mem) => mem.maximum(),
}
}
/// Returns whether or not the underlying storage of the memory is "static".
pub(crate) fn is_static(&self) -> bool {
if let MemoryStorage::Static { .. } = &self.0 {
if let MemoryStorage::Static { .. } = &self.storage {
true
} else {
false
@@ -239,8 +308,30 @@ impl Memory {
///
/// Returns `None` if memory can't be grown by the specified amount
/// of wasm pages.
pub fn grow(&self, delta: u32) -> Option<u32> {
match &self.0 {
///
/// # Safety
///
/// Resizing the memory can reallocate the memory buffer for dynamic memories.
/// An instance's `VMContext` may have pointers to the memory's base and will
/// need to be fixed up after growing the memory.
///
/// Generally, prefer using `InstanceHandle::memory_grow`, which encapsulates
/// this unsafety.
pub unsafe fn grow(&self, delta: u32) -> Option<u32> {
let old_size = self.size();
if delta == 0 {
return Some(old_size);
}
let new_size = old_size.checked_add(delta)?;
if let Some(limiter) = &self.limiter {
if !limiter.memory_growing(old_size, new_size, self.maximum()) {
return None;
}
}
match &self.storage {
MemoryStorage::Static {
base,
size,
@@ -252,13 +343,6 @@ impl Memory {
#[cfg(all(feature = "uffd", target_os = "linux"))]
self.reset_guard_pages().ok()?;
let old_size = size.get();
if delta == 0 {
return Some(old_size);
}
let new_size = old_size.checked_add(delta)?;
if new_size > *maximum || new_size >= WASM_MAX_PAGES {
return None;
}
@@ -266,7 +350,7 @@ impl Memory {
let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize;
let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize;
make_accessible(unsafe { base.add(start) }, len).ok()?;
make_accessible(base.add(start), len).ok()?;
size.set(new_size);
@@ -278,7 +362,7 @@ impl Memory {
/// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code.
pub fn vmmemory(&self) -> VMMemoryDefinition {
match &self.0 {
match &self.storage {
MemoryStorage::Static { base, size, .. } => VMMemoryDefinition {
base: *base,
current_length: size.get() as usize * WASM_PAGE_SIZE as usize,
@@ -299,7 +383,7 @@ impl Memory {
size: usize,
reset: fn(*mut u8, usize) -> Result<()>,
) {
match &self.0 {
match &self.storage {
MemoryStorage::Static {
guard_page_faults, ..
} => {
@@ -320,7 +404,7 @@ impl Memory {
/// This function will panic if called on a dynamic memory.
#[cfg(all(feature = "uffd", target_os = "linux"))]
pub(crate) fn reset_guard_pages(&self) -> Result<()> {
match &self.0 {
match &self.storage {
MemoryStorage::Static {
guard_page_faults, ..
} => {
@@ -345,13 +429,16 @@ impl Default for Memory {
unreachable!()
}
Self(MemoryStorage::Static {
base: ptr::null_mut(),
size: Cell::new(0),
maximum: 0,
make_accessible,
#[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: RefCell::new(Vec::new()),
})
Self {
storage: MemoryStorage::Static {
base: ptr::null_mut(),
size: Cell::new(0),
maximum: 0,
make_accessible,
#[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: RefCell::new(Vec::new()),
},
limiter: None,
}
}
}

133
crates/runtime/src/table.rs
View File

@@ -3,19 +3,21 @@
//! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition};
use crate::{Trap, VMExternRef};
use crate::{ResourceLimiter, Trap, VMExternRef};
use anyhow::{bail, Result};
use std::cell::{Cell, RefCell};
use std::cmp::min;
use std::convert::TryInto;
use std::convert::{TryFrom, TryInto};
use std::ops::Range;
use std::ptr;
use std::rc::Rc;
use wasmtime_environ::wasm::TableElementType;
use wasmtime_environ::{ir, TablePlan};
/// An element going into or coming out of a table.
///
/// Table elements are stored as pointers and are default-initialized with `ptr::null_mut`.
#[derive(Clone, Debug)]
#[derive(Clone)]
pub enum TableElement {
/// A `funcref`.
FuncRef(*mut VMCallerCheckedAnyfunc),
@@ -69,7 +71,7 @@ impl TableElement {
unsafe fn into_raw(self) -> *mut u8 {
match self {
Self::FuncRef(e) => e as _,
Self::ExternRef(e) => e.map(|e| e.into_raw()).unwrap_or(ptr::null_mut()),
Self::ExternRef(e) => e.map_or(ptr::null_mut(), |e| e.into_raw()),
}
}
}
@@ -92,7 +94,6 @@ impl From<VMExternRef> for TableElement {
}
}
#[derive(Debug)]
enum TableStorage {
Static {
data: *mut *mut u8,
@@ -108,38 +109,74 @@ enum TableStorage {
}
/// Represents an instance's table.
#[derive(Debug)]
pub struct Table(TableStorage);
pub struct Table {
storage: TableStorage,
limiter: Option<Rc<dyn ResourceLimiter>>,
}
impl Table {
/// Create a new dynamic (movable) table instance for the specified table plan.
pub fn new_dynamic(plan: &TablePlan) -> Self {
pub fn new_dynamic(
plan: &TablePlan,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<Self> {
let elements = RefCell::new(vec![ptr::null_mut(); plan.table.minimum as usize]);
let ty = plan.table.ty.clone();
let maximum = plan.table.maximum;
Self(TableStorage::Dynamic {
let storage = TableStorage::Dynamic {
elements,
ty,
maximum,
})
};
Self::new(plan, storage, limiter)
}
/// Create a new static (immovable) table instance for the specified table plan.
pub fn new_static(plan: &TablePlan, data: *mut *mut u8, maximum: u32) -> Self {
pub fn new_static(
plan: &TablePlan,
data: *mut *mut u8,
maximum: u32,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<Self> {
let size = Cell::new(plan.table.minimum);
let ty = plan.table.ty.clone();
let maximum = min(plan.table.maximum.unwrap_or(maximum), maximum);
Self(TableStorage::Static {
let storage = TableStorage::Static {
data,
size,
ty,
maximum,
};
Self::new(plan, storage, limiter)
}
fn new(
plan: &TablePlan,
storage: TableStorage,
limiter: Option<&Rc<dyn ResourceLimiter>>,
) -> Result<Self> {
if let Some(limiter) = limiter {
if !limiter.table_growing(0, plan.table.minimum, plan.table.maximum) {
bail!(
"table minimum size of {} elements exceeds table limits",
plan.table.minimum
);
}
}
Ok(Self {
storage,
limiter: limiter.cloned(),
})
}
/// Returns the type of the elements in this table.
pub fn element_type(&self) -> TableElementType {
match &self.0 {
match &self.storage {
TableStorage::Static { ty, .. } => *ty,
TableStorage::Dynamic { ty, .. } => *ty,
}
@@ -147,7 +184,7 @@ impl Table {
/// Returns whether or not the underlying storage of the table is "static".
pub(crate) fn is_static(&self) -> bool {
if let TableStorage::Static { .. } = &self.0 {
if let TableStorage::Static { .. } = &self.storage {
true
} else {
false
@@ -156,20 +193,51 @@ impl Table {
/// Returns the number of allocated elements.
pub fn size(&self) -> u32 {
match &self.0 {
match &self.storage {
TableStorage::Static { size, .. } => size.get(),
TableStorage::Dynamic { elements, .. } => elements.borrow().len().try_into().unwrap(),
}
}
/// Returns the maximum number of elements.
/// Returns the maximum number of elements at runtime.
///
/// Returns `None` if the table is unbounded.
///
/// The runtime maximum may not be equal to the maximum from the table's Wasm type
/// when it is being constrained by an instance allocator.
pub fn maximum(&self) -> Option<u32> {
match &self.0 {
match &self.storage {
TableStorage::Static { maximum, .. } => Some(*maximum),
TableStorage::Dynamic { maximum, .. } => maximum.clone(),
}
}
/// Fill `table[dst..]` with values from `items`
///
/// Returns a trap error on out-of-bounds accesses.
pub fn init_funcs(
&self,
dst: u32,
items: impl ExactSizeIterator<Item = *mut VMCallerCheckedAnyfunc>,
) -> Result<(), Trap> {
assert!(self.element_type() == TableElementType::Func);
self.with_elements_mut(|elements| {
let elements = match elements
.get_mut(usize::try_from(dst).unwrap()..)
.and_then(|s| s.get_mut(..items.len()))
{
Some(elements) => elements,
None => return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)),
};
for (item, slot) in items.zip(elements) {
*slot = item as *mut u8;
}
Ok(())
})
}
/// Fill `table[dst..dst + len]` with `val`.
///
/// Returns a trap error on out-of-bounds accesses.
@@ -218,8 +286,14 @@ impl Table {
/// this unsafety.
pub unsafe fn grow(&self, delta: u32, init_value: TableElement) -> Option<u32> {
let old_size = self.size();
let new_size = old_size.checked_add(delta)?;
if let Some(limiter) = &self.limiter {
if !limiter.table_growing(old_size, new_size, self.maximum()) {
return None;
}
}
if let Some(max) = self.maximum() {
if new_size > max {
return None;
@@ -229,7 +303,7 @@ impl Table {
debug_assert!(self.type_matches(&init_value));
// First resize the storage and then fill with the init value
match &self.0 {
match &self.storage {
TableStorage::Static { size, .. } => {
size.set(new_size);
}
@@ -319,7 +393,7 @@ impl Table {
/// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
pub fn vmtable(&self) -> VMTableDefinition {
match &self.0 {
match &self.storage {
TableStorage::Static { data, size, .. } => VMTableDefinition {
base: *data as _,
current_elements: size.get(),
@@ -346,7 +420,7 @@ impl Table {
where
F: FnOnce(&[*mut u8]) -> R,
{
match &self.0 {
match &self.storage {
TableStorage::Static { data, size, .. } => unsafe {
f(std::slice::from_raw_parts(*data, size.get() as usize))
},
@@ -361,7 +435,7 @@ impl Table {
where
F: FnOnce(&mut [*mut u8]) -> R,
{
match &self.0 {
match &self.storage {
TableStorage::Static { data, size, .. } => unsafe {
f(std::slice::from_raw_parts_mut(*data, size.get() as usize))
},
@@ -463,11 +537,14 @@ impl Drop for Table {
// The default table representation is an empty funcref table that cannot grow.
impl Default for Table {
fn default() -> Self {
Self(TableStorage::Static {
data: std::ptr::null_mut(),
size: Cell::new(0),
ty: TableElementType::Func,
maximum: 0,
})
Self {
storage: TableStorage::Static {
data: std::ptr::null_mut(),
size: Cell::new(0),
ty: TableElementType::Func,
maximum: 0,
},
limiter: None,
}
}
}

53
crates/runtime/src/traphandlers.rs
View File

@@ -58,13 +58,12 @@ static mut IS_WASM_PC: fn(usize) -> bool = |_| false;
/// program counter is the pc of an actual wasm trap or not. This is then used
/// to disambiguate faults that happen due to wasm and faults that happen due to
/// bugs in Rust or elsewhere.
pub fn init_traps(is_wasm_pc: fn(usize) -> bool) -> Result<(), Trap> {
pub fn init_traps(is_wasm_pc: fn(usize) -> bool) {
static INIT: Once = Once::new();
INIT.call_once(|| unsafe {
IS_WASM_PC = is_wasm_pc;
sys::platform_init();
});
sys::lazy_per_thread_init()
}
/// Raises a user-defined trap immediately.
@@ -256,7 +255,7 @@ impl<'a> CallThreadState<'a> {
}
fn with(self, closure: impl FnOnce(&CallThreadState) -> i32) -> Result<(), Trap> {
let ret = tls::set(&self, || closure(&self));
let ret = tls::set(&self, || closure(&self))?;
if ret != 0 {
return Ok(());
}
@@ -366,6 +365,7 @@ impl<T: Copy> Drop for ResetCell<'_, T> {
// the caller to the trap site.
mod tls {
use super::CallThreadState;
use crate::Trap;
use std::mem;
use std::ptr;
@@ -384,21 +384,38 @@ mod tls {
// these TLS values when the runtime may have crossed threads.
mod raw {
use super::CallThreadState;
use crate::Trap;
use std::cell::Cell;
use std::ptr;
pub type Ptr = *const CallThreadState<'static>;
thread_local!(static PTR: Cell<Ptr> = Cell::new(ptr::null()));
// The first entry here is the `Ptr` which is what's used as part of the
// public interface of this module. The second entry is a boolean which
// allows the runtime to perform per-thread initialization if necessary
// for handling traps (e.g. setting up ports on macOS and sigaltstack on
// Unix).
thread_local!(static PTR: Cell<(Ptr, bool)> = Cell::new((ptr::null(), false)));
#[inline(never)] // see module docs for why this is here
pub fn replace(val: Ptr) -> Ptr {
PTR.with(|p| p.replace(val))
pub fn replace(val: Ptr) -> Result<Ptr, Trap> {
PTR.with(|p| {
// When a new value is configured that means that we may be
// entering WebAssembly so check to see if this thread has
// performed per-thread initialization for traps.
let (prev, mut initialized) = p.get();
if !initialized {
super::super::sys::lazy_per_thread_init()?;
initialized = true;
}
p.set((val, initialized));
Ok(prev)
})
}
#[inline(never)] // see module docs for why this is here
pub fn get() -> Ptr {
PTR.with(|p| p.get())
PTR.with(|p| p.get().0)
}
}
@@ -412,7 +429,7 @@ mod tls {
///
/// This is not a safe operation since it's intended to only be used
/// with stack switching found with fibers and async wasmtime.
pub unsafe fn take() -> TlsRestore {
pub unsafe fn take() -> Result<TlsRestore, Trap> {
// Our tls pointer must be set at this time, and it must not be
// null. We need to restore the previous pointer since we're
// removing ourselves from the call-stack, and in the process we
@@ -421,8 +438,8 @@ mod tls {
let raw = raw::get();
assert!(!raw.is_null());
let prev = (*raw).prev.replace(ptr::null());
raw::replace(prev);
TlsRestore(raw)
raw::replace(prev)?;
Ok(TlsRestore(raw))
}
/// Restores a previous tls state back into this thread's TLS.
@@ -430,17 +447,12 @@ mod tls {
/// This is unsafe because it's intended to only be used within the
/// context of stack switching within wasmtime.
pub unsafe fn replace(self) -> Result<(), super::Trap> {
// When replacing to the previous value of TLS, we might have
// crossed a thread: make sure the trap-handling lazy initializer
// runs.
super::sys::lazy_per_thread_init()?;
// We need to configure our previous TLS pointer to whatever is in
// TLS at this time, and then we set the current state to ourselves.
let prev = raw::get();
assert!((*self.0).prev.get().is_null());
(*self.0).prev.set(prev);
raw::replace(self.0);
raw::replace(self.0)?;
Ok(())
}
}
@@ -448,13 +460,14 @@ mod tls {
/// Configures thread local state such that for the duration of the
/// execution of `closure` any call to `with` will yield `ptr`, unless this
/// is recursively called again.
pub fn set<R>(state: &CallThreadState<'_>, closure: impl FnOnce() -> R) -> R {
pub fn set<R>(state: &CallThreadState<'_>, closure: impl FnOnce() -> R) -> Result<R, Trap> {
struct Reset<'a, 'b>(&'a CallThreadState<'b>);
impl Drop for Reset<'_, '_> {
#[inline]
fn drop(&mut self) {
raw::replace(self.0.prev.replace(ptr::null()));
raw::replace(self.0.prev.replace(ptr::null()))
.expect("tls should be previously initialized");
}
}
@@ -464,10 +477,10 @@ mod tls {
let ptr = unsafe {
mem::transmute::<*const CallThreadState<'_>, *const CallThreadState<'static>>(state)
};
let prev = raw::replace(ptr);
let prev = raw::replace(ptr)?;
state.prev.set(prev);
let _reset = Reset(state);
closure()
Ok(closure())
}
/// Returns the last pointer configured with `set` above. Panics if `set`

33
crates/runtime/src/traphandlers/macos.rs
View File

@@ -25,7 +25,7 @@
//! use a thread-local to store information about how to unwind. Additionally
//! this requires that the check of whether a pc is a wasm trap or not is a
//! global check rather than a per-thread check. This necessitates the existence
//! of `GlobalFrameInfo` in the `wasmtime` crate.
//! of `GlobalModuleRegistry` in the `wasmtime` crate.
//!
//! Otherwise this file heavily uses the `mach` Rust crate for type and
//! function declarations. Many bits and pieces are copied or translated from
@@ -42,7 +42,6 @@ use mach::message::*;
use mach::port::*;
use mach::thread_act::*;
use mach::traps::*;
use std::cell::Cell;
use std::mem;
use std::thread;
@@ -425,26 +424,16 @@ impl Drop for ClosePort {
/// task-level port which is where we'd expected things like breakpad/crashpad
/// exception handlers to get registered.
pub fn lazy_per_thread_init() -> Result<(), Trap> {
thread_local! {
static PORTS_SET: Cell<bool> = Cell::new(false);
unsafe {
assert!(WASMTIME_PORT != MACH_PORT_NULL);
let kret = thread_set_exception_ports(
MY_PORT.with(|p| p.0),
EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION,
WASMTIME_PORT,
EXCEPTION_DEFAULT | MACH_EXCEPTION_CODES,
mach_addons::THREAD_STATE_NONE,
);
assert_eq!(kret, KERN_SUCCESS, "failed to set thread exception port");
}
PORTS_SET.with(|ports| {
if ports.replace(true) {
return;
}
unsafe {
assert!(WASMTIME_PORT != MACH_PORT_NULL);
let kret = thread_set_exception_ports(
MY_PORT.with(|p| p.0),
EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION,
WASMTIME_PORT,
EXCEPTION_DEFAULT | MACH_EXCEPTION_CODES,
mach_addons::THREAD_STATE_NONE,
);
assert_eq!(kret, KERN_SUCCESS, "failed to set thread exception port");
}
});
Ok(())
}

75
crates/runtime/src/traphandlers/unix.rs
View File

@@ -47,8 +47,8 @@ pub unsafe fn platform_init() {
// Handle `unreachable` instructions which execute `ud2` right now
register(&mut PREV_SIGILL, libc::SIGILL);
// x86 uses SIGFPE to report division by zero
if cfg!(target_arch = "x86") || cfg!(target_arch = "x86_64") {
// x86 and s390x use SIGFPE to report division by zero
if cfg!(target_arch = "x86") || cfg!(target_arch = "x86_64") || cfg!(target_arch = "s390x") {
register(&mut PREV_SIGFPE, libc::SIGFPE);
}
@@ -85,7 +85,7 @@ unsafe extern "C" fn trap_handler(
// Otherwise flag ourselves as handling a trap, do the trap
// handling, and reset our trap handling flag. Then we figure
// out what to do based on the result of the trap handling.
let pc = get_pc(context);
let pc = get_pc(context, signum);
let jmp_buf = info.jmp_buf_if_trap(pc, |handler| handler(signum, siginfo, context));
// Figure out what to do based on the result of this handling of
@@ -127,7 +127,7 @@ unsafe extern "C" fn trap_handler(
}
}
unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 {
unsafe fn get_pc(cx: *mut libc::c_void, _signum: libc::c_int) -> *const u8 {
cfg_if::cfg_if! {
if #[cfg(all(target_os = "linux", target_arch = "x86_64"))] {
let cx = &*(cx as *const libc::ucontext_t);
@@ -138,6 +138,23 @@ unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 {
} else if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "aarch64"))] {
let cx = &*(cx as *const libc::ucontext_t);
cx.uc_mcontext.pc as *const u8
} else if #[cfg(all(target_os = "linux", target_arch = "s390x"))] {
// On s390x, SIGILL and SIGFPE are delivered with the PSW address
// pointing *after* the faulting instruction, while SIGSEGV and
// SIGBUS are delivered with the PSW address pointing *to* the
// faulting instruction. To handle this, the code generator registers
// any trap that results in one of "late" signals on the last byte
// of the instruction, and any trap that results in one of the "early"
// signals on the first byte of the instruction (as usual). This
// means we simply need to decrement the reported PSW address by
// one in the case of a "late" signal here to ensure we always
// correctly find the associated trap handler.
let trap_offset = match _signum {
libc::SIGILL | libc::SIGFPE => 1,
_ => 0,
};
let cx = &*(cx as *const libc::ucontext_t);
(cx.uc_mcontext.psw.addr - trap_offset) as *const u8
} else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] {
let cx = &*(cx as *const libc::ucontext_t);
cx.uc_mcontext.mc_rip as *const u8
@@ -154,41 +171,35 @@ unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 {
/// and registering our own alternate stack that is large enough and has a guard
/// page.
pub fn lazy_per_thread_init() -> Result<(), Trap> {
// This thread local is purely used to register a `Stack` to get deallocated
// when the thread exists. Otherwise this function is only ever called at
// most once per-thread.
thread_local! {
/// Thread-local state is lazy-initialized on the first time it's used,
/// and dropped when the thread exits.
static TLS: RefCell<Tls> = RefCell::new(Tls::None);
static STACK: RefCell<Option<Stack>> = RefCell::new(None);
}
/// The size of the sigaltstack (not including the guard, which will be
/// added). Make this large enough to run our signal handlers.
const MIN_STACK_SIZE: usize = 16 * 4096;
enum Tls {
None,
Allocated {
mmap_ptr: *mut libc::c_void,
mmap_size: usize,
},
BigEnough,
struct Stack {
mmap_ptr: *mut libc::c_void,
mmap_size: usize,
}
return TLS.with(|slot| unsafe {
let mut slot = slot.borrow_mut();
match *slot {
Tls::None => {}
// already checked
_ => return Ok(()),
}
return STACK.with(|s| {
*s.borrow_mut() = unsafe { allocate_sigaltstack()? };
Ok(())
});
unsafe fn allocate_sigaltstack() -> Result<Option<Stack>, Trap> {
// Check to see if the existing sigaltstack, if it exists, is big
// enough. If so we don't need to allocate our own.
let mut old_stack = mem::zeroed();
let r = libc::sigaltstack(ptr::null(), &mut old_stack);
assert_eq!(r, 0, "learning about sigaltstack failed");
if old_stack.ss_flags & libc::SS_DISABLE == 0 && old_stack.ss_size >= MIN_STACK_SIZE {
*slot = Tls::BigEnough;
return Ok(());
return Ok(None);
}
// ... but failing that we need to allocate our own, so do all that
@@ -226,25 +237,17 @@ pub fn lazy_per_thread_init() -> Result<(), Trap> {
let r = libc::sigaltstack(&new_stack, ptr::null_mut());
assert_eq!(r, 0, "registering new sigaltstack failed");
*slot = Tls::Allocated {
Ok(Some(Stack {
mmap_ptr: ptr,
mmap_size: alloc_size,
};
Ok(())
});
}))
}
impl Drop for Tls {
impl Drop for Stack {
fn drop(&mut self) {
let (ptr, size) = match self {
Tls::Allocated {
mmap_ptr,
mmap_size,
} => (*mmap_ptr, *mmap_size),
_ => return,
};
unsafe {
// Deallocate the stack memory.
let r = libc::munmap(ptr, size);
let r = libc::munmap(self.mmap_ptr, self.mmap_size);
debug_assert_eq!(r, 0, "munmap failed during thread shutdown");
}
}

8
crates/wasi-common/cap-std-sync/Cargo.toml
View File

@@ -15,10 +15,10 @@ include = ["src/**/*", "LICENSE" ]
wasi-common = { path = "../", version = "0.26.0" }
async-trait = "0.1"
anyhow = "1.0"
cap-std = "0.13.7"
cap-fs-ext = "0.13.7"
cap-time-ext = "0.13.7"
cap-rand = "0.13.2"
cap-std = "0.13.9"
cap-fs-ext = "0.13.9"
cap-time-ext = "0.13.9"
cap-rand = "0.13.9"
fs-set-times = "0.3.1"
unsafe-io = "0.6.5"
system-interface = { version = "0.6.3", features = ["cap_std_impls"] }

2
crates/wasi-crypto/src/wiggle_interfaces/error.rs
View File

@@ -1,4 +1,4 @@
use super::{guest_types, WasiCryptoCtx};
use super::guest_types;
use std::num::TryFromIntError;
use wasi_crypto::CryptoError;

2
crates/wasi-nn/Cargo.toml
View File

@@ -22,7 +22,7 @@ wasmtime-wasi = { path = "../wasi", version = "0.26.0" }
wiggle = { path = "../wiggle", version = "0.26.0" }
# These dependencies are necessary for the wasi-nn implementation:
openvino = "0.1.5"
openvino = { version = "0.3.1", features = ["runtime-linking"] }
thiserror = "1.0"
[build-dependencies]

6
crates/wasi-nn/build.rs
View File

@@ -1,11 +1,9 @@
//! This build script:
//! - has the configuration necessary for the wiggle and witx macros.
use std::path::PathBuf;
fn main() {
// This is necessary for Wiggle/Witx macros.
let wasi_root = PathBuf::from("./spec").canonicalize().unwrap();
let cwd = std::env::current_dir().unwrap();
let wasi_root = cwd.join("spec");
println!("cargo:rustc-env=WASI_ROOT={}", wasi_root.display());
// Also automatically rebuild if the Witx files change

12
crates/wasi-nn/src/ctx.rs
View File

@@ -2,7 +2,7 @@
//! wasi-nn API.
use crate::r#impl::UsageError;
use crate::witx::types::{Graph, GraphExecutionContext};
use openvino::InferenceError;
use openvino::{InferenceError, SetupError};
use std::cell::RefCell;
use std::collections::HashMap;
use std::hash::Hash;
@@ -14,8 +14,10 @@ use wiggle::GuestError;
pub enum WasiNnError {
#[error("guest error")]
GuestError(#[from] GuestError),
#[error("openvino error")]
OpenvinoError(#[from] InferenceError),
#[error("openvino inference error")]
OpenvinoInferenceError(#[from] InferenceError),
#[error("openvino setup error")]
OpenvinoSetupError(#[from] SetupError),
#[error("usage error")]
UsageError(#[from] UsageError),
}
@@ -74,7 +76,7 @@ impl ExecutionContext {
/// Capture the state necessary for calling into `openvino`.
pub struct Ctx {
pub(crate) core: openvino::Core,
pub(crate) core: Option<openvino::Core>,
pub(crate) graphs: Table<Graph, (openvino::CNNNetwork, openvino::ExecutableNetwork)>,
pub(crate) executions: Table<GraphExecutionContext, ExecutionContext>,
}
@@ -83,7 +85,7 @@ impl Ctx {
/// Make a new `WasiNnCtx` with the default settings.
pub fn new() -> WasiNnResult<Self> {
Ok(Self {
core: openvino::Core::new(None)?,
core: Option::default(),
graphs: Table::default(),
executions: Table::default(),
})

20
crates/wasi-nn/src/impl.rs
View File

@@ -12,6 +12,8 @@ use wiggle::GuestPtr;
#[derive(Debug, Error)]
pub enum UsageError {
#[error("Invalid context; has the load function been called?")]
InvalidContext,
#[error("Only OpenVINO's IR is currently supported, passed encoding: {0:?}")]
InvalidEncoding(GraphEncoding),
#[error("OpenVINO expects only two buffers (i.e. [ir, weights]), passed: {0}")]
@@ -34,9 +36,21 @@ impl<'a> WasiEphemeralNn for WasiNnCtx {
if encoding != GraphEncoding::Openvino {
return Err(UsageError::InvalidEncoding(encoding).into());
}
if builders.len() != 2 {
return Err(UsageError::InvalidNumberOfBuilders(builders.len()).into());
}
// Construct the context if none is present; this is done lazily (i.e. upon actually loading
// a model) because it may fail to find and load the OpenVINO libraries. The laziness limits
// the extent of the error only to wasi-nn users, not all WASI users.
if self.ctx.borrow().core.is_none() {
self.ctx
.borrow_mut()
.core
.replace(openvino::Core::new(None)?);
}
let builders = builders.as_ptr();
let xml = builders.read()?.as_slice()?;
let weights = builders.add(1)?.read()?.as_slice()?;
@@ -44,11 +58,15 @@ impl<'a> WasiEphemeralNn for WasiNnCtx {
.ctx
.borrow_mut()
.core
.as_mut()
.ok_or(UsageError::InvalidContext)?
.read_network_from_buffer(&xml, &weights)?;
let executable_graph = self
.ctx
.borrow_mut()
.core
.as_mut()
.ok_or(UsageError::InvalidContext)?
.load_network(&graph, map_execution_target_to_string(target))?;
let id = self
.ctx
@@ -94,7 +112,7 @@ impl<'a> WasiEphemeralNn for WasiNnCtx {
.dimensions
.as_slice()?
.iter()
.map(|d| *d as u64)
.map(|d| *d as usize)
.collect::<Vec<_>>();
let precision = match tensor.type_ {
TensorType::F16 => Precision::FP16,

3
crates/wasi-nn/src/witx.rs
View File

@@ -14,7 +14,8 @@ impl<'a> types::UserErrorConversion for WasiNnCtx {
fn nn_errno_from_wasi_nn_error(&self, e: WasiNnError) -> Result<NnErrno, wiggle::Trap> {
eprintln!("Host error: {:?}", e);
match e {
WasiNnError::OpenvinoError(_) => unimplemented!(),
WasiNnError::OpenvinoSetupError(_) => unimplemented!(),
WasiNnError::OpenvinoInferenceError(_) => unimplemented!(),
WasiNnError::GuestError(_) => unimplemented!(),
WasiNnError::UsageError(_) => unimplemented!(),
}

47
crates/wasmtime/src/config.rs
View File

@@ -319,6 +319,10 @@ impl HostFuncMap {
fn async_required(&self) -> bool {
self.funcs.values().any(|f| f.1)
}
fn iter(&self) -> impl Iterator<Item = &HostFunc> {
self.funcs.values().map(|v| &*v.0)
}
}
macro_rules! generate_wrap_async_host_func {
@@ -379,9 +383,6 @@ pub struct Config {
pub(crate) max_wasm_stack: usize,
pub(crate) features: WasmFeatures,
pub(crate) wasm_backtrace_details_env_used: bool,
pub(crate) max_instances: usize,
pub(crate) max_tables: usize,
pub(crate) max_memories: usize,
#[cfg(feature = "async")]
pub(crate) async_stack_size: usize,
host_funcs: HostFuncMap,
@@ -418,9 +419,6 @@ impl Config {
max_wasm_stack: 1 << 20,
wasm_backtrace_details_env_used: false,
features: WasmFeatures::default(),
max_instances: 10_000,
max_tables: 10_000,
max_memories: 10_000,
#[cfg(feature = "async")]
async_stack_size: 2 << 20,
host_funcs: HostFuncMap::new(),
@@ -1192,39 +1190,6 @@ impl Config {
self
}
/// Configures the maximum number of instances which can be created within
/// this `Store`.
///
/// Instantiation will fail with an error if this limit is exceeded.
///
/// This value defaults to 10,000.
pub fn max_instances(&mut self, instances: usize) -> &mut Self {
self.max_instances = instances;
self
}
/// Configures the maximum number of tables which can be created within
/// this `Store`.
///
/// Instantiation will fail with an error if this limit is exceeded.
///
/// This value defaults to 10,000.
pub fn max_tables(&mut self, tables: usize) -> &mut Self {
self.max_tables = tables;
self
}
/// Configures the maximum number of memories which can be created within
/// this `Store`.
///
/// Instantiation will fail with an error if this limit is exceeded.
///
/// This value defaults to 10,000.
pub fn max_memories(&mut self, memories: usize) -> &mut Self {
self.max_memories = memories;
self
}
/// Defines a host function for the [`Config`] for the given callback.
///
/// Use [`Store::get_host_func`](crate::Store::get_host_func) to get a [`Func`](crate::Func) representing the function.
@@ -1318,6 +1283,10 @@ impl Config {
for_each_function_signature!(generate_wrap_async_host_func);
pub(crate) fn host_funcs(&self) -> impl Iterator<Item = &HostFunc> {
self.host_funcs.iter()
}
pub(crate) fn get_host_func(&self, module: &str, name: &str) -> Option<&HostFunc> {
self.host_funcs.get(module, name)
}

77
crates/wasmtime/src/engine.rs
View File

@@ -1,10 +1,41 @@
use crate::signatures::{SignatureCollection, SignatureRegistry};
use crate::Config;
use anyhow::Result;
use std::collections::HashMap;
use std::sync::Arc;
#[cfg(feature = "cache")]
use wasmtime_cache::CacheConfig;
use wasmtime_jit::Compiler;
use wasmtime_runtime::{debug_builtins, InstanceAllocator};
use wasmtime_runtime::{debug_builtins, InstanceAllocator, InstanceHandle, VMCallerCheckedAnyfunc};
/// This is used as a Send+Sync wrapper around two data structures relating to
/// host functions defined on `Config`:
///
/// * `anyfuncs` - this stores a mapping between the host function instance and
/// a `VMCallerCheckedAnyfunc` that can be used as the function's value in Wasmtime's ABI.
/// The address of the anyfunc needs to be stable, thus the boxed value.
///
/// * `signatures` - this stores the collection of shared signatures registered for every
/// usable host functions with this engine.
struct EngineHostFuncs {
anyfuncs: HashMap<InstanceHandle, Box<VMCallerCheckedAnyfunc>>,
signatures: SignatureCollection,
}
impl EngineHostFuncs {
fn new(registry: &SignatureRegistry) -> Self {
Self {
anyfuncs: HashMap::new(),
signatures: SignatureCollection::new(registry),
}
}
}
// This is safe for send and sync as it is read-only once the
// engine is constructed and the host functions live with the config,
// which the engine keeps a strong reference to.
unsafe impl Send for EngineHostFuncs {}
unsafe impl Sync for EngineHostFuncs {}
/// An `Engine` which is a global context for compilation and management of wasm
/// modules.
@@ -37,6 +68,8 @@ struct EngineInner {
config: Config,
compiler: Compiler,
allocator: Box<dyn InstanceAllocator>,
signatures: SignatureRegistry,
host_funcs: EngineHostFuncs,
}
impl Engine {
@@ -46,11 +79,29 @@ impl Engine {
debug_builtins::ensure_exported();
config.validate()?;
let allocator = config.build_allocator()?;
let registry = SignatureRegistry::new();
let mut host_funcs = EngineHostFuncs::new(&registry);
// Register all the host function signatures with the collection
for func in config.host_funcs() {
let sig = host_funcs
.signatures
.register(func.ty.as_wasm_func_type(), func.trampoline);
// Cloning the instance handle is safe as host functions outlive the engine
host_funcs.anyfuncs.insert(
unsafe { func.instance.clone() },
Box::new(func.anyfunc(sig)),
);
}
Ok(Engine {
inner: Arc::new(EngineInner {
config: config.clone(),
compiler: config.build_compiler(allocator.as_ref()),
allocator,
signatures: registry,
host_funcs,
}),
})
}
@@ -79,6 +130,25 @@ impl Engine {
Arc::ptr_eq(&a.inner, &b.inner)
}
pub(crate) fn signatures(&self) -> &SignatureRegistry {
&self.inner.signatures
}
pub(crate) fn host_func_signatures(&self) -> &SignatureCollection {
&self.inner.host_funcs.signatures
}
pub(crate) fn host_func_anyfunc(
&self,
instance: &InstanceHandle,
) -> Option<&VMCallerCheckedAnyfunc> {
self.inner
.host_funcs
.anyfuncs
.get(instance)
.map(AsRef::as_ref)
}
/// Ahead-of-time (AOT) compiles a WebAssembly module.
///
/// The `bytes` provided must be in one of two formats:
@@ -90,8 +160,9 @@ impl Engine {
/// Note that the `wat` feature is enabled by default.
///
/// This method may be used to compile a module for use with a different target
/// host. The output of this method may be used with [`Module::new`](crate::Module::new)
/// on hosts compatible with the [`Config`] associated with this [`Engine`].
/// host. The output of this method may be used with
/// [`Module::deserialize`](crate::Module::deserialize) on hosts compatible
/// with the [`Config`] associated with this [`Engine`].
///
/// The output of this method is safe to send to another host machine for later
/// execution. As the output is already a compiled module, translation and code

Some files were not shown because too many files have changed in this diff Show More