T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into pch/wasi_common_cap_std

Browse Source
This commit is contained in:
Pat Hickey
2020-12-14 16:17:10 -08:00
parent 1b8f9fd377 743529b4eb
commit 7ec9aac39f
160 changed files with 7328 additions and 1466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/actions/binary-compatible-builds/main.js vendored
View File

@@ -41,7 +41,6 @@ if (process.env.CENTOS !== undefined) {
let path = process.env.PATH; let path = process.env.PATH;
path = `${path}:/rust/bin`; path = `${path}:/rust/bin`;
path = `/opt/rh/devtoolset-8/root/usr/bin:${path}`; path = `/opt/rh/devtoolset-8/root/usr/bin:${path}`;
path = `/opt/rh/rh-python36/root/usr/bin:${path}`;
// Spawn a container daemonized in the background which we'll connect to via // Spawn a container daemonized in the background which we'll connect to via
// `docker exec`. This'll have access to the current directory. // `docker exec`. This'll have access to the current directory.
@@ -52,7 +51,7 @@ child_process.execFileSync('docker', [
'-v', `${process.cwd()}:${process.cwd()}`, '-v', `${process.cwd()}:${process.cwd()}`,
'-v', `${child_process.execSync('rustc --print sysroot').toString().trim()}:/rust:ro`, '-v', `${child_process.execSync('rustc --print sysroot').toString().trim()}:/rust:ro`,
'--env', `PATH=${path}`, '--env', `PATH=${path}`,
'centos:6', 'centos:7',
], stdio); ], stdio);
// Use ourselves to run future commands // Use ourselves to run future commands
@@ -63,7 +62,7 @@ const exec = s => {
child_process.execSync(`docker exec centos ${s}`, stdio); child_process.execSync(`docker exec centos ${s}`, stdio);
}; };
exec('yum install -y centos-release-scl cmake xz epel-release'); exec('yum install -y centos-release-scl cmake xz epel-release');
exec('yum install -y rh-python36 patchelf unzip'); exec('yum install -y python3 patchelf unzip');
exec('yum install -y devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++'); exec('yum install -y devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++');
exec('yum install -y git'); exec('yum install -y git');

13
.github/workflows/main.yml vendored
View File

@@ -60,7 +60,10 @@ jobs:
# nightly-only feature right now. # nightly-only feature right now.
- uses: ./.github/actions/install-rust - uses: ./.github/actions/install-rust
with: with:
toolchain: nightly # TODO (rust-lang/rust#79661): We are seeing an internal compiler error when
# building with the latest (2020-12-06) nightly; pin on a slightly older
# version for now.
toolchain: nightly-2020-11-29
- run: cargo doc --no-deps --all --exclude wasmtime-cli --exclude test-programs --exclude cranelift-codegen-meta - run: cargo doc --no-deps --all --exclude wasmtime-cli --exclude test-programs --exclude cranelift-codegen-meta
- run: cargo doc --package cranelift-codegen-meta --document-private-items - run: cargo doc --package cranelift-codegen-meta --document-private-items
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
@@ -145,7 +148,7 @@ jobs:
# flags to rustc. # flags to rustc.
- uses: ./.github/actions/install-rust - uses: ./.github/actions/install-rust
with: with:
toolchain: nightly toolchain: nightly-2020-11-29
- run: cargo install cargo-fuzz --vers "^0.8" - run: cargo install cargo-fuzz --vers "^0.8"
- run: cargo fetch - run: cargo fetch
working-directory: ./fuzz working-directory: ./fuzz
@@ -202,7 +205,7 @@ jobs:
rust: beta rust: beta
- build: nightly - build: nightly
os: ubuntu-latest os: ubuntu-latest
rust: nightly rust: nightly-2020-11-29
- build: macos - build: macos
os: macos-latest os: macos-latest
rust: stable rust: stable
@@ -292,7 +295,7 @@ jobs:
submodules: true submodules: true
- uses: ./.github/actions/install-rust - uses: ./.github/actions/install-rust
with: with:
toolchain: nightly toolchain: nightly-2020-11-29
- uses: ./.github/actions/define-llvm-env - uses: ./.github/actions/define-llvm-env
# Install wasm32 targets in order to build various tests throughout the # Install wasm32 targets in order to build various tests throughout the
@@ -303,7 +306,7 @@ jobs:
# Run the x64 CI script. # Run the x64 CI script.
- run: ./ci/run-experimental-x64-ci.sh - run: ./ci/run-experimental-x64-ci.sh
env: env:
CARGO_VERSION: "+nightly" CARGO_VERSION: "+nightly-2020-11-29"
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
# Build and test the wasi-nn module. # Build and test the wasi-nn module.

192
Cargo.lock generated
View File

@@ -391,6 +391,16 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "cpp_demangle"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44919ecaf6f99e8e737bc239408931c9a01e9a6c74814fee8242dd2506b65390"
dependencies = [
"cfg-if 1.0.0",
"glob",
]
[[package]] [[package]]
name = "cpu-time" name = "cpu-time"
version = "1.0.0" version = "1.0.0"
@@ -444,7 +454,7 @@ dependencies = [
"souper-ir", "souper-ir",
"target-lexicon", "target-lexicon",
"thiserror", "thiserror",
"wast 27.0.0", "wast 29.0.0",
] ]
[[package]] [[package]]
@@ -512,6 +522,26 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "cranelift-jit"
version = "0.68.0"
dependencies = [
"anyhow",
"cranelift",
"cranelift-codegen",
"cranelift-entity",
"cranelift-frontend",
"cranelift-module",
"cranelift-native",
"errno",
"libc",
"log",
"memmap",
"region",
"target-lexicon",
"winapi",
]
[[package]] [[package]]
name = "cranelift-module" name = "cranelift-module"
version = "0.68.0" version = "0.68.0"
@@ -577,25 +607,6 @@ dependencies = [
"serde_json", "serde_json",
] ]
[[package]]
name = "cranelift-simplejit"
version = "0.68.0"
dependencies = [
"cranelift",
"cranelift-codegen",
"cranelift-entity",
"cranelift-frontend",
"cranelift-module",
"cranelift-native",
"errno",
"libc",
"log",
"memmap",
"region",
"target-lexicon",
"winapi",
]
[[package]] [[package]]
name = "cranelift-tools" name = "cranelift-tools"
version = "0.66.0" version = "0.66.0"
@@ -609,13 +620,13 @@ dependencies = [
"cranelift-filetests", "cranelift-filetests",
"cranelift-frontend", "cranelift-frontend",
"cranelift-interpreter", "cranelift-interpreter",
"cranelift-jit",
"cranelift-module", "cranelift-module",
"cranelift-native", "cranelift-native",
"cranelift-object", "cranelift-object",
"cranelift-preopt", "cranelift-preopt",
"cranelift-reader", "cranelift-reader",
"cranelift-serde", "cranelift-serde",
"cranelift-simplejit",
"cranelift-wasm", "cranelift-wasm",
"file-per-thread-logger", "file-per-thread-logger",
"filecheck", "filecheck",
@@ -646,7 +657,7 @@ dependencies = [
"smallvec", "smallvec",
"target-lexicon", "target-lexicon",
"thiserror", "thiserror",
"wasmparser 0.68.0", "wasmparser 0.70.0",
"wat", "wat",
] ]
@@ -816,6 +827,12 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "downcast-rs"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
[[package]] [[package]]
name = "dynasm" name = "dynasm"
version = "1.0.0" version = "1.0.0"
@@ -1226,7 +1243,7 @@ dependencies = [
"smallvec", "smallvec",
"thiserror", "thiserror",
"typemap", "typemap",
"wasmparser 0.68.0", "wasmparser 0.70.0",
"wat", "wat",
] ]
@@ -1304,6 +1321,12 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "memory_units"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882"
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.4.3" version = "0.4.3"
@@ -1330,6 +1353,17 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "num-bigint"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.44" version = "0.1.44"
@@ -1340,6 +1374,18 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "num-rational"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg",
"num-bigint",
"num-integer",
"num-traits",
]
[[package]] [[package]]
name = "num-traits" name = "num-traits"
version = "0.2.14" version = "0.2.14"
@@ -1418,6 +1464,12 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "parity-wasm"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865"
[[package]] [[package]]
name = "peeking_take_while" name = "peeking_take_while"
version = "0.1.2" version = "0.1.2"
@@ -1435,7 +1487,7 @@ dependencies = [
"peepmatic-test-operator", "peepmatic-test-operator",
"peepmatic-traits", "peepmatic-traits",
"serde", "serde",
"wast 27.0.0", "wast 29.0.0",
"z3", "z3",
] ]
@@ -1463,7 +1515,7 @@ dependencies = [
"peepmatic-traits", "peepmatic-traits",
"rand", "rand",
"serde", "serde",
"wast 27.0.0", "wast 29.0.0",
] ]
[[package]] [[package]]
@@ -1488,7 +1540,7 @@ dependencies = [
"serde", "serde",
"serde_test", "serde_test",
"thiserror", "thiserror",
"wast 27.0.0", "wast 29.0.0",
] ]
[[package]] [[package]]
@@ -1500,7 +1552,7 @@ dependencies = [
"peepmatic", "peepmatic",
"peepmatic-test-operator", "peepmatic-test-operator",
"souper-ir", "souper-ir",
"wast 27.0.0", "wast 29.0.0",
] ]
[[package]] [[package]]
@@ -1521,7 +1573,7 @@ version = "0.68.0"
dependencies = [ dependencies = [
"peepmatic-traits", "peepmatic-traits",
"serde", "serde",
"wast 27.0.0", "wast 29.0.0",
] ]
[[package]] [[package]]
@@ -2027,9 +2079,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"
[[package]] [[package]]
name = "souper-ir" name = "souper-ir"
version = "1.0.0" version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "163cc2bdd8a66cbaccdf06a6b476689a97e928883e09bffbe06fd5945842a83f" checksum = "a50c18ce33988e1973003afbaa66e6a465ad7a614dc33f246879ccc209c2c044"
dependencies = [ dependencies = [
"id-arena", "id-arena",
] ]
@@ -2434,13 +2486,47 @@ dependencies = [
] ]
[[package]] [[package]]
name = "wasm-smith" name = "wasm-encoder"
version = "0.1.10" version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd5e4720bb44dc5e46a917139dc9dfa4bb6fee023bf9f77d2c55ec12eeaf9930" checksum = "ed89eaf99e08b84f96e477a16588a07dd3b51dc5f07291c3706782f62a10a5e1"
dependencies = [
"leb128",
]
[[package]]
name = "wasm-smith"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "509904d9c4c4659ac238a3f27c3656dd6d3931697eddd4b0f32e335769c298d0"
dependencies = [ dependencies = [
"arbitrary", "arbitrary",
"leb128", "leb128",
"wasm-encoder",
]
[[package]]
name = "wasmi"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bb6825d9b2147105789adb4c2d84b9b568719713f3ac39618b637b4dafc86c4"
dependencies = [
"downcast-rs",
"libc",
"memory_units",
"num-rational",
"num-traits",
"parity-wasm",
"wasmi-validation",
]
[[package]]
name = "wasmi-validation"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93"
dependencies = [
"parity-wasm",
] ]
[[package]] [[package]]
@@ -2451,18 +2537,18 @@ checksum = "32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6"
[[package]] [[package]]
name = "wasmparser" name = "wasmparser"
version = "0.68.0" version = "0.70.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29a00e14eed9c2ecbbdbdd4fb284f49d21b6808965de24769a6379a13ec47d4c" checksum = "ed1b3f9e9cf01a580b9f3281214dfdb1922b5dfb8494ee312ca03ae10036c2a2"
[[package]] [[package]]
name = "wasmprinter" name = "wasmprinter"
version = "0.2.15" version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f39a73b5f09cfcb1b568b61968d39b19e4ddec9b49040cfc091adf3b0788bca6" checksum = "f89b2b24dce17e27fe9c09c28331cbd77067fcde5c6ea2508ac84bcbd5d3e018"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"wasmparser 0.68.0", "wasmparser 0.70.0",
] ]
[[package]] [[package]]
@@ -2473,6 +2559,8 @@ dependencies = [
"backtrace", "backtrace",
"bincode", "bincode",
"cfg-if 1.0.0", "cfg-if 1.0.0",
"cpp_demangle",
"indexmap",
"libc", "libc",
"log", "log",
"region", "region",
@@ -2481,7 +2569,7 @@ dependencies = [
"smallvec", "smallvec",
"target-lexicon", "target-lexicon",
"tempfile", "tempfile",
"wasmparser 0.68.0", "wasmparser 0.70.0",
"wasmtime-cache", "wasmtime-cache",
"wasmtime-environ", "wasmtime-environ",
"wasmtime-jit", "wasmtime-jit",
@@ -2559,7 +2647,7 @@ dependencies = [
"test-programs", "test-programs",
"tracing-subscriber", "tracing-subscriber",
"wasi-common", "wasi-common",
"wasmparser 0.68.0", "wasmparser 0.70.0",
"wasmtime", "wasmtime",
"wasmtime-cache", "wasmtime-cache",
"wasmtime-debug", "wasmtime-debug",
@@ -2595,7 +2683,7 @@ dependencies = [
"object", "object",
"target-lexicon", "target-lexicon",
"thiserror", "thiserror",
"wasmparser 0.68.0", "wasmparser 0.70.0",
"wasmtime-environ", "wasmtime-environ",
] ]
@@ -2614,7 +2702,7 @@ dependencies = [
"more-asserts", "more-asserts",
"serde", "serde",
"thiserror", "thiserror",
"wasmparser 0.68.0", "wasmparser 0.70.0",
] ]
[[package]] [[package]]
@@ -2642,7 +2730,8 @@ dependencies = [
"log", "log",
"rayon", "rayon",
"wasm-smith", "wasm-smith",
"wasmparser 0.68.0", "wasmi",
"wasmparser 0.70.0",
"wasmprinter", "wasmprinter",
"wasmtime", "wasmtime",
"wasmtime-wast", "wasmtime-wast",
@@ -2653,6 +2742,7 @@ dependencies = [
name = "wasmtime-jit" name = "wasmtime-jit"
version = "0.21.0" version = "0.21.0"
dependencies = [ dependencies = [
"addr2line",
"anyhow", "anyhow",
"cfg-if 1.0.0", "cfg-if 1.0.0",
"cranelift-codegen", "cranelift-codegen",
@@ -2669,7 +2759,7 @@ dependencies = [
"serde", "serde",
"target-lexicon", "target-lexicon",
"thiserror", "thiserror",
"wasmparser 0.68.0", "wasmparser 0.70.0",
"wasmtime-cranelift", "wasmtime-cranelift",
"wasmtime-debug", "wasmtime-debug",
"wasmtime-environ", "wasmtime-environ",
@@ -2686,7 +2776,7 @@ version = "0.21.0"
dependencies = [ dependencies = [
"cranelift-codegen", "cranelift-codegen",
"lightbeam", "lightbeam",
"wasmparser 0.68.0", "wasmparser 0.70.0",
"wasmtime-environ", "wasmtime-environ",
] ]
@@ -2793,7 +2883,7 @@ version = "0.21.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"wasmtime", "wasmtime",
"wast 27.0.0", "wast 29.0.0",
] ]
[[package]] [[package]]
@@ -2829,20 +2919,20 @@ dependencies = [
[[package]] [[package]]
name = "wast" name = "wast"
version = "27.0.0" version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2c3ef5f6a72dffa44c24d5811123f704e18a1dbc83637d347b1852b41d3835c" checksum = "dcf2268937131d63c3d833242bf5e075406f9ed868b4265f3280e15dac29ac18"
dependencies = [ dependencies = [
"leb128", "leb128",
] ]
[[package]] [[package]]
name = "wat" name = "wat"
version = "1.0.28" version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "835cf59c907f67e2bbc20f50157e08f35006fe2a8444d8ec9f5683e22f937045" checksum = "0d11a88d953b298172d218d18f22853f4e6e12873b62755d05617b864d312c68"
dependencies = [ dependencies = [
"wast 27.0.0", "wast 29.0.0",
] ]
[[package]] [[package]]

4
Cargo.toml
View File

@@ -38,12 +38,12 @@ anyhow = "1.0.19"
target-lexicon = { version = "0.11.0", default-features = false } target-lexicon = { version = "0.11.0", default-features = false }
pretty_env_logger = "0.4.0" pretty_env_logger = "0.4.0"
file-per-thread-logger = "0.1.1" file-per-thread-logger = "0.1.1"
wat = "1.0.27" wat = "1.0.30"
libc = "0.2.60" libc = "0.2.60"
log = "0.4.8" log = "0.4.8"
rayon = "1.2.1" rayon = "1.2.1"
humantime = "2.0.0" humantime = "2.0.0"
wasmparser = "0.68" wasmparser = "0.70.0"
[dev-dependencies] [dev-dependencies]
env_logger = "0.8.1" env_logger = "0.8.1"

62
build.rs
View File

@@ -31,6 +31,7 @@ fn main() -> anyhow::Result<()> {
test_directory_module(out, "tests/misc_testsuite/bulk-memory-operations", strategy)?; test_directory_module(out, "tests/misc_testsuite/bulk-memory-operations", strategy)?;
test_directory_module(out, "tests/misc_testsuite/reference-types", strategy)?; test_directory_module(out, "tests/misc_testsuite/reference-types", strategy)?;
test_directory_module(out, "tests/misc_testsuite/multi-memory", strategy)?; test_directory_module(out, "tests/misc_testsuite/multi-memory", strategy)?;
test_directory_module(out, "tests/misc_testsuite/module-linking", strategy)?;
Ok(()) Ok(())
})?; })?;
@@ -174,46 +175,10 @@ fn write_testsuite_tests(
/// For experimental_x64 backend features that are not supported yet, mark tests as panicking, so /// For experimental_x64 backend features that are not supported yet, mark tests as panicking, so
/// they stop "passing" once the features are properly implemented. /// they stop "passing" once the features are properly implemented.
fn experimental_x64_should_panic(testsuite: &str, testname: &str, strategy: &str) -> bool { ///
if !cfg!(feature = "experimental_x64") || strategy != "Cranelift" { /// TODO(#2470): removed all tests from this set as we are disabling x64 SIMD tests unconditionally
return false; /// instead until we resolve a nondeterminism bug. Restore when fixed.
} fn experimental_x64_should_panic(_testsuite: &str, _testname: &str, _strategy: &str) -> bool {
match (testsuite, testname) {
("simd", "simd_address") => return false,
("simd", "simd_bitwise") => return false,
("simd", "simd_bit_shift") => return false,
("simd", "simd_boolean") => return false,
("simd", "simd_const") => return false,
("simd", "simd_i8x16_arith") => return false,
("simd", "simd_i8x16_arith2") => return false,
("simd", "simd_i8x16_cmp") => return false,
("simd", "simd_i8x16_sat_arith") => return false,
("simd", "simd_i16x8_arith") => return false,
("simd", "simd_i16x8_arith2") => return false,
("simd", "simd_i16x8_cmp") => return false,
("simd", "simd_i16x8_sat_arith") => return false,
("simd", "simd_i32x4_arith") => return false,
("simd", "simd_i32x4_arith2") => return false,
("simd", "simd_i32x4_cmp") => return false,
("simd", "simd_i64x2_arith") => return false,
("simd", "simd_f32x4") => return false,
("simd", "simd_f32x4_arith") => return false,
("simd", "simd_f32x4_cmp") => return false,
("simd", "simd_f32x4_pmin_pmax") => return false,
("simd", "simd_f64x2") => return false,
("simd", "simd_f64x2_arith") => return false,
("simd", "simd_f64x2_cmp") => return false,
("simd", "simd_f64x2_pmin_pmax") => return false,
("simd", "simd_lane") => return false,
("simd", "simd_load") => return false,
("simd", "simd_load_splat") => return false,
("simd", "simd_splat") => return false,
("simd", "simd_store") => return false,
("simd", _) => return true,
_ => {}
}
false false
} }
@@ -235,21 +200,22 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64"; return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64";
} }
// Ignore all x64 SIMD tests for now (#2470).
("simd", _) if cfg!(feature = "experimental_x64") => {
return env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64";
}
// These are only implemented on aarch64 and x64. // These are only implemented on aarch64 and x64.
("simd", "simd_boolean") ("simd", "simd_boolean")
| ("simd", "simd_f32x4_pmin_pmax") | ("simd", "simd_f32x4_pmin_pmax")
| ("simd", "simd_f64x2_pmin_pmax") => { | ("simd", "simd_f64x2_pmin_pmax")
| ("simd", "simd_f32x4_rounding")
| ("simd", "simd_f64x2_rounding")
| ("simd", "simd_i32x4_dot_i16x8") => {
return !(cfg!(feature = "experimental_x64") return !(cfg!(feature = "experimental_x64")
|| env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "aarch64") || env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "aarch64")
} }
// These are only implemented on aarch64.
("simd", "simd_f32x4_rounding") | ("simd", "simd_f64x2_rounding") => {
return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "aarch64";
}
// These tests have simd operators which aren't implemented yet.
// (currently none)
_ => {} _ => {}
}, },
_ => panic!("unrecognized strategy"), _ => panic!("unrecognized strategy"),

24
ci/run-wasi-nn-example.sh
View File

@@ -7,7 +7,16 @@
# executed with the Wasmtime CLI. # executed with the Wasmtime CLI.
set -e set -e
WASMTIME_DIR=$(dirname "$0" | xargs dirname) WASMTIME_DIR=$(dirname "$0" | xargs dirname)
FIXTURE=https://gist.github.com/abrown/c7847bf3701f9efbb2070da1878542c1/raw/07a9f163994b0ff8f0d7c5a5c9645ec3d8b24024 FIXTURE=https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/alexnet
if [ -z "${1+x}" ]; then
# If no temporary directory is specified, create one.
TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX)
REMOVE_TMP_DIR=1
else
# If a directory was specified, use it and avoid removing it.
TMP_DIR=$(realpath $1)
REMOVE_TMP_DIR=0
fi
# Inform the environment of OpenVINO library locations. Then we use OPENVINO_INSTALL_DIR below to avoid building all of # Inform the environment of OpenVINO library locations. Then we use OPENVINO_INSTALL_DIR below to avoid building all of
# OpenVINO from source (quite slow). # OpenVINO from source (quite slow).
@@ -17,10 +26,9 @@ source /opt/intel/openvino/bin/setupvars.sh
OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo build -p wasmtime-cli --features wasi-nn OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo build -p wasmtime-cli --features wasi-nn
# Download all necessary test fixtures to the temporary directory. # Download all necessary test fixtures to the temporary directory.
TMP_DIR=${1:-$(mktemp -d -t ci-XXXXXXXXXX)} wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/alexnet.bin
wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/frozen_inference_graph.bin wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/alexnet.xml
wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/frozen_inference_graph.xml wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/tensor-1x3x227x227-f32.bgr
wget --no-clobber --directory-prefix=$TMP_DIR $FIXTURE/tensor-1x3x300x300-f32.bgr
# Now build an example that uses the wasi-nn API. # Now build an example that uses the wasi-nn API.
pushd $WASMTIME_DIR/crates/wasi-nn/examples/classification-example pushd $WASMTIME_DIR/crates/wasi-nn/examples/classification-example
@@ -31,5 +39,7 @@ popd
# Run the example in Wasmtime (note that the example uses `fixture` as the expected location of the model/tensor files). # Run the example in Wasmtime (note that the example uses `fixture` as the expected location of the model/tensor files).
OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo run --features wasi-nn -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo run --features wasi-nn -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm
# Clean up. # Clean up the temporary directory only if it was not specified (users may want to keep the directory around).
rm -rf $TMP_DIR if [[ $REMOVE_TMP_DIR -eq 1 ]]; then
rm -rf $TMP_DIR
fi

2
cranelift/Cargo.toml
View File

@@ -26,7 +26,7 @@ cranelift-native = { path = "native", version = "0.68.0" }
cranelift-filetests = { path = "filetests", version = "0.66.0" } cranelift-filetests = { path = "filetests", version = "0.66.0" }
cranelift-module = { path = "module", version = "0.68.0" } cranelift-module = { path = "module", version = "0.68.0" }
cranelift-object = { path = "object", version = "0.68.0" } cranelift-object = { path = "object", version = "0.68.0" }
cranelift-simplejit = { path = "simplejit", version = "0.68.0" } cranelift-jit = { path = "jit", version = "0.68.0" }
cranelift-preopt = { path = "preopt", version = "0.68.0" } cranelift-preopt = { path = "preopt", version = "0.68.0" }
cranelift = { path = "umbrella", version = "0.68.0" } cranelift = { path = "umbrella", version = "0.68.0" }
filecheck = "0.5.0" filecheck = "0.5.0"

4
cranelift/README.md
View File

@@ -16,10 +16,10 @@ into executable machine code.
For more information, see [the documentation](docs/index.md). For more information, see [the documentation](docs/index.md).
For an example of how to use the JIT, see the [SimpleJIT Demo], which For an example of how to use the JIT, see the [JIT Demo], which
implements a toy language. implements a toy language.
[SimpleJIT Demo]: https://github.com/bytecodealliance/simplejit-demo [JIT Demo]: https://github.com/bytecodealliance/simplejit-demo
For an example of how to use Cranelift to run WebAssembly code, see For an example of how to use Cranelift to run WebAssembly code, see
[Wasmtime], which implements a standalone, embeddable, VM using Cranelift. [Wasmtime], which implements a standalone, embeddable, VM using Cranelift.

4
cranelift/codegen/Cargo.toml
View File

@@ -29,8 +29,8 @@ peepmatic = { path = "../peepmatic", optional = true, version = "0.68.0" }
peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.68.0" } peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.68.0" }
peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.68.0" } peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.68.0" }
regalloc = { version = "0.0.31" } regalloc = { version = "0.0.31" }
souper-ir = { version = "1", optional = true } souper-ir = { version = "2.1.0", optional = true }
wast = { version = "27.0.0", optional = true } wast = { version = "29.0.0", optional = true }
# It is a goal of the cranelift-codegen crate to have minimal external dependencies. # It is a goal of the cranelift-codegen crate to have minimal external dependencies.
# Please don't add any unless they are essential to the task of creating binary # Please don't add any unless they are essential to the task of creating binary
# machine code. Integration tests that need external dependencies can be # machine code. Integration tests that need external dependencies can be

5
cranelift/codegen/meta/src/isa/x86/encodings.rs
View File

@@ -549,10 +549,13 @@ fn define_moves(e: &mut PerCpuModeEncodings, shared_defs: &SharedDefinitions, r:
} }
e.enc64(bconst.bind(B64), rec_pu_id_bool.opcodes(&MOV_IMM).rex()); e.enc64(bconst.bind(B64), rec_pu_id_bool.opcodes(&MOV_IMM).rex());
// You may expect that i8 encodings would use 0x30 (XORB) to indicate that encodings should be
// on 8-bit operands (f.ex "xor %al, %al"). Cranelift currently does not know when it can
// safely drop the 0x66 prefix, so we explicitly select a wider but permissible opcode.
let is_zero_int = InstructionPredicate::new_is_zero_int(&formats.unary_imm, "imm"); let is_zero_int = InstructionPredicate::new_is_zero_int(&formats.unary_imm, "imm");
e.enc_both_instp( e.enc_both_instp(
iconst.bind(I8), iconst.bind(I8),
rec_u_id_z.opcodes(&XORB), rec_u_id_z.opcodes(&XOR),
is_zero_int.clone(), is_zero_int.clone(),
); );

3
cranelift/codegen/meta/src/isa/x86/opcodes.rs
View File

@@ -711,9 +711,6 @@ pub static XOR_IMM8_SIGN_EXTEND: [u8; 1] = [0x83];
/// r/m{16,32,64} XOR register of the same size. /// r/m{16,32,64} XOR register of the same size.
pub static XOR: [u8; 1] = [0x31]; pub static XOR: [u8; 1] = [0x31];
/// r/m8 XOR r8.
pub static XORB: [u8; 1] = [0x30];
/// Bitwise logical XOR of packed double-precision floating-point values. /// Bitwise logical XOR of packed double-precision floating-point values.
pub static XORPD: [u8; 3] = [0x66, 0x0f, 0x57]; pub static XORPD: [u8; 3] = [0x66, 0x0f, 0x57];

44
cranelift/codegen/src/ir/function.rs
View File

@@ -7,8 +7,9 @@ use crate::binemit::CodeOffset;
use crate::entity::{PrimaryMap, SecondaryMap}; use crate::entity::{PrimaryMap, SecondaryMap};
use crate::ir; use crate::ir;
use crate::ir::{ use crate::ir::{
Block, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, JumpTable, instructions::BranchInfo, Block, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap,
JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, HeapData, Inst, InstructionData, JumpTable, JumpTableData, Opcode, SigRef, StackSlot,
StackSlotData, Table, TableData,
}; };
use crate::ir::{BlockOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations}; use crate::ir::{BlockOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations};
use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature}; use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature};
@@ -270,6 +271,8 @@ impl Function {
/// Changes the destination of a jump or branch instruction. /// Changes the destination of a jump or branch instruction.
/// Does nothing if called with a non-jump or non-branch instruction. /// Does nothing if called with a non-jump or non-branch instruction.
///
/// Note that this method ignores multi-destination branches like `br_table`.
pub fn change_branch_destination(&mut self, inst: Inst, new_dest: Block) { pub fn change_branch_destination(&mut self, inst: Inst, new_dest: Block) {
match self.dfg[inst].branch_destination_mut() { match self.dfg[inst].branch_destination_mut() {
None => (), None => (),
@@ -277,6 +280,43 @@ impl Function {
} }
} }
/// Rewrite the branch destination to `new_dest` if the destination matches `old_dest`.
/// Does nothing if called with a non-jump or non-branch instruction.
///
/// Unlike [change_branch_destination](Function::change_branch_destination), this method rewrite the destinations of
/// multi-destination branches like `br_table`.
pub fn rewrite_branch_destination(&mut self, inst: Inst, old_dest: Block, new_dest: Block) {
match self.dfg.analyze_branch(inst) {
BranchInfo::SingleDest(dest, ..) => {
if dest == old_dest {
self.change_branch_destination(inst, new_dest);
}
}
BranchInfo::Table(table, default_dest) => {
self.jump_tables[table].iter_mut().for_each(|entry| {
if *entry == old_dest {
*entry = new_dest;
}
});
if default_dest == Some(old_dest) {
match &mut self.dfg[inst] {
InstructionData::BranchTable { destination, .. } => {
*destination = new_dest;
}
_ => panic!(
"Unexpected instruction {} having default destination",
self.dfg.display_inst(inst, None)
),
}
}
}
BranchInfo::NotABranch => {}
}
}
/// Checks that the specified block can be encoded as a basic block. /// Checks that the specified block can be encoded as a basic block.
/// ///
/// On error, returns the first invalid instruction and an error message. /// On error, returns the first invalid instruction and an error message.

2
cranelift/codegen/src/ir/instructions.rs
View File

@@ -277,7 +277,7 @@ impl InstructionData {
ref mut destination, ref mut destination,
.. ..
} => Some(destination), } => Some(destination),
Self::BranchTable { .. } => None, Self::BranchTable { .. } | Self::IndirectJump { .. } => None,
_ => { _ => {
debug_assert!(!self.opcode().is_branch()); debug_assert!(!self.opcode().is_branch());
None None

28
cranelift/codegen/src/ir/libcall.rs
View File

@@ -63,6 +63,7 @@ pub enum LibCall {
/// Elf __tls_get_addr /// Elf __tls_get_addr
ElfTlsGetAddr, ElfTlsGetAddr,
// When adding a new variant make sure to add it to `all_libcalls` too.
} }
impl fmt::Display for LibCall { impl fmt::Display for LibCall {
@@ -136,6 +137,33 @@ impl LibCall {
_ => return None, _ => return None,
}) })
} }
/// Get a list of all known `LibCall`'s.
pub fn all_libcalls() -> &'static [LibCall] {
use LibCall::*;
&[
Probestack,
UdivI64,
SdivI64,
UremI64,
SremI64,
IshlI64,
UshrI64,
SshrI64,
CeilF32,
CeilF64,
FloorF32,
FloorF64,
TruncF32,
TruncF64,
NearestF32,
NearestF64,
Memcpy,
Memset,
Memmove,
ElfTlsGetAddr,
]
}
} }
/// Get a function reference for `libcall` in `func`, following the signature /// Get a function reference for `libcall` in `func`, following the signature

52
cranelift/codegen/src/ir/memflags.rs
View File

@@ -6,15 +6,31 @@ enum FlagBit {
Notrap, Notrap,
Aligned, Aligned,
Readonly, Readonly,
LittleEndian,
BigEndian,
} }
const NAMES: [&str; 3] = ["notrap", "aligned", "readonly"]; const NAMES: [&str; 5] = ["notrap", "aligned", "readonly", "little", "big"];
/// Endianness of a memory access.
#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub enum Endianness {
/// Little-endian
Little,
/// Big-endian
Big,
}
/// Flags for memory operations like load/store. /// Flags for memory operations like load/store.
/// ///
/// Each of these flags introduce a limited form of undefined behavior. The flags each enable /// Each of these flags introduce a limited form of undefined behavior. The flags each enable
/// certain optimizations that need to make additional assumptions. Generally, the semantics of a /// certain optimizations that need to make additional assumptions. Generally, the semantics of a
/// program does not change when a flag is removed, but adding a flag will. /// program does not change when a flag is removed, but adding a flag will.
///
/// In addition, the flags determine the endianness of the memory access. By default,
/// any memory access uses the native endianness determined by the target ISA. This can
/// be overridden for individual accesses by explicitly specifying little- or big-endian
/// semantics via the flags.
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub struct MemFlags { pub struct MemFlags {
bits: u8, bits: u8,
@@ -48,16 +64,48 @@ impl MemFlags {
/// Set a flag bit by name. /// Set a flag bit by name.
/// ///
/// Returns true if the flag was found and set, false for an unknown flag name. /// Returns true if the flag was found and set, false for an unknown flag name.
/// Will also return false when trying to set inconsistent endianness flags.
pub fn set_by_name(&mut self, name: &str) -> bool { pub fn set_by_name(&mut self, name: &str) -> bool {
match NAMES.iter().position(|&s| s == name) { match NAMES.iter().position(|&s| s == name) {
Some(bit) => { Some(bit) => {
self.bits |= 1 << bit; let bits = self.bits | 1 << bit;
if (bits & (1 << FlagBit::LittleEndian as usize)) != 0
&& (bits & (1 << FlagBit::BigEndian as usize)) != 0
{
false
} else {
self.bits = bits;
true true
} }
}
None => false, None => false,
} }
} }
/// Return endianness of the memory access. This will return the endianness
/// explicitly specified by the flags if any, and will default to the native
/// endianness otherwise. The native endianness has to be provided by the
/// caller since it is not explicitly encoded in CLIF IR -- this allows a
/// front end to create IR without having to know the target endianness.
pub fn endianness(self, native_endianness: Endianness) -> Endianness {
if self.read(FlagBit::LittleEndian) {
Endianness::Little
} else if self.read(FlagBit::BigEndian) {
Endianness::Big
} else {
native_endianness
}
}
/// Set endianness of the memory access.
pub fn set_endianness(&mut self, endianness: Endianness) {
match endianness {
Endianness::Little => self.set(FlagBit::LittleEndian),
Endianness::Big => self.set(FlagBit::BigEndian),
};
assert!(!(self.read(FlagBit::LittleEndian) && self.read(FlagBit::BigEndian)));
}
/// Test if the `notrap` flag is set. /// Test if the `notrap` flag is set.
/// ///
/// Normally, trapping is part of the semantics of a load/store operation. If the platform /// Normally, trapping is part of the semantics of a load/store operation. If the platform

2
cranelift/codegen/src/ir/mod.rs
View File

@@ -50,7 +50,7 @@ pub use crate::ir::instructions::{
pub use crate::ir::jumptable::JumpTableData; pub use crate::ir::jumptable::JumpTableData;
pub use crate::ir::layout::Layout; pub use crate::ir::layout::Layout;
pub use crate::ir::libcall::{get_probestack_funcref, LibCall}; pub use crate::ir::libcall::{get_probestack_funcref, LibCall};
pub use crate::ir::memflags::MemFlags; pub use crate::ir::memflags::{Endianness, MemFlags};
pub use crate::ir::progpoint::{ExpandedProgramPoint, ProgramOrder, ProgramPoint}; pub use crate::ir::progpoint::{ExpandedProgramPoint, ProgramOrder, ProgramPoint};
pub use crate::ir::sourceloc::SourceLoc; pub use crate::ir::sourceloc::SourceLoc;
pub use crate::ir::stackslot::{StackLayoutInfo, StackSlotData, StackSlotKind, StackSlots}; pub use crate::ir::stackslot::{StackLayoutInfo, StackSlotData, StackSlotKind, StackSlots};

8
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -12,7 +12,7 @@ use crate::{CodegenError, CodegenResult};
use alloc::boxed::Box; use alloc::boxed::Box;
use alloc::vec::Vec; use alloc::vec::Vec;
use regalloc::{RealReg, Reg, RegClass, Set, Writable}; use regalloc::{RealReg, Reg, RegClass, Set, Writable};
use smallvec::SmallVec; use smallvec::{smallvec, SmallVec};
// We use a generic implementation that factors out AArch64 and x64 ABI commonalities, because // We use a generic implementation that factors out AArch64 and x64 ABI commonalities, because
// these ABIs are very similar. // these ABIs are very similar.
@@ -508,6 +508,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
insts insts
} }
fn gen_probestack(_: u32) -> SmallVec<[Self::I; 2]> {
// TODO: implement if we ever require stack probes on an AArch64 host
// (unlikely unless Lucet is ported)
smallvec![]
}
// Returns stack bytes used as well as instructions. Does not adjust // Returns stack bytes used as well as instructions. Does not adjust
// nominal SP offset; abi_impl generic code will do that. // nominal SP offset; abi_impl generic code will do that.
fn gen_clobber_save( fn gen_clobber_save(

18
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -1312,6 +1312,13 @@ impl MachInstEmit for Inst {
| machreg_to_vec(rd.to_reg()), | machreg_to_vec(rd.to_reg()),
); );
} }
&Inst::FpuExtend { rd, rn, size } => {
sink.put4(enc_fpurr(
0b000_11110_00_1_000000_10000 | (size.ftype() << 13),
rd,
rn,
));
}
&Inst::FpuRR { fpu_op, rd, rn } => { &Inst::FpuRR { fpu_op, rd, rn } => {
let top22 = match fpu_op { let top22 = match fpu_op {
FPUOp1::Abs32 => 0b000_11110_00_1_000001_10000, FPUOp1::Abs32 => 0b000_11110_00_1_000001_10000,
@@ -1746,6 +1753,17 @@ impl MachInstEmit for Inst {
| machreg_to_vec(rd.to_reg()), | machreg_to_vec(rd.to_reg()),
); );
} }
&Inst::VecDupFPImm { rd, imm, size } => {
let imm = imm.enc_bits();
let op = match size.lane_size() {
ScalarSize::Size32 => 0,
ScalarSize::Size64 => 1,
_ => unimplemented!(),
};
let q_op = op | ((size.is_128bits() as u32) << 1);
sink.put4(enc_asimd_mod_imm(rd, q_op, 0b1111, imm));
}
&Inst::VecDupImm { &Inst::VecDupImm {
rd, rd,
imm, imm,

110
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -2072,6 +2072,24 @@ fn test_aarch64_binemit() {
"5205084E", "5205084E",
"dup v18.2d, v10.d[0]", "dup v18.2d, v10.d[0]",
)); ));
insns.push((
Inst::VecDupFPImm {
rd: writable_vreg(31),
imm: ASIMDFPModImm::maybe_from_u64(1_f32.to_bits() as u64, ScalarSize::Size32).unwrap(),
size: VectorSize::Size32x2,
},
"1FF6030F",
"fmov v31.2s, #1",
));
insns.push((
Inst::VecDupFPImm {
rd: writable_vreg(0),
imm: ASIMDFPModImm::maybe_from_u64(2_f64.to_bits(), ScalarSize::Size64).unwrap(),
size: VectorSize::Size64x2,
},
"00F4006F",
"fmov v0.2d, #2",
));
insns.push(( insns.push((
Inst::VecDupImm { Inst::VecDupImm {
rd: writable_vreg(31), rd: writable_vreg(31),
@@ -2082,16 +2100,96 @@ fn test_aarch64_binemit() {
"FFE7074F", "FFE7074F",
"movi v31.16b, #255", "movi v31.16b, #255",
)); ));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(30),
imm: ASIMDMovModImm::maybe_from_u64(0, ScalarSize::Size16).unwrap(),
invert: false,
size: VectorSize::Size16x8,
},
"1E84004F",
"movi v30.8h, #0",
));
insns.push(( insns.push((
Inst::VecDupImm { Inst::VecDupImm {
rd: writable_vreg(0), rd: writable_vreg(0),
imm: ASIMDMovModImm::zero(), imm: ASIMDMovModImm::zero(ScalarSize::Size16),
invert: true, invert: true,
size: VectorSize::Size16x4, size: VectorSize::Size16x4,
}, },
"0084002F", "0084002F",
"mvni v0.4h, #0", "mvni v0.4h, #0",
)); ));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(0),
imm: ASIMDMovModImm::maybe_from_u64(256, ScalarSize::Size16).unwrap(),
invert: false,
size: VectorSize::Size16x8,
},
"20A4004F",
"movi v0.8h, #1, LSL #8",
));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(8),
imm: ASIMDMovModImm::maybe_from_u64(2228223, ScalarSize::Size32).unwrap(),
invert: false,
size: VectorSize::Size32x4,
},
"28D4014F",
"movi v8.4s, #33, MSL #16",
));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(16),
imm: ASIMDMovModImm::maybe_from_u64(35071, ScalarSize::Size32).unwrap(),
invert: true,
size: VectorSize::Size32x2,
},
"10C5042F",
"mvni v16.2s, #136, MSL #8",
));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(1),
imm: ASIMDMovModImm::maybe_from_u64(0, ScalarSize::Size32).unwrap(),
invert: false,
size: VectorSize::Size32x2,
},
"0104000F",
"movi v1.2s, #0",
));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(24),
imm: ASIMDMovModImm::maybe_from_u64(1107296256, ScalarSize::Size32).unwrap(),
invert: false,
size: VectorSize::Size32x4,
},
"5864024F",
"movi v24.4s, #66, LSL #24",
));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(8),
imm: ASIMDMovModImm::zero(ScalarSize::Size64),
invert: false,
size: VectorSize::Size64x2,
},
"08E4006F",
"movi v8.2d, #0",
));
insns.push((
Inst::VecDupImm {
rd: writable_vreg(7),
imm: ASIMDMovModImm::maybe_from_u64(18374687574904995840, ScalarSize::Size64).unwrap(),
invert: false,
size: VectorSize::Size64x2,
},
"87E6046F",
"movi v7.2d, #18374687574904995840",
));
insns.push(( insns.push((
Inst::VecExtend { Inst::VecExtend {
t: VecExtendOp::Sxtl8, t: VecExtendOp::Sxtl8,
@@ -4376,6 +4474,16 @@ fn test_aarch64_binemit() {
"mov d23, v11.d[0]", "mov d23, v11.d[0]",
)); ));
insns.push((
Inst::FpuExtend {
rd: writable_vreg(31),
rn: vreg(0),
size: ScalarSize::Size32,
},
"1F40201E",
"fmov s31, s0",
));
insns.push(( insns.push((
Inst::FpuRR { Inst::FpuRR {
fpu_op: FPUOp1::Abs32, fpu_op: FPUOp1::Abs32,

238
cranelift/codegen/src/isa/aarch64/inst/imms.rs
View File

@@ -668,39 +668,208 @@ impl MoveWideConst {
} }
/// Advanced SIMD modified immediate as used by MOVI/MVNI. /// Advanced SIMD modified immediate as used by MOVI/MVNI.
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug, PartialEq)]
pub struct ASIMDMovModImm { pub struct ASIMDMovModImm {
imm: u8, imm: u8,
shift: u8, shift: u8,
is_64bit: bool,
shift_ones: bool, shift_ones: bool,
} }
impl ASIMDMovModImm { impl ASIMDMovModImm {
/// Construct an ASIMDMovModImm from an arbitrary 64-bit constant, if possible.
/// Note that the bits in `value` outside of the range specified by `size` are
/// ignored; for example, in the case of `ScalarSize::Size8` all bits above the
/// lowest 8 are ignored.
pub fn maybe_from_u64(value: u64, size: ScalarSize) -> Option<ASIMDMovModImm> { pub fn maybe_from_u64(value: u64, size: ScalarSize) -> Option<ASIMDMovModImm> {
match size { match size {
ScalarSize::Size8 => Some(ASIMDMovModImm { ScalarSize::Size8 => Some(ASIMDMovModImm {
imm: value as u8, imm: value as u8,
shift: 0, shift: 0,
is_64bit: false,
shift_ones: false, shift_ones: false,
}), }),
ScalarSize::Size16 => {
let value = value as u16;
if value >> 8 == 0 {
Some(ASIMDMovModImm {
imm: value as u8,
shift: 0,
is_64bit: false,
shift_ones: false,
})
} else if value as u8 == 0 {
Some(ASIMDMovModImm {
imm: (value >> 8) as u8,
shift: 8,
is_64bit: false,
shift_ones: false,
})
} else {
None
}
}
ScalarSize::Size32 => {
let value = value as u32;
// Value is of the form 0x00MMFFFF.
if value & 0xFF00FFFF == 0x0000FFFF {
let imm = (value >> 16) as u8;
Some(ASIMDMovModImm {
imm,
shift: 16,
is_64bit: false,
shift_ones: true,
})
// Value is of the form 0x0000MMFF.
} else if value & 0xFFFF00FF == 0x000000FF {
let imm = (value >> 8) as u8;
Some(ASIMDMovModImm {
imm,
shift: 8,
is_64bit: false,
shift_ones: true,
})
} else {
// Of the 4 bytes, at most one is non-zero.
for shift in (0..32).step_by(8) {
if value & (0xFF << shift) == value {
return Some(ASIMDMovModImm {
imm: (value >> shift) as u8,
shift,
is_64bit: false,
shift_ones: false,
});
}
}
None
}
}
ScalarSize::Size64 => {
let mut imm = 0u8;
// Check if all bytes are either 0 or 0xFF.
for i in 0..8 {
let b = (value >> (i * 8)) as u8;
if b == 0 || b == 0xFF {
imm |= (b & 1) << i;
} else {
return None;
}
}
Some(ASIMDMovModImm {
imm,
shift: 0,
is_64bit: true,
shift_ones: false,
})
}
_ => None, _ => None,
} }
} }
/// Create a zero immediate of this format. /// Create a zero immediate of this format.
pub fn zero() -> Self { pub fn zero(size: ScalarSize) -> Self {
ASIMDMovModImm { ASIMDMovModImm {
imm: 0, imm: 0,
shift: 0, shift: 0,
is_64bit: size == ScalarSize::Size64,
shift_ones: false, shift_ones: false,
} }
} }
/// Returns the value that this immediate represents.
pub fn value(&self) -> (u8, u32, bool) { pub fn value(&self) -> (u8, u32, bool) {
(self.imm, self.shift as u32, self.shift_ones) (self.imm, self.shift as u32, self.shift_ones)
} }
} }
/// Advanced SIMD modified immediate as used by the vector variant of FMOV.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ASIMDFPModImm {
imm: u8,
is_64bit: bool,
}
impl ASIMDFPModImm {
/// Construct an ASIMDFPModImm from an arbitrary 64-bit constant, if possible.
pub fn maybe_from_u64(value: u64, size: ScalarSize) -> Option<ASIMDFPModImm> {
// In all cases immediates are encoded as an 8-bit number 0b_abcdefgh;
// let `D` be the inverse of the digit `d`.
match size {
ScalarSize::Size32 => {
// In this case the representable immediates are 32-bit numbers of the form
// 0b_aBbb_bbbc_defg_h000 shifted to the left by 16.
let value = value as u32;
let b0_5 = (value >> 19) & 0b111111;
let b6 = (value >> 19) & (1 << 6);
let b7 = (value >> 24) & (1 << 7);
let imm = (b0_5 | b6 | b7) as u8;
if value == Self::value32(imm) {
Some(ASIMDFPModImm {
imm,
is_64bit: false,
})
} else {
None
}
}
ScalarSize::Size64 => {
// In this case the representable immediates are 64-bit numbers of the form
// 0b_aBbb_bbbb_bbcd_efgh shifted to the left by 48.
let b0_5 = (value >> 48) & 0b111111;
let b6 = (value >> 48) & (1 << 6);
let b7 = (value >> 56) & (1 << 7);
let imm = (b0_5 | b6 | b7) as u8;
if value == Self::value64(imm) {
Some(ASIMDFPModImm {
imm,
is_64bit: true,
})
} else {
None
}
}
_ => None,
}
}
/// Returns bits ready for encoding.
pub fn enc_bits(&self) -> u8 {
self.imm
}
/// Returns the 32-bit value that corresponds to an 8-bit encoding.
fn value32(imm: u8) -> u32 {
let imm = imm as u32;
let b0_5 = imm & 0b111111;
let b6 = (imm >> 6) & 1;
let b6_inv = b6 ^ 1;
let b7 = (imm >> 7) & 1;
b0_5 << 19 | (b6 * 0b11111) << 25 | b6_inv << 30 | b7 << 31
}
/// Returns the 64-bit value that corresponds to an 8-bit encoding.
fn value64(imm: u8) -> u64 {
let imm = imm as u64;
let b0_5 = imm & 0b111111;
let b6 = (imm >> 6) & 1;
let b6_inv = b6 ^ 1;
let b7 = (imm >> 7) & 1;
b0_5 << 48 | (b6 * 0b11111111) << 54 | b6_inv << 62 | b7 << 63
}
}
impl PrettyPrint for NZCV { impl PrettyPrint for NZCV {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String { fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
let fmt = |c: char, v| if v { c.to_ascii_uppercase() } else { c }; let fmt = |c: char, v| if v { c.to_ascii_uppercase() } else { c };
@@ -782,7 +951,20 @@ impl PrettyPrint for MoveWideConst {
impl PrettyPrint for ASIMDMovModImm { impl PrettyPrint for ASIMDMovModImm {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String { fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
if self.shift == 0 { if self.is_64bit {
debug_assert_eq!(self.shift, 0);
let enc_imm = self.imm as i8;
let mut imm = 0u64;
for i in 0..8 {
let b = (enc_imm >> i) & 1;
imm |= (-b as u8 as u64) << (i * 8);
}
format!("#{}", imm)
} else if self.shift == 0 {
format!("#{}", self.imm) format!("#{}", self.imm)
} else { } else {
let shift_type = if self.shift_ones { "MSL" } else { "LSL" }; let shift_type = if self.shift_ones { "MSL" } else { "LSL" };
@@ -791,6 +973,16 @@ impl PrettyPrint for ASIMDMovModImm {
} }
} }
impl PrettyPrint for ASIMDFPModImm {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
if self.is_64bit {
format!("#{}", f64::from_bits(Self::value64(self.imm)))
} else {
format!("#{}", f32::from_bits(Self::value32(self.imm)))
}
}
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
@@ -1022,4 +1214,44 @@ mod test {
unreachable!(); unreachable!();
} }
} }
#[test]
fn asimd_fp_mod_imm_test() {
assert_eq!(None, ASIMDFPModImm::maybe_from_u64(0, ScalarSize::Size32));
assert_eq!(
None,
ASIMDFPModImm::maybe_from_u64(0.013671875_f32.to_bits() as u64, ScalarSize::Size32)
);
assert_eq!(None, ASIMDFPModImm::maybe_from_u64(0, ScalarSize::Size64));
assert_eq!(
None,
ASIMDFPModImm::maybe_from_u64(10000_f64.to_bits(), ScalarSize::Size64)
);
}
#[test]
fn asimd_mov_mod_imm_test() {
assert_eq!(
None,
ASIMDMovModImm::maybe_from_u64(513, ScalarSize::Size16)
);
assert_eq!(
None,
ASIMDMovModImm::maybe_from_u64(4278190335, ScalarSize::Size32)
);
assert_eq!(
None,
ASIMDMovModImm::maybe_from_u64(8388608, ScalarSize::Size64)
);
assert_eq!(
Some(ASIMDMovModImm {
imm: 66,
shift: 16,
is_64bit: false,
shift_ones: true,
}),
ASIMDMovModImm::maybe_from_u64(4390911, ScalarSize::Size32)
);
}
} }

86
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -755,6 +755,13 @@ pub enum Inst {
size: VectorSize, size: VectorSize,
}, },
/// Zero-extend a SIMD & FP scalar to the full width of a vector register.
FpuExtend {
rd: Writable<Reg>,
rn: Reg,
size: ScalarSize,
},
/// 1-op FPU instruction. /// 1-op FPU instruction.
FpuRR { FpuRR {
fpu_op: FPUOp1, fpu_op: FPUOp1,
@@ -928,6 +935,13 @@ pub enum Inst {
size: VectorSize, size: VectorSize,
}, },
/// Duplicate FP immediate to vector.
VecDupFPImm {
rd: Writable<Reg>,
imm: ASIMDFPModImm,
size: VectorSize,
},
/// Duplicate immediate to vector. /// Duplicate immediate to vector.
VecDupImm { VecDupImm {
rd: Writable<Reg>, rd: Writable<Reg>,
@@ -1295,12 +1309,15 @@ impl Inst {
value: u32, value: u32,
mut alloc_tmp: F, mut alloc_tmp: F,
) -> SmallVec<[Inst; 4]> { ) -> SmallVec<[Inst; 4]> {
// Note that we must make sure that all bits outside the lowest 32 are set to 0
// because this function is also used to load wider constants (that have zeros
// in their most significant bits).
if value == 0 { if value == 0 {
smallvec![Inst::VecDupImm { smallvec![Inst::VecDupImm {
rd, rd,
imm: ASIMDMovModImm::zero(), imm: ASIMDMovModImm::zero(ScalarSize::Size32),
invert: false, invert: false,
size: VectorSize::Size8x8 size: VectorSize::Size32x2
}] }]
} else { } else {
// TODO: use FMOV immediate form when `value` has sufficiently few mantissa/exponent // TODO: use FMOV immediate form when `value` has sufficiently few mantissa/exponent
@@ -1324,6 +1341,9 @@ impl Inst {
const_data: u64, const_data: u64,
mut alloc_tmp: F, mut alloc_tmp: F,
) -> SmallVec<[Inst; 4]> { ) -> SmallVec<[Inst; 4]> {
// Note that we must make sure that all bits outside the lowest 64 are set to 0
// because this function is also used to load wider constants (that have zeros
// in their most significant bits).
if let Ok(const_data) = u32::try_from(const_data) { if let Ok(const_data) = u32::try_from(const_data) {
Inst::load_fp_constant32(rd, const_data, alloc_tmp) Inst::load_fp_constant32(rd, const_data, alloc_tmp)
// TODO: use FMOV immediate form when `const_data` has sufficiently few mantissa/exponent // TODO: use FMOV immediate form when `const_data` has sufficiently few mantissa/exponent
@@ -1394,7 +1414,7 @@ impl Inst {
r r
} }
/// Create instructions that load a 128-bit vector constant consisting of elements with /// Create instructions that load a vector constant consisting of elements with
/// the same value. /// the same value.
pub fn load_replicated_vector_pattern<F: FnMut(RegClass, Type) -> Writable<Reg>>( pub fn load_replicated_vector_pattern<F: FnMut(RegClass, Type) -> Writable<Reg>>(
rd: Writable<Reg>, rd: Writable<Reg>,
@@ -1403,6 +1423,15 @@ impl Inst {
mut alloc_tmp: F, mut alloc_tmp: F,
) -> SmallVec<[Inst; 5]> { ) -> SmallVec<[Inst; 5]> {
let lane_size = size.lane_size(); let lane_size = size.lane_size();
let widen_32_bit_pattern = |pattern, lane_size| {
if lane_size == ScalarSize::Size32 {
let pattern = pattern as u32 as u64;
ASIMDMovModImm::maybe_from_u64(pattern | (pattern << 32), ScalarSize::Size64)
} else {
None
}
};
if let Some(imm) = ASIMDMovModImm::maybe_from_u64(pattern, lane_size) { if let Some(imm) = ASIMDMovModImm::maybe_from_u64(pattern, lane_size) {
smallvec![Inst::VecDupImm { smallvec![Inst::VecDupImm {
@@ -1421,6 +1450,27 @@ impl Inst {
invert: true, invert: true,
size size
}] }]
} else if let Some(imm) = widen_32_bit_pattern(pattern, lane_size) {
let mut insts = smallvec![Inst::VecDupImm {
rd,
imm,
invert: false,
size: VectorSize::Size64x2,
}];
// TODO: Implement support for 64-bit scalar MOVI; we zero-extend the
// lower 64 bits instead.
if !size.is_128bits() {
insts.push(Inst::FpuExtend {
rd,
rn: rd.to_reg(),
size: ScalarSize::Size64,
});
}
insts
} else if let Some(imm) = ASIMDFPModImm::maybe_from_u64(pattern, lane_size) {
smallvec![Inst::VecDupFPImm { rd, imm, size }]
} else { } else {
let tmp = alloc_tmp(RegClass::I64, I64); let tmp = alloc_tmp(RegClass::I64, I64);
let mut insts = SmallVec::from(&Inst::load_constant(tmp, pattern)[..]); let mut insts = SmallVec::from(&Inst::load_constant(tmp, pattern)[..]);
@@ -1721,6 +1771,10 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_def(rd); collector.add_def(rd);
collector.add_use(rn); collector.add_use(rn);
} }
&Inst::FpuExtend { rd, rn, .. } => {
collector.add_def(rd);
collector.add_use(rn);
}
&Inst::FpuRR { rd, rn, .. } => { &Inst::FpuRR { rd, rn, .. } => {
collector.add_def(rd); collector.add_def(rd);
collector.add_use(rn); collector.add_use(rn);
@@ -1870,6 +1924,9 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_def(rd); collector.add_def(rd);
collector.add_use(rn); collector.add_use(rn);
} }
&Inst::VecDupFPImm { rd, .. } => {
collector.add_def(rd);
}
&Inst::VecDupImm { rd, .. } => { &Inst::VecDupImm { rd, .. } => {
collector.add_def(rd); collector.add_def(rd);
} }
@@ -2299,6 +2356,14 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
map_def(mapper, rd); map_def(mapper, rd);
map_use(mapper, rn); map_use(mapper, rn);
} }
&mut Inst::FpuExtend {
ref mut rd,
ref mut rn,
..
} => {
map_def(mapper, rd);
map_use(mapper, rn);
}
&mut Inst::FpuRR { &mut Inst::FpuRR {
ref mut rd, ref mut rd,
ref mut rn, ref mut rn,
@@ -2582,6 +2647,9 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
map_def(mapper, rd); map_def(mapper, rd);
map_use(mapper, rn); map_use(mapper, rn);
} }
&mut Inst::VecDupFPImm { ref mut rd, .. } => {
map_def(mapper, rd);
}
&mut Inst::VecDupImm { ref mut rd, .. } => { &mut Inst::VecDupImm { ref mut rd, .. } => {
map_def(mapper, rd); map_def(mapper, rd);
} }
@@ -3229,6 +3297,12 @@ impl Inst {
let rn = show_vreg_element(rn, mb_rru, idx, size); let rn = show_vreg_element(rn, mb_rru, idx, size);
format!("mov {}, {}", rd, rn) format!("mov {}, {}", rd, rn)
} }
&Inst::FpuExtend { rd, rn, size } => {
let rd = show_vreg_scalar(rd.to_reg(), mb_rru, size);
let rn = show_vreg_scalar(rn, mb_rru, size);
format!("fmov {}, {}", rd, rn)
}
&Inst::FpuRR { fpu_op, rd, rn } => { &Inst::FpuRR { fpu_op, rd, rn } => {
let (op, sizesrc, sizedest) = match fpu_op { let (op, sizesrc, sizedest) = match fpu_op {
FPUOp1::Abs32 => ("fabs", ScalarSize::Size32, ScalarSize::Size32), FPUOp1::Abs32 => ("fabs", ScalarSize::Size32, ScalarSize::Size32),
@@ -3465,6 +3539,12 @@ impl Inst {
let rn = show_vreg_element(rn, mb_rru, 0, size); let rn = show_vreg_element(rn, mb_rru, 0, size);
format!("dup {}, {}", rd, rn) format!("dup {}, {}", rd, rn)
} }
&Inst::VecDupFPImm { rd, imm, size } => {
let imm = imm.show_rru(mb_rru);
let rd = show_vreg_vector(rd.to_reg(), mb_rru, size);
format!("fmov {}, {}", rd, imm)
}
&Inst::VecDupImm { &Inst::VecDupImm {
rd, rd,
imm, imm,

4
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -616,7 +616,7 @@ fn collect_address_addends<C: LowerCtx<I = Inst>>(
maybe_input_insn(ctx, extendee_input, Opcode::Iconst), maybe_input_insn(ctx, extendee_input, Opcode::Iconst),
extendop, extendop,
) { ) {
let value = ctx.get_constant(insn).unwrap() as i64; let value = (ctx.get_constant(insn).unwrap() & 0xFFFF_FFFF_u64) as i64;
offset += value; offset += value;
} else { } else {
let reg = put_input_in_reg(ctx, extendee_input, NarrowValueMode::None); let reg = put_input_in_reg(ctx, extendee_input, NarrowValueMode::None);
@@ -853,7 +853,7 @@ pub(crate) fn lower_constant_f128<C: LowerCtx<I = Inst>>(
// is potentially expensive. // is potentially expensive.
ctx.emit(Inst::VecDupImm { ctx.emit(Inst::VecDupImm {
rd, rd,
imm: ASIMDMovModImm::zero(), imm: ASIMDMovModImm::zero(ScalarSize::Size8),
invert: false, invert: false,
size: VectorSize::Size8x16, size: VectorSize::Size8x16,
}); });

9
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -2075,8 +2075,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// derivation of these sequences. Alternative sequences are discussed in // derivation of these sequences. Alternative sequences are discussed in
// https://github.com/bytecodealliance/wasmtime/issues/2296, although they are not // https://github.com/bytecodealliance/wasmtime/issues/2296, although they are not
// used here. // used here.
// Also .. FIXME: when https://github.com/bytecodealliance/wasmtime/pull/2310 is
// merged, use `lower_splat_constant` instead to generate the constants.
let tmp_r0 = ctx.alloc_tmp(RegClass::I64, I64); let tmp_r0 = ctx.alloc_tmp(RegClass::I64, I64);
let tmp_v0 = ctx.alloc_tmp(RegClass::V128, I8X16); let tmp_v0 = ctx.alloc_tmp(RegClass::V128, I8X16);
let tmp_v1 = ctx.alloc_tmp(RegClass::V128, I8X16); let tmp_v1 = ctx.alloc_tmp(RegClass::V128, I8X16);
@@ -2100,12 +2098,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
size: VectorSize::Size8x16, size: VectorSize::Size8x16,
imm: 7, imm: 7,
}); });
lower_constant_u64(ctx, tmp_r0, 0x8040201008040201u64); lower_splat_const(ctx, tmp_v0, 0x8040201008040201u64, VectorSize::Size64x2);
ctx.emit(Inst::VecDup {
rd: tmp_v0,
rn: tmp_r0.to_reg(),
size: VectorSize::Size64x2,
});
ctx.emit(Inst::VecRRR { ctx.emit(Inst::VecRRR {
alu_op: VecALUOp::And, alu_op: VecALUOp::And,
rd: tmp_v1, rd: tmp_v1,

8
cranelift/codegen/src/isa/arm32/abi.rs
View File

@@ -10,7 +10,7 @@ use crate::{CodegenError, CodegenResult};
use alloc::boxed::Box; use alloc::boxed::Box;
use alloc::vec::Vec; use alloc::vec::Vec;
use regalloc::{RealReg, Reg, RegClass, Set, Writable}; use regalloc::{RealReg, Reg, RegClass, Set, Writable};
use smallvec::SmallVec; use smallvec::{smallvec, SmallVec};
/// Support for the ARM ABI from the callee side (within a function body). /// Support for the ARM ABI from the callee side (within a function body).
pub(crate) type Arm32ABICallee = ABICalleeImpl<Arm32MachineDeps>; pub(crate) type Arm32ABICallee = ABICalleeImpl<Arm32MachineDeps>;
@@ -305,6 +305,12 @@ impl ABIMachineSpec for Arm32MachineDeps {
ret ret
} }
fn gen_probestack(_: u32) -> SmallVec<[Self::I; 2]> {
// TODO: implement if we ever require stack probes on ARM32 (unlikely
// unless Lucet is ported)
smallvec![]
}
/// Returns stack bytes used as well as instructions. Does not adjust /// Returns stack bytes used as well as instructions. Does not adjust
/// nominal SP offset; caller will do that. /// nominal SP offset; caller will do that.
fn gen_clobber_save( fn gen_clobber_save(

8
cranelift/codegen/src/isa/mod.rs
View File

@@ -235,6 +235,14 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
CallConv::triple_default(self.triple()) CallConv::triple_default(self.triple())
} }
/// Get the endianness of this ISA.
fn endianness(&self) -> ir::Endianness {
match self.triple().endianness().unwrap() {
target_lexicon::Endianness::Little => ir::Endianness::Little,
target_lexicon::Endianness::Big => ir::Endianness::Big,
}
}
/// Get the pointer type of this ISA. /// Get the pointer type of this ISA.
fn pointer_type(&self) -> ir::Type { fn pointer_type(&self) -> ir::Type {
ir::Type::int(u16::from(self.pointer_bits())).unwrap() ir::Type::int(u16::from(self.pointer_bits())).unwrap()

18
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -1,7 +1,7 @@
//! Implementation of the standard x64 ABI. //! Implementation of the standard x64 ABI.
use crate::ir::types::*; use crate::ir::types::*;
use crate::ir::{self, types, MemFlags, TrapCode, Type}; use crate::ir::{self, types, ExternalName, LibCall, MemFlags, Opcode, TrapCode, Type};
use crate::isa; use crate::isa;
use crate::isa::{x64::inst::*, CallConv}; use crate::isa::{x64::inst::*, CallConv};
use crate::machinst::abi_impl::*; use crate::machinst::abi_impl::*;
@@ -389,6 +389,22 @@ impl ABIMachineSpec for X64ABIMachineSpec {
insts insts
} }
fn gen_probestack(frame_size: u32) -> SmallVec<[Self::I; 2]> {
let mut insts = SmallVec::new();
insts.push(Inst::imm(
OperandSize::Size32,
frame_size as u64,
Writable::from_reg(regs::rax()),
));
insts.push(Inst::CallKnown {
dest: ExternalName::LibCall(LibCall::Probestack),
uses: vec![regs::rax()],
defs: vec![],
opcode: Opcode::Call,
});
insts
}
fn gen_clobber_save( fn gen_clobber_save(
call_conv: isa::CallConv, call_conv: isa::CallConv,
_: &settings::Flags, _: &settings::Flags,

78
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -467,7 +467,10 @@ pub enum SseOpcode {
Pabsb, Pabsb,
Pabsw, Pabsw,
Pabsd, Pabsd,
Packssdw,
Packsswb, Packsswb,
Packusdw,
Packuswb,
Paddb, Paddb,
Paddd, Paddd,
Paddq, Paddq,
@@ -476,6 +479,7 @@ pub enum SseOpcode {
Paddsw, Paddsw,
Paddusb, Paddusb,
Paddusw, Paddusw,
Palignr,
Pand, Pand,
Pandn, Pandn,
Pavgb, Pavgb,
@@ -507,6 +511,18 @@ pub enum SseOpcode {
Pminuw, Pminuw,
Pminud, Pminud,
Pmovmskb, Pmovmskb,
Pmovsxbd,
Pmovsxbw,
Pmovsxbq,
Pmovsxwd,
Pmovsxwq,
Pmovsxdq,
Pmovzxbd,
Pmovzxbw,
Pmovzxbq,
Pmovzxwd,
Pmovzxwq,
Pmovzxdq,
Pmulld, Pmulld,
Pmullw, Pmullw,
Pmuludq, Pmuludq,
@@ -534,6 +550,8 @@ pub enum SseOpcode {
Punpcklbw, Punpcklbw,
Pxor, Pxor,
Rcpss, Rcpss,
Roundps,
Roundpd,
Roundss, Roundss,
Roundsd, Roundsd,
Rsqrtss, Rsqrtss,
@@ -620,7 +638,9 @@ impl SseOpcode {
| SseOpcode::Mulpd | SseOpcode::Mulpd
| SseOpcode::Mulsd | SseOpcode::Mulsd
| SseOpcode::Orpd | SseOpcode::Orpd
| SseOpcode::Packssdw
| SseOpcode::Packsswb | SseOpcode::Packsswb
| SseOpcode::Packuswb
| SseOpcode::Paddb | SseOpcode::Paddb
| SseOpcode::Paddd | SseOpcode::Paddd
| SseOpcode::Paddq | SseOpcode::Paddq
@@ -676,9 +696,14 @@ impl SseOpcode {
| SseOpcode::Ucomisd | SseOpcode::Ucomisd
| SseOpcode::Xorpd => SSE2, | SseOpcode::Xorpd => SSE2,
SseOpcode::Pabsb | SseOpcode::Pabsw | SseOpcode::Pabsd | SseOpcode::Pshufb => SSSE3, SseOpcode::Pabsb
| SseOpcode::Pabsw
| SseOpcode::Pabsd
| SseOpcode::Palignr
| SseOpcode::Pshufb => SSSE3,
SseOpcode::Insertps SseOpcode::Insertps
| SseOpcode::Packusdw
| SseOpcode::Pcmpeqq | SseOpcode::Pcmpeqq
| SseOpcode::Pextrb | SseOpcode::Pextrb
| SseOpcode::Pextrd | SseOpcode::Pextrd
@@ -692,8 +717,22 @@ impl SseOpcode {
| SseOpcode::Pminsd | SseOpcode::Pminsd
| SseOpcode::Pminuw | SseOpcode::Pminuw
| SseOpcode::Pminud | SseOpcode::Pminud
| SseOpcode::Pmovsxbd
| SseOpcode::Pmovsxbw
| SseOpcode::Pmovsxbq
| SseOpcode::Pmovsxwd
| SseOpcode::Pmovsxwq
| SseOpcode::Pmovsxdq
| SseOpcode::Pmovzxbd
| SseOpcode::Pmovzxbw
| SseOpcode::Pmovzxbq
| SseOpcode::Pmovzxwd
| SseOpcode::Pmovzxwq
| SseOpcode::Pmovzxdq
| SseOpcode::Pmulld | SseOpcode::Pmulld
| SseOpcode::Ptest | SseOpcode::Ptest
| SseOpcode::Roundps
| SseOpcode::Roundpd
| SseOpcode::Roundss | SseOpcode::Roundss
| SseOpcode::Roundsd => SSE41, | SseOpcode::Roundsd => SSE41,
@@ -772,7 +811,10 @@ impl fmt::Debug for SseOpcode {
SseOpcode::Pabsb => "pabsb", SseOpcode::Pabsb => "pabsb",
SseOpcode::Pabsw => "pabsw", SseOpcode::Pabsw => "pabsw",
SseOpcode::Pabsd => "pabsd", SseOpcode::Pabsd => "pabsd",
SseOpcode::Packssdw => "packssdw",
SseOpcode::Packsswb => "packsswb", SseOpcode::Packsswb => "packsswb",
SseOpcode::Packusdw => "packusdw",
SseOpcode::Packuswb => "packuswb",
SseOpcode::Paddb => "paddb", SseOpcode::Paddb => "paddb",
SseOpcode::Paddd => "paddd", SseOpcode::Paddd => "paddd",
SseOpcode::Paddq => "paddq", SseOpcode::Paddq => "paddq",
@@ -781,6 +823,7 @@ impl fmt::Debug for SseOpcode {
SseOpcode::Paddsw => "paddsw", SseOpcode::Paddsw => "paddsw",
SseOpcode::Paddusb => "paddusb", SseOpcode::Paddusb => "paddusb",
SseOpcode::Paddusw => "paddusw", SseOpcode::Paddusw => "paddusw",
SseOpcode::Palignr => "palignr",
SseOpcode::Pand => "pand", SseOpcode::Pand => "pand",
SseOpcode::Pandn => "pandn", SseOpcode::Pandn => "pandn",
SseOpcode::Pavgb => "pavgb", SseOpcode::Pavgb => "pavgb",
@@ -812,6 +855,18 @@ impl fmt::Debug for SseOpcode {
SseOpcode::Pminuw => "pminuw", SseOpcode::Pminuw => "pminuw",
SseOpcode::Pminud => "pminud", SseOpcode::Pminud => "pminud",
SseOpcode::Pmovmskb => "pmovmskb", SseOpcode::Pmovmskb => "pmovmskb",
SseOpcode::Pmovsxbd => "pmovsxbd",
SseOpcode::Pmovsxbw => "pmovsxbw",
SseOpcode::Pmovsxbq => "pmovsxbq",
SseOpcode::Pmovsxwd => "pmovsxwd",
SseOpcode::Pmovsxwq => "pmovsxwq",
SseOpcode::Pmovsxdq => "pmovsxdq",
SseOpcode::Pmovzxbd => "pmovzxbd",
SseOpcode::Pmovzxbw => "pmovzxbw",
SseOpcode::Pmovzxbq => "pmovzxbq",
SseOpcode::Pmovzxwd => "pmovzxwd",
SseOpcode::Pmovzxwq => "pmovzxwq",
SseOpcode::Pmovzxdq => "pmovzxdq",
SseOpcode::Pmulld => "pmulld", SseOpcode::Pmulld => "pmulld",
SseOpcode::Pmullw => "pmullw", SseOpcode::Pmullw => "pmullw",
SseOpcode::Pmuludq => "pmuludq", SseOpcode::Pmuludq => "pmuludq",
@@ -839,6 +894,8 @@ impl fmt::Debug for SseOpcode {
SseOpcode::Punpcklbw => "punpcklbw", SseOpcode::Punpcklbw => "punpcklbw",
SseOpcode::Pxor => "pxor", SseOpcode::Pxor => "pxor",
SseOpcode::Rcpss => "rcpss", SseOpcode::Rcpss => "rcpss",
SseOpcode::Roundps => "roundps",
SseOpcode::Roundpd => "roundpd",
SseOpcode::Roundss => "roundss", SseOpcode::Roundss => "roundss",
SseOpcode::Roundsd => "roundsd", SseOpcode::Roundsd => "roundsd",
SseOpcode::Rsqrtss => "rsqrtss", SseOpcode::Rsqrtss => "rsqrtss",
@@ -1187,6 +1244,25 @@ impl From<FloatCC> for FcmpImm {
} }
} }
/// Encode the rounding modes used as part of the Rounding Control field.
/// Note, these rounding immediates only consider the rounding control field
/// (i.e. the rounding mode) which only take up the first two bits when encoded.
/// However the rounding immediate which this field helps make up, also includes
/// bits 3 and 4 which define the rounding select and precision mask respectively.
/// These two bits are not defined here and are implictly set to zero when encoded.
pub(crate) enum RoundImm {
RoundNearest = 0x00,
RoundDown = 0x01,
RoundUp = 0x02,
RoundZero = 0x03,
}
impl RoundImm {
pub(crate) fn encode(self) -> u8 {
self as u8
}
}
/// An operand's size in bits. /// An operand's size in bits.
#[derive(Clone, Copy, PartialEq)] #[derive(Clone, Copy, PartialEq)]
pub enum OperandSize { pub enum OperandSize {

114
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -182,6 +182,7 @@ impl LegacyPrefixes {
fn emit_std_enc_mem( fn emit_std_enc_mem(
sink: &mut MachBuffer<Inst>, sink: &mut MachBuffer<Inst>,
state: &EmitState, state: &EmitState,
info: &EmitInfo,
prefixes: LegacyPrefixes, prefixes: LegacyPrefixes,
opcodes: u32, opcodes: u32,
mut num_opcodes: usize, mut num_opcodes: usize,
@@ -194,7 +195,8 @@ fn emit_std_enc_mem(
// expression. But `enc_g` can be derived from a register of any class. // expression. But `enc_g` can be derived from a register of any class.
let srcloc = state.cur_srcloc(); let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() && mem_e.can_trap() { let can_trap = mem_e.can_trap();
if srcloc != SourceLoc::default() && can_trap {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
@@ -202,6 +204,12 @@ fn emit_std_enc_mem(
match mem_e { match mem_e {
Amode::ImmReg { simm32, base, .. } => { Amode::ImmReg { simm32, base, .. } => {
// If this is an access based off of RSP, it may trap with a stack overflow if it's the
// first touch of a new stack page.
if *base == regs::rsp() && !can_trap && info.flags().enable_probestack() {
sink.add_trap(srcloc, TrapCode::StackOverflow);
}
// First, the REX byte. // First, the REX byte.
let enc_e = int_reg_enc(*base); let enc_e = int_reg_enc(*base);
rex.emit_two_op(sink, enc_g, enc_e); rex.emit_two_op(sink, enc_g, enc_e);
@@ -262,6 +270,12 @@ fn emit_std_enc_mem(
shift, shift,
.. ..
} => { } => {
// If this is an access based off of RSP, it may trap with a stack overflow if it's the
// first touch of a new stack page.
if *reg_base == regs::rsp() && !can_trap && info.flags().enable_probestack() {
sink.add_trap(srcloc, TrapCode::StackOverflow);
}
let enc_base = int_reg_enc(*reg_base); let enc_base = int_reg_enc(*reg_base);
let enc_index = int_reg_enc(*reg_index); let enc_index = int_reg_enc(*reg_index);
@@ -350,6 +364,7 @@ fn emit_std_enc_enc(
fn emit_std_reg_mem( fn emit_std_reg_mem(
sink: &mut MachBuffer<Inst>, sink: &mut MachBuffer<Inst>,
state: &EmitState, state: &EmitState,
info: &EmitInfo,
prefixes: LegacyPrefixes, prefixes: LegacyPrefixes,
opcodes: u32, opcodes: u32,
num_opcodes: usize, num_opcodes: usize,
@@ -361,6 +376,7 @@ fn emit_std_reg_mem(
emit_std_enc_mem( emit_std_enc_mem(
sink, sink,
state, state,
info,
prefixes, prefixes,
opcodes, opcodes,
num_opcodes, num_opcodes,
@@ -538,6 +554,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0x0FAF, 0x0FAF,
2, 2,
@@ -597,6 +614,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
opcode_m, opcode_m,
1, 1,
@@ -654,6 +672,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
prefix, prefix,
opcode, opcode,
num_opcodes, num_opcodes,
@@ -717,7 +736,9 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr: src } => { RegMem::Mem { addr: src } => {
let amode = src.finalize(state, sink); let amode = src.finalize(state, sink);
emit_std_enc_mem(sink, state, prefix, opcode, 1, subopcode, &amode, rex_flags); emit_std_enc_mem(
sink, state, info, prefix, opcode, 1, subopcode, &amode, rex_flags,
);
} }
} }
} }
@@ -738,7 +759,9 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr: src } => { RegMem::Mem { addr: src } => {
let amode = src.finalize(state, sink); let amode = src.finalize(state, sink);
emit_std_enc_mem(sink, state, prefix, 0xF7, 1, subopcode, &amode, rex_flags); emit_std_enc_mem(
sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags,
);
} }
} }
} }
@@ -987,6 +1010,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
opcodes, opcodes,
num_opcodes, num_opcodes,
@@ -1004,6 +1028,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0x8B, 0x8B,
1, 1,
@@ -1019,6 +1044,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0x8D, 0x8D,
1, 1,
@@ -1081,6 +1107,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
opcodes, opcodes,
num_opcodes, num_opcodes,
@@ -1108,7 +1135,17 @@ pub(crate) fn emit(
}; };
// MOV r8, r/m8 is (REX.W==0) 88 /r // MOV r8, r/m8 is (REX.W==0) 88 /r
emit_std_reg_mem(sink, state, LegacyPrefixes::None, 0x88, 1, *src, dst, rex) emit_std_reg_mem(
sink,
state,
info,
LegacyPrefixes::None,
0x88,
1,
*src,
dst,
rex,
)
} }
2 => { 2 => {
@@ -1116,6 +1153,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::_66, LegacyPrefixes::_66,
0x89, 0x89,
1, 1,
@@ -1130,6 +1168,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0x89, 0x89,
1, 1,
@@ -1144,6 +1183,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0x89, 0x89,
1, 1,
@@ -1253,6 +1293,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
prefix, prefix,
opcode_bytes, opcode_bytes,
2, 2,
@@ -1311,7 +1352,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state, sink); let addr = &addr.finalize(state, sink);
// Whereas here we revert to the "normal" G-E ordering. // Whereas here we revert to the "normal" G-E ordering.
let opcode = if *size == 1 { 0x3A } else { 0x3B }; let opcode = if *size == 1 { 0x3A } else { 0x3B };
emit_std_reg_mem(sink, state, prefix, opcode, 1, *reg_g, addr, rex); emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *reg_g, addr, rex);
} }
RegMemImm::Imm { simm32 } => { RegMemImm::Imm { simm32 } => {
@@ -1372,6 +1413,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
prefix, prefix,
opcode, opcode,
2, 2,
@@ -1408,6 +1450,10 @@ pub(crate) fn emit(
} }
Inst::Push64 { src } => { Inst::Push64 { src } => {
if info.flags().enable_probestack() {
sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow);
}
match src { match src {
RegMemImm::Reg { reg } => { RegMemImm::Reg { reg } => {
let enc_reg = int_reg_enc(*reg); let enc_reg = int_reg_enc(*reg);
@@ -1423,6 +1469,7 @@ pub(crate) fn emit(
emit_std_enc_mem( emit_std_enc_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0xFF, 0xFF,
1, 1,
@@ -1454,6 +1501,9 @@ pub(crate) fn emit(
} }
Inst::CallKnown { dest, opcode, .. } => { Inst::CallKnown { dest, opcode, .. } => {
if info.flags().enable_probestack() {
sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow);
}
if let Some(s) = state.take_stack_map() { if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s); sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s);
} }
@@ -1469,6 +1519,9 @@ pub(crate) fn emit(
} }
Inst::CallUnknown { dest, opcode, .. } => { Inst::CallUnknown { dest, opcode, .. } => {
if info.flags().enable_probestack() {
sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow);
}
let start_offset = sink.cur_offset(); let start_offset = sink.cur_offset();
match dest { match dest {
RegMem::Reg { reg } => { RegMem::Reg { reg } => {
@@ -1489,6 +1542,7 @@ pub(crate) fn emit(
emit_std_enc_mem( emit_std_enc_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0xFF, 0xFF,
1, 1,
@@ -1587,6 +1641,7 @@ pub(crate) fn emit(
emit_std_enc_mem( emit_std_enc_mem(
sink, sink,
state, state,
info,
LegacyPrefixes::None, LegacyPrefixes::None,
0xFF, 0xFF,
1, 1,
@@ -1733,6 +1788,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
prefix, prefix,
opcode, opcode,
num_opcodes, num_opcodes,
@@ -1781,7 +1837,10 @@ pub(crate) fn emit(
SseOpcode::Mulsd => (LegacyPrefixes::_F2, 0x0F59, 2), SseOpcode::Mulsd => (LegacyPrefixes::_F2, 0x0F59, 2),
SseOpcode::Orpd => (LegacyPrefixes::_66, 0x0F56, 2), SseOpcode::Orpd => (LegacyPrefixes::_66, 0x0F56, 2),
SseOpcode::Orps => (LegacyPrefixes::None, 0x0F56, 2), SseOpcode::Orps => (LegacyPrefixes::None, 0x0F56, 2),
SseOpcode::Packssdw => (LegacyPrefixes::_66, 0x0F6B, 2),
SseOpcode::Packsswb => (LegacyPrefixes::_66, 0x0F63, 2), SseOpcode::Packsswb => (LegacyPrefixes::_66, 0x0F63, 2),
SseOpcode::Packusdw => (LegacyPrefixes::_66, 0x0F382B, 3),
SseOpcode::Packuswb => (LegacyPrefixes::_66, 0x0F67, 2),
SseOpcode::Paddb => (LegacyPrefixes::_66, 0x0FFC, 2), SseOpcode::Paddb => (LegacyPrefixes::_66, 0x0FFC, 2),
SseOpcode::Paddd => (LegacyPrefixes::_66, 0x0FFE, 2), SseOpcode::Paddd => (LegacyPrefixes::_66, 0x0FFE, 2),
SseOpcode::Paddq => (LegacyPrefixes::_66, 0x0FD4, 2), SseOpcode::Paddq => (LegacyPrefixes::_66, 0x0FD4, 2),
@@ -1802,6 +1861,18 @@ pub(crate) fn emit(
SseOpcode::Pcmpgtw => (LegacyPrefixes::_66, 0x0F65, 2), SseOpcode::Pcmpgtw => (LegacyPrefixes::_66, 0x0F65, 2),
SseOpcode::Pcmpgtd => (LegacyPrefixes::_66, 0x0F66, 2), SseOpcode::Pcmpgtd => (LegacyPrefixes::_66, 0x0F66, 2),
SseOpcode::Pcmpgtq => (LegacyPrefixes::_66, 0x0F3837, 3), SseOpcode::Pcmpgtq => (LegacyPrefixes::_66, 0x0F3837, 3),
SseOpcode::Pmovsxbd => (LegacyPrefixes::_66, 0x0F3821, 3),
SseOpcode::Pmovsxbw => (LegacyPrefixes::_66, 0x0F3820, 3),
SseOpcode::Pmovsxbq => (LegacyPrefixes::_66, 0x0F3822, 3),
SseOpcode::Pmovsxwd => (LegacyPrefixes::_66, 0x0F3823, 3),
SseOpcode::Pmovsxwq => (LegacyPrefixes::_66, 0x0F3824, 3),
SseOpcode::Pmovsxdq => (LegacyPrefixes::_66, 0x0F3825, 3),
SseOpcode::Pmovzxbd => (LegacyPrefixes::_66, 0x0F3831, 3),
SseOpcode::Pmovzxbw => (LegacyPrefixes::_66, 0x0F3830, 3),
SseOpcode::Pmovzxbq => (LegacyPrefixes::_66, 0x0F3832, 3),
SseOpcode::Pmovzxwd => (LegacyPrefixes::_66, 0x0F3833, 3),
SseOpcode::Pmovzxwq => (LegacyPrefixes::_66, 0x0F3834, 3),
SseOpcode::Pmovzxdq => (LegacyPrefixes::_66, 0x0F3835, 3),
SseOpcode::Pmaxsb => (LegacyPrefixes::_66, 0x0F383C, 3), SseOpcode::Pmaxsb => (LegacyPrefixes::_66, 0x0F383C, 3),
SseOpcode::Pmaxsw => (LegacyPrefixes::_66, 0x0FEE, 2), SseOpcode::Pmaxsw => (LegacyPrefixes::_66, 0x0FEE, 2),
SseOpcode::Pmaxsd => (LegacyPrefixes::_66, 0x0F383D, 3), SseOpcode::Pmaxsd => (LegacyPrefixes::_66, 0x0F383D, 3),
@@ -1848,6 +1919,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
prefix, prefix,
opcode, opcode,
length, length,
@@ -1958,6 +2030,7 @@ pub(crate) fn emit(
SseOpcode::Cmpss => (LegacyPrefixes::_F3, 0x0FC2, 2), SseOpcode::Cmpss => (LegacyPrefixes::_F3, 0x0FC2, 2),
SseOpcode::Cmpsd => (LegacyPrefixes::_F2, 0x0FC2, 2), SseOpcode::Cmpsd => (LegacyPrefixes::_F2, 0x0FC2, 2),
SseOpcode::Insertps => (LegacyPrefixes::_66, 0x0F3A21, 3), SseOpcode::Insertps => (LegacyPrefixes::_66, 0x0F3A21, 3),
SseOpcode::Palignr => (LegacyPrefixes::_66, 0x0F3A0F, 3),
SseOpcode::Pinsrb => (LegacyPrefixes::_66, 0x0F3A20, 3), SseOpcode::Pinsrb => (LegacyPrefixes::_66, 0x0F3A20, 3),
SseOpcode::Pinsrw => (LegacyPrefixes::_66, 0x0FC4, 2), SseOpcode::Pinsrw => (LegacyPrefixes::_66, 0x0FC4, 2),
SseOpcode::Pinsrd => (LegacyPrefixes::_66, 0x0F3A22, 3), SseOpcode::Pinsrd => (LegacyPrefixes::_66, 0x0F3A22, 3),
@@ -1965,6 +2038,8 @@ pub(crate) fn emit(
SseOpcode::Pextrw => (LegacyPrefixes::_66, 0x0FC5, 2), SseOpcode::Pextrw => (LegacyPrefixes::_66, 0x0FC5, 2),
SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3), SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3),
SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2), SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2),
SseOpcode::Roundps => (LegacyPrefixes::_66, 0x0F3A08, 3),
SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3),
_ => unimplemented!("Opcode {:?} not implemented", op), _ => unimplemented!("Opcode {:?} not implemented", op),
}; };
let rex = if *is64 { let rex = if *is64 {
@@ -1994,7 +2069,17 @@ pub(crate) fn emit(
!regs_swapped, !regs_swapped,
"No existing way to encode a mem argument in the ModRM r/m field." "No existing way to encode a mem argument in the ModRM r/m field."
); );
emit_std_reg_mem(sink, state, prefix, opcode, len, dst.to_reg(), addr, rex); emit_std_reg_mem(
sink,
state,
info,
prefix,
opcode,
len,
dst.to_reg(),
addr,
rex,
);
} }
} }
sink.put1(*imm); sink.put1(*imm);
@@ -2027,6 +2112,7 @@ pub(crate) fn emit(
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state, state,
info,
prefix, prefix,
opcode, opcode,
2, 2,
@@ -2091,7 +2177,17 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink); let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, prefix, opcode, 2, reg_g.to_reg(), addr, rex); emit_std_reg_mem(
sink,
state,
info,
prefix,
opcode,
2,
reg_g.to_reg(),
addr,
rex,
);
} }
} }
} }
@@ -2111,7 +2207,7 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink); let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, prefix, opcode, len, *dst, addr, rex); emit_std_reg_mem(sink, state, info, prefix, opcode, len, *dst, addr, rex);
} }
} }
} }
@@ -2625,7 +2721,7 @@ pub(crate) fn emit(
_ => unreachable!(), _ => unreachable!(),
}; };
let amode = dst.finalize(state, sink); let amode = dst.finalize(state, sink);
emit_std_reg_mem(sink, state, prefix, opcodes, 2, *src, &amode, rex); emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex);
} }
Inst::AtomicRmwSeq { ty, op } => { Inst::AtomicRmwSeq { ty, op } => {

119
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

@@ -3151,12 +3151,30 @@ fn test_x64_emit() {
"pshufb %xmm11, %xmm2", "pshufb %xmm11, %xmm2",
)); ));
insns.push((
Inst::xmm_rm_r(SseOpcode::Packssdw, RegMem::reg(xmm11), w_xmm12),
"66450F6BE3",
"packssdw %xmm11, %xmm12",
));
insns.push(( insns.push((
Inst::xmm_rm_r(SseOpcode::Packsswb, RegMem::reg(xmm11), w_xmm2), Inst::xmm_rm_r(SseOpcode::Packsswb, RegMem::reg(xmm11), w_xmm2),
"66410F63D3", "66410F63D3",
"packsswb %xmm11, %xmm2", "packsswb %xmm11, %xmm2",
)); ));
insns.push((
Inst::xmm_rm_r(SseOpcode::Packusdw, RegMem::reg(xmm13), w_xmm6),
"66410F382BF5",
"packusdw %xmm13, %xmm6",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Packuswb, RegMem::reg(xmm9), w_xmm4),
"66410F67E1",
"packuswb %xmm9, %xmm4",
));
insns.push(( insns.push((
Inst::xmm_rm_r(SseOpcode::Punpckhbw, RegMem::reg(xmm3), w_xmm2), Inst::xmm_rm_r(SseOpcode::Punpckhbw, RegMem::reg(xmm3), w_xmm2),
"660F68D3", "660F68D3",
@@ -3183,6 +3201,81 @@ fn test_x64_emit() {
"cvttps2dq %xmm9, %xmm8", "cvttps2dq %xmm9, %xmm8",
)); ));
// ========================================================
// XMM_RM_R: Packed Move
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovsxbd, RegMem::reg(xmm6), w_xmm8),
"66440F3821C6",
"pmovsxbd %xmm6, %xmm8",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovsxbw, RegMem::reg(xmm9), w_xmm10),
"66450F3820D1",
"pmovsxbw %xmm9, %xmm10",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovsxbq, RegMem::reg(xmm1), w_xmm1),
"660F3822C9",
"pmovsxbq %xmm1, %xmm1",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovsxwd, RegMem::reg(xmm13), w_xmm10),
"66450F3823D5",
"pmovsxwd %xmm13, %xmm10",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovsxwq, RegMem::reg(xmm12), w_xmm12),
"66450F3824E4",
"pmovsxwq %xmm12, %xmm12",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovsxdq, RegMem::reg(xmm10), w_xmm8),
"66450F3825C2",
"pmovsxdq %xmm10, %xmm8",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovzxbd, RegMem::reg(xmm5), w_xmm6),
"660F3831F5",
"pmovzxbd %xmm5, %xmm6",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovzxbw, RegMem::reg(xmm5), w_xmm13),
"66440F3830ED",
"pmovzxbw %xmm5, %xmm13",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovzxbq, RegMem::reg(xmm10), w_xmm11),
"66450F3832DA",
"pmovzxbq %xmm10, %xmm11",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovzxwd, RegMem::reg(xmm2), w_xmm10),
"66440F3833D2",
"pmovzxwd %xmm2, %xmm10",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovzxwq, RegMem::reg(xmm7), w_xmm4),
"660F3834E7",
"pmovzxwq %xmm7, %xmm4",
));
insns.push((
Inst::xmm_rm_r(SseOpcode::Pmovzxdq, RegMem::reg(xmm3), w_xmm4),
"660F3835E3",
"pmovzxdq %xmm3, %xmm4",
));
// XMM_Mov_R_M: float stores // XMM_Mov_R_M: float stores
insns.push(( insns.push((
Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12)), Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12)),
@@ -3406,6 +3499,32 @@ fn test_x64_emit() {
"410FC2FF00", "410FC2FF00",
"cmpps $0, %xmm15, %xmm7", "cmpps $0, %xmm15, %xmm7",
)); ));
insns.push((
Inst::xmm_rm_r_imm(SseOpcode::Palignr, RegMem::reg(xmm1), w_xmm9, 3, false),
"66440F3A0FC903",
"palignr $3, %xmm1, %xmm9",
));
insns.push((
Inst::xmm_rm_r_imm(SseOpcode::Roundps, RegMem::reg(xmm7), w_xmm8, 3, false),
"66440F3A08C703",
"roundps $3, %xmm7, %xmm8",
));
insns.push((
Inst::xmm_rm_r_imm(SseOpcode::Roundpd, RegMem::reg(xmm10), w_xmm7, 2, false),
"66410F3A09FA02",
"roundpd $2, %xmm10, %xmm7",
));
insns.push((
Inst::xmm_rm_r_imm(SseOpcode::Roundps, RegMem::reg(xmm4), w_xmm8, 1, false),
"66440F3A08C401",
"roundps $1, %xmm4, %xmm8",
));
insns.push((
Inst::xmm_rm_r_imm(SseOpcode::Roundpd, RegMem::reg(xmm15), w_xmm15, 0, false),
"66450F3A09FF00",
"roundpd $0, %xmm15, %xmm15",
));
// ======================================================== // ========================================================
// Pertaining to atomics. // Pertaining to atomics.

78
cranelift/codegen/src/isa/x64/inst/regs.rs
View File

@@ -1,14 +1,20 @@
//! Registers, the Universe thereof, and printing. //! Registers, the Universe thereof, and printing.
//! //!
//! These are ordered by sequence number, as required in the Universe. The strange ordering is //! These are ordered by sequence number, as required in the Universe.
//! intended to make callee-save registers available before caller-saved ones. This is a net win
//! provided that each function makes at least one onward call. It'll be a net loss for leaf
//! functions, and we should change the ordering in that case, so as to make caller-save regs
//! available first.
//! //!
//! TODO Maybe have two different universes, one for leaf functions and one for non-leaf functions? //! The caller-saved registers are placed first in order to prefer not to clobber (requiring
//! Also, they will have to be ABI dependent. Need to find a way to avoid constructing a universe //! saves/restores in prologue/epilogue code) when possible. Note that there is no other heuristic
//! for each function we compile. //! in the backend that will apply such pressure; the register allocator's cost heuristics are not
//! aware of the cost of clobber-save/restore code.
//!
//! One might worry that this pessimizes code with many callsites, where using caller-saves causes
//! us to have to save them (as we are the caller) frequently. However, the register allocator
//! *should be* aware of *this* cost, because it sees that the call instruction modifies all of the
//! caller-saved (i.e., callee-clobbered) registers.
//!
//! Hence, this ordering encodes pressure in one direction (prefer not to clobber registers that we
//! ourselves have to save) and this is balanaced against the RA's pressure in the other direction
//! at callsites.
use crate::settings; use crate::settings;
use alloc::vec::Vec; use alloc::vec::Vec;
@@ -31,44 +37,44 @@ fn gpr(enc: u8, index: u8) -> Reg {
Reg::new_real(RegClass::I64, enc, index) Reg::new_real(RegClass::I64, enc, index)
} }
pub(crate) fn r12() -> Reg {
gpr(ENC_R12, 16)
}
pub(crate) fn r13() -> Reg {
gpr(ENC_R13, 17)
}
pub(crate) fn r14() -> Reg {
gpr(ENC_R14, 18)
}
pub(crate) fn rbx() -> Reg {
gpr(ENC_RBX, 19)
}
pub(crate) fn rsi() -> Reg { pub(crate) fn rsi() -> Reg {
gpr(6, 20) gpr(6, 16)
} }
pub(crate) fn rdi() -> Reg { pub(crate) fn rdi() -> Reg {
gpr(7, 21) gpr(7, 17)
} }
pub(crate) fn rax() -> Reg { pub(crate) fn rax() -> Reg {
gpr(0, 22) gpr(0, 18)
} }
pub(crate) fn rcx() -> Reg { pub(crate) fn rcx() -> Reg {
gpr(1, 23) gpr(1, 19)
} }
pub(crate) fn rdx() -> Reg { pub(crate) fn rdx() -> Reg {
gpr(2, 24) gpr(2, 20)
} }
pub(crate) fn r8() -> Reg { pub(crate) fn r8() -> Reg {
gpr(8, 25) gpr(8, 21)
} }
pub(crate) fn r9() -> Reg { pub(crate) fn r9() -> Reg {
gpr(9, 26) gpr(9, 22)
} }
pub(crate) fn r10() -> Reg { pub(crate) fn r10() -> Reg {
gpr(10, 27) gpr(10, 23)
} }
pub(crate) fn r11() -> Reg { pub(crate) fn r11() -> Reg {
gpr(11, 28) gpr(11, 24)
}
pub(crate) fn r12() -> Reg {
gpr(ENC_R12, 25)
}
pub(crate) fn r13() -> Reg {
gpr(ENC_R13, 26)
}
pub(crate) fn r14() -> Reg {
gpr(ENC_R14, 27)
}
pub(crate) fn rbx() -> Reg {
gpr(ENC_RBX, 28)
} }
pub(crate) fn r15() -> Reg { pub(crate) fn r15() -> Reg {
@@ -176,13 +182,6 @@ pub(crate) fn create_reg_universe_systemv(flags: &settings::Flags) -> RealRegUni
// Integer regs. // Integer regs.
let first_gpr = regs.len(); let first_gpr = regs.len();
// Callee-saved, in the SystemV x86_64 ABI.
regs.push((r12().to_real_reg(), "%r12".into()));
regs.push((r13().to_real_reg(), "%r13".into()));
regs.push((r14().to_real_reg(), "%r14".into()));
regs.push((rbx().to_real_reg(), "%rbx".into()));
// Caller-saved, in the SystemV x86_64 ABI. // Caller-saved, in the SystemV x86_64 ABI.
regs.push((rsi().to_real_reg(), "%rsi".into())); regs.push((rsi().to_real_reg(), "%rsi".into()));
regs.push((rdi().to_real_reg(), "%rdi".into())); regs.push((rdi().to_real_reg(), "%rdi".into()));
@@ -194,6 +193,13 @@ pub(crate) fn create_reg_universe_systemv(flags: &settings::Flags) -> RealRegUni
regs.push((r10().to_real_reg(), "%r10".into())); regs.push((r10().to_real_reg(), "%r10".into()));
regs.push((r11().to_real_reg(), "%r11".into())); regs.push((r11().to_real_reg(), "%r11".into()));
// Callee-saved, in the SystemV x86_64 ABI.
regs.push((r12().to_real_reg(), "%r12".into()));
regs.push((r13().to_real_reg(), "%r13".into()));
regs.push((r14().to_real_reg(), "%r14".into()));
regs.push((rbx().to_real_reg(), "%rbx".into()));
// Other regs, not available to the allocator. // Other regs, not available to the allocator.
debug_assert_eq!(r15(), pinned_reg()); debug_assert_eq!(r15(), pinned_reg());
let allocable = if use_pinned_reg { let allocable = if use_pinned_reg {

288
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -1530,7 +1530,10 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let src = if let Some(ext_spec) = ext_spec { let src = if let Some(ext_spec) = ext_spec {
RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec)) RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))
} else { } else {
input_to_reg_mem(ctx, inputs[0]) // N.B.: explicitly put input in a reg here because the width of the instruction
// into which this RM op goes may not match the width of the input type (in fact,
// it won't for i32.popcnt), and we don't want a larger than necessary load.
RegMem::reg(put_input_in_reg(ctx, inputs[0]))
}; };
let dst = get_output_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
@@ -2722,6 +2725,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else { } else {
if op == Opcode::FcvtToSintSat { if op == Opcode::FcvtToSintSat {
// Sets destination to zero if float is NaN // Sets destination to zero if float is NaN
assert_eq!(types::F32X4, ctx.input_ty(insn, 0));
let tmp = ctx.alloc_tmp(RegClass::V128, types::I32X4); let tmp = ctx.alloc_tmp(RegClass::V128, types::I32X4);
ctx.emit(Inst::xmm_unary_rm_r( ctx.emit(Inst::xmm_unary_rm_r(
SseOpcode::Movapd, SseOpcode::Movapd,
@@ -2776,7 +2780,118 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
dst, dst,
)); ));
} else if op == Opcode::FcvtToUintSat { } else if op == Opcode::FcvtToUintSat {
unimplemented!("f32x4.convert_i32x4_u"); // The algorithm for converting floats to unsigned ints is a little tricky. The
// complication arises because we are converting from a signed 64-bit int with a positive
// integer range from 1..INT_MAX (0x1..0x7FFFFFFF) to an unsigned integer with an extended
// range from (INT_MAX+1)..UINT_MAX. It's this range from (INT_MAX+1)..UINT_MAX
// (0x80000000..0xFFFFFFFF) that needs to be accounted for as a special case since our
// conversion instruction (cvttps2dq) only converts as high as INT_MAX (0x7FFFFFFF), but
// which conveniently setting underflows and overflows (smaller than MIN_INT or larger than
// MAX_INT) to be INT_MAX+1 (0x80000000). Nothing that the range (INT_MAX+1)..UINT_MAX includes
// precisely INT_MAX values we can correctly account for and convert every value in this range
// if we simply subtract INT_MAX+1 before doing the cvttps2dq conversion. After the subtraction
// every value originally (INT_MAX+1)..UINT_MAX is now the range (0..INT_MAX).
// After the conversion we add INT_MAX+1 back to this converted value, noting again that
// values we are trying to account for were already set to INT_MAX+1 during the original conversion.
// We simply have to create a mask and make sure we are adding together only the lanes that need
// to be accounted for. Digesting it all the steps then are:
//
// Step 1 - Account for NaN and negative floats by setting these src values to zero.
// Step 2 - Make a copy (tmp1) of the src value since we need to convert twice for
// reasons described above.
// Step 3 - Convert the original src values. This will convert properly all floats up to INT_MAX
// Step 4 - Subtract INT_MAX from the copy set (tmp1). Note, all zero and negative values are those
// values that were originally in the range (0..INT_MAX). This will come in handy during
// step 7 when we zero negative lanes.
// Step 5 - Create a bit mask for tmp1 that will correspond to all lanes originally less than
// UINT_MAX that are now less than INT_MAX thanks to the subtraction.
// Step 6 - Convert the second set of values (tmp1)
// Step 7 - Prep the converted second set by zeroing out negative lanes (these have already been
// converted correctly with the first set) and by setting overflow lanes to 0x7FFFFFFF
// as this will allow us to properly saturate overflow lanes when adding to 0x80000000
// Step 8 - Add the orginal converted src and the converted tmp1 where float values originally less
// than and equal to INT_MAX will be unchanged, float values originally between INT_MAX+1 and
// UINT_MAX will add together (INT_MAX) + (SRC - INT_MAX), and float values originally
// greater than UINT_MAX will be saturated to UINT_MAX (0xFFFFFFFF) after adding (0x8000000 + 0x7FFFFFFF).
//
//
// The table below illustrates the result after each step where it matters for the converted set.
// Note the original value range (original src set) is the final dst in Step 8:
//
// Original src set:
// | Original Value Range | Step 1 | Step 3 | Step 8 |
// | -FLT_MIN..FLT_MAX | 0.0..FLT_MAX | 0..INT_MAX(w/overflow) | 0..UINT_MAX(w/saturation) |
//
// Copied src set (tmp1):
// | Step 2 | Step 4 |
// | 0.0..FLT_MAX | (0.0-(INT_MAX+1))..(FLT_MAX-(INT_MAX+1)) |
//
// | Step 6 | Step 7 |
// | (0-(INT_MAX+1))..(UINT_MAX-(INT_MAX+1))(w/overflow) | ((INT_MAX+1)-(INT_MAX+1))..(INT_MAX+1) |
// Create temporaries
assert_eq!(types::F32X4, ctx.input_ty(insn, 0));
let tmp1 = ctx.alloc_tmp(RegClass::V128, types::I32X4);
let tmp2 = ctx.alloc_tmp(RegClass::V128, types::I32X4);
// Converting to unsigned int so if float src is negative or NaN
// will first set to zero.
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp2));
ctx.emit(Inst::gen_move(dst, src, input_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Maxps, RegMem::from(tmp2), dst));
// Set tmp2 to INT_MAX+1. It is important to note here that after it looks
// like we are only converting INT_MAX (0x7FFFFFFF) but in fact because
// single precision IEEE-754 floats can only accurately represent contingous
// integers up to 2^23 and outside of this range it rounds to the closest
// integer that it can represent. In the case of INT_MAX, this value gets
// represented as 0x4f000000 which is the integer value (INT_MAX+1).
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pcmpeqd, RegMem::from(tmp2), tmp2));
ctx.emit(Inst::xmm_rmi_reg(SseOpcode::Psrld, RegMemImm::imm(1), tmp2));
ctx.emit(Inst::xmm_rm_r(
SseOpcode::Cvtdq2ps,
RegMem::from(tmp2),
tmp2,
));
// Make a copy of these lanes and then do the first conversion.
// Overflow lanes greater than the maximum allowed signed value will
// set to 0x80000000. Negative and NaN lanes will be 0x0
ctx.emit(Inst::xmm_mov(SseOpcode::Movaps, RegMem::from(dst), tmp1));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Cvttps2dq, RegMem::from(dst), dst));
// Set lanes to src - max_signed_int
ctx.emit(Inst::xmm_rm_r(SseOpcode::Subps, RegMem::from(tmp2), tmp1));
// Create mask for all positive lanes to saturate (i.e. greater than
// or equal to the maxmimum allowable unsigned int).
let cond = FcmpImm::from(FloatCC::LessThanOrEqual);
ctx.emit(Inst::xmm_rm_r_imm(
SseOpcode::Cmpps,
RegMem::from(tmp1),
tmp2,
cond.encode(),
false,
));
// Convert those set of lanes that have the max_signed_int factored out.
ctx.emit(Inst::xmm_rm_r(
SseOpcode::Cvttps2dq,
RegMem::from(tmp1),
tmp1,
));
// Prepare converted lanes by zeroing negative lanes and prepping lanes
// that have positive overflow (based on the mask) by setting these lanes
// to 0x7FFFFFFF
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp1));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp2));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmaxsd, RegMem::from(tmp2), tmp1));
// Add this second set of converted lanes to the original to properly handle
// values greater than max signed int.
ctx.emit(Inst::xmm_rm_r(SseOpcode::Paddd, RegMem::from(tmp1), dst));
} else { } else {
// Since this branch is also guarded by a check for vector types // Since this branch is also guarded by a check for vector types
// neither Opcode::FcvtToUint nor Opcode::FcvtToSint can reach here // neither Opcode::FcvtToUint nor Opcode::FcvtToSint can reach here
@@ -2786,7 +2901,127 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
} }
} }
Opcode::UwidenHigh | Opcode::UwidenLow | Opcode::SwidenHigh | Opcode::SwidenLow => {
let input_ty = ctx.input_ty(insn, 0);
let output_ty = ctx.output_ty(insn, 0);
let src = put_input_in_reg(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]);
if output_ty.is_vector() {
match op {
Opcode::SwidenLow => match (input_ty, output_ty) {
(types::I8X16, types::I16X8) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxbw, RegMem::from(dst), dst));
}
(types::I16X8, types::I32X4) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxwd, RegMem::from(dst), dst));
}
_ => unreachable!(),
},
Opcode::SwidenHigh => match (input_ty, output_ty) {
(types::I8X16, types::I16X8) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r_imm(
SseOpcode::Palignr,
RegMem::reg(src),
dst,
8,
false,
));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxbw, RegMem::from(dst), dst));
}
(types::I16X8, types::I32X4) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r_imm(
SseOpcode::Palignr,
RegMem::reg(src),
dst,
8,
false,
));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovsxwd, RegMem::from(dst), dst));
}
_ => unreachable!(),
},
Opcode::UwidenLow => match (input_ty, output_ty) {
(types::I8X16, types::I16X8) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxbw, RegMem::from(dst), dst));
}
(types::I16X8, types::I32X4) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxwd, RegMem::from(dst), dst));
}
_ => unreachable!(),
},
Opcode::UwidenHigh => match (input_ty, output_ty) {
(types::I8X16, types::I16X8) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r_imm(
SseOpcode::Palignr,
RegMem::reg(src),
dst,
8,
false,
));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxbw, RegMem::from(dst), dst));
}
(types::I16X8, types::I32X4) => {
ctx.emit(Inst::gen_move(dst, src, output_ty));
ctx.emit(Inst::xmm_rm_r_imm(
SseOpcode::Palignr,
RegMem::reg(src),
dst,
8,
false,
));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmovzxwd, RegMem::from(dst), dst));
}
_ => unreachable!(),
},
_ => unreachable!(),
}
} else {
panic!("Unsupported non-vector type for widen instruction {:?}", ty);
}
}
Opcode::Snarrow | Opcode::Unarrow => {
let input_ty = ctx.input_ty(insn, 0);
let output_ty = ctx.output_ty(insn, 0);
let src1 = put_input_in_reg(ctx, inputs[0]);
let src2 = put_input_in_reg(ctx, inputs[1]);
let dst = get_output_reg(ctx, outputs[0]);
if output_ty.is_vector() {
match op {
Opcode::Snarrow => match (input_ty, output_ty) {
(types::I16X8, types::I8X16) => {
ctx.emit(Inst::gen_move(dst, src1, input_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Packsswb, RegMem::reg(src2), dst));
}
(types::I32X4, types::I16X8) => {
ctx.emit(Inst::gen_move(dst, src1, input_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Packssdw, RegMem::reg(src2), dst));
}
_ => unreachable!(),
},
Opcode::Unarrow => match (input_ty, output_ty) {
(types::I16X8, types::I8X16) => {
ctx.emit(Inst::gen_move(dst, src1, input_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Packuswb, RegMem::reg(src2), dst));
}
(types::I32X4, types::I16X8) => {
ctx.emit(Inst::gen_move(dst, src1, input_ty));
ctx.emit(Inst::xmm_rm_r(SseOpcode::Packusdw, RegMem::reg(src2), dst));
}
_ => unreachable!(),
},
_ => unreachable!(),
}
} else {
panic!("Unsupported non-vector type for widen instruction {:?}", ty);
}
}
Opcode::Bitcast => { Opcode::Bitcast => {
let input_ty = ctx.input_ty(insn, 0); let input_ty = ctx.input_ty(insn, 0);
let output_ty = ctx.output_ty(insn, 0); let output_ty = ctx.output_ty(insn, 0);
@@ -2975,22 +3210,45 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// Lower to VM calls when there's no access to SSE4.1. // Lower to VM calls when there's no access to SSE4.1.
let ty = ty.unwrap(); let ty = ty.unwrap();
let libcall = match (ty, op) { if !ty.is_vector() {
(types::F32, Opcode::Ceil) => LibCall::CeilF32, let libcall = match (op, ty) {
(types::F64, Opcode::Ceil) => LibCall::CeilF64, (Opcode::Ceil, types::F32) => LibCall::CeilF32,
(types::F32, Opcode::Floor) => LibCall::FloorF32, (Opcode::Ceil, types::F64) => LibCall::CeilF64,
(types::F64, Opcode::Floor) => LibCall::FloorF64, (Opcode::Floor, types::F32) => LibCall::FloorF32,
(types::F32, Opcode::Nearest) => LibCall::NearestF32, (Opcode::Floor, types::F64) => LibCall::FloorF64,
(types::F64, Opcode::Nearest) => LibCall::NearestF64, (Opcode::Nearest, types::F32) => LibCall::NearestF32,
(types::F32, Opcode::Trunc) => LibCall::TruncF32, (Opcode::Nearest, types::F64) => LibCall::NearestF64,
(types::F64, Opcode::Trunc) => LibCall::TruncF64, (Opcode::Trunc, types::F32) => LibCall::TruncF32,
(Opcode::Trunc, types::F64) => LibCall::TruncF64,
_ => panic!( _ => panic!(
"unexpected type/opcode {:?}/{:?} in Ceil/Floor/Nearest/Trunc", "unexpected type/opcode {:?}/{:?} in Ceil/Floor/Nearest/Trunc",
ty, op ty, op
), ),
}; };
emit_vm_call(ctx, flags, triple, libcall, insn, inputs, outputs)?; emit_vm_call(ctx, flags, triple, libcall, insn, inputs, outputs)?;
} else {
let (op, mode) = match (op, ty) {
(Opcode::Ceil, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundUp),
(Opcode::Ceil, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundUp),
(Opcode::Floor, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundDown),
(Opcode::Floor, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundDown),
(Opcode::Trunc, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundZero),
(Opcode::Trunc, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundZero),
(Opcode::Nearest, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundNearest),
(Opcode::Nearest, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundNearest),
_ => panic!("Unknown op/ty combination (vector){:?}", ty),
};
let src = put_input_in_reg(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::gen_move(dst, src, ty));
ctx.emit(Inst::xmm_rm_r_imm(
op,
RegMem::from(dst),
dst,
mode.encode(),
false,
));
}
} }
Opcode::Load Opcode::Load
@@ -4079,8 +4337,8 @@ impl LowerBackend for X64Backend {
let ty = ctx.input_ty(ifcmp_sp, 0); let ty = ctx.input_ty(ifcmp_sp, 0);
ctx.emit(Inst::cmp_rmi_r( ctx.emit(Inst::cmp_rmi_r(
ty.bytes() as u8, ty.bytes() as u8,
RegMemImm::reg(operand), RegMemImm::reg(regs::rsp()),
regs::rsp(), operand,
)); ));
let cond_code = ctx.data(branches[0]).cond_code().unwrap(); let cond_code = ctx.data(branches[0]).cond_code().unwrap();
let cc = CC::from_intcc(cond_code); let cc = CC::from_intcc(cond_code);

12
cranelift/codegen/src/isa/x64/mod.rs
View File

@@ -92,15 +92,15 @@ impl MachBackend for X64Backend {
} }
fn unsigned_add_overflow_condition(&self) -> IntCC { fn unsigned_add_overflow_condition(&self) -> IntCC {
// Unsigned `>=`; this corresponds to the carry flag set on x86, which happens on // Unsigned `<`; this corresponds to the carry flag set on x86, which
// overflow of an add. // indicates an add has overflowed.
IntCC::UnsignedGreaterThanOrEqual IntCC::UnsignedLessThan
} }
fn unsigned_sub_overflow_condition(&self) -> IntCC { fn unsigned_sub_overflow_condition(&self) -> IntCC {
// unsigned `>=`; this corresponds to the carry flag set on x86, which happens on // unsigned `<`; this corresponds to the carry flag set on x86, which
// underflow of a subtract (carry is borrow for subtract). // indicates a sub has underflowed (carry is borrow for subtract).
IntCC::UnsignedGreaterThanOrEqual IntCC::UnsignedLessThan
} }
#[cfg(feature = "unwind")] #[cfg(feature = "unwind")]

12
cranelift/codegen/src/legalizer/mod.rs
View File

@@ -659,7 +659,7 @@ fn narrow_load(
inst: ir::Inst, inst: ir::Inst,
func: &mut ir::Function, func: &mut ir::Function,
_cfg: &mut ControlFlowGraph, _cfg: &mut ControlFlowGraph,
_isa: &dyn TargetIsa, isa: &dyn TargetIsa,
) { ) {
let mut pos = FuncCursor::new(func).at_inst(inst); let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst); pos.use_srcloc(inst);
@@ -684,6 +684,10 @@ fn narrow_load(
ptr, ptr,
offset.try_add_i64(8).expect("load offset overflow"), offset.try_add_i64(8).expect("load offset overflow"),
); );
let (al, ah) = match flags.endianness(isa.endianness()) {
ir::Endianness::Little => (al, ah),
ir::Endianness::Big => (ah, al),
};
pos.func.dfg.replace(inst).iconcat(al, ah); pos.func.dfg.replace(inst).iconcat(al, ah);
} }
@@ -692,7 +696,7 @@ fn narrow_store(
inst: ir::Inst, inst: ir::Inst,
func: &mut ir::Function, func: &mut ir::Function,
_cfg: &mut ControlFlowGraph, _cfg: &mut ControlFlowGraph,
_isa: &dyn TargetIsa, isa: &dyn TargetIsa,
) { ) {
let mut pos = FuncCursor::new(func).at_inst(inst); let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst); pos.use_srcloc(inst);
@@ -708,6 +712,10 @@ fn narrow_store(
}; };
let (al, ah) = pos.ins().isplit(val); let (al, ah) = pos.ins().isplit(val);
let (al, ah) = match flags.endianness(isa.endianness()) {
ir::Endianness::Little => (al, ah),
ir::Endianness::Big => (ah, al),
};
pos.ins().store(flags, al, ptr, offset); pos.ins().store(flags, al, ptr, offset);
pos.ins().store( pos.ins().store(
flags, flags,

29
cranelift/codegen/src/licm.rs
View File

@@ -61,8 +61,8 @@ pub fn do_licm(
domtree.compute(func, cfg); domtree.compute(func, cfg);
} }
// Insert a pre-header before the header, modifying the function layout and CFG to reflect it. /// Insert a pre-header before the header, modifying the function layout and CFG to reflect it.
// A jump instruction to the header is placed at the end of the pre-header. /// A jump instruction to the header is placed at the end of the pre-header.
fn create_pre_header( fn create_pre_header(
isa: &dyn TargetIsa, isa: &dyn TargetIsa,
header: Block, header: Block,
@@ -81,30 +81,31 @@ fn create_pre_header(
for typ in header_args_types { for typ in header_args_types {
pre_header_args_value.push(func.dfg.append_block_param(pre_header, typ), pool); pre_header_args_value.push(func.dfg.append_block_param(pre_header, typ), pool);
} }
for BlockPredecessor { for BlockPredecessor {
inst: last_inst, .. inst: last_inst, ..
} in cfg.pred_iter(header) } in cfg.pred_iter(header)
{ {
// We only follow normal edges (not the back edges) // We only follow normal edges (not the back edges)
if !domtree.dominates(header, last_inst, &func.layout) { if !domtree.dominates(header, last_inst, &func.layout) {
func.change_branch_destination(last_inst, pre_header); func.rewrite_branch_destination(last_inst, header, pre_header);
} }
} }
{
let mut pos = EncCursor::new(func, isa).at_top(header);
// Inserts the pre-header at the right place in the layout. // Inserts the pre-header at the right place in the layout.
let mut pos = EncCursor::new(func, isa).at_top(header);
pos.insert_block(pre_header); pos.insert_block(pre_header);
pos.next_inst(); pos.next_inst();
pos.ins().jump(header, pre_header_args_value.as_slice(pool)); pos.ins().jump(header, pre_header_args_value.as_slice(pool));
}
pre_header pre_header
} }
// Detects if a loop header has a natural pre-header. /// Detects if a loop header has a natural pre-header.
// ///
// A loop header has a pre-header if there is only one predecessor that the header doesn't /// A loop header has a pre-header if there is only one predecessor that the header doesn't
// dominate. /// dominate.
// Returns the pre-header Block and the instruction jumping to the header. /// Returns the pre-header Block and the instruction jumping to the header.
fn has_pre_header( fn has_pre_header(
layout: &Layout, layout: &Layout,
cfg: &ControlFlowGraph, cfg: &ControlFlowGraph,
@@ -176,9 +177,9 @@ fn is_loop_invariant(inst: Inst, dfg: &DataFlowGraph, loop_values: &FxHashSet<Va
true true
} }
// Traverses a loop in reverse post-order from a header block and identify loop-invariant /// Traverses a loop in reverse post-order from a header block and identify loop-invariant
// instructions. These loop-invariant instructions are then removed from the code and returned /// instructions. These loop-invariant instructions are then removed from the code and returned
// (in reverse post-order) for later use. /// (in reverse post-order) for later use.
fn remove_loop_invariant_instructions( fn remove_loop_invariant_instructions(
lp: Loop, lp: Loop,
func: &mut Function, func: &mut Function,

24
cranelift/codegen/src/machinst/abi_impl.rs
View File

@@ -314,6 +314,9 @@ pub trait ABIMachineSpec {
/// Generate the usual frame-restore sequence for this architecture. /// Generate the usual frame-restore sequence for this architecture.
fn gen_epilogue_frame_restore() -> SmallVec<[Self::I; 2]>; fn gen_epilogue_frame_restore() -> SmallVec<[Self::I; 2]>;
/// Generate a probestack call.
fn gen_probestack(_frame_size: u32) -> SmallVec<[Self::I; 2]>;
/// Generate a clobber-save sequence. This takes the list of *all* registers /// Generate a clobber-save sequence. This takes the list of *all* registers
/// written/modified by the function body. The implementation here is /// written/modified by the function body. The implementation here is
/// responsible for determining which of these are callee-saved according to /// responsible for determining which of these are callee-saved according to
@@ -481,6 +484,9 @@ pub struct ABICalleeImpl<M: ABIMachineSpec> {
/// manually register-allocated and carefully only use caller-saved /// manually register-allocated and carefully only use caller-saved
/// registers and keep nothing live after this sequence of instructions. /// registers and keep nothing live after this sequence of instructions.
stack_limit: Option<(Reg, Vec<M::I>)>, stack_limit: Option<(Reg, Vec<M::I>)>,
/// Are we to invoke the probestack function in the prologue? If so,
/// what is the minimum size at which we must invoke it?
probestack_min_frame: Option<u32>,
_mach: PhantomData<M>, _mach: PhantomData<M>,
} }
@@ -536,6 +542,18 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
.map(|reg| (reg, Vec::new())) .map(|reg| (reg, Vec::new()))
.or_else(|| f.stack_limit.map(|gv| gen_stack_limit::<M>(f, &sig, gv))); .or_else(|| f.stack_limit.map(|gv| gen_stack_limit::<M>(f, &sig, gv)));
// Determine whether a probestack call is required for large enough
// frames (and the minimum frame size if so).
let probestack_min_frame = if flags.enable_probestack() {
assert!(
!flags.probestack_func_adjusts_sp(),
"SP-adjusting probestack not supported in new backends"
);
Some(1 << flags.probestack_size_log2())
} else {
None
};
Ok(Self { Ok(Self {
sig, sig,
stackslots, stackslots,
@@ -550,6 +568,7 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
flags, flags,
is_leaf: f.is_leaf(), is_leaf: f.is_leaf(),
stack_limit, stack_limit,
probestack_min_frame,
_mach: PhantomData, _mach: PhantomData,
}) })
} }
@@ -978,6 +997,11 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
insts.extend_from_slice(stack_limit_load); insts.extend_from_slice(stack_limit_load);
self.insert_stack_check(*reg, total_stacksize, &mut insts); self.insert_stack_check(*reg, total_stacksize, &mut insts);
} }
if let Some(min_frame) = &self.probestack_min_frame {
if total_stacksize >= *min_frame {
insts.extend(M::gen_probestack(total_stacksize));
}
}
} }
if total_stacksize > 0 { if total_stacksize > 0 {
self.fixed_frame_storage_size += total_stacksize; self.fixed_frame_storage_size += total_stacksize;

11
cranelift/codegen/src/machinst/buffer.rs
View File

@@ -674,6 +674,12 @@ impl<I: VCodeInst> MachBuffer<I> {
// (end of buffer) // (end of buffer)
self.data.truncate(b.start as usize); self.data.truncate(b.start as usize);
self.fixup_records.truncate(b.fixup); self.fixup_records.truncate(b.fixup);
while let Some(last_srcloc) = self.srclocs.last() {
if last_srcloc.end <= b.start {
break;
}
self.srclocs.pop();
}
// State: // State:
// [PRE CODE] // [PRE CODE]
// cur_off, Offset b.start, b.labels_at_this_branch: // cur_off, Offset b.start, b.labels_at_this_branch:
@@ -1184,12 +1190,15 @@ impl<I: VCodeInst> MachBuffer<I> {
// incorrect. // incorrect.
assert!(self.fixup_records.is_empty()); assert!(self.fixup_records.is_empty());
let mut srclocs = self.srclocs;
srclocs.sort_by_key(|entry| entry.start);
MachBufferFinalized { MachBufferFinalized {
data: self.data, data: self.data,
relocs: self.relocs, relocs: self.relocs,
traps: self.traps, traps: self.traps,
call_sites: self.call_sites, call_sites: self.call_sites,
srclocs: self.srclocs, srclocs,
stack_maps: self.stack_maps, stack_maps: self.stack_maps,
} }
} }

1
cranelift/codegen/src/machinst/lower.rs
View File

@@ -913,6 +913,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
// the code-motion. // the code-motion.
if self.cur_scan_entry_color.is_some() if self.cur_scan_entry_color.is_some()
&& self.value_uses[val] == 1 && self.value_uses[val] == 1
&& self.value_lowered_uses[val] == 0
&& self.num_outputs(src_inst) == 1 && self.num_outputs(src_inst) == 1
&& self && self
.side_effect_inst_entry_colors .side_effect_inst_entry_colors

22
cranelift/codegen/src/peepmatic.rs
View File

@@ -1312,19 +1312,31 @@ unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
} }
#[cfg(test)] #[cfg(test)]
#[cfg(feature = "x86")] #[cfg(any(feature = "x64", feature = "x86", feature = "arm64"))]
mod tests { mod tests {
use super::*; use super::*;
use crate::isa::lookup; use crate::isa::{lookup, TargetIsa};
use crate::settings::{builder, Flags}; use crate::settings::{builder, Flags};
use std::str::FromStr; use std::str::FromStr;
use target_lexicon::triple; use target_lexicon::triple;
fn isa() -> Box<dyn TargetIsa> {
// We need a triple to instantiate and run the peephole optimizer, but we
// don't care which one when we're just trying to trigger a rebuild of the
// peephole optimizer (it doesn't affect the serialized bytes at all).
let triple = if cfg!(any(feature = "x64", feature = "x86")) {
triple!("x86_64")
} else if cfg!(feature = "arm64") {
triple!("aarch64")
} else {
panic!("unknown arch")
};
lookup(triple).unwrap().finish(Flags::new(builder()))
}
#[test] #[test]
fn get_peepmatic_preopt() { fn get_peepmatic_preopt() {
let isa = lookup(triple!("x86_64")) let isa = isa();
.expect("expect x86 ISA")
.finish(Flags::new(builder()));
let _ = preopt(&*isa); let _ = preopt(&*isa);
} }
} }

90
cranelift/codegen/src/souper_harvest.rs
View File

@@ -142,7 +142,49 @@ fn harvest_candidate_lhs(
let souper_assignment_rhs = match func.dfg.value_def(val) { let souper_assignment_rhs = match func.dfg.value_def(val) {
ir::ValueDef::Result(inst, 0) => { ir::ValueDef::Result(inst, 0) => {
let args = func.dfg.inst_args(inst); let args = func.dfg.inst_args(inst);
let arg = |allocs: &mut Allocs, n| allocs.ir_to_souper_val[&args[n]].into();
// Get the n^th argument as a souper operand.
let arg = |allocs: &mut Allocs, n| {
let arg = args[n];
if let Some(a) = allocs.ir_to_souper_val.get(&arg).copied() {
a.into()
} else {
// The only arguments we get that we haven't already
// converted into a souper instruction are `iconst`s and
// `bconst`s. This is because souper only allows
// constants as operands, and it doesn't allow assigning
// constants to a variable name. So we lazily convert
// `iconst`s and `bconst`s into souper operands here,
// when they are actually used.
match func.dfg.value_def(arg) {
ir::ValueDef::Result(inst, 0) => match func.dfg[inst] {
ir::InstructionData::UnaryImm { opcode, imm } => {
debug_assert_eq!(opcode, ir::Opcode::Iconst);
let imm: i64 = imm.into();
ast::Operand::Constant(ast::Constant {
value: imm.into(),
r#type: souper_type_of(&func.dfg, arg),
})
}
ir::InstructionData::UnaryBool { opcode, imm } => {
debug_assert_eq!(opcode, ir::Opcode::Iconst);
ast::Operand::Constant(ast::Constant {
value: imm.into(),
r#type: souper_type_of(&func.dfg, arg),
})
}
_ => unreachable!(
"only iconst and bconst instructions \
aren't in `ir_to_souper_val`"
),
},
_ => unreachable!(
"only iconst and bconst instructions \
aren't in `ir_to_souper_val`"
),
}
}
};
match (func.dfg[inst].opcode(), &func.dfg[inst]) { match (func.dfg[inst].opcode(), &func.dfg[inst]) {
(ir::Opcode::Iadd, _) => { (ir::Opcode::Iadd, _) => {
@@ -350,6 +392,28 @@ fn harvest_candidate_lhs(
} }
(ir::Opcode::Select, _) => { (ir::Opcode::Select, _) => {
let a = arg(allocs, 0); let a = arg(allocs, 0);
// While Cranelift allows any width condition for
// `select`, Souper requires an `i1`.
let a = match a {
ast::Operand::Value(id) => match lhs.get_value(id).r#type {
Some(ast::Type { width: 1 }) => a,
_ => lhs
.assignment(
None,
Some(ast::Type { width: 1 }),
ast::Instruction::Trunc { a },
vec![],
)
.into(),
},
ast::Operand::Constant(ast::Constant { value, .. }) => ast::Constant {
value: (value != 0) as _,
r#type: Some(ast::Type { width: 1 }),
}
.into(),
};
let b = arg(allocs, 1); let b = arg(allocs, 1);
let c = arg(allocs, 2); let c = arg(allocs, 2);
ast::Instruction::Select { a, b, c }.into() ast::Instruction::Select { a, b, c }.into()
@@ -421,23 +485,13 @@ fn harvest_candidate_lhs(
let b = arg(allocs, 1); let b = arg(allocs, 1);
ast::Instruction::UsubSat { a, b }.into() ast::Instruction::UsubSat { a, b }.into()
} }
(ir::Opcode::Iconst, ir::InstructionData::UnaryImm { imm, .. }) => { // Because Souper doesn't allow constants to be on the right
let value: i64 = (*imm).into(); // hand side of an assignment (i.e. `%0:i32 = 1234` is
let value: i128 = value.into(); // disallowed) we have to ignore `iconst` and `bconst`
ast::Constant { // instructions until we process them as operands for some
value, // other instruction. See the `arg` closure above for
r#type: souper_type_of(&func.dfg, val), // details.
} (ir::Opcode::Iconst, _) | (ir::Opcode::Bconst, _) => return,
.into()
}
(ir::Opcode::Bconst, ir::InstructionData::UnaryBool { imm, .. }) => {
let value = *imm as i128;
ast::Constant {
value,
r#type: souper_type_of(&func.dfg, val),
}
.into()
}
_ => ast::AssignmentRhs::Var, _ => ast::AssignmentRhs::Var,
} }
} }

4
cranelift/docs/index.md
View File

@@ -52,6 +52,6 @@
emits native object files using the emits native object files using the
`object <https://github.com/gimli-rs/object>`_ library. `object <https://github.com/gimli-rs/object>`_ library.
- [cranelift-simplejit](https://docs.rs/cranelift-simplejit) - [cranelift-jit](https://docs.rs/cranelift-jit)
This crate provides a simple JIT backend for `cranelift-module`, which This crate provides a JIT backend for `cranelift-module`, which
emits code and data into memory. emits code and data into memory.

66
cranelift/filetests/filetests/isa/aarch64/amodes.clif
View File

@@ -344,3 +344,69 @@ block0(v0: i64, v1: i32):
; nextln: mov sp, fp ; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16 ; nextln: ldp fp, lr, [sp], #16
; nextln: ret ; nextln: ret
function %f18(i64, i64, i64) -> i32 {
block0(v0: i64, v1: i64, v2: i64):
v3 = iconst.i32 -4098
v6 = uextend.i64 v3
v5 = sload16.i32 v6+0
return v5
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: movn w0, #4097
; nextln: ldrsh x0, [x0]
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f19(i64, i64, i64) -> i32 {
block0(v0: i64, v1: i64, v2: i64):
v3 = iconst.i32 4098
v6 = uextend.i64 v3
v5 = sload16.i32 v6+0
return v5
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: movz x0, #4098
; nextln: ldrsh x0, [x0]
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f20(i64, i64, i64) -> i32 {
block0(v0: i64, v1: i64, v2: i64):
v3 = iconst.i32 -4098
v6 = sextend.i64 v3
v5 = sload16.i32 v6+0
return v5
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: movn w0, #4097
; nextln: sxtw x0, w0
; nextln: ldrsh x0, [x0]
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f21(i64, i64, i64) -> i32 {
block0(v0: i64, v1: i64, v2: i64):
v3 = iconst.i32 4098
v6 = sextend.i64 v3
v5 = sload16.i32 v6+0
return v5
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: movz x0, #4098
; nextln: sxtw x0, w0
; nextln: ldrsh x0, [x0]
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret

16
cranelift/filetests/filetests/isa/aarch64/floating-point.clif
View File

@@ -715,7 +715,7 @@ block0(v0: f32):
; nextln: movz x0, #20352, LSL #16 ; nextln: movz x0, #20352, LSL #16
; nextln: fmov d1, x0 ; nextln: fmov d1, x0
; nextln: fmin s2, s0, s1 ; nextln: fmin s2, s0, s1
; nextln: movi v1.8b, #0 ; nextln: movi v1.2s, #0
; nextln: fmax s2, s2, s1 ; nextln: fmax s2, s2, s1
; nextln: fcmp s0, s0 ; nextln: fcmp s0, s0
; nextln: fcsel s0, s1, s2, ne ; nextln: fcsel s0, s1, s2, ne
@@ -738,7 +738,7 @@ block0(v0: f32):
; nextln: movz x0, #52992, LSL #16 ; nextln: movz x0, #52992, LSL #16
; nextln: fmov d2, x0 ; nextln: fmov d2, x0
; nextln: fmax s1, s1, s2 ; nextln: fmax s1, s1, s2
; nextln: movi v2.8b, #0 ; nextln: movi v2.2s, #0
; nextln: fcmp s0, s0 ; nextln: fcmp s0, s0
; nextln: fcsel s0, s2, s1, ne ; nextln: fcsel s0, s2, s1, ne
; nextln: fcvtzs w0, s0 ; nextln: fcvtzs w0, s0
@@ -757,7 +757,7 @@ block0(v0: f32):
; nextln: movz x0, #24448, LSL #16 ; nextln: movz x0, #24448, LSL #16
; nextln: fmov d1, x0 ; nextln: fmov d1, x0
; nextln: fmin s2, s0, s1 ; nextln: fmin s2, s0, s1
; nextln: movi v1.8b, #0 ; nextln: movi v1.2s, #0
; nextln: fmax s2, s2, s1 ; nextln: fmax s2, s2, s1
; nextln: fcmp s0, s0 ; nextln: fcmp s0, s0
; nextln: fcsel s0, s1, s2, ne ; nextln: fcsel s0, s1, s2, ne
@@ -780,7 +780,7 @@ block0(v0: f32):
; nextln: movz x0, #57088, LSL #16 ; nextln: movz x0, #57088, LSL #16
; nextln: fmov d2, x0 ; nextln: fmov d2, x0
; nextln: fmax s1, s1, s2 ; nextln: fmax s1, s1, s2
; nextln: movi v2.8b, #0 ; nextln: movi v2.2s, #0
; nextln: fcmp s0, s0 ; nextln: fcmp s0, s0
; nextln: fcsel s0, s2, s1, ne ; nextln: fcsel s0, s2, s1, ne
; nextln: fcvtzs x0, s0 ; nextln: fcvtzs x0, s0
@@ -798,7 +798,7 @@ block0(v0: f64):
; nextln: mov fp, sp ; nextln: mov fp, sp
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 4294967295 ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 4294967295
; nextln: fmin d2, d0, d1 ; nextln: fmin d2, d0, d1
; nextln: movi v1.8b, #0 ; nextln: movi v1.2s, #0
; nextln: fmax d2, d2, d1 ; nextln: fmax d2, d2, d1
; nextln: fcmp d0, d0 ; nextln: fcmp d0, d0
; nextln: fcsel d0, d1, d2, ne ; nextln: fcsel d0, d1, d2, ne
@@ -820,7 +820,7 @@ block0(v0: f64):
; nextln: movz x0, #49632, LSL #48 ; nextln: movz x0, #49632, LSL #48
; nextln: fmov d2, x0 ; nextln: fmov d2, x0
; nextln: fmax d1, d1, d2 ; nextln: fmax d1, d1, d2
; nextln: movi v2.8b, #0 ; nextln: movi v2.2s, #0
; nextln: fcmp d0, d0 ; nextln: fcmp d0, d0
; nextln: fcsel d0, d2, d1, ne ; nextln: fcsel d0, d2, d1, ne
; nextln: fcvtzs w0, d0 ; nextln: fcvtzs w0, d0
@@ -839,7 +839,7 @@ block0(v0: f64):
; nextln: movz x0, #17392, LSL #48 ; nextln: movz x0, #17392, LSL #48
; nextln: fmov d1, x0 ; nextln: fmov d1, x0
; nextln: fmin d2, d0, d1 ; nextln: fmin d2, d0, d1
; nextln: movi v1.8b, #0 ; nextln: movi v1.2s, #0
; nextln: fmax d2, d2, d1 ; nextln: fmax d2, d2, d1
; nextln: fcmp d0, d0 ; nextln: fcmp d0, d0
; nextln: fcsel d0, d1, d2, ne ; nextln: fcsel d0, d1, d2, ne
@@ -862,7 +862,7 @@ block0(v0: f64):
; nextln: movz x0, #50144, LSL #48 ; nextln: movz x0, #50144, LSL #48
; nextln: fmov d2, x0 ; nextln: fmov d2, x0
; nextln: fmax d1, d1, d2 ; nextln: fmax d1, d1, d2
; nextln: movi v2.8b, #0 ; nextln: movi v2.2s, #0
; nextln: fcmp d0, d0 ; nextln: fcmp d0, d0
; nextln: fcsel d0, d2, d1, ne ; nextln: fcsel d0, d2, d1, ne
; nextln: fcvtzs x0, d0 ; nextln: fcvtzs x0, d0

43
cranelift/filetests/filetests/isa/aarch64/simd.clif
View File

@@ -127,3 +127,46 @@ block0(v0: i64, v1: i64):
; nextln: mov sp, fp ; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16 ; nextln: ldp fp, lr, [sp], #16
; nextln: ret ; nextln: ret
function %f9() -> i32x2 {
block0:
v0 = iconst.i32 4278190335
v1 = splat.i32x2 v0
return v1
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: movi v0.2d, #18374687579166474495
; nextln: fmov d0, d0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f10() -> i32x4 {
block0:
v0 = iconst.i32 4293918720
v1 = splat.i32x4 v0
return v1
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: mvni v0.4s, #15, MSL #16
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f11() -> f32x4 {
block0:
v0 = f32const 0x1.5
v1 = splat.f32x4 v0
return v1
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fmov v0.4s, #1.3125
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret

8
cranelift/filetests/filetests/isa/x64/amode-opt.clif
View File

@@ -7,7 +7,7 @@ block0(v0: i64, v1: i64):
v2 = iadd v0, v1 v2 = iadd v0, v1
v3 = load.i64 v2 v3 = load.i64 v2
return v3 return v3
; check: movq 0(%rdi,%rsi,1), %r12 ; check: movq 0(%rdi,%rsi,1), %rsi
} }
function %amode_add_imm(i64) -> i64 { function %amode_add_imm(i64) -> i64 {
@@ -16,7 +16,7 @@ block0(v0: i64):
v2 = iadd v0, v1 v2 = iadd v0, v1
v3 = load.i64 v2 v3 = load.i64 v2
return v3 return v3
; check: movq 42(%rdi), %r12 ; check: movq 42(%rdi), %rsi
} }
;; Same as above, but add operands have been reversed. ;; Same as above, but add operands have been reversed.
@@ -26,7 +26,7 @@ block0(v0: i64):
v2 = iadd v1, v0 v2 = iadd v1, v0
v3 = load.i64 v2 v3 = load.i64 v2
return v3 return v3
; check: movq 42(%rdi), %r12 ; check: movq 42(%rdi), %rsi
} }
;; Make sure that uextend(cst) are ignored when the cst will naturally sign-extend. ;; Make sure that uextend(cst) are ignored when the cst will naturally sign-extend.
@@ -37,5 +37,5 @@ block0(v0: i64):
v3 = iadd v2, v0 v3 = iadd v2, v0
v4 = load.i64 v3 v4 = load.i64 v3
return v4 return v4
; check: movq 42(%rdi), %r12 ; check: movq 42(%rdi), %rsi
} }

23
cranelift/filetests/filetests/isa/x64/heap.clif Normal file
View File

@@ -0,0 +1,23 @@
test compile
target x86_64
feature "experimental_x64"
function %f(i32, i64 vmctx) -> i64 {
gv0 = vmctx
gv1 = load.i64 notrap aligned gv0+0
gv2 = load.i32 notrap aligned gv0+8
heap0 = dynamic gv1, bound gv2, offset_guard 0x1000, index_type i32
block0(v0: i32, v1: i64):
v2 = heap_addr.i64 heap0, v0, 0x8000
; check: movl 8(%rsi), %ecx
; nextln: movq %rdi, %rax
; nextln: addl $$32768, %eax
; nextln: jnb ; ud2 heap_oob ;
; nextln: cmpl %ecx, %eax
; nextln: jbe label1; j label2
; check: Block 1:
return v2
}

27
cranelift/filetests/filetests/isa/x64/load-op.clif
View File

@@ -6,7 +6,7 @@ function %add_from_mem_u32_1(i64, i32) -> i32 {
block0(v0: i64, v1: i32): block0(v0: i64, v1: i32):
v2 = load.i32 v0 v2 = load.i32 v0
v3 = iadd.i32 v2, v1 v3 = iadd.i32 v2, v1
; check: addl 0(%rdi), %r12d ; check: addl 0(%rdi), %esi
return v3 return v3
} }
@@ -14,7 +14,7 @@ function %add_from_mem_u32_2(i64, i32) -> i32 {
block0(v0: i64, v1: i32): block0(v0: i64, v1: i32):
v2 = load.i32 v0 v2 = load.i32 v0
v3 = iadd.i32 v1, v2 v3 = iadd.i32 v1, v2
; check: addl 0(%rdi), %r12d ; check: addl 0(%rdi), %esi
return v3 return v3
} }
@@ -22,7 +22,7 @@ function %add_from_mem_u64_1(i64, i64) -> i64 {
block0(v0: i64, v1: i64): block0(v0: i64, v1: i64):
v2 = load.i64 v0 v2 = load.i64 v0
v3 = iadd.i64 v2, v1 v3 = iadd.i64 v2, v1
; check: addq 0(%rdi), %r12 ; check: addq 0(%rdi), %rsi
return v3 return v3
} }
@@ -30,7 +30,7 @@ function %add_from_mem_u64_2(i64, i64) -> i64 {
block0(v0: i64, v1: i64): block0(v0: i64, v1: i64):
v2 = load.i64 v0 v2 = load.i64 v0
v3 = iadd.i64 v1, v2 v3 = iadd.i64 v1, v2
; check: addq 0(%rdi), %r12 ; check: addq 0(%rdi), %rsi
return v3 return v3
} }
@@ -40,7 +40,22 @@ function %add_from_mem_not_narrow(i64, i8) -> i8 {
block0(v0: i64, v1: i8): block0(v0: i64, v1: i8):
v2 = load.i8 v0 v2 = load.i8 v0
v3 = iadd.i8 v2, v1 v3 = iadd.i8 v2, v1
; check: movzbq 0(%rdi), %r12 ; check: movzbq 0(%rdi), %rdi
; nextln: addl %esi, %r12d ; nextln: addl %esi, %edi
return v3 return v3
} }
function %no_merge_if_lookback_use(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = load.i64 v0
v3 = iadd.i64 v2, v0
store.i64 v3, v1
v4 = load.i64 v3
return v4
; check: movq 0(%rdi), %rax
; nextln: movq %rax, %rcx
; nextln: addq %rdi, %rcx
; nextln: movq %rcx, 0(%rsi)
; nextln: movq 0(%rax,%rdi,1), %rsi
; nextln: movq %rsi, %rax
}

113
cranelift/filetests/filetests/isa/x64/popcnt.clif Normal file
View File

@@ -0,0 +1,113 @@
test compile
target x86_64
feature "experimental_x64"
; TODO: test with popcnt feature available too, once new backend supports that.
function %popcnt64(i64) -> i64 {
block0(v0: i64):
v1 = popcnt v0
; check: movq %rdi, %rsi
; nextln: shrq $$1, %rsi
; nextln: movabsq $$8608480567731124087, %rax
; nextln: andq %rax, %rsi
; nextln: subq %rsi, %rdi
; nextln: shrq $$1, %rsi
; nextln: andq %rax, %rsi
; nextln: subq %rsi, %rdi
; nextln: shrq $$1, %rsi
; nextln: andq %rax, %rsi
; nextln: subq %rsi, %rdi
; nextln: movq %rdi, %rsi
; nextln: shrq $$4, %rsi
; nextln: addq %rdi, %rsi
; nextln: movabsq $$1085102592571150095, %rdi
; nextln: andq %rdi, %rsi
; nextln: movabsq $$72340172838076673, %rdi
; nextln: imulq %rdi, %rsi
; nextln: shrq $$56, %rsi
; nextln: movq %rsi, %rax
return v1
}
function %popcnt64load(i64) -> i64 {
block0(v0: i64):
v1 = load.i64 v0
v2 = popcnt v1
return v2
; check: movq 0(%rdi), %rdi
; nextln: movq %rdi, %rsi
; nextln: shrq $$1, %rsi
; nextln: movabsq $$8608480567731124087, %rax
; nextln: andq %rax, %rsi
; nextln: subq %rsi, %rdi
; nextln: shrq $$1, %rsi
; nextln: andq %rax, %rsi
; nextln: subq %rsi, %rdi
; nextln: shrq $$1, %rsi
; nextln: andq %rax, %rsi
; nextln: subq %rsi, %rdi
; nextln: movq %rdi, %rsi
; nextln: shrq $$4, %rsi
; nextln: addq %rdi, %rsi
; nextln: movabsq $$1085102592571150095, %rdi
; nextln: andq %rdi, %rsi
; nextln: movabsq $$72340172838076673, %rdi
; nextln: imulq %rdi, %rsi
; nextln: shrq $$56, %rsi
; nextln: movq %rsi, %rax
}
function %popcnt32(i32) -> i32 {
block0(v0: i32):
v1 = popcnt v0
return v1
; check: movq %rdi, %rsi
; nextln: shrl $$1, %esi
; nextln: andl $$2004318071, %esi
; nextln: subl %esi, %edi
; nextln: shrl $$1, %esi
; nextln: andl $$2004318071, %esi
; nextln: subl %esi, %edi
; nextln: shrl $$1, %esi
; nextln: andl $$2004318071, %esi
; nextln: subl %esi, %edi
; nextln: movq %rdi, %rsi
; nextln: shrl $$4, %esi
; nextln: addl %edi, %esi
; nextln: andl $$252645135, %esi
; nextln: imull $$16843009, %esi
; nextln: shrl $$24, %esi
; nextln: movq %rsi, %rax
; nextln: movq %rbp, %rsp
; nextln: popq %rbp
; nextln: ret
}
function %popcnt32load(i64) -> i32 {
block0(v0: i64):
v1 = load.i32 v0
v2 = popcnt v1
return v2
; check: movl 0(%rdi), %edi
; nextln: movq %rdi, %rsi
; nextln: shrl $$1, %esi
; nextln: andl $$2004318071, %esi
; nextln: subl %esi, %edi
; nextln: shrl $$1, %esi
; nextln: andl $$2004318071, %esi
; nextln: subl %esi, %edi
; nextln: shrl $$1, %esi
; nextln: andl $$2004318071, %esi
; nextln: subl %esi, %edi
; nextln: movq %rdi, %rsi
; nextln: shrl $$4, %esi
; nextln: addl %edi, %esi
; nextln: andl $$252645135, %esi
; nextln: imull $$16843009, %esi
; nextln: shrl $$24, %esi
; nextln: movq %rsi, %rax
; nextln: movq %rbp, %rsp
; nextln: popq %rbp
; nextln: ret
}

17
cranelift/filetests/filetests/isa/x64/probestack.clif Normal file
View File

@@ -0,0 +1,17 @@
test compile
set enable_probestack=true
target x86_64
feature "experimental_x64"
function %f1() -> i64 {
ss0 = explicit_slot 100000
block0:
v1 = stack_addr.i64 ss0
return v1
}
; check: pushq %rbp
; nextln: movq %rsp, %rbp
; nextln: movl $$100000, %eax
; nextln: call LibCall(Probestack)

16
cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
View File

@@ -28,9 +28,9 @@ block0(v0: i32):
} }
; check: movd %edi, %xmm1 ; check: movd %edi, %xmm1
; nextln: psllw %xmm1, %xmm0 ; nextln: psllw %xmm1, %xmm0
; nextln: lea const(VCodeConstant(0)), %r12 ; nextln: lea const(VCodeConstant(0)), %rsi
; nextln: shlq $$4, %rdi ; nextln: shlq $$4, %rdi
; nextln: movdqu 0(%r12,%rdi,1), %xmm1 ; nextln: movdqu 0(%rsi,%rdi,1), %xmm1
; nextln: pand %xmm1, %xmm0 ; nextln: pand %xmm1, %xmm0
function %ushr_i8x16_imm() -> i8x16 { function %ushr_i8x16_imm() -> i8x16 {
@@ -81,12 +81,12 @@ block0(v0: i64x2, v1: i32):
v2 = sshr v0, v1 v2 = sshr v0, v1
return v2 return v2
} }
; check: pextrd.w $$0, %xmm0, %r12 ; check: pextrd.w $$0, %xmm0, %rsi
; nextln: pextrd.w $$1, %xmm0, %r13 ; nextln: pextrd.w $$1, %xmm0, %rax
; nextln: movq %rdi, %rcx ; nextln: movq %rdi, %rcx
; nextln: sarq %cl, %r12 ; nextln: sarq %cl, %rsi
; nextln: movq %rdi, %rcx ; nextln: movq %rdi, %rcx
; nextln: sarq %cl, %r13 ; nextln: sarq %cl, %rax
; nextln: pinsrd.w $$0, %r12, %xmm1 ; nextln: pinsrd.w $$0, %rsi, %xmm1
; nextln: pinsrd.w $$1, %r13, %xmm1 ; nextln: pinsrd.w $$1, %rax, %xmm1
; nextln: movdqa %xmm1, %xmm0 ; nextln: movdqa %xmm1, %xmm0

4
cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif
View File

@@ -70,8 +70,8 @@ block0:
return v1 return v1
} }
; check: uninit %xmm0 ; check: uninit %xmm0
; nextln: pinsrw $$0, %r12, %xmm0 ; nextln: pinsrw $$0, %rsi, %xmm0
; nextln: pinsrw $$1, %r12, %xmm0 ; nextln: pinsrw $$1, %rsi, %xmm0
; nextln: pshufd $$0, %xmm0, %xmm0 ; nextln: pshufd $$0, %xmm0, %xmm0
function %splat_i32(i32) -> i32x4 { function %splat_i32(i32) -> i32x4 {

4
cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif
View File

@@ -17,7 +17,7 @@ block0(v0: b32x4):
return v1 return v1
} }
; check: ptest %xmm0, %xmm0 ; check: ptest %xmm0, %xmm0
; nextln: setnz %r12b ; nextln: setnz %sil
function %vall_true_i64x2(i64x2) -> b1 { function %vall_true_i64x2(i64x2) -> b1 {
block0(v0: i64x2): block0(v0: i64x2):
@@ -27,4 +27,4 @@ block0(v0: i64x2):
; check: pxor %xmm1, %xmm1 ; check: pxor %xmm1, %xmm1
; nextln: pcmpeqq %xmm0, %xmm1 ; nextln: pcmpeqq %xmm0, %xmm1
; nextln: ptest %xmm1, %xmm1 ; nextln: ptest %xmm1, %xmm1
; nextln: setz %r12b ; nextln: setz %sil

8
cranelift/filetests/filetests/isa/x86/optimized-zero-constants-32bit.clif
View File

@@ -44,9 +44,9 @@ block0:
function %zero_byte() -> i8 fast { function %zero_byte() -> i8 fast {
block0: block0:
; asm: xor %al, %al ; asm: xor %eax, %eax
[-,%rax] v0 = iconst.i8 0 ; bin: 30 c0 [-,%rax] v0 = iconst.i8 0 ; bin: 31 c0
; asm: xor %dh, %dh ; asm: xor %edi, %edi
[-,%rdi] v1 = iconst.i8 0 ; bin: 30 ff [-,%rdi] v1 = iconst.i8 0 ; bin: 31 ff
return v0 return v0
} }

12
cranelift/filetests/filetests/isa/x86/optimized-zero-constants.clif
View File

@@ -62,11 +62,11 @@ block0:
function %zero_byte() -> i8 fast { function %zero_byte() -> i8 fast {
block0: block0:
; asm: xor %r8b, %r8b ; asm: xor %r8d, %r8d
[-,%r15] v0 = iconst.i8 0 ; bin: 45 30 ff [-,%r15] v0 = iconst.i8 0 ; bin: 45 31 ff
; asm: xor %al, %al ; asm: xor %eax, eax
[-,%rax] v1 = iconst.i8 0 ; bin: 30 c0 [-,%rax] v1 = iconst.i8 0 ; bin: 31 c0
; asm: xor %dh, %dh ; asm: xor %edi, %edi
[-,%rdi] v2 = iconst.i8 0 ; bin: 30 ff [-,%rdi] v2 = iconst.i8 0 ; bin: 31 ff
return v0 return v0
} }

19
cranelift/filetests/filetests/licm/br-table.clif Normal file
View File

@@ -0,0 +1,19 @@
test compile
set opt_level=speed_and_size
target x86_64
function %br_table_opt() {
jt0 = jump_table [block1, block2]
block0:
v0 = iconst.i32 1
br_table v0, block2, jt0
block1:
return
block2:
v1 = iconst.i32 1
jump block2
}

28
cranelift/filetests/filetests/licm/rewrite-jump-table.clif Normal file
View File

@@ -0,0 +1,28 @@
test licm
target aarch64
function %rewrite_jump_table() {
jt0 = jump_table [block1, block2]
block0:
v0 = iconst.i64 1
v1 = jump_table_base.i64 jt0
v2 = jump_table_entry.i64 v0, v1, 4, jt0
v3 = iadd v1, v2
indirect_jump_table_br v3, jt0
block1:
return
block2:
v4 = bconst.b1 false
jump block2
}
; sameln: function
; nextln: jt0 = jump_table [block1, block3]
; check: block3:
; nextln: v4 = bconst.b1 false
; nextln: jump block2
; check: block2:
; nextln: jump block2

2
cranelift/filetests/src/function_runner.rs
View File

@@ -21,7 +21,7 @@ use thiserror::Error;
/// `CompiledFunction`s and subsequently calling them through the use of a `Trampoline`. As its /// `CompiledFunction`s and subsequently calling them through the use of a `Trampoline`. As its
/// name indicates, this compiler is limited: any functionality that requires knowledge of things /// name indicates, this compiler is limited: any functionality that requires knowledge of things
/// outside the [Function] will likely not work (e.g. global values, calls). For an example of this /// outside the [Function] will likely not work (e.g. global values, calls). For an example of this
/// "outside-of-function" functionality, see `cranelift_simplejit::backend::SimpleJITBackend`. /// "outside-of-function" functionality, see `cranelift_jit::backend::JITBackend`.
/// ///
/// ``` /// ```
/// use cranelift_filetests::SingleFunctionCompiler; /// use cranelift_filetests::SingleFunctionCompiler;

74
cranelift/interpreter/src/environment.rs
View File

@@ -1,52 +1,74 @@
//! Implements the function environment (e.g. a name-to-function mapping) for interpretation. //! Implements the function environment (e.g. a name-to-function mapping) for interpretation.
use cranelift_codegen::ir::{FuncRef, Function}; use cranelift_codegen::ir::{FuncRef, Function};
use cranelift_entity::{entity_impl, PrimaryMap};
use std::collections::HashMap; use std::collections::HashMap;
/// A function store contains all of the functions that are accessible to an interpreter.
#[derive(Default, Clone)] #[derive(Default, Clone)]
pub struct FunctionStore<'a> { pub struct FunctionStore<'a> {
functions: HashMap<FuncRef, &'a Function>, functions: PrimaryMap<FuncIndex, &'a Function>,
function_name_to_func_ref: HashMap<String, FuncRef>, function_names: HashMap<String, FuncIndex>,
} }
/// An opaque reference to a [`Function`](Function) stored in the [FunctionStore].
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct FuncIndex(u32);
entity_impl!(FuncIndex, "fn");
/// This is a helpful conversion for instantiating a store from a single [Function].
impl<'a> From<&'a Function> for FunctionStore<'a> { impl<'a> From<&'a Function> for FunctionStore<'a> {
fn from(f: &'a Function) -> Self { fn from(function: &'a Function) -> Self {
let func_ref = FuncRef::from_u32(0); let mut store = FunctionStore::default();
let mut function_name_to_func_ref = HashMap::new(); store.add(function.name.to_string(), function);
function_name_to_func_ref.insert(f.name.to_string(), func_ref); store
let mut functions = HashMap::new();
functions.insert(func_ref, f);
Self {
functions,
function_name_to_func_ref,
}
} }
} }
impl<'a> FunctionStore<'a> { impl<'a> FunctionStore<'a> {
/// Add a function by name. /// Add a function by name.
pub fn add(&mut self, name: String, function: &'a Function) { pub fn add(&mut self, name: String, function: &'a Function) {
let func_ref = FuncRef::with_number(self.function_name_to_func_ref.len() as u32) assert!(!self.function_names.contains_key(&name));
.expect("a valid function reference"); let index = self.functions.push(function);
self.function_name_to_func_ref.insert(name, func_ref); self.function_names.insert(name, index);
self.functions.insert(func_ref, function);
} }
/// Retrieve a reference to a function in the environment by its name. /// Retrieve the index of a function in the function store by its `name`.
pub fn index_of(&self, name: &str) -> Option<FuncRef> { pub fn index_of(&self, name: &str) -> Option<FuncIndex> {
self.function_name_to_func_ref.get(name).cloned() self.function_names.get(name).cloned()
} }
/// Retrieve a function by its function reference. /// Retrieve a function by its index in the function store.
pub fn get_by_func_ref(&self, func_ref: FuncRef) -> Option<&'a Function> { pub fn get_by_index(&self, index: FuncIndex) -> Option<&'a Function> {
self.functions.get(&func_ref).cloned() self.functions.get(index).cloned()
} }
/// Retrieve a function by its name. /// Retrieve a function by its name.
pub fn get_by_name(&self, name: &str) -> Option<&'a Function> { pub fn get_by_name(&self, name: &str) -> Option<&'a Function> {
let func_ref = self.index_of(name)?; let index = self.index_of(name)?;
self.get_by_func_ref(func_ref) self.get_by_index(index)
} }
/// Retrieve a function from a [FuncRef] within a [Function]. TODO this should be optimized, if possible, as
/// currently it retrieves the function name as a string and performs string matching.
pub fn get_from_func_ref(
&self,
func_ref: FuncRef,
function: &Function,
) -> Option<&'a Function> {
self.get_by_name(&get_function_name(func_ref, function))
}
}
/// Retrieve a function name from a [FuncRef] within a [Function]. TODO this should be optimized, if possible, as
/// currently it retrieves the function name as a string and performs string matching.
fn get_function_name(func_ref: FuncRef, function: &Function) -> String {
function
.dfg
.ext_funcs
.get(func_ref)
.expect("function to exist")
.name
.to_string()
} }
#[cfg(test)] #[cfg(test)]
@@ -77,6 +99,6 @@ mod tests {
let signature = Signature::new(CallConv::Fast); let signature = Signature::new(CallConv::Fast);
let func = &Function::with_name_signature(name, signature); let func = &Function::with_name_signature(name, signature);
let env: FunctionStore = func.into(); let env: FunctionStore = func.into();
assert_eq!(env.index_of("%test"), FuncRef::with_number(0)); assert_eq!(env.index_of("%test"), Some(FuncIndex::from_u32(0)));
} }
} }

169
cranelift/interpreter/src/frame.rs
View File

@@ -2,41 +2,45 @@
use cranelift_codegen::data_value::DataValue; use cranelift_codegen::data_value::DataValue;
use cranelift_codegen::ir::{Function, Value as ValueRef}; use cranelift_codegen::ir::{Function, Value as ValueRef};
use cranelift_entity::EntityRef;
use log::trace; use log::trace;
use std::collections::HashMap;
/// Holds the mutable elements of an interpretation. At some point I thought about using /// The type used for ensuring [Frame](crate::frame::Frame) entries conform to the expected memory layout.
/// Cell/RefCell to do field-level mutability, thinking that otherwise I would have to pub(crate) type Entries = Vec<Option<DataValue>>;
/// pass around a mutable object (for inst and registers) and an immutable one (for function,
/// could be self)--in the end I decided to do exactly that but perhaps one day that will become /// Holds the mutable elements of an interpreted function call.
/// untenable.
#[derive(Debug)] #[derive(Debug)]
pub struct Frame<'a> { pub struct Frame<'a> {
/// The currently executing function. /// The currently executing function.
pub function: &'a Function, pub(crate) function: &'a Function,
/// The current mapping of SSA value-references to their actual values. /// The current mapping of SSA value-references to their actual values. For efficiency, each SSA value is used as an
registers: HashMap<ValueRef, DataValue>, /// index into the Vec, meaning some slots may be unused.
registers: Entries,
} }
impl<'a> Frame<'a> { impl<'a> Frame<'a> {
/// Construct a new [Frame] for a function. This allocates a slot in the hash map for each SSA /// Construct a new [Frame] for a function. This allocates a slot in the hash map for each SSA `Value` (renamed to
/// `Value` (renamed to `ValueRef` here) which should mean that no additional allocations are /// `ValueRef` here) which should mean that no additional allocations are needed while interpreting the frame.
/// needed while interpreting the frame.
pub fn new(function: &'a Function) -> Self { pub fn new(function: &'a Function) -> Self {
let num_slots = function.dfg.num_values();
trace!("Create new frame for function: {}", function.signature); trace!("Create new frame for function: {}", function.signature);
Self { Self {
function, function,
registers: HashMap::with_capacity(function.dfg.num_values()), registers: vec![None; num_slots],
} }
} }
/// Retrieve the actual value associated with an SSA reference. /// Retrieve the actual value associated with an SSA reference.
#[inline] #[inline]
pub fn get(&self, name: ValueRef) -> &DataValue { pub fn get(&self, name: ValueRef) -> &DataValue {
assert!(name.index() < self.registers.len());
trace!("Get {}", name); trace!("Get {}", name);
self.registers &self
.get(&name) .registers
.get(name.index())
.unwrap_or_else(|| panic!("unknown value: {}", name)) .unwrap_or_else(|| panic!("unknown value: {}", name))
.as_ref()
.unwrap_or_else(|| panic!("empty slot: {}", name))
} }
/// Retrieve multiple SSA references; see `get`. /// Retrieve multiple SSA references; see `get`.
@@ -47,8 +51,9 @@ impl<'a> Frame<'a> {
/// Assign `value` to the SSA reference `name`. /// Assign `value` to the SSA reference `name`.
#[inline] #[inline]
pub fn set(&mut self, name: ValueRef, value: DataValue) -> Option<DataValue> { pub fn set(&mut self, name: ValueRef, value: DataValue) -> Option<DataValue> {
assert!(name.index() < self.registers.len());
trace!("Set {} -> {}", name, value); trace!("Set {} -> {}", name, value);
self.registers.insert(name, value) std::mem::replace(&mut self.registers[name.index()], Some(value))
} }
/// Assign to multiple SSA references; see `set`. /// Assign to multiple SSA references; see `set`.
@@ -66,12 +71,18 @@ impl<'a> Frame<'a> {
pub fn rename(&mut self, old_names: &[ValueRef], new_names: &[ValueRef]) { pub fn rename(&mut self, old_names: &[ValueRef], new_names: &[ValueRef]) {
trace!("Renaming {:?} -> {:?}", old_names, new_names); trace!("Renaming {:?} -> {:?}", old_names, new_names);
assert_eq!(old_names.len(), new_names.len()); assert_eq!(old_names.len(), new_names.len());
let mut registers = HashMap::with_capacity(self.registers.len()); let new_registers = vec![None; self.registers.len()];
for (on, nn) in old_names.iter().zip(new_names) { let mut old_registers = std::mem::replace(&mut self.registers, new_registers);
let v = self.registers.get(on).unwrap().clone(); self.registers = vec![None; self.registers.len()];
registers.insert(*nn, v); for (&on, &nn) in old_names.iter().zip(new_names) {
let value = std::mem::replace(&mut old_registers[on.index()], None);
self.registers[nn.index()] = value;
} }
self.registers = registers; }
/// Accessor for the current entries in the frame.
pub fn entries_mut(&mut self) -> &mut [Option<DataValue>] {
&mut self.registers
} }
} }
@@ -79,8 +90,15 @@ impl<'a> Frame<'a> {
mod tests { mod tests {
use super::*; use super::*;
use cranelift_codegen::data_value::DataValue; use cranelift_codegen::data_value::DataValue;
use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
use cranelift_codegen::ir::InstBuilder; use cranelift_codegen::ir::InstBuilder;
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_reader::parse_functions;
/// Helper to create a function from CLIF IR.
fn function(code: &str) -> Function {
parse_functions(code).unwrap().into_iter().next().unwrap()
}
/// Build an empty function with a single return. /// Build an empty function with a single return.
fn empty_function() -> Function { fn empty_function() -> Function {
@@ -101,23 +119,112 @@ mod tests {
} }
#[test] #[test]
fn assignment() { fn assignment_and_retrieval() {
let func = empty_function(); let func = function("function %test(i32) -> i32 { block0(v0:i32): return v0 }");
let mut frame = Frame::new(&func); let mut frame = Frame::new(&func);
let ssa_value_ref = ValueRef::from_u32(0);
let a = ValueRef::with_number(1).unwrap();
let fortytwo = DataValue::I32(42); let fortytwo = DataValue::I32(42);
frame.set(a, fortytwo.clone());
assert_eq!(frame.get(a), &fortytwo); // Verify that setting a valid SSA ref will make the value retrievable.
frame.set(ssa_value_ref, fortytwo.clone());
assert_eq!(frame.get(ssa_value_ref), &fortytwo);
} }
#[test] #[test]
#[should_panic] fn assignment_to_extra_slots() {
fn no_existing_value() { let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }");
let mut frame = Frame::new(&func);
let ssa_value_ref = ValueRef::from_u32(5);
let fortytwo = DataValue::I32(42);
// Due to how Cranelift organizes its SSA values, the use of v10 defines 11 slots for values
// to fit in--the following should work.
frame.set(ssa_value_ref, fortytwo.clone());
assert_eq!(frame.get(ssa_value_ref), &fortytwo);
}
#[test]
#[should_panic(expected = "assertion failed: name.index() < self.registers.len()")]
fn invalid_assignment() {
let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }");
let mut frame = Frame::new(&func);
let fortytwo = DataValue::I32(42);
// Since the SSA value ref points to 42 and the function only has 11 slots, this should
// fail. TODO currently this is a panic under the assumption we will not set indexes outside
// of the valid SSA value range but it might be better as a result.
frame.set(ValueRef::from_u32(11), fortytwo.clone());
}
#[test]
#[should_panic(expected = "assertion failed: name.index() < self.registers.len()")]
fn retrieve_nonexistent_value() {
let func = empty_function(); let func = empty_function();
let frame = Frame::new(&func); let frame = Frame::new(&func);
let ssa_value_ref = ValueRef::from_u32(1);
let a = ValueRef::with_number(1).unwrap(); // Retrieving a non-existent value should return an error.
frame.get(a); frame.get(ssa_value_ref);
}
#[test]
#[should_panic(expected = "empty slot: v5")]
fn retrieve_and_assign_multiple_values() {
let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }");
let mut frame = Frame::new(&func);
let ssa_value_refs = [
ValueRef::from_u32(2),
ValueRef::from_u32(4),
ValueRef::from_u32(6),
];
let values = vec![
DataValue::B(true),
DataValue::I8(42),
DataValue::F32(Ieee32::from(0.42)),
];
// We can assign and retrieve multiple (cloned) values.
frame.set_all(&ssa_value_refs, values.clone());
let retrieved_values = frame.get_all(&ssa_value_refs);
assert_eq!(values, retrieved_values);
// But if we attempt to retrieve an invalid value we should get an error:
frame.get_all(&[ValueRef::from_u32(2), ValueRef::from_u32(5)]);
}
#[test]
#[should_panic(expected = "empty slot: v10")]
fn rename() {
let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }");
let mut frame = Frame::new(&func);
let old_ssa_value_refs = [ValueRef::from_u32(9), ValueRef::from_u32(10)];
let values = vec![DataValue::B(true), DataValue::F64(Ieee64::from(0.0))];
frame.set_all(&old_ssa_value_refs, values.clone());
// Rename the old SSA values to the new values.
let new_ssa_value_refs = [ValueRef::from_u32(4), ValueRef::from_u32(2)];
frame.rename(&old_ssa_value_refs, &new_ssa_value_refs);
// Now we should be able to retrieve new values and the old ones should fail.
assert_eq!(frame.get_all(&new_ssa_value_refs), values);
frame.get(ValueRef::from_u32(10));
}
#[test]
#[should_panic(expected = "empty slot: v2")]
fn rename_duplicates_causes_inconsistency() {
let func = function("function %test(i32) -> i32 { block0(v10:i32): return v10 }");
let mut frame = Frame::new(&func);
let old_ssa_value_refs = [ValueRef::from_u32(1), ValueRef::from_u32(9)];
let values = vec![DataValue::B(true), DataValue::F64(Ieee64::from(f64::NAN))];
frame.set_all(&old_ssa_value_refs, values.clone());
// Rename the old SSA values to the new values.
let old_duplicated_ssa_value_refs = [ValueRef::from_u32(1), ValueRef::from_u32(1)];
let new_ssa_value_refs = [ValueRef::from_u32(4), ValueRef::from_u32(2)];
frame.rename(&old_duplicated_ssa_value_refs, &new_ssa_value_refs);
// If we use duplicates then subsequent renamings (v1 -> v2) will be empty.
frame.get(ValueRef::from_u32(2));
} }
} }

58
cranelift/interpreter/src/interpreter.rs
View File

@@ -2,7 +2,7 @@
//! //!
//! This module partially contains the logic for interpreting Cranelift IR. //! This module partially contains the logic for interpreting Cranelift IR.
use crate::environment::FunctionStore; use crate::environment::{FuncIndex, FunctionStore};
use crate::frame::Frame; use crate::frame::Frame;
use crate::instruction::DfgInstructionContext; use crate::instruction::DfgInstructionContext;
use crate::state::{MemoryError, State}; use crate::state::{MemoryError, State};
@@ -34,22 +34,22 @@ impl<'a> Interpreter<'a> {
func_name: &str, func_name: &str,
arguments: &[DataValue], arguments: &[DataValue],
) -> Result<ControlFlow<'a, DataValue>, InterpreterError> { ) -> Result<ControlFlow<'a, DataValue>, InterpreterError> {
let func_ref = self let index = self
.state .state
.functions .functions
.index_of(func_name) .index_of(func_name)
.ok_or_else(|| InterpreterError::UnknownFunctionName(func_name.to_string()))?; .ok_or_else(|| InterpreterError::UnknownFunctionName(func_name.to_string()))?;
self.call_by_index(func_ref, arguments) self.call_by_index(index, arguments)
} }
/// Call a function by its index in the [FunctionStore]; this is a proxy for [Interpreter::call]. /// Call a function by its index in the [FunctionStore]; this is a proxy for [Interpreter::call].
pub fn call_by_index( pub fn call_by_index(
&mut self, &mut self,
func_ref: FuncRef, index: FuncIndex,
arguments: &[DataValue], arguments: &[DataValue],
) -> Result<ControlFlow<'a, DataValue>, InterpreterError> { ) -> Result<ControlFlow<'a, DataValue>, InterpreterError> {
match self.state.get_function(func_ref) { match self.state.functions.get_by_index(index) {
None => Err(InterpreterError::UnknownFunctionReference(func_ref)), None => Err(InterpreterError::UnknownFunctionIndex(index)),
Some(func) => self.call(func, arguments), Some(func) => self.call(func, arguments),
} }
} }
@@ -98,8 +98,9 @@ impl<'a> Interpreter<'a> {
.set_all(function.dfg.block_params(block), block_arguments.to_vec()); .set_all(function.dfg.block_params(block), block_arguments.to_vec());
maybe_inst = layout.first_inst(block) maybe_inst = layout.first_inst(block)
} }
ControlFlow::Call(function, arguments) => { ControlFlow::Call(called_function, arguments) => {
let returned_arguments = self.call(function, &arguments)?.unwrap_return(); let returned_arguments =
self.call(called_function, &arguments)?.unwrap_return();
self.state self.state
.current_frame_mut() .current_frame_mut()
.set_all(function.dfg.inst_results(inst), returned_arguments); .set_all(function.dfg.inst_results(inst), returned_arguments);
@@ -123,8 +124,8 @@ pub enum InterpreterError {
StepError(#[from] StepError), StepError(#[from] StepError),
#[error("reached an unreachable statement")] #[error("reached an unreachable statement")]
Unreachable, Unreachable,
#[error("unknown function reference (has it been added to the function store?): {0}")] #[error("unknown function index (has it been added to the function store?): {0}")]
UnknownFunctionReference(FuncRef), UnknownFunctionIndex(FuncIndex),
#[error("unknown function with name (has it been added to the function store?): {0}")] #[error("unknown function with name (has it been added to the function store?): {0}")]
UnknownFunctionName(String), UnknownFunctionName(String),
#[error("value error")] #[error("value error")]
@@ -176,7 +177,8 @@ impl<'a> InterpreterState<'a> {
impl<'a> State<'a, DataValue> for InterpreterState<'a> { impl<'a> State<'a, DataValue> for InterpreterState<'a> {
fn get_function(&self, func_ref: FuncRef) -> Option<&'a Function> { fn get_function(&self, func_ref: FuncRef) -> Option<&'a Function> {
self.functions.get_by_func_ref(func_ref) self.functions
.get_from_func_ref(func_ref, self.frame_stack.last().unwrap().function)
} }
fn push_frame(&mut self, function: &'a Function) { fn push_frame(&mut self, function: &'a Function) {
self.frame_stack.push(Frame::new(function)); self.frame_stack.push(Frame::new(function));
@@ -273,6 +275,40 @@ mod tests {
assert_eq!(result, vec![DataValue::B(true)]) assert_eq!(result, vec![DataValue::B(true)])
} }
// This test verifies that functions can refer to each other using the function store. A double indirection is
// required, which is tricky to get right: a referenced function is a FuncRef when called but a FuncIndex inside the
// function store. This test would preferably be a CLIF filetest but the filetest infrastructure only looks at a
// single function at a time--we need more than one function in the store for this test.
#[test]
fn function_references() {
let code = "
function %child(i32) -> i32 {
block0(v0: i32):
v1 = iadd_imm v0, -1
return v1
}
function %parent(i32) -> i32 {
fn42 = %child(i32) -> i32
block0(v0: i32):
v1 = iadd_imm v0, 1
v2 = call fn42(v1)
return v2
}";
let mut env = FunctionStore::default();
let funcs = parse_functions(code).unwrap().to_vec();
funcs.iter().for_each(|f| env.add(f.name.to_string(), f));
let state = InterpreterState::default().with_function_store(env);
let result = Interpreter::new(state)
.call_by_name("%parent", &[DataValue::I32(0)])
.unwrap()
.unwrap_return();
assert_eq!(result, vec![DataValue::I32(0)])
}
#[test] #[test]
fn state_heap_roundtrip() -> Result<(), MemoryError> { fn state_heap_roundtrip() -> Result<(), MemoryError> {
let mut state = InterpreterState::default(); let mut state = InterpreterState::default();

7
cranelift/simplejit/Cargo.toml → cranelift/jit/Cargo.toml
View File

@@ -1,10 +1,10 @@
[package] [package]
name = "cranelift-simplejit" name = "cranelift-jit"
version = "0.68.0" version = "0.68.0"
authors = ["The Cranelift Project Developers"] authors = ["The Cranelift Project Developers"]
description = "A simple JIT library backed by Cranelift" description = "A JIT library backed by Cranelift"
repository = "https://github.com/bytecodealliance/wasmtime" repository = "https://github.com/bytecodealliance/wasmtime"
documentation = "https://docs.rs/cranelift-simplejit" documentation = "https://docs.rs/cranelift-jit"
license = "Apache-2.0 WITH LLVM-exception" license = "Apache-2.0 WITH LLVM-exception"
readme = "README.md" readme = "README.md"
edition = "2018" edition = "2018"
@@ -14,6 +14,7 @@ cranelift-module = { path = "../module", version = "0.68.0" }
cranelift-native = { path = "../native", version = "0.68.0" } cranelift-native = { path = "../native", version = "0.68.0" }
cranelift-codegen = { path = "../codegen", version = "0.68.0", default-features = false, features = ["std"] } cranelift-codegen = { path = "../codegen", version = "0.68.0", default-features = false, features = ["std"] }
cranelift-entity = { path = "../entity", version = "0.68.0" } cranelift-entity = { path = "../entity", version = "0.68.0" }
anyhow = "1.0"
region = "2.2.0" region = "2.2.0"
libc = { version = "0.2.42" } libc = { version = "0.2.42" }
errno = "0.2.4" errno = "0.2.4"

0
cranelift/simplejit/LICENSE → cranelift/jit/LICENSE
View File

4
cranelift/simplejit/README.md → cranelift/jit/README.md
View File

@@ -1,8 +1,8 @@
This crate provides a simple JIT library that uses This crate provides a JIT library that uses
[Cranelift](https://crates.io/crates/cranelift). [Cranelift](https://crates.io/crates/cranelift).
This crate is extremely experimental. This crate is extremely experimental.
See the [example program] for a brief overview of how to use this. See the [example program] for a brief overview of how to use this.
[example program]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/simplejit/examples/simplejit-minimal.rs [example program]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/jit/examples/jit-minimal.rs

15
cranelift/simplejit/examples/simplejit-minimal.rs → cranelift/jit/examples/jit-minimal.rs
View File

@@ -1,12 +1,21 @@
use cranelift::prelude::*; use cranelift::prelude::*;
use cranelift_codegen::binemit::NullTrapSink; use cranelift_codegen::binemit::NullTrapSink;
use cranelift_codegen::settings::{self, Configurable};
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{default_libcall_names, Linkage, Module}; use cranelift_module::{default_libcall_names, Linkage, Module};
use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule};
use std::mem; use std::mem;
fn main() { fn main() {
let mut module: SimpleJITModule = let mut flag_builder = settings::builder();
SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); flag_builder.set("use_colocated_libcalls", "false").unwrap();
// FIXME set back to true once the x64 backend supports it.
flag_builder.set("is_pic", "false").unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {}", msg);
});
let isa = isa_builder.finish(settings::Flags::new(flag_builder));
let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names()));
let mut ctx = module.make_context(); let mut ctx = module.make_context();
let mut func_ctx = FunctionBuilderContext::new(); let mut func_ctx = FunctionBuilderContext::new();

378
cranelift/simplejit/src/backend.rs → cranelift/jit/src/backend.rs
View File

@@ -1,4 +1,4 @@
//! Defines `SimpleJITModule`. //! Defines `JITModule`.
use crate::{compiled_blob::CompiledBlob, memory::Memory}; use crate::{compiled_blob::CompiledBlob, memory::Memory};
use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::isa::TargetIsa;
@@ -18,10 +18,11 @@ use cranelift_native;
use libc; use libc;
use log::info; use log::info;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::TryInto; use std::convert::{TryFrom, TryInto};
use std::ffi::CString; use std::ffi::CString;
use std::io::Write; use std::io::Write;
use std::ptr; use std::ptr;
use std::ptr::NonNull;
use target_lexicon::PointerWidth; use target_lexicon::PointerWidth;
#[cfg(windows)] #[cfg(windows)]
use winapi; use winapi;
@@ -30,15 +31,16 @@ const EXECUTABLE_DATA_ALIGNMENT: u64 = 0x10;
const WRITABLE_DATA_ALIGNMENT: u64 = 0x8; const WRITABLE_DATA_ALIGNMENT: u64 = 0x8;
const READONLY_DATA_ALIGNMENT: u64 = 0x1; const READONLY_DATA_ALIGNMENT: u64 = 0x1;
/// A builder for `SimpleJITModule`. /// A builder for `JITModule`.
pub struct SimpleJITBuilder { pub struct JITBuilder {
isa: Box<dyn TargetIsa>, isa: Box<dyn TargetIsa>,
symbols: HashMap<String, *const u8>, symbols: HashMap<String, *const u8>,
libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>, libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
hotswap_enabled: bool,
} }
impl SimpleJITBuilder { impl JITBuilder {
/// Create a new `SimpleJITBuilder`. /// Create a new `JITBuilder`.
/// ///
/// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall` /// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall`
/// enum to symbols. LibCalls are inserted in the IR as part of the legalization for certain /// enum to symbols. LibCalls are inserted in the IR as part of the legalization for certain
@@ -50,6 +52,7 @@ impl SimpleJITBuilder {
// which might not reach all definitions; we can't handle that here, so // which might not reach all definitions; we can't handle that here, so
// we require long-range relocation types. // we require long-range relocation types.
flag_builder.set("use_colocated_libcalls", "false").unwrap(); flag_builder.set("use_colocated_libcalls", "false").unwrap();
flag_builder.set("is_pic", "true").unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {}", msg); panic!("host machine is not supported: {}", msg);
}); });
@@ -57,12 +60,10 @@ impl SimpleJITBuilder {
Self::with_isa(isa, libcall_names) Self::with_isa(isa, libcall_names)
} }
/// Create a new `SimpleJITBuilder` with an arbitrary target. This is mainly /// Create a new `JITBuilder` with an arbitrary target. This is mainly
/// useful for testing. /// useful for testing.
/// ///
/// SimpleJIT requires a `TargetIsa` configured for non-PIC. /// To create a `JITBuilder` for native use, use the `new` constructor
///
/// To create a `SimpleJITBuilder` for native use, use the `new` constructor
/// instead. /// instead.
/// ///
/// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall` /// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall`
@@ -73,12 +74,12 @@ impl SimpleJITBuilder {
isa: Box<dyn TargetIsa>, isa: Box<dyn TargetIsa>,
libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>, libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
) -> Self { ) -> Self {
debug_assert!(!isa.flags().is_pic(), "SimpleJIT requires non-PIC code");
let symbols = HashMap::new(); let symbols = HashMap::new();
Self { Self {
isa, isa,
symbols, symbols,
libcall_names, libcall_names,
hotswap_enabled: false,
} }
} }
@@ -117,18 +118,33 @@ impl SimpleJITBuilder {
} }
self self
} }
/// Enable or disable hotswap support. See [`JITModule::prepare_for_function_redefine`]
/// for more information.
///
/// Enabling hotswap support requires PIC code.
pub fn hotswap(&mut self, enabled: bool) -> &mut Self {
self.hotswap_enabled = enabled;
self
}
} }
/// A `SimpleJITModule` implements `Module` and emits code and data into memory where it can be /// A `JITModule` implements `Module` and emits code and data into memory where it can be
/// directly called and accessed. /// directly called and accessed.
/// ///
/// See the `SimpleJITBuilder` for a convenient way to construct `SimpleJITModule` instances. /// See the `JITBuilder` for a convenient way to construct `JITModule` instances.
pub struct SimpleJITModule { pub struct JITModule {
isa: Box<dyn TargetIsa>, isa: Box<dyn TargetIsa>,
hotswap_enabled: bool,
symbols: HashMap<String, *const u8>, symbols: HashMap<String, *const u8>,
libcall_names: Box<dyn Fn(ir::LibCall) -> String>, libcall_names: Box<dyn Fn(ir::LibCall) -> String>,
memory: MemoryHandle, memory: MemoryHandle,
declarations: ModuleDeclarations, declarations: ModuleDeclarations,
function_got_entries: SecondaryMap<FuncId, Option<NonNull<*const u8>>>,
function_plt_entries: SecondaryMap<FuncId, Option<NonNull<[u8; 16]>>>,
data_object_got_entries: SecondaryMap<DataId, Option<NonNull<*const u8>>>,
libcall_got_entries: HashMap<ir::LibCall, NonNull<*const u8>>,
libcall_plt_entries: HashMap<ir::LibCall, NonNull<[u8; 16]>>,
compiled_functions: SecondaryMap<FuncId, Option<CompiledBlob>>, compiled_functions: SecondaryMap<FuncId, Option<CompiledBlob>>,
compiled_data_objects: SecondaryMap<DataId, Option<CompiledBlob>>, compiled_data_objects: SecondaryMap<DataId, Option<CompiledBlob>>,
functions_to_finalize: Vec<FuncId>, functions_to_finalize: Vec<FuncId>,
@@ -142,7 +158,7 @@ struct MemoryHandle {
writable: Memory, writable: Memory,
} }
impl SimpleJITModule { impl JITModule {
/// Free memory allocated for code and data segments of compiled functions. /// Free memory allocated for code and data segments of compiled functions.
/// ///
/// # Safety /// # Safety
@@ -164,10 +180,28 @@ impl SimpleJITModule {
.or_else(|| lookup_with_dlsym(name)) .or_else(|| lookup_with_dlsym(name))
} }
fn get_definition(&self, name: &ir::ExternalName) -> *const u8 { unsafe fn write_plt_entry_bytes(plt_ptr: *mut [u8; 16], got_ptr: *mut *const u8) {
assert!(
cfg!(target_arch = "x86_64"),
"PLT is currently only supported on x86_64"
);
// jmp *got_ptr; ud2; ud2; ud2; ud2; ud2
let mut plt_val = [
0xff, 0x25, 0, 0, 0, 0, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b, 0x0f, 0x0b,
];
let what = got_ptr as isize - 4;
let at = plt_ptr as isize + 2;
plt_val[2..6].copy_from_slice(&i32::to_ne_bytes(i32::try_from(what - at).unwrap()));
std::ptr::write(plt_ptr, plt_val);
}
fn get_address(&self, name: &ir::ExternalName) -> *const u8 {
match *name { match *name {
ir::ExternalName::User { .. } => { ir::ExternalName::User { .. } => {
let (name, linkage) = if ModuleDeclarations::is_function(name) { let (name, linkage) = if ModuleDeclarations::is_function(name) {
if self.hotswap_enabled {
return self.get_plt_address(name);
} else {
let func_id = FuncId::from_name(name); let func_id = FuncId::from_name(name);
match &self.compiled_functions[func_id] { match &self.compiled_functions[func_id] {
Some(compiled) => return compiled.ptr, Some(compiled) => return compiled.ptr,
@@ -176,6 +210,7 @@ impl SimpleJITModule {
(&decl.name, decl.linkage) (&decl.name, decl.linkage)
} }
} }
}
} else { } else {
let data_id = DataId::from_name(name); let data_id = DataId::from_name(name);
match &self.compiled_data_objects[data_id] { match &self.compiled_data_objects[data_id] {
@@ -203,10 +238,60 @@ impl SimpleJITModule {
} }
} }
fn get_got_address(&self, name: &ir::ExternalName) -> *const u8 {
match *name {
ir::ExternalName::User { .. } => {
if ModuleDeclarations::is_function(name) {
let func_id = FuncId::from_name(name);
self.function_got_entries[func_id]
.unwrap()
.as_ptr()
.cast::<u8>()
} else {
let data_id = DataId::from_name(name);
self.data_object_got_entries[data_id]
.unwrap()
.as_ptr()
.cast::<u8>()
}
}
ir::ExternalName::LibCall(ref libcall) => self
.libcall_got_entries
.get(libcall)
.unwrap_or_else(|| panic!("can't resolve libcall {}", libcall))
.as_ptr()
.cast::<u8>(),
_ => panic!("invalid ExternalName {}", name),
}
}
fn get_plt_address(&self, name: &ir::ExternalName) -> *const u8 {
match *name {
ir::ExternalName::User { .. } => {
if ModuleDeclarations::is_function(name) {
let func_id = FuncId::from_name(name);
self.function_plt_entries[func_id]
.unwrap()
.as_ptr()
.cast::<u8>()
} else {
unreachable!("PLT relocations can only have functions as target");
}
}
ir::ExternalName::LibCall(ref libcall) => self
.libcall_plt_entries
.get(libcall)
.unwrap_or_else(|| panic!("can't resolve libcall {}", libcall))
.as_ptr()
.cast::<u8>(),
_ => panic!("invalid ExternalName {}", name),
}
}
/// Returns the address of a finalized function. /// Returns the address of a finalized function.
pub fn get_finalized_function(&self, func_id: FuncId) -> *const u8 { pub fn get_finalized_function(&self, func_id: FuncId) -> *const u8 {
let info = &self.compiled_functions[func_id]; let info = &self.compiled_functions[func_id];
debug_assert!( assert!(
!self.functions_to_finalize.iter().any(|x| *x == func_id), !self.functions_to_finalize.iter().any(|x| *x == func_id),
"function not yet finalized" "function not yet finalized"
); );
@@ -218,7 +303,7 @@ impl SimpleJITModule {
/// Returns the address and size of a finalized data object. /// Returns the address and size of a finalized data object.
pub fn get_finalized_data(&self, data_id: DataId) -> (*const u8, usize) { pub fn get_finalized_data(&self, data_id: DataId) -> (*const u8, usize) {
let info = &self.compiled_data_objects[data_id]; let info = &self.compiled_data_objects[data_id];
debug_assert!( assert!(
!self.data_objects_to_finalize.iter().any(|x| *x == data_id), !self.data_objects_to_finalize.iter().any(|x| *x == data_id),
"data object not yet finalized" "data object not yet finalized"
); );
@@ -255,19 +340,28 @@ impl SimpleJITModule {
pub fn finalize_definitions(&mut self) { pub fn finalize_definitions(&mut self) {
for func in std::mem::take(&mut self.functions_to_finalize) { for func in std::mem::take(&mut self.functions_to_finalize) {
let decl = self.declarations.get_function_decl(func); let decl = self.declarations.get_function_decl(func);
debug_assert!(decl.linkage.is_definable()); assert!(decl.linkage.is_definable());
let func = self.compiled_functions[func] let func = self.compiled_functions[func]
.as_ref() .as_ref()
.expect("function must be compiled before it can be finalized"); .expect("function must be compiled before it can be finalized");
func.perform_relocations(|name| self.get_definition(name)); func.perform_relocations(
|name| self.get_address(name),
|name| self.get_got_address(name),
|name| self.get_plt_address(name),
);
} }
for data in std::mem::take(&mut self.data_objects_to_finalize) { for data in std::mem::take(&mut self.data_objects_to_finalize) {
let decl = self.declarations.get_data_decl(data); let decl = self.declarations.get_data_decl(data);
debug_assert!(decl.linkage.is_definable()); assert!(decl.linkage.is_definable());
let data = self.compiled_data_objects[data] let data = self.compiled_data_objects[data]
.as_ref() .as_ref()
.expect("data object must be compiled before it can be finalized"); .expect("data object must be compiled before it can be finalized");
data.perform_relocations(|name| self.get_definition(name)); data.perform_relocations(
|name| self.get_address(name),
|name| self.get_got_address(name),
|name| self.get_plt_address(name),
);
} }
// Now that we're done patching, prepare the memory for execution! // Now that we're done patching, prepare the memory for execution!
@@ -275,29 +369,111 @@ impl SimpleJITModule {
self.memory.code.set_readable_and_executable(); self.memory.code.set_readable_and_executable();
} }
/// Create a new `SimpleJITModule`. /// Create a new `JITModule`.
pub fn new(builder: SimpleJITBuilder) -> Self { pub fn new(builder: JITBuilder) -> Self {
let memory = MemoryHandle { if builder.hotswap_enabled {
assert!(
builder.isa.flags().is_pic(),
"Hotswapping requires PIC code"
);
}
let mut memory = MemoryHandle {
code: Memory::new(), code: Memory::new(),
readonly: Memory::new(), readonly: Memory::new(),
writable: Memory::new(), writable: Memory::new(),
}; };
let mut libcall_got_entries = HashMap::new();
let mut libcall_plt_entries = HashMap::new();
// Pre-create a GOT and PLT entry for each libcall.
let all_libcalls = if builder.isa.flags().is_pic() {
ir::LibCall::all_libcalls()
} else {
&[] // Not PIC, so no GOT and PLT entries necessary
};
for &libcall in all_libcalls {
let got_entry = memory
.writable
.allocate(
std::mem::size_of::<*const u8>(),
std::mem::align_of::<*const u8>().try_into().unwrap(),
)
.unwrap()
.cast::<*const u8>();
libcall_got_entries.insert(libcall, NonNull::new(got_entry).unwrap());
let sym = (builder.libcall_names)(libcall);
let addr = if let Some(addr) = builder
.symbols
.get(&sym)
.copied()
.or_else(|| lookup_with_dlsym(&sym))
{
addr
} else {
continue;
};
unsafe {
std::ptr::write(got_entry, addr);
}
let plt_entry = memory
.code
.allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT)
.unwrap()
.cast::<[u8; 16]>();
libcall_plt_entries.insert(libcall, NonNull::new(plt_entry).unwrap());
unsafe {
Self::write_plt_entry_bytes(plt_entry, got_entry);
}
}
Self { Self {
isa: builder.isa, isa: builder.isa,
hotswap_enabled: builder.hotswap_enabled,
symbols: builder.symbols, symbols: builder.symbols,
libcall_names: builder.libcall_names, libcall_names: builder.libcall_names,
memory, memory,
declarations: ModuleDeclarations::default(), declarations: ModuleDeclarations::default(),
function_got_entries: SecondaryMap::new(),
function_plt_entries: SecondaryMap::new(),
data_object_got_entries: SecondaryMap::new(),
libcall_got_entries,
libcall_plt_entries,
compiled_functions: SecondaryMap::new(), compiled_functions: SecondaryMap::new(),
compiled_data_objects: SecondaryMap::new(), compiled_data_objects: SecondaryMap::new(),
functions_to_finalize: Vec::new(), functions_to_finalize: Vec::new(),
data_objects_to_finalize: Vec::new(), data_objects_to_finalize: Vec::new(),
} }
} }
/// Allow a single future `define_function` on a previously defined function. This allows for
/// hot code swapping and lazy compilation of functions.
///
/// This requires hotswap support to be enabled first using [`JITBuilder::hotswap`].
pub fn prepare_for_function_redefine(&mut self, func_id: FuncId) -> ModuleResult<()> {
assert!(self.hotswap_enabled, "Hotswap support is not enabled");
let decl = self.declarations.get_function_decl(func_id);
if !decl.linkage.is_definable() {
return Err(ModuleError::InvalidImportDefinition(decl.name.clone()));
}
if self.compiled_functions[func_id].is_none() {
return Err(ModuleError::Backend(anyhow::anyhow!(
"Tried to redefine not yet defined function {}",
decl.name
)));
}
self.compiled_functions[func_id] = None;
// FIXME return some kind of handle that allows for deallocating the function
Ok(())
}
} }
impl<'simple_jit_backend> Module for SimpleJITModule { impl Module for JITModule {
fn isa(&self) -> &dyn TargetIsa { fn isa(&self) -> &dyn TargetIsa {
&*self.isa &*self.isa
} }
@@ -315,6 +491,38 @@ impl<'simple_jit_backend> Module for SimpleJITModule {
let (id, _decl) = self let (id, _decl) = self
.declarations .declarations
.declare_function(name, linkage, signature)?; .declare_function(name, linkage, signature)?;
if self.function_got_entries[id].is_none() && self.isa.flags().is_pic() {
let got_entry = self
.memory
.writable
.allocate(
std::mem::size_of::<*const u8>(),
std::mem::align_of::<*const u8>().try_into().unwrap(),
)
.unwrap()
.cast::<*const u8>();
self.function_got_entries[id] = Some(NonNull::new(got_entry).unwrap());
// FIXME populate got entries with a null pointer when defined
let val = self.lookup_symbol(name).unwrap_or(std::ptr::null());
unsafe {
std::ptr::write(got_entry, val);
}
let plt_entry = self
.memory
.code
.allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT)
.unwrap()
.cast::<[u8; 16]>();
self.record_function_for_perf(
plt_entry as *mut _,
std::mem::size_of::<[u8; 16]>(),
&format!("{}@plt", name),
);
self.function_plt_entries[id] = Some(NonNull::new(plt_entry).unwrap());
unsafe {
Self::write_plt_entry_bytes(plt_entry, got_entry);
}
}
Ok(id) Ok(id)
} }
@@ -325,13 +533,69 @@ impl<'simple_jit_backend> Module for SimpleJITModule {
writable: bool, writable: bool,
tls: bool, tls: bool,
) -> ModuleResult<DataId> { ) -> ModuleResult<DataId> {
assert!(!tls, "SimpleJIT doesn't yet support TLS"); assert!(!tls, "JIT doesn't yet support TLS");
let (id, _decl) = self let (id, _decl) = self
.declarations .declarations
.declare_data(name, linkage, writable, tls)?; .declare_data(name, linkage, writable, tls)?;
if self.data_object_got_entries[id].is_none() && self.isa.flags().is_pic() {
let got_entry = self
.memory
.writable
.allocate(
std::mem::size_of::<*const u8>(),
std::mem::align_of::<*const u8>().try_into().unwrap(),
)
.unwrap()
.cast::<*const u8>();
self.data_object_got_entries[id] = Some(NonNull::new(got_entry).unwrap());
// FIXME populate got entries with a null pointer when defined
let val = self.lookup_symbol(name).unwrap_or(std::ptr::null());
unsafe {
std::ptr::write(got_entry, val);
}
}
Ok(id) Ok(id)
} }
/// Use this when you're building the IR of a function to reference a function.
///
/// TODO: Coalesce redundant decls and signatures.
/// TODO: Look into ways to reduce the risk of using a FuncRef in the wrong function.
fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef {
let decl = self.declarations.get_function_decl(func);
let signature = in_func.import_signature(decl.signature.clone());
let colocated = !self.hotswap_enabled && decl.linkage.is_final();
in_func.import_function(ir::ExtFuncData {
name: ir::ExternalName::user(0, func.as_u32()),
signature,
colocated,
})
}
/// Use this when you're building the IR of a function to reference a data object.
///
/// TODO: Same as above.
fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue {
let decl = self.declarations.get_data_decl(data);
let colocated = !self.hotswap_enabled && decl.linkage.is_final();
func.create_global_value(ir::GlobalValueData::Symbol {
name: ir::ExternalName::user(1, data.as_u32()),
offset: ir::immediates::Imm64::new(0),
colocated,
tls: decl.tls,
})
}
/// TODO: Same as above.
fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef {
ctx.import_function(ir::ExternalName::user(0, func.as_u32()))
}
/// TODO: Same as above.
fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue {
ctx.import_global_value(ir::ExternalName::user(1, data.as_u32()))
}
fn define_function<TS>( fn define_function<TS>(
&mut self, &mut self,
id: FuncId, id: FuncId,
@@ -363,7 +627,7 @@ impl<'simple_jit_backend> Module for SimpleJITModule {
.allocate(size, EXECUTABLE_DATA_ALIGNMENT) .allocate(size, EXECUTABLE_DATA_ALIGNMENT)
.expect("TODO: handle OOM etc."); .expect("TODO: handle OOM etc.");
let mut reloc_sink = SimpleJITRelocSink::default(); let mut reloc_sink = JITRelocSink::default();
let mut stack_map_sink = binemit::NullStackMapSink {}; let mut stack_map_sink = binemit::NullStackMapSink {};
unsafe { unsafe {
ctx.emit_to_memory( ctx.emit_to_memory(
@@ -381,7 +645,36 @@ impl<'simple_jit_backend> Module for SimpleJITModule {
size, size,
relocs: reloc_sink.relocs, relocs: reloc_sink.relocs,
}); });
if self.isa.flags().is_pic() {
unsafe {
std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr);
}
}
if self.hotswap_enabled {
self.compiled_functions[id]
.as_ref()
.unwrap()
.perform_relocations(
|name| match *name {
ir::ExternalName::User { .. } => {
unreachable!("non GOT or PLT relocation in function {} to {}", id, name)
}
ir::ExternalName::LibCall(ref libcall) => self
.libcall_plt_entries
.get(libcall)
.unwrap_or_else(|| panic!("can't resolve libcall {}", libcall))
.as_ptr()
.cast::<u8>(),
_ => panic!("invalid ExternalName {}", name),
},
|name| self.get_got_address(name),
|name| self.get_plt_address(name),
);
} else {
self.functions_to_finalize.push(id); self.functions_to_finalize.push(id);
}
Ok(ModuleCompiledFunction { size: code_size }) Ok(ModuleCompiledFunction { size: code_size })
} }
@@ -424,7 +717,25 @@ impl<'simple_jit_backend> Module for SimpleJITModule {
size, size,
relocs: relocs.to_vec(), relocs: relocs.to_vec(),
}); });
if self.isa.flags().is_pic() {
unsafe {
std::ptr::write(self.function_got_entries[id].unwrap().as_ptr(), ptr);
}
}
if self.hotswap_enabled {
self.compiled_functions[id]
.as_ref()
.unwrap()
.perform_relocations(
|name| unreachable!("non GOT or PLT relocation in function {} to {}", id, name),
|name| self.get_got_address(name),
|name| self.get_plt_address(name),
);
} else {
self.functions_to_finalize.push(id); self.functions_to_finalize.push(id);
}
Ok(ModuleCompiledFunction { size: total_size }) Ok(ModuleCompiledFunction { size: total_size })
} }
@@ -439,7 +750,7 @@ impl<'simple_jit_backend> Module for SimpleJITModule {
return Err(ModuleError::DuplicateDefinition(decl.name.to_owned())); return Err(ModuleError::DuplicateDefinition(decl.name.to_owned()));
} }
assert!(!decl.tls, "SimpleJIT doesn't yet support TLS"); assert!(!decl.tls, "JIT doesn't yet support TLS");
let &DataDescription { let &DataDescription {
ref init, ref init,
@@ -489,6 +800,11 @@ impl<'simple_jit_backend> Module for SimpleJITModule {
self.compiled_data_objects[id] = Some(CompiledBlob { ptr, size, relocs }); self.compiled_data_objects[id] = Some(CompiledBlob { ptr, size, relocs });
self.data_objects_to_finalize.push(id); self.data_objects_to_finalize.push(id);
if self.isa.flags().is_pic() {
unsafe {
std::ptr::write(self.data_object_got_entries[id].unwrap().as_ptr(), ptr);
}
}
Ok(()) Ok(())
} }
@@ -534,11 +850,11 @@ fn lookup_with_dlsym(name: &str) -> Option<*const u8> {
} }
#[derive(Default)] #[derive(Default)]
struct SimpleJITRelocSink { struct JITRelocSink {
relocs: Vec<RelocRecord>, relocs: Vec<RelocRecord>,
} }
impl RelocSink for SimpleJITRelocSink { impl RelocSink for JITRelocSink {
fn reloc_external( fn reloc_external(
&mut self, &mut self,
offset: CodeOffset, offset: CodeOffset,

79
cranelift/jit/src/compiled_blob.rs Normal file
View File

@@ -0,0 +1,79 @@
use cranelift_codegen::binemit::Reloc;
use cranelift_codegen::ir::ExternalName;
use cranelift_module::RelocRecord;
use std::convert::TryFrom;
#[derive(Clone)]
pub(crate) struct CompiledBlob {
pub(crate) ptr: *mut u8,
pub(crate) size: usize,
pub(crate) relocs: Vec<RelocRecord>,
}
impl CompiledBlob {
pub(crate) fn perform_relocations(
&self,
get_address: impl Fn(&ExternalName) -> *const u8,
get_got_entry: impl Fn(&ExternalName) -> *const u8,
get_plt_entry: impl Fn(&ExternalName) -> *const u8,
) {
use std::ptr::write_unaligned;
for &RelocRecord {
reloc,
offset,
ref name,
addend,
} in &self.relocs
{
debug_assert!((offset as usize) < self.size);
let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) };
match reloc {
Reloc::Abs4 => {
let base = get_address(name);
let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut u32, u32::try_from(what as usize).unwrap())
};
}
Reloc::Abs8 => {
let base = get_address(name);
let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut u64, u64::try_from(what as usize).unwrap())
};
}
Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
let base = get_address(name);
let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut i32, pcrel)
};
}
Reloc::X86GOTPCRel4 => {
let base = get_got_entry(name);
let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut i32, pcrel)
};
}
Reloc::X86CallPLTRel4 => {
let base = get_plt_entry(name);
let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut i32, pcrel)
};
}
_ => unimplemented!(),
}
}
}
}

4
cranelift/simplejit/src/lib.rs → cranelift/jit/src/lib.rs
View File

@@ -1,4 +1,4 @@
//! Top-level lib.rs for `cranelift_simplejit`. //! Top-level lib.rs for `cranelift_jit`.
#![deny( #![deny(
missing_docs, missing_docs,
@@ -27,7 +27,7 @@ mod backend;
mod compiled_blob; mod compiled_blob;
mod memory; mod memory;
pub use crate::backend::{SimpleJITBuilder, SimpleJITModule}; pub use crate::backend::{JITBuilder, JITModule};
/// Version number of this crate. /// Version number of this crate.
pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const VERSION: &str = env!("CARGO_PKG_VERSION");

0
cranelift/simplejit/src/memory.rs → cranelift/jit/src/memory.rs
View File

39
cranelift/simplejit/tests/basic.rs → cranelift/jit/tests/basic.rs
View File

@@ -1,16 +1,25 @@
use cranelift_codegen::binemit::NullTrapSink; use cranelift_codegen::binemit::NullTrapSink;
use cranelift_codegen::ir::*; use cranelift_codegen::ir::*;
use cranelift_codegen::isa::CallConv; use cranelift_codegen::isa::CallConv;
use cranelift_codegen::settings::{self, Configurable};
use cranelift_codegen::{ir::types::I16, Context}; use cranelift_codegen::{ir::types::I16, Context};
use cranelift_entity::EntityRef; use cranelift_entity::EntityRef;
use cranelift_frontend::*; use cranelift_frontend::*;
use cranelift_jit::*;
use cranelift_module::*; use cranelift_module::*;
use cranelift_simplejit::*;
#[test] #[test]
fn error_on_incompatible_sig_in_declare_function() { fn error_on_incompatible_sig_in_declare_function() {
let mut module: SimpleJITModule = let mut flag_builder = settings::builder();
SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); flag_builder.set("use_colocated_libcalls", "false").unwrap();
// FIXME set back to true once the x64 backend supports it.
flag_builder.set("is_pic", "false").unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {}", msg);
});
let isa = isa_builder.finish(settings::Flags::new(flag_builder));
let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names()));
let mut sig = Signature { let mut sig = Signature {
params: vec![AbiParam::new(types::I64)], params: vec![AbiParam::new(types::I64)],
returns: vec![], returns: vec![],
@@ -26,7 +35,7 @@ fn error_on_incompatible_sig_in_declare_function() {
.unwrap(); // Make sure this is an error .unwrap(); // Make sure this is an error
} }
fn define_simple_function(module: &mut SimpleJITModule) -> FuncId { fn define_simple_function(module: &mut JITModule) -> FuncId {
let sig = Signature { let sig = Signature {
params: vec![], params: vec![],
returns: vec![], returns: vec![],
@@ -58,8 +67,15 @@ fn define_simple_function(module: &mut SimpleJITModule) -> FuncId {
#[test] #[test]
#[should_panic(expected = "Result::unwrap()` on an `Err` value: DuplicateDefinition(\"abc\")")] #[should_panic(expected = "Result::unwrap()` on an `Err` value: DuplicateDefinition(\"abc\")")]
fn panic_on_define_after_finalize() { fn panic_on_define_after_finalize() {
let mut module: SimpleJITModule = let mut flag_builder = settings::builder();
SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); flag_builder.set("use_colocated_libcalls", "false").unwrap();
// FIXME set back to true once the x64 backend supports it.
flag_builder.set("is_pic", "false").unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {}", msg);
});
let isa = isa_builder.finish(settings::Flags::new(flag_builder));
let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names()));
define_simple_function(&mut module); define_simple_function(&mut module);
define_simple_function(&mut module); define_simple_function(&mut module);
@@ -140,8 +156,15 @@ fn switch_error() {
#[test] #[test]
fn libcall_function() { fn libcall_function() {
let mut module: SimpleJITModule = let mut flag_builder = settings::builder();
SimpleJITModule::new(SimpleJITBuilder::new(default_libcall_names())); flag_builder.set("use_colocated_libcalls", "false").unwrap();
// FIXME set back to true once the x64 backend supports it.
flag_builder.set("is_pic", "false").unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {}", msg);
});
let isa = isa_builder.finish(settings::Flags::new(flag_builder));
let mut module = JITModule::new(JITBuilder::with_isa(isa, default_libcall_names()));
let sig = Signature { let sig = Signature {
params: vec![], params: vec![],

8
cranelift/module/README.md
View File

@@ -10,10 +10,8 @@ A module is a collection of functions and data objects that are linked
together. The `Module` trait that defines a common interface for various kinds together. The `Module` trait that defines a common interface for various kinds
of modules. Most users will use one of the following `Module` implementations: of modules. Most users will use one of the following `Module` implementations:
- `SimpleJITModule`, provided by [cranelift-simplejit], which JITs - `JITModule`, provided by [cranelift-jit], which JITs code to memory for direct execution.
code to memory for direct execution. - `ObjectModule`, provided by [cranelift-object], which emits native object files.
- `ObjectModule`, provided by [cranelift-object], which emits native
object files.
[cranelift-simplejit]: https://crates.io/crates/cranelift-simplejit [cranelift-jit]: https://crates.io/crates/cranelift-jit
[cranelift-object]: https://crates.io/crates/cranelift-object [cranelift-object]: https://crates.io/crates/cranelift-object

94
cranelift/module/src/module.rs
View File

@@ -490,3 +490,97 @@ pub trait Module {
/// Define a data object, producing the data contents from the given `DataContext`. /// Define a data object, producing the data contents from the given `DataContext`.
fn define_data(&mut self, data: DataId, data_ctx: &DataContext) -> ModuleResult<()>; fn define_data(&mut self, data: DataId, data_ctx: &DataContext) -> ModuleResult<()>;
} }
impl<M: Module> Module for &mut M {
fn isa(&self) -> &dyn isa::TargetIsa {
(**self).isa()
}
fn declarations(&self) -> &ModuleDeclarations {
(**self).declarations()
}
fn get_name(&self, name: &str) -> Option<FuncOrDataId> {
(**self).get_name(name)
}
fn target_config(&self) -> isa::TargetFrontendConfig {
(**self).target_config()
}
fn make_context(&self) -> Context {
(**self).make_context()
}
fn clear_context(&self, ctx: &mut Context) {
(**self).clear_context(ctx)
}
fn make_signature(&self) -> ir::Signature {
(**self).make_signature()
}
fn clear_signature(&self, sig: &mut ir::Signature) {
(**self).clear_signature(sig)
}
fn declare_function(
&mut self,
name: &str,
linkage: Linkage,
signature: &ir::Signature,
) -> ModuleResult<FuncId> {
(**self).declare_function(name, linkage, signature)
}
fn declare_data(
&mut self,
name: &str,
linkage: Linkage,
writable: bool,
tls: bool,
) -> ModuleResult<DataId> {
(**self).declare_data(name, linkage, writable, tls)
}
fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef {
(**self).declare_func_in_func(func, in_func)
}
fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue {
(**self).declare_data_in_func(data, func)
}
fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef {
(**self).declare_func_in_data(func, ctx)
}
fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue {
(**self).declare_data_in_data(data, ctx)
}
fn define_function<TS>(
&mut self,
func: FuncId,
ctx: &mut Context,
trap_sink: &mut TS,
) -> ModuleResult<ModuleCompiledFunction>
where
TS: binemit::TrapSink,
{
(**self).define_function(func, ctx, trap_sink)
}
fn define_function_bytes(
&mut self,
func: FuncId,
bytes: &[u8],
relocs: &[RelocRecord],
) -> ModuleResult<ModuleCompiledFunction> {
(**self).define_function_bytes(func, bytes, relocs)
}
fn define_data(&mut self, data: DataId, data_ctx: &DataContext) -> ModuleResult<()> {
(**self).define_data(data, data_ctx)
}
}

2
cranelift/peepmatic/Cargo.toml
View File

@@ -15,7 +15,7 @@ peepmatic-macro = { version = "0.68.0", path = "crates/macro" }
peepmatic-runtime = { version = "0.68.0", path = "crates/runtime", features = ["construct"] } peepmatic-runtime = { version = "0.68.0", path = "crates/runtime", features = ["construct"] }
peepmatic-traits = { version = "0.68.0", path = "crates/traits" } peepmatic-traits = { version = "0.68.0", path = "crates/traits" }
serde = { version = "1.0.105", features = ["derive"] } serde = { version = "1.0.105", features = ["derive"] }
wast = "27.0.0" wast = "29.0.0"
z3 = { version = "0.7.1", features = ["static-link-z3"] } z3 = { version = "0.7.1", features = ["static-link-z3"] }
[dev-dependencies] [dev-dependencies]

2
cranelift/peepmatic/crates/fuzzing/Cargo.toml
View File

@@ -21,4 +21,4 @@ peepmatic-test-operator = { path = "../test-operator" }
peepmatic-traits = { path = "../traits" } peepmatic-traits = { path = "../traits" }
rand = { version = "0.7.3", features = ["small_rng"] } rand = { version = "0.7.3", features = ["small_rng"] }
serde = "1.0.106" serde = "1.0.106"
wast = "27.0.0" wast = "29.0.0"

2
cranelift/peepmatic/crates/runtime/Cargo.toml
View File

@@ -16,7 +16,7 @@ peepmatic-automata = { version = "0.68.0", path = "../automata", features = ["se
peepmatic-traits = { version = "0.68.0", path = "../traits" } peepmatic-traits = { version = "0.68.0", path = "../traits" }
serde = { version = "1.0.105", features = ["derive"] } serde = { version = "1.0.105", features = ["derive"] }
thiserror = "1.0.15" thiserror = "1.0.15"
wast = { version = "27.0.0", optional = true } wast = { version = "29.0.0", optional = true }
[dev-dependencies] [dev-dependencies]
peepmatic-test-operator = { version = "0.68.0", path = "../test-operator" } peepmatic-test-operator = { version = "0.68.0", path = "../test-operator" }

4
cranelift/peepmatic/crates/souper/Cargo.toml
View File

@@ -10,10 +10,10 @@ description = "Converting Souper optimizations into Peepmatic DSL"
[dependencies] [dependencies]
anyhow = "1" anyhow = "1"
souper-ir = { version = "1", features = ["parse"] } souper-ir = { version = "2.1.0", features = ["parse"] }
log = "0.4.8" log = "0.4.8"
[dev-dependencies] [dev-dependencies]
peepmatic = { path = "../..", version = "0.68.0" } peepmatic = { path = "../..", version = "0.68.0" }
peepmatic-test-operator = { version = "0.68.0", path = "../test-operator" } peepmatic-test-operator = { version = "0.68.0", path = "../test-operator" }
wast = "27.0.0" wast = "29.0.0"

19
cranelift/peepmatic/crates/souper/src/lib.rs
View File

@@ -174,7 +174,6 @@ fn convert_operand(
} }
Some(format!("{}", convert_name(&assn.name))) Some(format!("{}", convert_name(&assn.name)))
} }
ast::AssignmentRhs::Constant(c) => Some(format!("{}", c.value)),
ast::AssignmentRhs::Instruction(inst) => match inst { ast::AssignmentRhs::Instruction(inst) => match inst {
// Unsupported instructions. // Unsupported instructions.
ast::Instruction::Bswap { .. } ast::Instruction::Bswap { .. }
@@ -619,8 +618,7 @@ mod tests {
" "
%0:i64 = var %0:i64 = var
%1:i32 = trunc %0 %1:i32 = trunc %0
%2:i32 = 0 cand %1 0
cand %1 %2
", ",
"\ "\
(=> (when (ireduce {i32} $v0) (=> (when (ireduce {i32} $v0)
@@ -631,8 +629,7 @@ mod tests {
" "
%0:i32 = var %0:i32 = var
%1:i64 = sext %0 %1:i64 = sext %0
%2:i64 = 0 cand %1 0
cand %1 %2
", ",
"\ "\
(=> (when (sextend {i64} $v0) (=> (when (sextend {i64} $v0)
@@ -643,8 +640,7 @@ mod tests {
" "
%0:i32 = var %0:i32 = var
%1:i64 = zext %0 %1:i64 = zext %0
%2:i64 = 0 cand %1 0
cand %1 %2
", ",
"\ "\
(=> (when (uextend {i64} $v0) (=> (when (uextend {i64} $v0)
@@ -677,8 +673,7 @@ mod tests {
%1:i32 = var %1:i32 = var
%2:i1 = eq %0, %1 %2:i1 = eq %0, %1
%3:i32 = zext %2 %3:i32 = zext %2
%4:i32 = 0 cand %3 0
cand %3 %4
", ",
"\ "\
(=> (when (bint (icmp eq $v0 $v1)) (=> (when (bint (icmp eq $v0 $v1))
@@ -693,8 +688,7 @@ mod tests {
" "
%0:i32 = var %0:i32 = var
%1:i32 = add %0, 1 %1:i32 = add %0, 1
%2:i32 = 0 cand %1 0
cand %1 %2
", ",
"\ "\
(=> (when (iadd_imm 1 $v0) (=> (when (iadd_imm 1 $v0)
@@ -705,8 +699,7 @@ mod tests {
" "
%0:i32 = var %0:i32 = var
%1:i32 = add 1, %0 %1:i32 = add 1, %0
%2:i32 = 0 cand %1 0
cand %1 %2
", ",
"\ "\
(=> (when (iadd_imm 1 $v0) (=> (when (iadd_imm 1 $v0)

2
cranelift/peepmatic/crates/test-operator/Cargo.toml
View File

@@ -9,4 +9,4 @@ edition = "2018"
[dependencies] [dependencies]
peepmatic-traits = { version = "0.68.0", path = "../traits" } peepmatic-traits = { version = "0.68.0", path = "../traits" }
serde = { version = "1.0.105", features = ["derive"] } serde = { version = "1.0.105", features = ["derive"] }
wast = "27.0.0" wast = "29.0.0"

55
cranelift/simplejit/src/compiled_blob.rs
View File

@@ -1,55 +0,0 @@
use cranelift_codegen::binemit::Reloc;
use cranelift_codegen::ir::ExternalName;
use cranelift_module::RelocRecord;
#[derive(Clone)]
pub(crate) struct CompiledBlob {
pub(crate) ptr: *mut u8,
pub(crate) size: usize,
pub(crate) relocs: Vec<RelocRecord>,
}
impl CompiledBlob {
pub(crate) fn perform_relocations(&self, get_definition: impl Fn(&ExternalName) -> *const u8) {
use std::ptr::write_unaligned;
for &RelocRecord {
reloc,
offset,
ref name,
addend,
} in &self.relocs
{
debug_assert!((offset as usize) < self.size);
let at = unsafe { self.ptr.offset(offset as isize) };
let base = get_definition(name);
// TODO: Handle overflow.
let what = unsafe { base.offset(addend as isize) };
match reloc {
Reloc::Abs4 => {
// TODO: Handle overflow.
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut u32, what as u32)
};
}
Reloc::Abs8 => {
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut u64, what as u64)
};
}
Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
// TODO: Handle overflow.
let pcrel = ((what as isize) - (at as isize)) as i32;
#[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
unsafe {
write_unaligned(at as *mut i32, pcrel)
};
}
Reloc::X86GOTPCRel4 | Reloc::X86CallPLTRel4 => panic!("unexpected PIC relocation"),
_ => unimplemented!(),
}
}
}
}

2
cranelift/wasm/Cargo.toml
View File

@@ -12,7 +12,7 @@ keywords = ["webassembly", "wasm"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
wasmparser = { version = "0.68.0", default-features = false } wasmparser = { version = "0.70", default-features = false }
cranelift-codegen = { path = "../codegen", version = "0.68.0", default-features = false } cranelift-codegen = { path = "../codegen", version = "0.68.0", default-features = false }
cranelift-entity = { path = "../entity", version = "0.68.0" } cranelift-entity = { path = "../entity", version = "0.68.0" }
cranelift-frontend = { path = "../frontend", version = "0.68.0", default-features = false } cranelift-frontend = { path = "../frontend", version = "0.68.0", default-features = false }

25
cranelift/wasm/src/code_translator.rs
View File

@@ -538,10 +538,10 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
} }
/********************************** Exception handing **********************************/ /********************************** Exception handing **********************************/
Operator::Try { .. } Operator::Try { .. }
| Operator::Catch | Operator::Catch { .. }
| Operator::BrOnExn { .. }
| Operator::Throw { .. } | Operator::Throw { .. }
| Operator::Rethrow => { | Operator::Unwind
| Operator::Rethrow { .. } => {
return Err(wasm_unsupported!( return Err(wasm_unsupported!(
"proposed exception handling operator {:?}", "proposed exception handling operator {:?}",
op op
@@ -2056,7 +2056,9 @@ fn prepare_load<FE: FuncEnvironment + ?Sized>(
// Note that we don't set `is_aligned` here, even if the load instruction's // Note that we don't set `is_aligned` here, even if the load instruction's
// alignment immediate says it's aligned, because WebAssembly's immediate // alignment immediate says it's aligned, because WebAssembly's immediate
// field is just a hint, while Cranelift's aligned flag needs a guarantee. // field is just a hint, while Cranelift's aligned flag needs a guarantee.
let flags = MemFlags::new(); // WebAssembly memory accesses are always little-endian.
let mut flags = MemFlags::new();
flags.set_endianness(ir::Endianness::Little);
Ok((flags, base, offset.into())) Ok((flags, base, offset.into()))
} }
@@ -2103,7 +2105,8 @@ fn translate_store<FE: FuncEnvironment + ?Sized>(
builder, builder,
); );
// See the comments in `prepare_load` about the flags. // See the comments in `prepare_load` about the flags.
let flags = MemFlags::new(); let mut flags = MemFlags::new();
flags.set_endianness(ir::Endianness::Little);
builder builder
.ins() .ins()
.Store(opcode, val_ty, flags, offset.into(), val, base); .Store(opcode, val_ty, flags, offset.into(), val, base);
@@ -2207,7 +2210,8 @@ fn translate_atomic_rmw<FE: FuncEnvironment + ?Sized>(
finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
// See the comments in `prepare_load` about the flags. // See the comments in `prepare_load` about the flags.
let flags = MemFlags::new(); let mut flags = MemFlags::new();
flags.set_endianness(ir::Endianness::Little);
let mut res = builder let mut res = builder
.ins() .ins()
.atomic_rmw(access_ty, flags, op, final_effective_address, arg2); .atomic_rmw(access_ty, flags, op, final_effective_address, arg2);
@@ -2260,7 +2264,8 @@ fn translate_atomic_cas<FE: FuncEnvironment + ?Sized>(
finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
// See the comments in `prepare_load` about the flags. // See the comments in `prepare_load` about the flags.
let flags = MemFlags::new(); let mut flags = MemFlags::new();
flags.set_endianness(ir::Endianness::Little);
let mut res = builder let mut res = builder
.ins() .ins()
.atomic_cas(flags, final_effective_address, expected, replacement); .atomic_cas(flags, final_effective_address, expected, replacement);
@@ -2302,7 +2307,8 @@ fn translate_atomic_load<FE: FuncEnvironment + ?Sized>(
finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
// See the comments in `prepare_load` about the flags. // See the comments in `prepare_load` about the flags.
let flags = MemFlags::new(); let mut flags = MemFlags::new();
flags.set_endianness(ir::Endianness::Little);
let mut res = builder let mut res = builder
.ins() .ins()
.atomic_load(access_ty, flags, final_effective_address); .atomic_load(access_ty, flags, final_effective_address);
@@ -2348,7 +2354,8 @@ fn translate_atomic_store<FE: FuncEnvironment + ?Sized>(
finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?;
// See the comments in `prepare_load` about the flags. // See the comments in `prepare_load` about the flags.
let flags = MemFlags::new(); let mut flags = MemFlags::new();
flags.set_endianness(ir::Endianness::Little);
builder builder
.ins() .ins()
.atomic_store(flags, data, final_effective_address); .atomic_store(flags, data, final_effective_address);

4
cranelift/wasm/src/environ/mod.rs
View File

@@ -6,6 +6,6 @@ mod spec;
pub use crate::environ::dummy::DummyEnvironment; pub use crate::environ::dummy::DummyEnvironment;
pub use crate::environ::spec::{ pub use crate::environ::spec::{
FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode, TargetEnvironment, WasmError, Alias, FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode, TargetEnvironment,
WasmFuncType, WasmResult, WasmType, WasmError, WasmFuncType, WasmResult, WasmType,
}; };

95
cranelift/wasm/src/environ/spec.rs
View File

@@ -8,8 +8,9 @@
use crate::state::FuncTranslationState; use crate::state::FuncTranslationState;
use crate::translation_utils::{ use crate::translation_utils::{
DataIndex, ElemIndex, EntityType, Event, EventIndex, FuncIndex, Global, GlobalIndex, Memory, DataIndex, ElemIndex, EntityIndex, EntityType, Event, EventIndex, FuncIndex, Global,
MemoryIndex, Table, TableIndex, TypeIndex, GlobalIndex, InstanceIndex, InstanceTypeIndex, Memory, MemoryIndex, ModuleIndex,
ModuleTypeIndex, SignatureIndex, Table, TableIndex, TypeIndex,
}; };
use core::convert::From; use core::convert::From;
use core::convert::TryFrom; use core::convert::TryFrom;
@@ -22,6 +23,7 @@ use cranelift_frontend::FunctionBuilder;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::boxed::Box; use std::boxed::Box;
use std::string::ToString; use std::string::ToString;
use std::vec::Vec;
use thiserror::Error; use thiserror::Error;
use wasmparser::ValidatorResources; use wasmparser::ValidatorResources;
use wasmparser::{BinaryReaderError, FuncValidator, FunctionBody, Operator, WasmFeatures}; use wasmparser::{BinaryReaderError, FuncValidator, FunctionBody, Operator, WasmFeatures};
@@ -201,6 +203,30 @@ pub enum ReturnMode {
FallthroughReturn, FallthroughReturn,
} }
/// An entry in the alias section of a wasm module (from the module linking
/// proposal)
pub enum Alias {
/// A parent's module is being aliased into our own index space.
///
/// Note that the index here is in the parent's index space, not our own.
ParentModule(ModuleIndex),
/// A parent's type is being aliased into our own index space
///
/// Note that the index here is in the parent's index space, not our own.
ParentType(TypeIndex),
/// A previously created instance is having one of its exports aliased into
/// our index space.
Child {
/// The index we're aliasing.
instance: InstanceIndex,
/// The nth export that we're inserting into our own index space
/// locally.
export: usize,
},
}
/// Environment affecting the translation of a WebAssembly. /// Environment affecting the translation of a WebAssembly.
pub trait TargetEnvironment { pub trait TargetEnvironment {
/// Get the information needed to produce Cranelift IR for the given target. /// Get the information needed to produce Cranelift IR for the given target.
@@ -683,6 +709,27 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
Err(WasmError::Unsupported("module linking".to_string())) Err(WasmError::Unsupported("module linking".to_string()))
} }
/// Translates a type index to its signature index, only called for type
/// indices which point to functions.
fn type_to_signature(&self, index: TypeIndex) -> WasmResult<SignatureIndex> {
drop(index);
Err(WasmError::Unsupported("module linking".to_string()))
}
/// Translates a type index to its module type index, only called for type
/// indices which point to modules.
fn type_to_module_type(&self, index: TypeIndex) -> WasmResult<ModuleTypeIndex> {
drop(index);
Err(WasmError::Unsupported("module linking".to_string()))
}
/// Translates a type index to its instance type index, only called for type
/// indices which point to instances.
fn type_to_instance_type(&self, index: TypeIndex) -> WasmResult<InstanceTypeIndex> {
drop(index);
Err(WasmError::Unsupported("module linking".to_string()))
}
/// Provides the number of imports up front. By default this does nothing, but /// Provides the number of imports up front. By default this does nothing, but
/// implementations can use this to preallocate memory if desired. /// implementations can use this to preallocate memory if desired.
fn reserve_imports(&mut self, _num: u32) -> WasmResult<()> { fn reserve_imports(&mut self, _num: u32) -> WasmResult<()> {
@@ -844,6 +891,22 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
name: &'data str, name: &'data str,
) -> WasmResult<()>; ) -> WasmResult<()>;
/// Declares an instance export to the environment.
fn declare_instance_export(
&mut self,
index: InstanceIndex,
name: &'data str,
) -> WasmResult<()> {
drop((index, name));
Err(WasmError::Unsupported("module linking".to_string()))
}
/// Declares an instance export to the environment.
fn declare_module_export(&mut self, index: ModuleIndex, name: &'data str) -> WasmResult<()> {
drop((index, name));
Err(WasmError::Unsupported("module linking".to_string()))
}
/// Notifies the implementation that all exports have been declared. /// Notifies the implementation that all exports have been declared.
fn finish_exports(&mut self) -> WasmResult<()> { fn finish_exports(&mut self) -> WasmResult<()> {
Ok(()) Ok(())
@@ -951,6 +1014,12 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
drop(amount); drop(amount);
} }
/// Declares that a module will come later with the type signature provided.
fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> {
drop(ty);
Err(WasmError::Unsupported("module linking".to_string()))
}
/// Called at the beginning of translating a module. /// Called at the beginning of translating a module.
/// ///
/// The `index` argument is a monotonically increasing index which /// The `index` argument is a monotonically increasing index which
@@ -969,4 +1038,26 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
fn module_end(&mut self, index: usize) { fn module_end(&mut self, index: usize) {
drop(index); drop(index);
} }
/// Indicates that this module will have `amount` instances.
fn reserve_instances(&mut self, amount: u32) {
drop(amount);
}
/// Declares a new instance which this module will instantiate before it's
/// instantiated.
fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
drop((module, args));
Err(WasmError::Unsupported("wasm instance".to_string()))
}
/// Declares a new alias being added to this module.
///
/// The alias comes from the `instance` specified (or the parent if `None`
/// is supplied) and the index is either in the module's own index spaces
/// for the parent or an index into the exports for nested instances.
fn declare_alias(&mut self, alias: Alias) -> WasmResult<()> {
drop(alias);
Err(WasmError::Unsupported("wasm alias".to_string()))
}
} }

2
cranelift/wasm/src/lib.rs
View File

@@ -57,7 +57,7 @@ mod state;
mod translation_utils; mod translation_utils;
pub use crate::environ::{ pub use crate::environ::{
DummyEnvironment, FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode, Alias, DummyEnvironment, FuncEnvironment, GlobalVariable, ModuleEnvironment, ReturnMode,
TargetEnvironment, WasmError, WasmFuncType, WasmResult, WasmType, TargetEnvironment, WasmError, WasmFuncType, WasmResult, WasmType,
}; };
pub use crate::func_translator::FuncTranslator; pub use crate::func_translator::FuncTranslator;

13
cranelift/wasm/src/module_translator.rs
View File

@@ -2,9 +2,10 @@
//! to deal with each part of it. //! to deal with each part of it.
use crate::environ::{ModuleEnvironment, WasmResult}; use crate::environ::{ModuleEnvironment, WasmResult};
use crate::sections_translator::{ use crate::sections_translator::{
parse_data_section, parse_element_section, parse_event_section, parse_export_section, parse_alias_section, parse_data_section, parse_element_section, parse_event_section,
parse_function_section, parse_global_section, parse_import_section, parse_memory_section, parse_export_section, parse_function_section, parse_global_section, parse_import_section,
parse_name_section, parse_start_section, parse_table_section, parse_type_section, parse_instance_section, parse_memory_section, parse_module_section, parse_name_section,
parse_start_section, parse_table_section, parse_type_section,
}; };
use crate::state::ModuleTranslationState; use crate::state::ModuleTranslationState;
use cranelift_codegen::timing; use cranelift_codegen::timing;
@@ -112,15 +113,15 @@ pub fn translate_module<'data>(
Payload::ModuleSection(s) => { Payload::ModuleSection(s) => {
validator.module_section(&s)?; validator.module_section(&s)?;
environ.reserve_modules(s.get_count()); parse_module_section(s, environ)?;
} }
Payload::InstanceSection(s) => { Payload::InstanceSection(s) => {
validator.instance_section(&s)?; validator.instance_section(&s)?;
unimplemented!("module linking not implemented yet") parse_instance_section(s, environ)?;
} }
Payload::AliasSection(s) => { Payload::AliasSection(s) => {
validator.alias_section(&s)?; validator.alias_section(&s)?;
unimplemented!("module linking not implemented yet") parse_alias_section(s, environ)?;
} }
Payload::ModuleCodeSectionStart { Payload::ModuleCodeSectionStart {
count, count,

143
cranelift/wasm/src/sections_translator.rs
View File

@@ -7,12 +7,12 @@
//! The special case of the initialize expressions for table elements offsets or global variables //! The special case of the initialize expressions for table elements offsets or global variables
//! is handled, according to the semantics of WebAssembly, to only specific expressions that are //! is handled, according to the semantics of WebAssembly, to only specific expressions that are
//! interpreted on the fly. //! interpreted on the fly.
use crate::environ::{ModuleEnvironment, WasmError, WasmResult}; use crate::environ::{Alias, ModuleEnvironment, WasmError, WasmResult};
use crate::state::ModuleTranslationState; use crate::state::ModuleTranslationState;
use crate::translation_utils::{ use crate::translation_utils::{
tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityType, Event, EventIndex, tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityIndex, EntityType, Event,
FuncIndex, Global, GlobalIndex, GlobalInit, Memory, MemoryIndex, Table, TableElementType, EventIndex, FuncIndex, Global, GlobalIndex, GlobalInit, InstanceIndex, Memory, MemoryIndex,
TableIndex, TypeIndex, ModuleIndex, Table, TableElementType, TableIndex, TypeIndex,
}; };
use crate::wasm_unsupported; use crate::wasm_unsupported;
use core::convert::TryFrom; use core::convert::TryFrom;
@@ -36,9 +36,15 @@ fn entity_type(
environ: &mut dyn ModuleEnvironment<'_>, environ: &mut dyn ModuleEnvironment<'_>,
) -> WasmResult<EntityType> { ) -> WasmResult<EntityType> {
Ok(match ty { Ok(match ty {
ImportSectionEntryType::Function(sig) => EntityType::Function(TypeIndex::from_u32(sig)), ImportSectionEntryType::Function(sig) => {
ImportSectionEntryType::Module(sig) => EntityType::Module(TypeIndex::from_u32(sig)), EntityType::Function(environ.type_to_signature(TypeIndex::from_u32(sig))?)
ImportSectionEntryType::Instance(sig) => EntityType::Instance(TypeIndex::from_u32(sig)), }
ImportSectionEntryType::Module(sig) => {
EntityType::Module(environ.type_to_module_type(TypeIndex::from_u32(sig))?)
}
ImportSectionEntryType::Instance(sig) => {
EntityType::Instance(environ.type_to_instance_type(TypeIndex::from_u32(sig))?)
}
ImportSectionEntryType::Memory(ty) => EntityType::Memory(memory(ty)), ImportSectionEntryType::Memory(ty) => EntityType::Memory(memory(ty)),
ImportSectionEntryType::Event(evt) => EntityType::Event(event(evt)), ImportSectionEntryType::Event(evt) => EntityType::Event(event(evt)),
ImportSectionEntryType::Global(ty) => { ImportSectionEntryType::Global(ty) => {
@@ -156,24 +162,40 @@ pub fn parse_import_section<'data>(
for entry in imports { for entry in imports {
let import = entry?; let import = entry?;
match entity_type(import.ty, environ)? { match import.ty {
EntityType::Function(idx) => { ImportSectionEntryType::Function(sig) => {
environ.declare_func_import(idx, import.module, import.field)?; environ.declare_func_import(
TypeIndex::from_u32(sig),
import.module,
import.field,
)?;
} }
EntityType::Module(idx) => { ImportSectionEntryType::Module(sig) => {
environ.declare_module_import(idx, import.module, import.field)?; environ.declare_module_import(
TypeIndex::from_u32(sig),
import.module,
import.field,
)?;
} }
EntityType::Instance(idx) => { ImportSectionEntryType::Instance(sig) => {
environ.declare_instance_import(idx, import.module, import.field)?; environ.declare_instance_import(
TypeIndex::from_u32(sig),
import.module,
import.field,
)?;
} }
EntityType::Memory(ty) => { ImportSectionEntryType::Memory(ty) => {
environ.declare_memory_import(ty, import.module, import.field)?; environ.declare_memory_import(memory(ty), import.module, import.field)?;
} }
EntityType::Event(e) => environ.declare_event_import(e, import.module, import.field)?, ImportSectionEntryType::Event(e) => {
EntityType::Global(ty) => { environ.declare_event_import(event(e), import.module, import.field)?;
}
ImportSectionEntryType::Global(ty) => {
let ty = global(ty, environ, GlobalInit::Import)?;
environ.declare_global_import(ty, import.module, import.field)?; environ.declare_global_import(ty, import.module, import.field)?;
} }
EntityType::Table(ty) => { ImportSectionEntryType::Table(ty) => {
let ty = table(ty, environ)?;
environ.declare_table_import(ty, import.module, import.field)?; environ.declare_table_import(ty, import.module, import.field)?;
} }
} }
@@ -316,9 +338,15 @@ pub fn parse_export_section<'data>(
ExternalKind::Global => { ExternalKind::Global => {
environ.declare_global_export(GlobalIndex::new(index), field)? environ.declare_global_export(GlobalIndex::new(index), field)?
} }
ExternalKind::Type | ExternalKind::Module | ExternalKind::Instance => { ExternalKind::Module => {
unimplemented!("module linking not implemented yet") environ.declare_module_export(ModuleIndex::new(index), field)?
} }
ExternalKind::Instance => {
environ.declare_instance_export(InstanceIndex::new(index), field)?
}
// this never gets past validation
ExternalKind::Type => unreachable!(),
} }
} }
@@ -475,3 +503,76 @@ pub fn parse_name_section<'data>(
} }
Ok(()) Ok(())
} }
/// Parses the Module section of the wasm module.
pub fn parse_module_section<'data>(
section: wasmparser::ModuleSectionReader<'data>,
environ: &mut dyn ModuleEnvironment<'data>,
) -> WasmResult<()> {
environ.reserve_modules(section.get_count());
for module_ty in section {
environ.declare_module(TypeIndex::from_u32(module_ty?))?;
}
Ok(())
}
/// Parses the Instance section of the wasm module.
pub fn parse_instance_section<'data>(
section: wasmparser::InstanceSectionReader<'data>,
environ: &mut dyn ModuleEnvironment<'data>,
) -> WasmResult<()> {
environ.reserve_instances(section.get_count());
for instance in section {
let instance = instance?;
let module = ModuleIndex::from_u32(instance.module());
let args = instance
.args()?
.into_iter()
.map(|result| {
let (kind, idx) = result?;
Ok(match kind {
ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(idx)),
ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(idx)),
ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(idx)),
ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(idx)),
ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(idx)),
ExternalKind::Instance => EntityIndex::Instance(InstanceIndex::from_u32(idx)),
ExternalKind::Event => unimplemented!(),
// this won't pass validation
ExternalKind::Type => unreachable!(),
})
})
.collect::<WasmResult<Vec<_>>>()?;
environ.declare_instance(module, args)?;
}
Ok(())
}
/// Parses the Alias section of the wasm module.
pub fn parse_alias_section<'data>(
section: wasmparser::AliasSectionReader<'data>,
environ: &mut dyn ModuleEnvironment<'data>,
) -> WasmResult<()> {
for alias in section {
let alias = alias?;
let alias = match alias.instance {
wasmparser::AliasedInstance::Parent => {
match alias.kind {
ExternalKind::Module => Alias::ParentModule(ModuleIndex::from_u32(alias.index)),
ExternalKind::Type => Alias::ParentType(TypeIndex::from_u32(alias.index)),
// shouldn't get past validation
_ => unreachable!(),
}
}
wasmparser::AliasedInstance::Child(i) => Alias::Child {
instance: InstanceIndex::from_u32(i),
export: alias.index as usize,
},
};
environ.declare_alias(alias)?;
}
Ok(())
}

20
cranelift/wasm/src/translation_utils.rs
View File

@@ -97,8 +97,20 @@ entity_impl!(InstanceIndex);
pub struct EventIndex(u32); pub struct EventIndex(u32);
entity_impl!(EventIndex); entity_impl!(EventIndex);
/// Specialized index for just module types.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct ModuleTypeIndex(u32);
entity_impl!(ModuleTypeIndex);
/// Specialized index for just instance types.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct InstanceTypeIndex(u32);
entity_impl!(InstanceTypeIndex);
/// An index of an entity. /// An index of an entity.
#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub enum EntityIndex { pub enum EntityIndex {
/// Function index. /// Function index.
@@ -131,13 +143,13 @@ pub enum EntityType {
Table(Table), Table(Table),
/// A function type where the index points to the type section and records a /// A function type where the index points to the type section and records a
/// function signature. /// function signature.
Function(TypeIndex), Function(SignatureIndex),
/// An instance where the index points to the type section and records a /// An instance where the index points to the type section and records a
/// instance's exports. /// instance's exports.
Instance(TypeIndex), Instance(InstanceTypeIndex),
/// A module where the index points to the type section and records a /// A module where the index points to the type section and records a
/// module's imports and exports. /// module's imports and exports.
Module(TypeIndex), Module(ModuleTypeIndex),
} }
/// A WebAssembly global. /// A WebAssembly global.

7
crates/c-api/include/doc-wasm.h
View File

@@ -793,8 +793,15 @@
* \typedef wasm_externkind_t * \typedef wasm_externkind_t
* \brief Classifier for #wasm_externtype_t, defined by #wasm_externkind_enum * \brief Classifier for #wasm_externtype_t, defined by #wasm_externkind_enum
* *
* This is returned from #wasm_extern_kind and #wasm_externtype_kind to
* determine what kind of type is wrapped.
*
* \enum wasm_externkind_enum * \enum wasm_externkind_enum
* \brief Kinds of external items for a wasm module. * \brief Kinds of external items for a wasm module.
*
* Note that this also includes #WASM_EXTERN_INSTANCE as well as
* #WASM_EXTERN_MODULE and is intended to be used when #wasm_externkind_t is
* used.
*/ */
/** /**

293
crates/c-api/include/wasmtime.h
View File

@@ -208,6 +208,14 @@ WASMTIME_CONFIG_PROP(void, wasm_bulk_memory, bool)
*/ */
WASMTIME_CONFIG_PROP(void, wasm_multi_value, bool) WASMTIME_CONFIG_PROP(void, wasm_multi_value, bool)
/**
* \brief Configures whether the WebAssembly module linking proposal is
* enabled.
*
* This setting is `false` by default.
*/
WASMTIME_CONFIG_PROP(void, wasm_module_linking, bool)
/** /**
* \brief Configures how JIT code will be compiled. * \brief Configures how JIT code will be compiled.
* *
@@ -961,6 +969,291 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize(
own wasm_module_t **ret own wasm_module_t **ret
); );
/**
* \struct wasm_instancetype_t
* \brief An opaque object representing the type of a function.
*
* \typedef wasm_instancetype_t
* \brief Convenience alias for #wasm_instancetype_t
*
* \struct wasm_instancetype_vec_t
* \brief A list of #wasm_instancetype_t values.
*
* \var wasm_instancetype_vec_t::size
* \brief Length of this vector.
*
* \var wasm_instancetype_vec_t::data
* \brief Pointer to the base of this vector
*
* \typedef wasm_instancetype_vec_t
* \brief Convenience alias for #wasm_instancetype_vec_t
*
* \fn void wasm_instancetype_delete(own wasm_instancetype_t *);
* \brief Deletes a type.
*
* \fn void wasm_instancetype_vec_new_empty(own wasm_instancetype_vec_t *out);
* \brief Creates an empty vector.
*
* See #wasm_byte_vec_new_empty for more information.
*
* \fn void wasm_instancetype_vec_new_uninitialized(own wasm_instancetype_vec_t *out, size_t);
* \brief Creates a vector with the given capacity.
*
* See #wasm_byte_vec_new_uninitialized for more information.
*
* \fn void wasm_instancetype_vec_new(own wasm_instancetype_vec_t *out, size_t, own wasm_instancetype_t *const[]);
* \brief Creates a vector with the provided contents.
*
* See #wasm_byte_vec_new for more information.
*
* \fn void wasm_instancetype_vec_copy(own wasm_instancetype_vec_t *out, const wasm_instancetype_vec_t *)
* \brief Copies one vector to another
*
* See #wasm_byte_vec_copy for more information.
*
* \fn void wasm_instancetype_vec_delete(own wasm_instancetype_vec_t *out)
* \brief Deallocates memory for a vector.
*
* See #wasm_byte_vec_delete for more information.
*
* \fn own wasm_instancetype_t* wasm_instancetype_copy(wasm_instancetype_t *)
* \brief Creates a new value which matches the provided one.
*
* The caller is responsible for deleting the returned value.
*/
WASM_DECLARE_TYPE(instancetype)
/**
* \brief Returns the list of exports that this instance type provides.
*
* This function does not take ownership of the provided instance type but
* ownership of `out` is passed to the caller. Note that `out` is treated as
* uninitialized when passed to this function.
*/
WASM_API_EXTERN void wasm_instancetype_exports(const wasm_instancetype_t*, own wasm_exporttype_vec_t* out);
/**
* \brief Converts a #wasm_instancetype_t to a #wasm_externtype_t
*
* The returned value is owned by the #wasm_instancetype_t argument and should not
* be deleted.
*/
WASM_API_EXTERN wasm_externtype_t* wasm_instancetype_as_externtype(wasm_instancetype_t*);
/**
* \brief Attempts to convert a #wasm_externtype_t to a #wasm_instancetype_t
*
* The returned value is owned by the #wasm_instancetype_t argument and should not
* be deleted. Returns `NULL` if the provided argument is not a
* #wasm_instancetype_t.
*/
WASM_API_EXTERN wasm_instancetype_t* wasm_externtype_as_instancetype(wasm_externtype_t*);
/**
* \brief Converts a #wasm_instancetype_t to a #wasm_externtype_t
*
* The returned value is owned by the #wasm_instancetype_t argument and should not
* be deleted.
*/
WASM_API_EXTERN const wasm_externtype_t* wasm_instancetype_as_externtype_const(const wasm_instancetype_t*);
/**
* \brief Attempts to convert a #wasm_externtype_t to a #wasm_instancetype_t
*
* The returned value is owned by the #wasm_instancetype_t argument and should not
* be deleted. Returns `NULL` if the provided argument is not a
* #wasm_instancetype_t.
*/
WASM_API_EXTERN const wasm_instancetype_t* wasm_externtype_as_instancetype_const(const wasm_externtype_t*);
/**
* \struct wasm_moduletype_t
* \brief An opaque object representing the type of a function.
*
* \typedef wasm_moduletype_t
* \brief Convenience alias for #wasm_moduletype_t
*
* \struct wasm_moduletype_vec_t
* \brief A list of #wasm_moduletype_t values.
*
* \var wasm_moduletype_vec_t::size
* \brief Length of this vector.
*
* \var wasm_moduletype_vec_t::data
* \brief Pointer to the base of this vector
*
* \typedef wasm_moduletype_vec_t
* \brief Convenience alias for #wasm_moduletype_vec_t
*
* \fn void wasm_moduletype_delete(own wasm_moduletype_t *);
* \brief Deletes a type.
*
* \fn void wasm_moduletype_vec_new_empty(own wasm_moduletype_vec_t *out);
* \brief Creates an empty vector.
*
* See #wasm_byte_vec_new_empty for more information.
*
* \fn void wasm_moduletype_vec_new_uninitialized(own wasm_moduletype_vec_t *out, size_t);
* \brief Creates a vector with the given capacity.
*
* See #wasm_byte_vec_new_uninitialized for more information.
*
* \fn void wasm_moduletype_vec_new(own wasm_moduletype_vec_t *out, size_t, own wasm_moduletype_t *const[]);
* \brief Creates a vector with the provided contents.
*
* See #wasm_byte_vec_new for more information.
*
* \fn void wasm_moduletype_vec_copy(own wasm_moduletype_vec_t *out, const wasm_moduletype_vec_t *)
* \brief Copies one vector to another
*
* See #wasm_byte_vec_copy for more information.
*
* \fn void wasm_moduletype_vec_delete(own wasm_moduletype_vec_t *out)
* \brief Deallocates memory for a vector.
*
* See #wasm_byte_vec_delete for more information.
*
* \fn own wasm_moduletype_t* wasm_moduletype_copy(wasm_moduletype_t *)
* \brief Creates a new value which matches the provided one.
*
* The caller is responsible for deleting the returned value.
*/
WASM_DECLARE_TYPE(moduletype)
/**
* \brief Returns the list of imports that this module type requires.
*
* This function does not take ownership of the provided module type but
* ownership of `out` is passed to the caller. Note that `out` is treated as
* uninitialized when passed to this function.
*/
WASM_API_EXTERN void wasm_moduletype_imports(const wasm_moduletype_t*, own wasm_importtype_vec_t* out);
/**
* \brief Returns the list of exports that this module type provides.
*
* This function does not take ownership of the provided module type but
* ownership of `out` is passed to the caller. Note that `out` is treated as
* uninitialized when passed to this function.
*/
WASM_API_EXTERN void wasm_moduletype_exports(const wasm_moduletype_t*, own wasm_exporttype_vec_t* out);
/**
* \brief Converts a #wasm_moduletype_t to a #wasm_externtype_t
*
* The returned value is owned by the #wasm_moduletype_t argument and should not
* be deleted.
*/
WASM_API_EXTERN wasm_externtype_t* wasm_moduletype_as_externtype(wasm_moduletype_t*);
/**
* \brief Attempts to convert a #wasm_externtype_t to a #wasm_moduletype_t
*
* The returned value is owned by the #wasm_moduletype_t argument and should not
* be deleted. Returns `NULL` if the provided argument is not a
* #wasm_moduletype_t.
*/
WASM_API_EXTERN wasm_moduletype_t* wasm_externtype_as_moduletype(wasm_externtype_t*);
/**
* \brief Converts a #wasm_moduletype_t to a #wasm_externtype_t
*
* The returned value is owned by the #wasm_moduletype_t argument and should not
* be deleted.
*/
WASM_API_EXTERN const wasm_externtype_t* wasm_moduletype_as_externtype_const(const wasm_moduletype_t*);
/**
* \brief Attempts to convert a #wasm_externtype_t to a #wasm_moduletype_t
*
* The returned value is owned by the #wasm_moduletype_t argument and should not
* be deleted. Returns `NULL` if the provided argument is not a
* #wasm_moduletype_t.
*/
WASM_API_EXTERN const wasm_moduletype_t* wasm_externtype_as_moduletype_const(const wasm_externtype_t*);
/**
* \brief Converts a #wasm_module_t to #wasm_extern_t.
*
* The returned #wasm_extern_t is owned by the #wasm_module_t argument. Callers
* should not delete the returned value, and it only lives as long as the
* #wasm_module_t argument.
*/
WASM_API_EXTERN wasm_extern_t* wasm_module_as_extern(wasm_module_t*);
/**
* \brief Converts a #wasm_extern_t to #wasm_module_t.
*
* The returned #wasm_module_t is owned by the #wasm_extern_t argument. Callers
* should not delete the returned value, and it only lives as long as the
* #wasm_extern_t argument.
*
* If the #wasm_extern_t argument isn't a #wasm_module_t then `NULL` is returned.
*/
WASM_API_EXTERN wasm_module_t* wasm_extern_as_module(wasm_extern_t*);
/**
* \brief Converts a #wasm_extern_t to #wasm_instance_t.
*
* The returned #wasm_instance_t is owned by the #wasm_extern_t argument. Callers
* should not delete the returned value, and it only lives as long as the
* #wasm_extern_t argument.
*/
WASM_API_EXTERN const wasm_module_t* wasm_extern_as_module_const(const wasm_extern_t*);
/**
* \brief Converts a #wasm_instance_t to #wasm_extern_t.
*
* The returned #wasm_extern_t is owned by the #wasm_instance_t argument. Callers
* should not delete the returned value, and it only lives as long as the
* #wasm_instance_t argument.
*/
WASM_API_EXTERN wasm_extern_t* wasm_instance_as_extern(wasm_instance_t*);
/**
* \brief Converts a #wasm_extern_t to #wasm_instance_t.
*
* The returned #wasm_instance_t is owned by the #wasm_extern_t argument. Callers
* should not delete the returned value, and it only lives as long as the
* #wasm_extern_t argument.
*
* If the #wasm_extern_t argument isn't a #wasm_instance_t then `NULL` is returned.
*/
WASM_API_EXTERN wasm_instance_t* wasm_extern_as_instance(wasm_extern_t*);
/**
* \brief Converts a #wasm_extern_t to #wasm_instance_t.
*
* The returned #wasm_instance_t is owned by the #wasm_extern_t argument. Callers
* should not delete the returned value, and it only lives as long as the
* #wasm_extern_t argument.
*/
WASM_API_EXTERN const wasm_instance_t* wasm_extern_as_instance_const(const wasm_extern_t*);
/**
* \brief Returns the type of this instance.
*
* The returned #wasm_instancetype_t is expected to be deallocated by the caller.
*/
WASM_API_EXTERN own wasm_instancetype_t* wasm_instance_type(const wasm_instance_t*);
/**
* \brief Returns the type of this module.
*
* The returned #wasm_moduletype_t is expected to be deallocated by the caller.
*/
WASM_API_EXTERN own wasm_moduletype_t* wasm_module_type(const wasm_module_t*);
/**
* \brief Value of #wasm_externkind_enum corresponding to a wasm module.
*/
#define WASM_EXTERN_MODULE 4
/**
* \brief Value of #wasm_externkind_enum corresponding to a wasm instance.
*/
#define WASM_EXTERN_INSTANCE 5
#undef own #undef own
#ifdef __cplusplus #ifdef __cplusplus

5
crates/c-api/src/config.rs
View File

@@ -85,6 +85,11 @@ pub extern "C" fn wasmtime_config_wasm_multi_value_set(c: &mut wasm_config_t, en
c.config.wasm_multi_value(enable); c.config.wasm_multi_value(enable);
} }
#[no_mangle]
pub extern "C" fn wasmtime_config_wasm_module_linking_set(c: &mut wasm_config_t, enable: bool) {
c.config.wasm_module_linking(enable);
}
#[no_mangle] #[no_mangle]
pub extern "C" fn wasmtime_config_strategy_set( pub extern "C" fn wasmtime_config_strategy_set(
c: &mut wasm_config_t, c: &mut wasm_config_t,

28
crates/c-api/src/extern.rs
View File

@@ -1,5 +1,7 @@
use crate::wasm_externkind_t; use crate::{
use crate::{wasm_externtype_t, wasm_func_t, wasm_global_t, wasm_memory_t, wasm_table_t}; wasm_externkind_t, wasm_externtype_t, wasm_func_t, wasm_global_t, wasm_instance_t,
wasm_memory_t, wasm_module_t, wasm_table_t,
};
use wasmtime::Extern; use wasmtime::Extern;
#[derive(Clone)] #[derive(Clone)]
@@ -16,6 +18,8 @@ pub extern "C" fn wasm_extern_kind(e: &wasm_extern_t) -> wasm_externkind_t {
Extern::Global(_) => crate::WASM_EXTERN_GLOBAL, Extern::Global(_) => crate::WASM_EXTERN_GLOBAL,
Extern::Table(_) => crate::WASM_EXTERN_TABLE, Extern::Table(_) => crate::WASM_EXTERN_TABLE,
Extern::Memory(_) => crate::WASM_EXTERN_MEMORY, Extern::Memory(_) => crate::WASM_EXTERN_MEMORY,
Extern::Instance(_) => crate::WASM_EXTERN_INSTANCE,
Extern::Module(_) => crate::WASM_EXTERN_MODULE,
} }
} }
@@ -63,3 +67,23 @@ pub extern "C" fn wasm_extern_as_memory(e: &wasm_extern_t) -> Option<&wasm_memor
pub extern "C" fn wasm_extern_as_memory_const(e: &wasm_extern_t) -> Option<&wasm_memory_t> { pub extern "C" fn wasm_extern_as_memory_const(e: &wasm_extern_t) -> Option<&wasm_memory_t> {
wasm_extern_as_memory(e) wasm_extern_as_memory(e)
} }
#[no_mangle]
pub extern "C" fn wasm_extern_as_module(e: &wasm_extern_t) -> Option<&wasm_module_t> {
wasm_module_t::try_from(e)
}
#[no_mangle]
pub extern "C" fn wasm_extern_as_module_const(e: &wasm_extern_t) -> Option<&wasm_module_t> {
wasm_extern_as_module(e)
}
#[no_mangle]
pub extern "C" fn wasm_extern_as_instance(e: &wasm_extern_t) -> Option<&wasm_instance_t> {
wasm_instance_t::try_from(e)
}
#[no_mangle]
pub extern "C" fn wasm_extern_as_instance_const(e: &wasm_extern_t) -> Option<&wasm_instance_t> {
wasm_extern_as_instance(e)
}

44
crates/c-api/src/instance.rs
View File

@@ -1,20 +1,38 @@
use crate::{wasm_extern_t, wasm_extern_vec_t, wasm_module_t, wasm_trap_t}; use crate::{wasm_extern_t, wasm_extern_vec_t, wasm_module_t, wasm_trap_t};
use crate::{wasm_store_t, wasmtime_error_t}; use crate::{wasm_instancetype_t, wasm_store_t, wasmtime_error_t};
use anyhow::Result; use anyhow::Result;
use std::ptr; use std::ptr;
use wasmtime::{Instance, Trap}; use wasmtime::{Extern, Instance, Trap};
#[repr(C)]
#[derive(Clone)] #[derive(Clone)]
#[repr(transparent)]
pub struct wasm_instance_t { pub struct wasm_instance_t {
pub(crate) instance: Instance, ext: wasm_extern_t,
} }
wasmtime_c_api_macros::declare_ref!(wasm_instance_t); wasmtime_c_api_macros::declare_ref!(wasm_instance_t);
impl wasm_instance_t { impl wasm_instance_t {
pub(crate) fn new(instance: Instance) -> wasm_instance_t { pub(crate) fn new(instance: Instance) -> wasm_instance_t {
wasm_instance_t { instance: instance } wasm_instance_t {
ext: wasm_extern_t {
which: instance.into(),
},
}
}
pub(crate) fn try_from(e: &wasm_extern_t) -> Option<&wasm_instance_t> {
match &e.which {
Extern::Instance(_) => Some(unsafe { &*(e as *const _ as *const _) }),
_ => None,
}
}
pub(crate) fn instance(&self) -> &Instance {
match &self.ext.which {
Extern::Instance(i) => i,
_ => unreachable!(),
}
} }
} }
@@ -31,7 +49,7 @@ pub unsafe extern "C" fn wasm_instance_new(
store, store,
wasm_module, wasm_module,
imports, imports,
wasm_module.imports.len(), wasm_module.module().imports().len(),
&mut instance, &mut instance,
&mut trap, &mut trap,
); );
@@ -92,7 +110,7 @@ fn _wasmtime_instance_new(
.map(|import| import.which.clone()) .map(|import| import.which.clone())
.collect::<Vec<_>>(); .collect::<Vec<_>>();
handle_instantiate( handle_instantiate(
Instance::new(store, &module.module, &imports), Instance::new(store, module.module(), &imports),
instance_ptr, instance_ptr,
trap_ptr, trap_ptr,
) )
@@ -122,11 +140,16 @@ pub fn handle_instantiate(
} }
} }
#[no_mangle]
pub extern "C" fn wasm_instance_as_extern(m: &wasm_instance_t) -> &wasm_extern_t {
&m.ext
}
#[no_mangle] #[no_mangle]
pub extern "C" fn wasm_instance_exports(instance: &wasm_instance_t, out: &mut wasm_extern_vec_t) { pub extern "C" fn wasm_instance_exports(instance: &wasm_instance_t, out: &mut wasm_extern_vec_t) {
out.set_buffer( out.set_buffer(
instance instance
.instance .instance()
.exports() .exports()
.map(|e| { .map(|e| {
Some(Box::new(wasm_extern_t { Some(Box::new(wasm_extern_t {
@@ -136,3 +159,8 @@ pub extern "C" fn wasm_instance_exports(instance: &wasm_instance_t, out: &mut wa
.collect(), .collect(),
); );
} }
#[no_mangle]
pub extern "C" fn wasm_instance_type(f: &wasm_instance_t) -> Box<wasm_instancetype_t> {
Box::new(wasm_instancetype_t::new(f.instance().ty()))
}

6
crates/c-api/src/linker.rs
View File

@@ -68,7 +68,7 @@ pub extern "C" fn wasmtime_linker_define_instance(
Ok(s) => s, Ok(s) => s,
Err(_) => return bad_utf8(), Err(_) => return bad_utf8(),
}; };
handle_result(linker.instance(name, &instance.instance), |_linker| ()) handle_result(linker.instance(name, instance.instance()), |_linker| ())
} }
#[no_mangle] #[no_mangle]
@@ -78,7 +78,7 @@ pub extern "C" fn wasmtime_linker_instantiate(
instance_ptr: &mut *mut wasm_instance_t, instance_ptr: &mut *mut wasm_instance_t,
trap_ptr: &mut *mut wasm_trap_t, trap_ptr: &mut *mut wasm_trap_t,
) -> Option<Box<wasmtime_error_t>> { ) -> Option<Box<wasmtime_error_t>> {
let result = linker.linker.instantiate(&module.module); let result = linker.linker.instantiate(module.module());
super::instance::handle_instantiate(result, instance_ptr, trap_ptr) super::instance::handle_instantiate(result, instance_ptr, trap_ptr)
} }
@@ -93,7 +93,7 @@ pub extern "C" fn wasmtime_linker_module(
Ok(s) => s, Ok(s) => s,
Err(_) => return bad_utf8(), Err(_) => return bad_utf8(),
}; };
handle_result(linker.module(name, &module.module), |_linker| ()) handle_result(linker.module(name, module.module()), |_linker| ())
} }
#[no_mangle] #[no_mangle]

140
crates/c-api/src/module.rs
View File

@@ -1,20 +1,43 @@
use crate::{ use crate::{
handle_result, wasm_byte_vec_t, wasm_engine_t, wasm_exporttype_t, wasm_exporttype_vec_t, handle_result, wasm_byte_vec_t, wasm_engine_t, wasm_exporttype_t, wasm_exporttype_vec_t,
wasm_importtype_t, wasm_importtype_vec_t, wasm_store_t, wasmtime_error_t, wasm_extern_t, wasm_importtype_t, wasm_importtype_vec_t, wasm_moduletype_t, wasm_store_t,
wasmtime_error_t,
}; };
use std::ptr; use std::ptr;
use wasmtime::{Engine, Module}; use wasmtime::{Engine, Extern, Module};
#[repr(C)]
#[derive(Clone)] #[derive(Clone)]
#[repr(transparent)]
pub struct wasm_module_t { pub struct wasm_module_t {
pub(crate) module: Module, ext: wasm_extern_t,
pub(crate) imports: Vec<wasm_importtype_t>,
pub(crate) exports: Vec<wasm_exporttype_t>,
} }
wasmtime_c_api_macros::declare_ref!(wasm_module_t); wasmtime_c_api_macros::declare_ref!(wasm_module_t);
impl wasm_module_t {
pub(crate) fn new(module: Module) -> wasm_module_t {
wasm_module_t {
ext: wasm_extern_t {
which: module.into(),
},
}
}
pub(crate) fn try_from(e: &wasm_extern_t) -> Option<&wasm_module_t> {
match &e.which {
Extern::Module(_) => Some(unsafe { &*(e as *const _ as *const _) }),
_ => None,
}
}
pub(crate) fn module(&self) -> &Module {
match &self.ext.which {
Extern::Module(i) => i,
_ => unreachable!(),
}
}
}
#[repr(C)] #[repr(C)]
#[derive(Clone)] #[derive(Clone)]
pub struct wasm_shared_module_t { pub struct wasm_shared_module_t {
@@ -49,25 +72,7 @@ pub extern "C" fn wasmtime_module_new(
) -> Option<Box<wasmtime_error_t>> { ) -> Option<Box<wasmtime_error_t>> {
let binary = binary.as_slice(); let binary = binary.as_slice();
handle_result(Module::from_binary(&engine.engine, binary), |module| { handle_result(Module::from_binary(&engine.engine, binary), |module| {
let imports = module let module = Box::new(wasm_module_t::new(module));
.imports()
.map(|i| {
wasm_importtype_t::new(
i.module().to_owned(),
i.name().map(|s| s.to_owned()),
i.ty(),
)
})
.collect::<Vec<_>>();
let exports = module
.exports()
.map(|e| wasm_exporttype_t::new(e.name().to_owned(), e.ty()))
.collect::<Vec<_>>();
let module = Box::new(wasm_module_t {
module: module,
imports,
exports,
});
*ret = Box::into_raw(module); *ret = Box::into_raw(module);
}) })
} }
@@ -86,30 +91,46 @@ pub extern "C" fn wasmtime_module_validate(
handle_result(Module::validate(store.store.engine(), binary), |()| {}) handle_result(Module::validate(store.store.engine(), binary), |()| {})
} }
#[no_mangle]
pub extern "C" fn wasm_module_as_extern(m: &wasm_module_t) -> &wasm_extern_t {
&m.ext
}
#[no_mangle] #[no_mangle]
pub extern "C" fn wasm_module_exports(module: &wasm_module_t, out: &mut wasm_exporttype_vec_t) { pub extern "C" fn wasm_module_exports(module: &wasm_module_t, out: &mut wasm_exporttype_vec_t) {
let buffer = module let exports = module
.exports .module()
.iter() .exports()
.map(|et| Some(Box::new(et.clone()))) .map(|e| {
Some(Box::new(wasm_exporttype_t::new(
e.name().to_owned(),
e.ty(),
)))
})
.collect::<Vec<_>>(); .collect::<Vec<_>>();
out.set_buffer(buffer); out.set_buffer(exports);
} }
#[no_mangle] #[no_mangle]
pub extern "C" fn wasm_module_imports(module: &wasm_module_t, out: &mut wasm_importtype_vec_t) { pub extern "C" fn wasm_module_imports(module: &wasm_module_t, out: &mut wasm_importtype_vec_t) {
let buffer = module let imports = module
.imports .module()
.iter() .imports()
.map(|it| Some(Box::new(it.clone()))) .map(|i| {
Some(Box::new(wasm_importtype_t::new(
i.module().to_owned(),
i.name().map(|s| s.to_owned()),
i.ty(),
)))
})
.collect::<Vec<_>>(); .collect::<Vec<_>>();
out.set_buffer(buffer); out.set_buffer(imports);
} }
#[no_mangle] #[no_mangle]
pub extern "C" fn wasm_module_share(module: &wasm_module_t) -> Box<wasm_shared_module_t> { pub extern "C" fn wasm_module_share(module: &wasm_module_t) -> Box<wasm_shared_module_t> {
Box::new(wasm_shared_module_t { Box::new(wasm_shared_module_t {
module: module.module.clone(), module: module.module().clone(),
}) })
} }
@@ -122,25 +143,7 @@ pub extern "C" fn wasm_module_obtain(
if !Engine::same(store.store.engine(), module.engine()) { if !Engine::same(store.store.engine(), module.engine()) {
return None; return None;
} }
let imports = module Some(Box::new(wasm_module_t::new(module)))
.imports()
.map(|i| {
wasm_importtype_t::new(
i.module().to_owned(),
i.name().map(|s| s.to_owned()),
i.ty(),
)
})
.collect::<Vec<_>>();
let exports = module
.exports()
.map(|e| wasm_exporttype_t::new(e.name().to_owned(), e.ty()))
.collect::<Vec<_>>();
Some(Box::new(wasm_module_t {
module: module,
imports,
exports,
}))
} }
#[no_mangle] #[no_mangle]
@@ -171,7 +174,7 @@ pub extern "C" fn wasmtime_module_serialize(
module: &wasm_module_t, module: &wasm_module_t,
ret: &mut wasm_byte_vec_t, ret: &mut wasm_byte_vec_t,
) -> Option<Box<wasmtime_error_t>> { ) -> Option<Box<wasmtime_error_t>> {
handle_result(module.module.serialize(), |buf| { handle_result(module.module().serialize(), |buf| {
ret.set_buffer(buf); ret.set_buffer(buf);
}) })
} }
@@ -185,26 +188,13 @@ pub extern "C" fn wasmtime_module_deserialize(
handle_result( handle_result(
Module::deserialize(&engine.engine, binary.as_slice()), Module::deserialize(&engine.engine, binary.as_slice()),
|module| { |module| {
let imports = module let module = Box::new(wasm_module_t::new(module));
.imports()
.map(|i| {
wasm_importtype_t::new(
i.module().to_owned(),
i.name().map(|s| s.to_owned()),
i.ty(),
)
})
.collect::<Vec<_>>();
let exports = module
.exports()
.map(|e| wasm_exporttype_t::new(e.name().to_owned(), e.ty()))
.collect::<Vec<_>>();
let module = Box::new(wasm_module_t {
module: module,
imports,
exports,
});
*ret = Box::into_raw(module); *ret = Box::into_raw(module);
}, },
) )
} }
#[no_mangle]
pub extern "C" fn wasm_module_type(f: &wasm_module_t) -> Box<wasm_moduletype_t> {
Box::new(wasm_moduletype_t::new(f.module().ty()))
}

8
crates/c-api/src/types/extern.rs
View File

@@ -27,8 +27,8 @@ pub const WASM_EXTERN_FUNC: wasm_externkind_t = 0;
pub const WASM_EXTERN_GLOBAL: wasm_externkind_t = 1; pub const WASM_EXTERN_GLOBAL: wasm_externkind_t = 1;
pub const WASM_EXTERN_TABLE: wasm_externkind_t = 2; pub const WASM_EXTERN_TABLE: wasm_externkind_t = 2;
pub const WASM_EXTERN_MEMORY: wasm_externkind_t = 3; pub const WASM_EXTERN_MEMORY: wasm_externkind_t = 3;
pub const WASMTIME_EXTERN_MODULE: wasm_externkind_t = 4; pub const WASM_EXTERN_MODULE: wasm_externkind_t = 4;
pub const WASMTIME_EXTERN_INSTANCE: wasm_externkind_t = 5; pub const WASM_EXTERN_INSTANCE: wasm_externkind_t = 5;
impl wasm_externtype_t { impl wasm_externtype_t {
pub(crate) fn new(ty: ExternType) -> wasm_externtype_t { pub(crate) fn new(ty: ExternType) -> wasm_externtype_t {
@@ -63,8 +63,8 @@ pub extern "C" fn wasm_externtype_kind(et: &wasm_externtype_t) -> wasm_externkin
CExternType::Table(_) => WASM_EXTERN_TABLE, CExternType::Table(_) => WASM_EXTERN_TABLE,
CExternType::Global(_) => WASM_EXTERN_GLOBAL, CExternType::Global(_) => WASM_EXTERN_GLOBAL,
CExternType::Memory(_) => WASM_EXTERN_MEMORY, CExternType::Memory(_) => WASM_EXTERN_MEMORY,
CExternType::Instance(_) => WASMTIME_EXTERN_INSTANCE, CExternType::Instance(_) => WASM_EXTERN_INSTANCE,
CExternType::Module(_) => WASMTIME_EXTERN_MODULE, CExternType::Module(_) => WASM_EXTERN_MODULE,
} }
} }

41
crates/c-api/src/types/instance.rs
View File

@@ -1,5 +1,4 @@
use crate::{wasm_externtype_t, wasm_limits_t, CExternType}; use crate::{wasm_exporttype_t, wasm_exporttype_vec_t, wasm_externtype_t, CExternType};
use once_cell::unsync::OnceCell;
use wasmtime::InstanceType; use wasmtime::InstanceType;
#[repr(transparent)] #[repr(transparent)]
@@ -13,24 +12,33 @@ wasmtime_c_api_macros::declare_ty!(wasm_instancetype_t);
#[derive(Clone)] #[derive(Clone)]
pub(crate) struct CInstanceType { pub(crate) struct CInstanceType {
pub(crate) ty: InstanceType, pub(crate) ty: InstanceType,
limits_cache: OnceCell<wasm_limits_t>,
} }
impl wasm_instancetype_t { impl wasm_instancetype_t {
pub(crate) fn new(ty: InstanceType) -> wasm_instancetype_t {
wasm_instancetype_t {
ext: wasm_externtype_t::new(ty.into()),
}
}
pub(crate) fn try_from(e: &wasm_externtype_t) -> Option<&wasm_instancetype_t> { pub(crate) fn try_from(e: &wasm_externtype_t) -> Option<&wasm_instancetype_t> {
match &e.which { match &e.which {
CExternType::Instance(_) => Some(unsafe { &*(e as *const _ as *const _) }), CExternType::Instance(_) => Some(unsafe { &*(e as *const _ as *const _) }),
_ => None, _ => None,
} }
} }
pub(crate) fn ty(&self) -> &CInstanceType {
match &self.ext.which {
CExternType::Instance(f) => &f,
_ => unreachable!(),
}
}
} }
impl CInstanceType { impl CInstanceType {
pub(crate) fn new(ty: InstanceType) -> CInstanceType { pub(crate) fn new(ty: InstanceType) -> CInstanceType {
CInstanceType { CInstanceType { ty }
ty,
limits_cache: OnceCell::new(),
}
} }
} }
#[no_mangle] #[no_mangle]
@@ -44,3 +52,22 @@ pub extern "C" fn wasm_instancetype_as_externtype_const(
) -> &wasm_externtype_t { ) -> &wasm_externtype_t {
&ty.ext &ty.ext
} }
#[no_mangle]
pub extern "C" fn wasm_instancetype_exports(
instance: &wasm_instancetype_t,
out: &mut wasm_exporttype_vec_t,
) {
let exports = instance
.ty()
.ty
.exports()
.map(|e| {
Some(Box::new(wasm_exporttype_t::new(
e.name().to_owned(),
e.ty(),
)))
})
.collect::<Vec<_>>();
out.set_buffer(exports);
}

64
crates/c-api/src/types/module.rs
View File

@@ -1,5 +1,7 @@
use crate::{wasm_externtype_t, wasm_limits_t, CExternType}; use crate::{
use once_cell::unsync::OnceCell; wasm_exporttype_t, wasm_exporttype_vec_t, wasm_externtype_t, wasm_importtype_t,
wasm_importtype_vec_t, CExternType,
};
use wasmtime::ModuleType; use wasmtime::ModuleType;
#[repr(transparent)] #[repr(transparent)]
@@ -13,24 +15,33 @@ wasmtime_c_api_macros::declare_ty!(wasm_moduletype_t);
#[derive(Clone)] #[derive(Clone)]
pub(crate) struct CModuleType { pub(crate) struct CModuleType {
pub(crate) ty: ModuleType, pub(crate) ty: ModuleType,
limits_cache: OnceCell<wasm_limits_t>,
} }
impl wasm_moduletype_t { impl wasm_moduletype_t {
pub(crate) fn new(ty: ModuleType) -> wasm_moduletype_t {
wasm_moduletype_t {
ext: wasm_externtype_t::new(ty.into()),
}
}
pub(crate) fn try_from(e: &wasm_externtype_t) -> Option<&wasm_moduletype_t> { pub(crate) fn try_from(e: &wasm_externtype_t) -> Option<&wasm_moduletype_t> {
match &e.which { match &e.which {
CExternType::Module(_) => Some(unsafe { &*(e as *const _ as *const _) }), CExternType::Module(_) => Some(unsafe { &*(e as *const _ as *const _) }),
_ => None, _ => None,
} }
} }
pub(crate) fn ty(&self) -> &CModuleType {
match &self.ext.which {
CExternType::Module(f) => &f,
_ => unreachable!(),
}
}
} }
impl CModuleType { impl CModuleType {
pub(crate) fn new(ty: ModuleType) -> CModuleType { pub(crate) fn new(ty: ModuleType) -> CModuleType {
CModuleType { CModuleType { ty }
ty,
limits_cache: OnceCell::new(),
}
} }
} }
@@ -45,3 +56,42 @@ pub extern "C" fn wasm_moduletype_as_externtype_const(
) -> &wasm_externtype_t { ) -> &wasm_externtype_t {
&ty.ext &ty.ext
} }
#[no_mangle]
pub extern "C" fn wasm_moduletype_exports(
module: &wasm_moduletype_t,
out: &mut wasm_exporttype_vec_t,
) {
let exports = module
.ty()
.ty
.exports()
.map(|e| {
Some(Box::new(wasm_exporttype_t::new(
e.name().to_owned(),
e.ty(),
)))
})
.collect::<Vec<_>>();
out.set_buffer(exports);
}
#[no_mangle]
pub extern "C" fn wasm_moduletype_imports(
module: &wasm_moduletype_t,
out: &mut wasm_importtype_vec_t,
) {
let imports = module
.ty()
.ty
.imports()
.map(|i| {
Some(Box::new(wasm_importtype_t::new(
i.module().to_owned(),
i.name().map(|s| s.to_owned()),
i.ty(),
)))
})
.collect::<Vec<_>>();
out.set_buffer(imports);
}

27
crates/c-api/src/vec.rs
View File

@@ -1,7 +1,8 @@
use crate::wasm_valtype_t; use crate::{
use crate::{wasm_exporttype_t, wasm_extern_t, wasm_frame_t, wasm_val_t}; wasm_exporttype_t, wasm_extern_t, wasm_externtype_t, wasm_frame_t, wasm_functype_t,
use crate::{wasm_externtype_t, wasm_importtype_t, wasm_memorytype_t}; wasm_globaltype_t, wasm_importtype_t, wasm_instancetype_t, wasm_memorytype_t,
use crate::{wasm_functype_t, wasm_globaltype_t, wasm_tabletype_t}; wasm_moduletype_t, wasm_tabletype_t, wasm_val_t, wasm_valtype_t,
};
use std::mem; use std::mem;
use std::ptr; use std::ptr;
use std::slice; use std::slice;
@@ -172,6 +173,24 @@ declare_vecs! {
copy: wasm_memorytype_vec_copy, copy: wasm_memorytype_vec_copy,
delete: wasm_memorytype_vec_delete, delete: wasm_memorytype_vec_delete,
) )
(
name: wasm_instancetype_vec_t,
ty: Option<Box<wasm_instancetype_t>>,
new: wasm_instancetype_vec_new,
empty: wasm_instancetype_vec_new_empty,
uninit: wasm_instancetype_vec_new_uninitialized,
copy: wasm_instancetype_vec_copy,
delete: wasm_instancetype_vec_delete,
)
(
name: wasm_moduletype_vec_t,
ty: Option<Box<wasm_moduletype_t>>,
new: wasm_moduletype_vec_new,
empty: wasm_moduletype_vec_new_empty,
uninit: wasm_moduletype_vec_new_uninitialized,
copy: wasm_moduletype_vec_copy,
delete: wasm_moduletype_vec_delete,
)
( (
name: wasm_externtype_vec_t, name: wasm_externtype_vec_t,
ty: Option<Box<wasm_externtype_t>>, ty: Option<Box<wasm_externtype_t>>,

3
crates/cranelift/src/func_environ.rs
View File

@@ -1039,7 +1039,6 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
callee: ir::Value, callee: ir::Value,
call_args: &[ir::Value], call_args: &[ir::Value],
) -> WasmResult<ir::Inst> { ) -> WasmResult<ir::Inst> {
let sig_index = self.module.types[ty_index].unwrap_function();
let pointer_type = self.pointer_type(); let pointer_type = self.pointer_type();
let table_entry_addr = pos.ins().table_addr(pointer_type, table, callee, 0); let table_entry_addr = pos.ins().table_addr(pointer_type, table, callee, 0);
@@ -1071,7 +1070,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
let vmctx = self.vmctx(pos.func); let vmctx = self.vmctx(pos.func);
let base = pos.ins().global_value(pointer_type, vmctx); let base = pos.ins().global_value(pointer_type, vmctx);
let offset = let offset =
i32::try_from(self.offsets.vmctx_vmshared_signature_id(sig_index)).unwrap(); i32::try_from(self.offsets.vmctx_vmshared_signature_id(ty_index)).unwrap();
// Load the caller ID. // Load the caller ID.
let mut mem_flags = ir::MemFlags::trusted(); let mut mem_flags = ir::MemFlags::trusted();

11
crates/cranelift/src/lib.rs
View File

@@ -99,7 +99,7 @@ use std::sync::Mutex;
use wasmtime_environ::{ use wasmtime_environ::{
CompileError, CompiledFunction, Compiler, FunctionAddressMap, FunctionBodyData, CompileError, CompiledFunction, Compiler, FunctionAddressMap, FunctionBodyData,
InstructionAddressMap, ModuleTranslation, Relocation, RelocationTarget, StackMapInformation, InstructionAddressMap, ModuleTranslation, Relocation, RelocationTarget, StackMapInformation,
TrapInformation, Tunables, TrapInformation, Tunables, TypeTables,
}; };
mod func_environ; mod func_environ;
@@ -348,21 +348,22 @@ impl Compiler for Cranelift {
mut input: FunctionBodyData<'_>, mut input: FunctionBodyData<'_>,
isa: &dyn isa::TargetIsa, isa: &dyn isa::TargetIsa,
tunables: &Tunables, tunables: &Tunables,
types: &TypeTables,
) -> Result<CompiledFunction, CompileError> { ) -> Result<CompiledFunction, CompileError> {
let module = &translation.module; let module = &translation.module;
let func_index = module.func_index(func_index); let func_index = module.func_index(func_index);
let mut context = Context::new(); let mut context = Context::new();
context.func.name = get_func_name(func_index); context.func.name = get_func_name(func_index);
let sig_index = module.functions[func_index]; let sig_index = module.functions[func_index];
context.func.signature = translation.native_signatures[sig_index].clone(); context.func.signature = types.native_signatures[sig_index].clone();
if tunables.debug_info { if tunables.generate_native_debuginfo {
context.func.collect_debug_info(); context.func.collect_debug_info();
} }
let mut func_env = FuncEnvironment::new( let mut func_env = FuncEnvironment::new(
isa.frontend_config(), isa.frontend_config(),
module, module,
&translation.native_signatures, &types.native_signatures,
tunables, tunables,
); );
@@ -434,7 +435,7 @@ impl Compiler for Cranelift {
let address_transform = let address_transform =
get_function_address_map(&context, &input, code_buf.len() as u32, isa); get_function_address_map(&context, &input, code_buf.len() as u32, isa);
let ranges = if tunables.debug_info { let ranges = if tunables.generate_native_debuginfo {
let ranges = context.build_value_labels_ranges(isa).map_err(|error| { let ranges = context.build_value_labels_ranges(isa).map_err(|error| {
CompileError::Codegen(pretty_error(&context.func, Some(isa), error)) CompileError::Codegen(pretty_error(&context.func, Some(isa), error))
})?; })?;

2
crates/debug/Cargo.toml
View File

@@ -13,7 +13,7 @@ edition = "2018"
[dependencies] [dependencies]
gimli = "0.23.0" gimli = "0.23.0"
wasmparser = "0.68.0" wasmparser = "0.70"
object = { version = "0.22.0", default-features = false, features = ["read", "write"] } object = { version = "0.22.0", default-features = false, features = ["read", "write"] }
wasmtime-environ = { path = "../environ", version = "0.21.0" } wasmtime-environ = { path = "../environ", version = "0.21.0" }
target-lexicon = { version = "0.11.0", default-features = false } target-lexicon = { version = "0.11.0", default-features = false }

2
crates/environ/Cargo.toml
View File

@@ -16,7 +16,7 @@ anyhow = "1.0"
cranelift-codegen = { path = "../../cranelift/codegen", version = "0.68.0", features = ["enable-serde"] } cranelift-codegen = { path = "../../cranelift/codegen", version = "0.68.0", features = ["enable-serde"] }
cranelift-entity = { path = "../../cranelift/entity", version = "0.68.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.68.0", features = ["enable-serde"] }
cranelift-wasm = { path = "../../cranelift/wasm", version = "0.68.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.68.0", features = ["enable-serde"] }
wasmparser = "0.68.0" wasmparser = "0.70"
indexmap = { version = "1.0.2", features = ["serde-1"] } indexmap = { version = "1.0.2", features = ["serde-1"] }
thiserror = "1.0.4" thiserror = "1.0.4"
serde = { version = "1.0.94", features = ["derive"] } serde = { version = "1.0.94", features = ["derive"] }

Some files were not shown because too many files have changed in this diff Show More