Cranelift: Add heap_load and heap_store instructions (#5300)

* Cranelift: Define `heap_load` and `heap_store` instructions

* Cranelift: Implement interpreter support for `heap_load` and `heap_store`

* Cranelift: Add a suite runtests for `heap_{load,store}`

There are so many knobs we can twist for heaps and I wanted to exhaustively test
all of them, so I wrote a script to generate the tests. I've checked in the
script in case we want to make any changes in the future, but I don't think it
is worth adding this to CI to check that scripts are up to date or anything like
that.

* Review feedback

cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif

@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd

cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# This script generates the `heap_load_store_*.clif` test files.
+#
+# Usage:
+#
+#     $ ./make-heap-load-store-tests.sh
+
+set -e
+cd $(dirname "$0")
+
+function generate_one_test() {
+    local kind=$1
+    local index_type=$2
+    local guard=$3
+    local spectre=$4
+
+    local enable_spectre=true
+    if [[ spectre == "no" ]]; then
+        enable_spectre=false
+    fi
+
+    local have_guards=yes
+    if [[ guard == "0" ]]; then
+        have_guards=no
+    fi
+
+    local gv2=""
+    local bound=0x1000
+    if [[ $kind == "dynamic" ]]; then
+        gv2="gv2 = load.i64 notrap aligned gv0+8"
+        bound=gv2
+    fi
+
+    local filename="heap_load_store_${kind}_${index_type}_${have_guards}_guards_${spectre}_spectre.clif"
+    echo "Generating $filename"
+
+    cat <<EOF > "$filename"
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=${enable_spectre}
+
+function %do_store(i64 vmctx, ${index_type}, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    $gv2
+    heap0 = ${kind} gv1, min 0x1000, bound ${bound}, offset_guard ${guard}, index_type ${index_type}
+
+block0(v0: i64, v1: ${index_type}, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, ${index_type}, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    $gv2
+    heap0 = ${kind} gv1, min 0x1000, bound ${bound}, offset_guard ${guard}, index_type ${index_type}
+    fn0 = %do_store(i64, ${index_type}, i32)
+
+block0(v0: i64, v1: ${index_type}, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: ${kind}, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
+EOF
+}
+
+for spectre in yes no; do
+    for guard in 0 0xffff_ffff; do
+        for index_type in i32 i64; do
+            for kind in static dynamic; do
+                generate_one_test $kind $index_type $guard $spectre
+            done
+        done
+    done
+done