Cranelift: implement heap_{load,store} instruction legalization (#5351)

* Cranelift: implement `heap_{load,store}` instruction legalization This does not remove `heap_addr` yet, but it does factor out the common bounds-check-and-compute-the-native-address functionality that is shared between all of `heap_{addr,load,store}`. Finally, this adds a missing optimization for when we can dedupe explicit bounds checks for static memories and Spectre mitigations. * Cranelift: Enable `heap_load_store_*` run tests on all targets
2022-11-30 11:12:49 -08:00
parent 830885383f
commit 79f7fa6079
27 changed files with 566 additions and 471 deletions
--- a/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
@@ -35,18 +35,12 @@ block0(v0: i64, v1: i32):

 ; block0:
 ;   mov w6, w1
+;   add x7, x0, x1, UXTW
+;   movz x5, #0
 ;   subs xzr, x6, #65536
-;   b.ls label1 ; b label2
-; block1:
-;   add x8, x0, x1, UXTW
-;   movz x7, #0
-;   subs xzr, x6, #65536
-;   csel x0, x7, x8, hi
+;   csel x0, x5, x7, hi
 ;   csdb
 ;   ret
-; block2:
-;   udf #0xc11f
-

 function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
    gv0 = vmctx
@@ -59,16 +53,16 @@ block0(v0: i64, v1: i32):
 }

 ; block0:
-;   mov w11, w1
-;   ldr x10, [x0]
+;   mov w10, w1
 ;   movz x9, #24
-;   adds x11, x11, x9
+;   adds x11, x10, x9
 ;   b.lo 8 ; udf
-;   add x12, x0, x1, UXTW
-;   add x12, x12, #16
-;   movz x13, #0
-;   subs xzr, x11, x10
-;   csel x0, x13, x12, hi
+;   ldr x12, [x0]
+;   add x13, x0, x1, UXTW
+;   add x13, x13, #16
+;   movz x10, #0
+;   subs xzr, x11, x12
+;   csel x0, x10, x13, hi
 ;   csdb
 ;   ret

@@ -82,19 +76,13 @@ block0(v0: i64, v1: i32):
 }

 ; block0:
-;   mov w9, w1
-;   movz w10, #65512
-;   subs xzr, x9, x10
-;   b.ls label1 ; b label2
-; block1:
-;   add x11, x0, x1, UXTW
-;   add x11, x11, #16
-;   movz w10, #65512
-;   movz x12, #0
-;   subs xzr, x9, x10
-;   csel x0, x12, x11, hi
+;   mov w8, w1
+;   add x9, x0, x1, UXTW
+;   add x9, x9, #16
+;   movz w6, #65512
+;   movz x10, #0
+;   subs xzr, x8, x6
+;   csel x0, x10, x9, hi
 ;   csdb
 ;   ret
-; block2:
-;   udf #0xc11f

--- a/cranelift/filetests/filetests/isa/riscv64/heap-addr.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/heap-addr.clif
@@ -32,19 +32,13 @@ block0(v0: i64, v1: i32):
 }

 ; block0:
-;   uext.w a7,a1
-;   lui a6,16
-;   ule t3,a7,a6##ty=i64
-;   bne t3,zero,taken(label1),not_taken(label2)
-; block1:
-;   add t3,a0,a7
-;   lui a6,16
-;   ugt t4,a7,a6##ty=i64
-;   li t0,0
-;   selectif_spectre_guard a0,t0,t3##test=t4
+;   uext.w a6,a1
+;   add a5,a0,a6
+;   lui a3,16
+;   ugt a6,a6,a3##ty=i64
+;   li a7,0
+;   selectif_spectre_guard a0,a7,a5##test=a6
 ;   ret
-; block2:
-;   udf##trap_code=heap_oob

 function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
    gv0 = vmctx
@@ -57,17 +51,17 @@ block0(v0: i64, v1: i32):
 }

 ; block0:
-;   uext.w t0,a1
-;   ld t4,0(a0)
+;   uext.w t4,a1
 ;   li a7,24
-;   add t1,t0,a7
-;   ult t2,t1,t0##ty=i64
-;   trap_if t2,heap_oob
-;   add t0,a0,t0
-;   addi t0,t0,16
-;   ugt t4,t1,t4##ty=i64
+;   add t0,t4,a7
+;   ult t1,t0,t4##ty=i64
+;   trap_if t1,heap_oob
+;   ld t1,0(a0)
+;   add t2,a0,t4
+;   addi t2,t2,16
+;   ugt t4,t0,t1##ty=i64
 ;   li t1,0
-;   selectif_spectre_guard a0,t1,t0##test=t4
+;   selectif_spectre_guard a0,t1,t2##test=t4
 ;   ret

 function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
@@ -80,20 +74,13 @@ block0(v0: i64, v1: i32):
 }

 ; block0:
-;   uext.w t3,a1
-;   lui a7,16
-;   addi a7,a7,4072
-;   ule t0,t3,a7##ty=i64
-;   bne t0,zero,taken(label1),not_taken(label2)
-; block1:
-;   add t0,a0,t3
-;   addi t0,t0,16
-;   lui t4,16
-;   addi t4,t4,4072
-;   ugt t1,t3,t4##ty=i64
-;   li a0,0
-;   selectif_spectre_guard a0,a0,t0##test=t1
+;   uext.w a6,a1
+;   add a7,a0,a6
+;   addi a7,a7,16
+;   lui a4,16
+;   addi a4,a4,4072
+;   ugt t3,a6,a4##ty=i64
+;   li t4,0
+;   selectif_spectre_guard a0,t4,a7##test=t3
 ;   ret
-; block2:
-;   udf##trap_code=heap_oob

--- a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
+++ b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
@@ -32,16 +32,11 @@ block0(v0: i64, v1: i32):

 ; block0:
 ;   llgfr %r4, %r3
-;   clgfi %r4, 65536
-;   jgnh label1 ; jg label2
-; block1:
 ;   agr %r2, %r4
-;   lghi %r5, 0
+;   lghi %r3, 0
 ;   clgfi %r4, 65536
-;   locgrh %r2, %r5
+;   locgrh %r2, %r3
 ;   br %r14
-; block2:
-;   trap

 function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
    gv0 = vmctx
@@ -53,19 +48,17 @@ block0(v0: i64, v1: i32):
    return v2
 }

-;   stmg %r7, %r15, 56(%r15)
 ; block0:
-;   llgfr %r7, %r3
-;   lg %r4, 0(%r2)
-;   lghi %r5, 24
-;   algfr %r5, %r3
+;   llgfr %r5, %r3
+;   lghi %r4, 24
+;   algfr %r4, %r3
 ;   jle 6 ; trap
-;   agr %r2, %r7
-;   aghi %r2, 16
-;   lghi %r3, 0
-;   clgr %r5, %r4
-;   locgrh %r2, %r3
-;   lmg %r7, %r15, 56(%r15)
+;   lg %r3, 0(%r2)
+;   agrk %r5, %r2, %r5
+;   aghik %r2, %r5, 16
+;   lghi %r5, 0
+;   clgr %r4, %r3
+;   locgrh %r2, %r5
 ;   br %r14

 function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
@@ -79,15 +72,10 @@ block0(v0: i64, v1: i32):

 ; block0:
 ;   llgfr %r5, %r3
+;   agr %r2, %r5
+;   aghi %r2, 16
+;   lghi %r4, 0
 ;   clgfi %r5, 65512
-;   jgnh label1 ; jg label2
-; block1:
-;   agrk %r3, %r2, %r5
-;   aghik %r2, %r3, 16
-;   lghi %r3, 0
-;   clgfi %r5, 65512
-;   locgrh %r2, %r3
+;   locgrh %r2, %r4
 ;   br %r14
-; block2:
-;   trap

--- a/cranelift/filetests/filetests/isa/x64/heap-no-spectre.clif
+++ b/cranelift/filetests/filetests/isa/x64/heap-no-spectre.clif
@@ -20,11 +20,11 @@ block0(v0: i32, v1: i64):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %edi, %eax
-;   movq    8(%rsi), %r10
-;   movq    %rax, %r11
-;   addq    %r11, $32768, %r11
+;   movq    %rax, %r10
+;   addq    %r10, $32768, %r10
 ;   jnb ; ud2 heap_oob ;
-;   cmpq    %r10, %r11
+;   movq    8(%rsi), %r11
+;   cmpq    %r11, %r10
 ;   jbe     label1; j label2
 ; block1:
 ;   addq    %rax, 0(%rsi), %rax
--- a/cranelift/filetests/filetests/isa/x64/heap.clif
+++ b/cranelift/filetests/filetests/isa/x64/heap.clif
@@ -33,14 +33,14 @@ block0(v0: i32, v1: i64):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %edi, %eax
-;   movq    8(%rsi), %rdi
-;   movq    %rax, %rcx
-;   addq    %rcx, $32768, %rcx
+;   movq    %rax, %rdi
+;   addq    %rdi, $32768, %rdi
 ;   jnb ; ud2 heap_oob ;
+;   movq    8(%rsi), %rcx
 ;   addq    %rax, 0(%rsi), %rax
 ;   addq    %rax, $32768, %rax
 ;   xorq    %rsi, %rsi, %rsi
-;   cmpq    %rdi, %rcx
+;   cmpq    %rcx, %rdi
 ;   cmovnbeq %rsi, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -64,19 +64,14 @@ block0(v0: i64, v1: i32):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %esi, %r9d
-;   cmpq    $4096, %r9
-;   jbe     label1; j label2
-; block1:
 ;   movq    %r9, %rax
 ;   addq    %rax, 0(%rdi), %rax
-;   xorq    %r10, %r10, %r10
+;   xorq    %r8, %r8, %r8
 ;   cmpq    $4096, %r9
-;   cmovnbeq %r10, %rax, %rax
+;   cmovnbeq %r8, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
-; block2:
-;   ud2 heap_oob

 ;; When a static memory is the "right" size (4GB memory, 2GB guard regions), the
 ;; Spectre mitigation is not present. Cranelift relies on the memory permissions
@@ -113,17 +108,16 @@ block0(v0: i64, v1: i32):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movq    %rdi, %rax
-;   movl    %esi, %edi
-;   movq    0(%rax), %rsi
-;   movq    %rdi, %rcx
-;   addq    %rcx, $24, %rcx
+;   movl    %esi, %esi
+;   movq    %rsi, %r11
+;   addq    %r11, $24, %r11
 ;   jnb ; ud2 heap_oob ;
-;   addq    %rax, %rdi, %rax
+;   movq    %rdi, %rax
+;   addq    %rax, %rsi, %rax
 ;   addq    %rax, $16, %rax
-;   xorq    %rdi, %rdi, %rdi
-;   cmpq    %rsi, %rcx
-;   cmovnbeq %rdi, %rax, %rax
+;   xorq    %rsi, %rsi, %rsi
+;   cmpq    0(%rdi), %r11
+;   cmovnbeq %rsi, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -141,18 +135,13 @@ block0(v0: i64, v1: i32):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %esi, %r10d
-;   cmpq    $65512, %r10
-;   jbe     label1; j label2
-; block1:
 ;   movq    %rdi, %rax
 ;   addq    %rax, %r10, %rax
 ;   addq    %rax, $16, %rax
-;   xorq    %r11, %r11, %r11
+;   xorq    %r9, %r9, %r9
 ;   cmpq    $65512, %r10
-;   cmovnbeq %r11, %rax, %rax
+;   cmovnbeq %r9, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
-; block2:
-;   ud2 heap_oob

--- a/cranelift/filetests/filetests/legalizer/bounds-checks.clif
+++ b/cranelift/filetests/filetests/legalizer/bounds-checks.clif
@@ -20,13 +20,14 @@ block0(v0: i64, v1: i32):

 ; check:  block0(v0: i64, v1: i32):
 ; nextln:     v4 = uextend.i64 v1
-; nextln:     v5 = load.i64 notrap aligned v0+88
-; nextln:     v6 = iconst.i64 4
-; nextln:     v7 = uadd_overflow_trap v4, v6, heap_oob  ; v6 = 4
+; nextln:     v5 = iconst.i64 4
+; nextln:     v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 4
+; nextln:     v7 = load.i64 notrap aligned v0+88
 ; nextln:     v8 = load.i64 notrap aligned v0+80
 ; nextln:     v9 = iadd v8, v4
 ; nextln:     v10 = iconst.i64 0
-; nextln:     v11 = icmp ugt v7, v5
-; nextln:     v2 = select_spectre_guard v11, v10, v9  ; v10 = 0
+; nextln:     v11 = icmp ugt v6, v7
+; nextln:     v12 = select_spectre_guard v11, v10, v9  ; v10 = 0
+; nextln:     v2 -> v12
 ; nextln:     v3 = load.i32 little heap v2
 ; nextln:     return v3
--- a/cranelift/filetests/filetests/legalizer/static-heap-with-guard-pages.clif
+++ b/cranelift/filetests/filetests/legalizer/static-heap-with-guard-pages.clif
@@ -17,5 +17,6 @@ block0(v0: i64, v1: i32):
 ; check:  block0(v0: i64, v1: i32):
 ; nextln:     v3 = uextend.i64 v1
 ; nextln:     v4 = load.i64 notrap aligned v0
-; nextln:     v2 = iadd v4, v3
+; nextln:     v5 = iadd v4, v3
+; nextln:     v2 -> v5
 ; nextln:     return v2
--- a/cranelift/filetests/filetests/legalizer/static-heap-without-guard-pages.clif
+++ b/cranelift/filetests/filetests/legalizer/static-heap-without-guard-pages.clif
@@ -3,6 +3,8 @@ set enable_heap_access_spectre_mitigation=true
 target x86_64

 ;; The offset guard is not large enough to avoid explicit bounds checks.
+;; Additionally, the explicit bounds check gets deduped with the Spectre
+;; mitigation.

 function %test(i64 vmctx, i32) -> i64 {
    gv0 = vmctx
@@ -16,19 +18,11 @@ block0(v0: i64, v1: i32):

 ; check:  block0(v0: i64, v1: i32):
 ; nextln:     v3 = uextend.i64 v1
-; nextln:     v10 = iconst.i64 4092
-; nextln:     v4 = icmp ugt v3, v10  ; v10 = 4092
-; nextln:     brz v4, block2
-; nextln:     jump block1
-; nextln: 
-; nextln: block1:
-; nextln:     trap heap_oob
-; nextln: 
-; nextln: block2:
-; nextln:     v5 = iconst.i64 4092
-; nextln:     v6 = load.i64 notrap aligned v0
-; nextln:     v7 = iadd v6, v3
-; nextln:     v8 = iconst.i64 0
-; nextln:     v9 = icmp.i64 ugt v3, v5  ; v5 = 4092
-; nextln:     v2 = select_spectre_guard v9, v8, v7  ; v8 = 0
+; nextln:     v4 = iconst.i64 4092
+; nextln:     v5 = load.i64 notrap aligned v0
+; nextln:     v6 = iadd v5, v3
+; nextln:     v7 = iconst.i64 0
+; nextln:     v8 = icmp ugt v3, v4  ; v4 = 4092
+; nextln:     v9 = select_spectre_guard v8, v7, v6  ; v7 = 0
+; nextln:     v2 -> v9
 ; nextln:     return v2
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=false

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=true

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0
--- a/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh
+++ b/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh
@@ -39,14 +39,15 @@ function generate_one_test() {
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!

 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run

 set enable_heap_access_spectre_mitigation=${enable_spectre}

+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, ${index_type}, i32) {
    gv0 = vmctx
    gv1 = load.i64 notrap aligned gv0+0