Cranelift: Make heap_addr return calculated base + index + offset (#5231)

* Cranelift: Make `heap_addr` return calculated `base + index + offset`

Rather than return just the `base + index`.

(Note: I've chosen to use the nomenclature "index" for the dynamic operand and
"offset" for the static immediate.)

This move the addition of the `offset` into `heap_addr`, instead of leaving it
for the subsequent memory operation, so that we can Spectre-guard the full
address, and not allow speculative execution to read the first 4GiB of memory.

Before this commit, we were effectively doing

    load(spectre_guard(base + index) + offset)

Now we are effectively doing

    load(spectre_guard(base + index + offset))

Finally, this also corrects `heap_addr`'s documented semantics to say that it
returns an address that will trap on access if `index + offset + access_size` is
out of bounds for the given heap, rather than saying that the `heap_addr` itself
will trap. This matches the implemented behavior for static memories, and after
https://github.com/bytecodealliance/wasmtime/pull/5190 lands (which is blocked
on this commit) will also match the implemented behavior for dynamic memories.

* Update heap_addr docs

* Factor out `offset + size` to a helper

cranelift/filetests/filetests/isa/aarch64/heap_addr.clif

@@ -9,7 +9,7 @@ function %dynamic_heap_check(i64 vmctx, i32) -> i64 {
     heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v2 = heap_addr.i64 heap0, v1, 0
+    v2 = heap_addr.i64 heap0, v1, 0, 0
     return v2
 }
 
@@ -34,7 +34,7 @@ function %static_heap_check(i64 vmctx, i32) -> i64 {
     heap0 = static gv0, bound 0x1_0000, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v2 = heap_addr.i64 heap0, v1, 0
+    v2 = heap_addr.i64 heap0, v1, 0, 0
     return v2
 }
 
@@ -52,3 +52,59 @@ block0(v0: i64, v1: i32):
 ; block2:
 ;   udf #0xc11f
 
+
+function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0
+    heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+; block0:
+;   mov w11, w1
+;   ldr x10, [x0]
+;   movz x9, #24
+;   adds x11, x11, x9
+;   b.lo 8 ; udf
+;   subs xzr, x11, x10
+;   b.ls label1 ; b label2
+; block1:
+;   add x13, x0, x1, UXTW
+;   add x13, x13, #16
+;   movz x12, #0
+;   subs xzr, x11, x10
+;   csel x0, x12, x13, hi
+;   csdb
+;   ret
+; block2:
+;   udf #0xc11f
+
+function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    heap0 = static gv0, bound 0x1_0000, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+; block0:
+;   mov w9, w1
+;   movz x10, #65512
+;   subs xzr, x9, x10
+;   b.ls label1 ; b label2
+; block1:
+;   add x11, x0, x1, UXTW
+;   add x11, x11, #16
+;   movz x10, #65512
+;   movz x12, #0
+;   subs xzr, x9, x10
+;   csel x0, x12, x11, hi
+;   csdb
+;   ret
+; block2:
+;   udf #0xc11f
+

cranelift/filetests/filetests/isa/riscv64/heap-addr.clif

@@ -8,7 +8,7 @@ function %dynamic_heap_check(i64 vmctx, i32) -> i64 {
     heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v2 = heap_addr.i64 heap0, v1, 0
+    v2 = heap_addr.i64 heap0, v1, 0, 0
     return v2
 }
 
@@ -32,7 +32,7 @@ function %static_heap_check(i64 vmctx, i32) -> i64 {
     heap0 = static gv0, bound 0x1_0000, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v2 = heap_addr.i64 heap0, v1, 0
+    v2 = heap_addr.i64 heap0, v1, 0, 0
     return v2
 }
 
@@ -51,3 +51,59 @@ block0(v0: i64, v1: i32):
 ; block2:
 ;   udf##trap_code=heap_oob
 
+function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0
+    heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+; block0:
+;   uext.w t1,a1
+;   ld t0,0(a0)
+;   li t3,24
+;   add t2,t1,t3
+;   ult a1,t2,t1##ty=i64
+;   trap_if a1,heap_oob
+;   ule a1,t2,t0##ty=i64
+;   bne a1,zero,taken(label1),not_taken(label2)
+; block1:
+;   add a0,a0,t1
+;   addi a0,a0,16
+;   ugt t1,t2,t0##ty=i64
+;   li a1,0
+;   selectif_spectre_guard a0,a1,a0##test=t1
+;   ret
+; block2:
+;   udf##trap_code=heap_oob
+
+function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    heap0 = static gv0, bound 0x1_0000, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+; block0:
+;   uext.w t3,a1
+;   lui a7,16
+;   addi a7,a7,4072
+;   ule t0,t3,a7##ty=i64
+;   bne t0,zero,taken(label1),not_taken(label2)
+; block1:
+;   add t0,a0,t3
+;   addi t0,t0,16
+;   lui t4,16
+;   addi t4,t4,4072
+;   ugt t1,t3,t4##ty=i64
+;   li a0,0
+;   selectif_spectre_guard a0,a0,t0##test=t1
+;   ret
+; block2:
+;   udf##trap_code=heap_oob
+

cranelift/filetests/filetests/isa/s390x/heap_addr.clif

@@ -7,7 +7,7 @@ function %dynamic_heap_check(i64 vmctx, i32) -> i64 {
     heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v2 = heap_addr.i64 heap0, v1, 0
+    v2 = heap_addr.i64 heap0, v1, 0, 0
     return v2
 }
 
@@ -32,7 +32,7 @@ function %static_heap_check(i64 vmctx, i32) -> i64 {
     heap0 = static gv0, bound 0x1_0000, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v2 = heap_addr.i64 heap0, v1, 0
+    v2 = heap_addr.i64 heap0, v1, 0, 0
     return v2
 }
 
@@ -49,3 +49,56 @@ block0(v0: i64, v1: i32):
 ; block2:
 ;   trap
 
+function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0
+    heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+;   stmg %r7, %r15, 56(%r15)
+; block0:
+;   llgfr %r7, %r3
+;   lg %r4, 0(%r2)
+;   lghi %r5, 24
+;   algfr %r5, %r3
+;   jle 6 ; trap
+;   clgr %r5, %r4
+;   jgnh label1 ; jg label2
+; block1:
+;   agrk %r3, %r2, %r7
+;   aghik %r2, %r3, 16
+;   lghi %r3, 0
+;   clgr %r5, %r4
+;   locgrh %r2, %r3
+;   lmg %r7, %r15, 56(%r15)
+;   br %r14
+; block2:
+;   trap
+
+function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    heap0 = static gv0, bound 0x1_0000, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+; block0:
+;   llgfr %r5, %r3
+;   clgfi %r5, 65512
+;   jgnh label1 ; jg label2
+; block1:
+;   agrk %r3, %r2, %r5
+;   aghik %r2, %r3, 16
+;   lghi %r3, 0
+;   clgfi %r5, 65512
+;   locgrh %r2, %r3
+;   br %r14
+; block2:
+;   trap
+

cranelift/filetests/filetests/isa/x64/heap-no-spectre.clif

@@ -12,7 +12,7 @@ function %f(i32, i64 vmctx) -> i64 {
     heap0 = dynamic gv1, bound gv2, offset_guard 0x1000, index_type i32
 
 block0(v0: i32, v1: i64):
-    v2 = heap_addr.i64 heap0, v0, 0x8000
+    v2 = heap_addr.i64 heap0, v0, 0x8000, 0
     return v2
 }
 
@@ -20,14 +20,15 @@ block0(v0: i32, v1: i64):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %edi, %eax
-;   movq    8(%rsi), %r9
-;   movq    %rax, %r10
-;   addq    %r10, $32768, %r10
+;   movq    8(%rsi), %r10
+;   movq    %rax, %r11
+;   addq    %r11, $32768, %r11
 ;   jnb ; ud2 heap_oob ;
-;   cmpq    %r9, %r10
+;   cmpq    %r10, %r11
 ;   jbe     label1; j label2
 ; block1:
 ;   addq    %rax, 0(%rsi), %rax
+;   addq    %rax, $32768, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -43,7 +44,7 @@ function %f(i64 vmctx, i32) -> i64 system_v {
     heap0 = static gv1, bound 0x1000, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v10 = heap_addr.i64 heap0, v1, 0
+    v10 = heap_addr.i64 heap0, v1, 0, 0
     return v10
 }
 
@@ -70,7 +71,7 @@ function %f(i64 vmctx, i32) -> i64 system_v {
     heap0 = static gv1, bound 0x1_0000_0000, offset_guard 0x8000_0000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v10 = heap_addr.i64 heap0, v1, 0
+    v10 = heap_addr.i64 heap0, v1, 0, 0
     return v10
 }
 

cranelift/filetests/filetests/isa/x64/heap.clif

@@ -25,7 +25,7 @@ function %f(i32, i64 vmctx) -> i64 {
     heap0 = dynamic gv1, bound gv2, offset_guard 0x1000, index_type i32
 
 block0(v0: i32, v1: i64):
-    v2 = heap_addr.i64 heap0, v0, 0x8000
+    v2 = heap_addr.i64 heap0, v0, 0x8000, 0
     return v2
 }
 
@@ -33,16 +33,17 @@ block0(v0: i32, v1: i64):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %edi, %eax
-;   movq    8(%rsi), %r11
+;   movq    8(%rsi), %rdx
 ;   movq    %rax, %rdi
 ;   addq    %rdi, $32768, %rdi
 ;   jnb ; ud2 heap_oob ;
-;   cmpq    %r11, %rdi
+;   cmpq    %rdx, %rdi
 ;   jbe     label1; j label2
 ; block1:
 ;   addq    %rax, 0(%rsi), %rax
+;   addq    %rax, $32768, %rax
 ;   xorq    %rcx, %rcx, %rcx
-;   cmpq    %r11, %rdi
+;   cmpq    %rdx, %rdi
 ;   cmovnbeq %rcx, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -60,7 +61,7 @@ function %f(i64 vmctx, i32) -> i64 system_v {
     heap0 = static gv1, bound 0x1000, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v10 = heap_addr.i64 heap0, v1, 0
+    v10 = heap_addr.i64 heap0, v1, 0, 0
     return v10
 }
 
@@ -91,7 +92,7 @@ function %f(i64 vmctx, i32) -> i64 system_v {
     heap0 = static gv1, bound 0x1_0000_0000, offset_guard 0x8000_0000, index_type i32
 
 block0(v0: i64, v1: i32):
-    v10 = heap_addr.i64 heap0, v1, 0
+    v10 = heap_addr.i64 heap0, v1, 0, 0
     return v10
 }
 
@@ -104,3 +105,66 @@ block0(v0: i64, v1: i32):
 ;   popq    %rbp
 ;   ret
 
+function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0
+    heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movq    %rdi, %rax
+;   movl    %esi, %edi
+;   movq    %rax, %rcx
+;   movq    0(%rcx), %rsi
+;   movq    %rdi, %rdx
+;   addq    %rdx, $24, %rdx
+;   jnb ; ud2 heap_oob ;
+;   cmpq    %rsi, %rdx
+;   jbe     label1; j label2
+; block1:
+;   movq    %rcx, %rax
+;   addq    %rax, %rdi, %rax
+;   addq    %rax, $16, %rax
+;   xorq    %rcx, %rcx, %rcx
+;   cmpq    %rsi, %rdx
+;   cmovnbeq %rcx, %rax, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; block2:
+;   ud2 heap_oob
+
+function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
+    gv0 = vmctx
+    heap0 = static gv0, bound 0x1_0000, offset_guard 0x1000, index_type i32
+
+block0(v0: i64, v1: i32):
+    v2 = heap_addr.i64 heap0, v1, 16, 8
+    return v2
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movl    %esi, %r10d
+;   cmpq    $65512, %r10
+;   jbe     label1; j label2
+; block1:
+;   movq    %rdi, %rax
+;   addq    %rax, %r10, %rax
+;   addq    %rax, $16, %rax
+;   xorq    %r11, %r11, %r11
+;   cmpq    $65512, %r10
+;   cmovnbeq %r11, %rax, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; block2:
+;   ud2 heap_oob
+