x64 and aarch64: carry MemFlags on loads/stores; don't emit trap info unless an op can trap.

This end result was previously enacted by carrying a `SourceLoc` on
every load/store, which was somewhat cumbersome, and only indirectly
encoded metadata about a memory reference (can it trap) by its presence
or absence. We have a type for this -- `MemFlags` -- that tells us
everything we might want to know about a load or store, and we should
plumb it through to code emission instead.

This PR attaches a `MemFlags` to an `Amode` on x64, and puts it on load
and store `Inst` variants on aarch64. These two choices seem to factor
things out in the nicest way: there are relatively few load/store insts
on aarch64 but many addressing modes, while the opposite is true on x64.

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -3,6 +3,7 @@
 use crate::ir;
 use crate::ir::types;
 use crate::ir::types::*;
+use crate::ir::MemFlags;
 use crate::isa;
 use crate::isa::aarch64::{inst::EmitState, inst::*};
 use crate::machinst::*;
@@ -312,11 +313,11 @@ impl ABIMachineSpec for AArch64MachineDeps {
     }
 
     fn gen_load_stack(mem: StackAMode, into_reg: Writable<Reg>, ty: Type) -> Inst {
-        Inst::gen_load(into_reg, mem.into(), ty)
+        Inst::gen_load(into_reg, mem.into(), ty, MemFlags::trusted())
     }
 
     fn gen_store_stack(mem: StackAMode, from_reg: Reg, ty: Type) -> Inst {
-        Inst::gen_store(mem.into(), from_reg, ty)
+        Inst::gen_store(mem.into(), from_reg, ty, MemFlags::trusted())
     }
 
     fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
@@ -402,12 +403,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
 
     fn gen_load_base_offset(into_reg: Writable<Reg>, base: Reg, offset: i32, ty: Type) -> Inst {
         let mem = AMode::RegOffset(base, offset as i64, ty);
-        Inst::gen_load(into_reg, mem, ty)
+        Inst::gen_load(into_reg, mem, ty, MemFlags::trusted())
     }
 
     fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst {
         let mem = AMode::RegOffset(base, offset as i64, ty);
-        Inst::gen_store(mem, from_reg, ty)
+        Inst::gen_store(mem, from_reg, ty, MemFlags::trusted())
     }
 
     fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Inst; 2]> {
@@ -464,6 +465,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 writable_stack_reg(),
                 SImm7Scaled::maybe_from_i64(-16, types::I64).unwrap(),
             ),
+            flags: MemFlags::trusted(),
         });
         // mov fp (x29), sp. This uses the ADDI rd, rs, 0 form of `MOV` because
         // the usual encoding (`ORR`) does not work with SP.
@@ -500,6 +502,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 writable_stack_reg(),
                 SImm7Scaled::maybe_from_i64(16, types::I64).unwrap(),
             ),
+            flags: MemFlags::trusted(),
         });
 
         insts
@@ -542,6 +545,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     stack_reg(),
                     SImm7Scaled::maybe_from_i64((i * 16) as i64, types::I64).unwrap(),
                 ),
+                flags: MemFlags::trusted(),
             });
         }
 
@@ -553,6 +557,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     stack_reg(),
                     SImm9::maybe_from_i64((vec_offset + (i * 16)) as i64).unwrap(),
                 ),
+                flags: MemFlags::trusted(),
             });
         }
 
@@ -591,6 +596,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     stack_reg(),
                     SImm7Scaled::maybe_from_i64((i * 16) as i64, types::I64).unwrap(),
                 ),
+                flags: MemFlags::trusted(),
             });
         }
 
@@ -601,6 +607,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     stack_reg(),
                     SImm9::maybe_from_i64(((i * 16) + int_save_bytes) as i64).unwrap(),
                 ),
+                flags: MemFlags::trusted(),
             });
         }
 
@@ -621,6 +628,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 writable_xreg(BALDRDASH_TLS_REG),
                 AMode::UnsignedOffset(fp_reg(), UImm12Scaled::maybe_from_i64(off, I64).unwrap()),
                 I64,
+                MemFlags::trusted(),
             ));
         }
 

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -3,7 +3,7 @@
 use crate::binemit::{CodeOffset, Reloc, StackMap};
 use crate::ir::constant::ConstantData;
 use crate::ir::types::*;
-use crate::ir::TrapCode;
+use crate::ir::{MemFlags, TrapCode};
 use crate::isa::aarch64::inst::*;
 use crate::machinst::ty_bits;
 
@@ -741,16 +741,18 @@ impl MachInstEmit for Inst {
                 sink.put4(enc_bit_rr(size, op1, op2, rn, rd))
             }
 
-            &Inst::ULoad8 { rd, ref mem }
+            &Inst::ULoad8 { rd, ref mem, flags }
-            | &Inst::SLoad8 { rd, ref mem }
+            | &Inst::SLoad8 { rd, ref mem, flags }
-            | &Inst::ULoad16 { rd, ref mem }
+            | &Inst::ULoad16 { rd, ref mem, flags }
-            | &Inst::SLoad16 { rd, ref mem }
+            | &Inst::SLoad16 { rd, ref mem, flags }
-            | &Inst::ULoad32 { rd, ref mem }
+            | &Inst::ULoad32 { rd, ref mem, flags }
-            | &Inst::SLoad32 { rd, ref mem }
+            | &Inst::SLoad32 { rd, ref mem, flags }
-            | &Inst::ULoad64 { rd, ref mem, .. }
+            | &Inst::ULoad64 {
-            | &Inst::FpuLoad32 { rd, ref mem }
+                rd, ref mem, flags, ..
-            | &Inst::FpuLoad64 { rd, ref mem }
+            }
-            | &Inst::FpuLoad128 { rd, ref mem } => {
+            | &Inst::FpuLoad32 { rd, ref mem, flags }
+            | &Inst::FpuLoad64 { rd, ref mem, flags }
+            | &Inst::FpuLoad128 { rd, ref mem, flags } => {
                 let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
 
                 for inst in mem_insts.into_iter() {
@@ -778,7 +780,7 @@ impl MachInstEmit for Inst {
                 };
 
                 let srcloc = state.cur_srcloc();
-                if srcloc != SourceLoc::default() {
+                if srcloc != SourceLoc::default() && !flags.notrap() {
                     // Register the offset at which the actual load instruction starts.
                     sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
                 }
@@ -861,13 +863,13 @@ impl MachInstEmit for Inst {
                 }
             }
 
-            &Inst::Store8 { rd, ref mem }
+            &Inst::Store8 { rd, ref mem, flags }
-            | &Inst::Store16 { rd, ref mem }
+            | &Inst::Store16 { rd, ref mem, flags }
-            | &Inst::Store32 { rd, ref mem }
+            | &Inst::Store32 { rd, ref mem, flags }
-            | &Inst::Store64 { rd, ref mem, .. }
+            | &Inst::Store64 { rd, ref mem, flags }
-            | &Inst::FpuStore32 { rd, ref mem }
+            | &Inst::FpuStore32 { rd, ref mem, flags }
-            | &Inst::FpuStore64 { rd, ref mem }
+            | &Inst::FpuStore64 { rd, ref mem, flags }
-            | &Inst::FpuStore128 { rd, ref mem } => {
+            | &Inst::FpuStore128 { rd, ref mem, flags } => {
                 let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
 
                 for inst in mem_insts.into_iter() {
@@ -886,7 +888,7 @@ impl MachInstEmit for Inst {
                 };
 
                 let srcloc = state.cur_srcloc();
-                if srcloc != SourceLoc::default() {
+                if srcloc != SourceLoc::default() && !flags.notrap() {
                     // Register the offset at which the actual load instruction starts.
                     sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
                 }
@@ -943,7 +945,18 @@ impl MachInstEmit for Inst {
                 }
             }
 
-            &Inst::StoreP64 { rt, rt2, ref mem } => match mem {
+            &Inst::StoreP64 {
+                rt,
+                rt2,
+                ref mem,
+                flags,
+            } => {
+                let srcloc = state.cur_srcloc();
+                if srcloc != SourceLoc::default() && !flags.notrap() {
+                    // Register the offset at which the actual load instruction starts.
+                    sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
+                }
+                match mem {
                     &PairAMode::SignedOffset(reg, simm7) => {
                         assert_eq!(simm7.scale_ty, I64);
                         sink.put4(enc_ldst_pair(0b1010100100, simm7, reg, rt, rt2));
@@ -956,8 +969,20 @@ impl MachInstEmit for Inst {
                         assert_eq!(simm7.scale_ty, I64);
                         sink.put4(enc_ldst_pair(0b1010100010, simm7, reg.to_reg(), rt, rt2));
                     }
-            },
+                }
-            &Inst::LoadP64 { rt, rt2, ref mem } => {
+            }
+            &Inst::LoadP64 {
+                rt,
+                rt2,
+                ref mem,
+                flags,
+            } => {
+                let srcloc = state.cur_srcloc();
+                if srcloc != SourceLoc::default() && !flags.notrap() {
+                    // Register the offset at which the actual load instruction starts.
+                    sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
+                }
+
                 let rt = rt.to_reg();
                 let rt2 = rt2.to_reg();
                 match mem {
@@ -1546,6 +1571,7 @@ impl MachInstEmit for Inst {
                 let inst = Inst::FpuLoad64 {
                     rd,
                     mem: AMode::Label(MemLabel::PCRel(8)),
+                    flags: MemFlags::trusted(),
                 };
                 inst.emit(sink, emit_info, state);
                 let inst = Inst::Jump {
@@ -1558,6 +1584,7 @@ impl MachInstEmit for Inst {
                 let inst = Inst::FpuLoad128 {
                     rd,
                     mem: AMode::Label(MemLabel::PCRel(8)),
+                    flags: MemFlags::trusted(),
                 };
                 inst.emit(sink, emit_info, state);
                 let inst = Inst::Jump {
@@ -2169,6 +2196,7 @@ impl MachInstEmit for Inst {
                         I32,
                         ExtendOp::UXTW,
                     ),
+                    flags: MemFlags::trusted(),
                 };
                 inst.emit(sink, emit_info, state);
                 // Add base of jump table to jump-table-sourced block offset
@@ -2215,6 +2243,7 @@ impl MachInstEmit for Inst {
                 let inst = Inst::ULoad64 {
                     rd,
                     mem: AMode::Label(MemLabel::PCRel(8)),
+                    flags: MemFlags::trusted(),
                 };
                 inst.emit(sink, emit_info, state);
                 let inst = Inst::Jump {

cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs

@@ -1079,6 +1079,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad8 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "41004038",
         "ldurb w1, [x2]",
@@ -1087,6 +1088,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad8 {
             rd: writable_xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)),
+            flags: MemFlags::trusted(),
         },
         "41004039",
         "ldrb w1, [x2]",
@@ -1095,6 +1097,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad8 {
             rd: writable_xreg(1),
             mem: AMode::RegReg(xreg(2), xreg(5)),
+            flags: MemFlags::trusted(),
         },
         "41686538",
         "ldrb w1, [x2, x5]",
@@ -1103,6 +1106,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad8 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "41008038",
         "ldursb x1, [x2]",
@@ -1111,6 +1115,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad8 {
             rd: writable_xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41FC8039",
         "ldrsb x1, [x2, #63]",
@@ -1119,6 +1124,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad8 {
             rd: writable_xreg(1),
             mem: AMode::RegReg(xreg(2), xreg(5)),
+            flags: MemFlags::trusted(),
         },
         "4168A538",
         "ldrsb x1, [x2, x5]",
@@ -1127,6 +1133,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad16 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41504078",
         "ldurh w1, [x2, #5]",
@@ -1135,6 +1142,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad16 {
             rd: writable_xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41104079",
         "ldrh w1, [x2, #8]",
@@ -1143,6 +1151,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad16 {
             rd: writable_xreg(1),
             mem: AMode::RegScaled(xreg(2), xreg(3), I16),
+            flags: MemFlags::trusted(),
         },
         "41786378",
         "ldrh w1, [x2, x3, LSL #1]",
@@ -1151,6 +1160,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad16 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "41008078",
         "ldursh x1, [x2]",
@@ -1159,6 +1169,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad16 {
             rd: writable_xreg(28),
             mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "9C328079",
         "ldrsh x28, [x20, #24]",
@@ -1167,6 +1178,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad16 {
             rd: writable_xreg(28),
             mem: AMode::RegScaled(xreg(20), xreg(20), I16),
+            flags: MemFlags::trusted(),
         },
         "9C7AB478",
         "ldrsh x28, [x20, x20, LSL #1]",
@@ -1175,6 +1187,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad32 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "410040B8",
         "ldur w1, [x2]",
@@ -1183,6 +1196,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad32 {
             rd: writable_xreg(12),
             mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "0CCC40B9",
         "ldr w12, [x0, #204]",
@@ -1191,6 +1205,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad32 {
             rd: writable_xreg(1),
             mem: AMode::RegScaled(xreg(2), xreg(12), I32),
+            flags: MemFlags::trusted(),
         },
         "41786CB8",
         "ldr w1, [x2, x12, LSL #2]",
@@ -1199,6 +1214,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad32 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "410080B8",
         "ldursw x1, [x2]",
@@ -1207,6 +1223,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad32 {
             rd: writable_xreg(12),
             mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "2CFCBFB9",
         "ldrsw x12, [x1, #16380]",
@@ -1215,6 +1232,7 @@ fn test_aarch64_binemit() {
         Inst::SLoad32 {
             rd: writable_xreg(1),
             mem: AMode::RegScaled(xreg(5), xreg(1), I32),
+            flags: MemFlags::trusted(),
         },
         "A178A1B8",
         "ldrsw x1, [x5, x1, LSL #2]",
@@ -1223,6 +1241,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "410040F8",
         "ldur x1, [x2]",
@@ -1231,6 +1250,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "410050F8",
         "ldur x1, [x2, #-256]",
@@ -1239,6 +1259,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41F04FF8",
         "ldur x1, [x2, #255]",
@@ -1247,6 +1268,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41FC7FF9",
         "ldr x1, [x2, #32760]",
@@ -1255,6 +1277,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::RegReg(xreg(2), xreg(3)),
+            flags: MemFlags::trusted(),
         },
         "416863F8",
         "ldr x1, [x2, x3]",
@@ -1263,6 +1286,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::RegScaled(xreg(2), xreg(3), I64),
+            flags: MemFlags::trusted(),
         },
         "417863F8",
         "ldr x1, [x2, x3, LSL #3]",
@@ -1271,6 +1295,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW),
+            flags: MemFlags::trusted(),
         },
         "41D863F8",
         "ldr x1, [x2, w3, SXTW #3]",
@@ -1279,6 +1304,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW),
+            flags: MemFlags::trusted(),
         },
         "41C863F8",
         "ldr x1, [x2, w3, SXTW]",
@@ -1287,6 +1313,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::Label(MemLabel::PCRel(64)),
+            flags: MemFlags::trusted(),
         },
         "01020058",
         "ldr x1, pc+64",
@@ -1295,6 +1322,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "410C41F8",
         "ldr x1, [x2, #16]!",
@@ -1303,6 +1331,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "410441F8",
         "ldr x1, [x2], #16",
@@ -1311,6 +1340,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::FPOffset(32768, I8),
+            flags: MemFlags::trusted(),
         },
         "100090D2B063308B010240F9",
         "movz x16, #32768 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1319,6 +1349,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::FPOffset(-32768, I8),
+            flags: MemFlags::trusted(),
         },
         "F0FF8F92B063308B010240F9",
         "movn x16, #32767 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1327,6 +1358,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::FPOffset(1048576, I8), // 2^20
+            flags: MemFlags::trusted(),
         },
         "1002A0D2B063308B010240F9",
         "movz x16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1335,6 +1367,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1
+            flags: MemFlags::trusted(),
         },
         "300080521002A072B063308B010240F9",
         "movz w16, #1 ; movk w16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1344,6 +1377,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::RegOffset(xreg(7), 8, I64),
+            flags: MemFlags::trusted(),
         },
         "E18040F8",
         "ldur x1, [x7, #8]",
@@ -1353,6 +1387,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::RegOffset(xreg(7), 1024, I64),
+            flags: MemFlags::trusted(),
         },
         "E10042F9",
         "ldr x1, [x7, #1024]",
@@ -1362,6 +1397,7 @@ fn test_aarch64_binemit() {
         Inst::ULoad64 {
             rd: writable_xreg(1),
             mem: AMode::RegOffset(xreg(7), 1048576, I64),
+            flags: MemFlags::trusted(),
         },
         "1002A0D2F060308B010240F9",
         "movz x16, #16, LSL #16 ; add x16, x7, x16, UXTX ; ldr x1, [x16]",
@@ -1371,6 +1407,7 @@ fn test_aarch64_binemit() {
         Inst::Store8 {
             rd: xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "41000038",
         "sturb w1, [x2]",
@@ -1379,6 +1416,7 @@ fn test_aarch64_binemit() {
         Inst::Store8 {
             rd: xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41FC3F39",
         "strb w1, [x2, #4095]",
@@ -1387,6 +1425,7 @@ fn test_aarch64_binemit() {
         Inst::Store16 {
             rd: xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "41000078",
         "sturh w1, [x2]",
@@ -1395,6 +1434,7 @@ fn test_aarch64_binemit() {
         Inst::Store16 {
             rd: xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41FC3F79",
         "strh w1, [x2, #8190]",
@@ -1403,6 +1443,7 @@ fn test_aarch64_binemit() {
         Inst::Store32 {
             rd: xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "410000B8",
         "stur w1, [x2]",
@@ -1411,6 +1452,7 @@ fn test_aarch64_binemit() {
         Inst::Store32 {
             rd: xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41FC3FB9",
         "str w1, [x2, #16380]",
@@ -1419,6 +1461,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::Unscaled(xreg(2), SImm9::zero()),
+            flags: MemFlags::trusted(),
         },
         "410000F8",
         "stur x1, [x2]",
@@ -1427,6 +1470,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "41FC3FF9",
         "str x1, [x2, #32760]",
@@ -1435,6 +1479,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::RegReg(xreg(2), xreg(3)),
+            flags: MemFlags::trusted(),
         },
         "416823F8",
         "str x1, [x2, x3]",
@@ -1443,6 +1488,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::RegScaled(xreg(2), xreg(3), I64),
+            flags: MemFlags::trusted(),
         },
         "417823F8",
         "str x1, [x2, x3, LSL #3]",
@@ -1451,6 +1497,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW),
+            flags: MemFlags::trusted(),
         },
         "415823F8",
         "str x1, [x2, w3, UXTW #3]",
@@ -1459,6 +1506,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW),
+            flags: MemFlags::trusted(),
         },
         "414823F8",
         "str x1, [x2, w3, UXTW]",
@@ -1467,6 +1515,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "410C01F8",
         "str x1, [x2, #16]!",
@@ -1475,6 +1524,7 @@ fn test_aarch64_binemit() {
         Inst::Store64 {
             rd: xreg(1),
             mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "410401F8",
         "str x1, [x2], #16",
@@ -1485,6 +1535,7 @@ fn test_aarch64_binemit() {
             rt: xreg(8),
             rt2: xreg(9),
             mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::zero(I64)),
+            flags: MemFlags::trusted(),
         },
         "482500A9",
         "stp x8, x9, [x10]",
@@ -1494,6 +1545,7 @@ fn test_aarch64_binemit() {
             rt: xreg(8),
             rt2: xreg(9),
             mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(504, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "48A51FA9",
         "stp x8, x9, [x10, #504]",
@@ -1503,6 +1555,7 @@ fn test_aarch64_binemit() {
             rt: xreg(8),
             rt2: xreg(9),
             mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "48253CA9",
         "stp x8, x9, [x10, #-64]",
@@ -1512,6 +1565,7 @@ fn test_aarch64_binemit() {
             rt: xreg(21),
             rt2: xreg(28),
             mem: PairAMode::SignedOffset(xreg(1), SImm7Scaled::maybe_from_i64(-512, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "357020A9",
         "stp x21, x28, [x1, #-512]",
@@ -1524,6 +1578,7 @@ fn test_aarch64_binemit() {
                 writable_xreg(10),
                 SImm7Scaled::maybe_from_i64(-64, I64).unwrap(),
             ),
+            flags: MemFlags::trusted(),
         },
         "4825BCA9",
         "stp x8, x9, [x10, #-64]!",
@@ -1536,6 +1591,7 @@ fn test_aarch64_binemit() {
                 writable_xreg(20),
                 SImm7Scaled::maybe_from_i64(504, I64).unwrap(),
             ),
+            flags: MemFlags::trusted(),
         },
         "8FC29FA8",
         "stp x15, x16, [x20], #504",
@@ -1546,6 +1602,7 @@ fn test_aarch64_binemit() {
             rt: writable_xreg(8),
             rt2: writable_xreg(9),
             mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::zero(I64)),
+            flags: MemFlags::trusted(),
         },
         "482540A9",
         "ldp x8, x9, [x10]",
@@ -1555,6 +1612,7 @@ fn test_aarch64_binemit() {
             rt: writable_xreg(8),
             rt2: writable_xreg(9),
             mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(504, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "48A55FA9",
         "ldp x8, x9, [x10, #504]",
@@ -1564,6 +1622,7 @@ fn test_aarch64_binemit() {
             rt: writable_xreg(8),
             rt2: writable_xreg(9),
             mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "48257CA9",
         "ldp x8, x9, [x10, #-64]",
@@ -1573,6 +1632,7 @@ fn test_aarch64_binemit() {
             rt: writable_xreg(8),
             rt2: writable_xreg(9),
             mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-512, I64).unwrap()),
+            flags: MemFlags::trusted(),
         },
         "482560A9",
         "ldp x8, x9, [x10, #-512]",
@@ -1585,6 +1645,7 @@ fn test_aarch64_binemit() {
                 writable_xreg(10),
                 SImm7Scaled::maybe_from_i64(-64, I64).unwrap(),
             ),
+            flags: MemFlags::trusted(),
         },
         "4825FCA9",
         "ldp x8, x9, [x10, #-64]!",
@@ -1597,6 +1658,7 @@ fn test_aarch64_binemit() {
                 writable_xreg(12),
                 SImm7Scaled::maybe_from_i64(504, I64).unwrap(),
             ),
+            flags: MemFlags::trusted(),
         },
         "88E5DFA8",
         "ldp x8, x25, [x12], #504",
@@ -4756,6 +4818,7 @@ fn test_aarch64_binemit() {
         Inst::FpuLoad32 {
             rd: writable_vreg(16),
             mem: AMode::RegScaled(xreg(8), xreg(9), F32),
+            flags: MemFlags::trusted(),
         },
         "107969BC",
         "ldr s16, [x8, x9, LSL #2]",
@@ -4765,6 +4828,7 @@ fn test_aarch64_binemit() {
         Inst::FpuLoad64 {
             rd: writable_vreg(16),
             mem: AMode::RegScaled(xreg(8), xreg(9), F64),
+            flags: MemFlags::trusted(),
         },
         "107969FC",
         "ldr d16, [x8, x9, LSL #3]",
@@ -4774,6 +4838,7 @@ fn test_aarch64_binemit() {
         Inst::FpuLoad128 {
             rd: writable_vreg(16),
             mem: AMode::RegScaled(xreg(8), xreg(9), I128),
+            flags: MemFlags::trusted(),
         },
         "1079E93C",
         "ldr q16, [x8, x9, LSL #4]",
@@ -4783,6 +4848,7 @@ fn test_aarch64_binemit() {
         Inst::FpuLoad32 {
             rd: writable_vreg(16),
             mem: AMode::Label(MemLabel::PCRel(8)),
+            flags: MemFlags::trusted(),
         },
         "5000001C",
         "ldr s16, pc+8",
@@ -4792,6 +4858,7 @@ fn test_aarch64_binemit() {
         Inst::FpuLoad64 {
             rd: writable_vreg(16),
             mem: AMode::Label(MemLabel::PCRel(8)),
+            flags: MemFlags::trusted(),
         },
         "5000005C",
         "ldr d16, pc+8",
@@ -4801,6 +4868,7 @@ fn test_aarch64_binemit() {
         Inst::FpuLoad128 {
             rd: writable_vreg(16),
             mem: AMode::Label(MemLabel::PCRel(8)),
+            flags: MemFlags::trusted(),
         },
         "5000009C",
         "ldr q16, pc+8",
@@ -4810,6 +4878,7 @@ fn test_aarch64_binemit() {
         Inst::FpuStore32 {
             rd: vreg(16),
             mem: AMode::RegScaled(xreg(8), xreg(9), F32),
+            flags: MemFlags::trusted(),
         },
         "107929BC",
         "str s16, [x8, x9, LSL #2]",
@@ -4819,6 +4888,7 @@ fn test_aarch64_binemit() {
         Inst::FpuStore64 {
             rd: vreg(16),
             mem: AMode::RegScaled(xreg(8), xreg(9), F64),
+            flags: MemFlags::trusted(),
         },
         "107929FC",
         "str d16, [x8, x9, LSL #3]",
@@ -4828,6 +4898,7 @@ fn test_aarch64_binemit() {
         Inst::FpuStore128 {
             rd: vreg(16),
             mem: AMode::RegScaled(xreg(8), xreg(9), I128),
+            flags: MemFlags::trusted(),
         },
         "1079A93C",
         "str q16, [x8, x9, LSL #4]",

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -9,7 +9,7 @@ use crate::ir::types::{
     F64X2, FFLAGS, I16, I16X4, I16X8, I32, I32X2, I32X4, I64, I64X2, I8, I8X16, I8X8, IFLAGS, R32,
     R64,
 };
-use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type};
+use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type};
 use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::{settings, CodegenError, CodegenResult};
@@ -523,57 +523,68 @@ pub enum Inst {
     ULoad8 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// A signed (sign-extending) 8-bit load.
     SLoad8 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// An unsigned (zero-extending) 16-bit load.
     ULoad16 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// A signed (sign-extending) 16-bit load.
     SLoad16 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// An unsigned (zero-extending) 32-bit load.
     ULoad32 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// A signed (sign-extending) 32-bit load.
     SLoad32 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// A 64-bit load.
     ULoad64 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
 
     /// An 8-bit store.
     Store8 {
         rd: Reg,
         mem: AMode,
+        flags: MemFlags,
     },
     /// A 16-bit store.
     Store16 {
         rd: Reg,
         mem: AMode,
+        flags: MemFlags,
     },
     /// A 32-bit store.
     Store32 {
         rd: Reg,
         mem: AMode,
+        flags: MemFlags,
     },
     /// A 64-bit store.
     Store64 {
         rd: Reg,
         mem: AMode,
+        flags: MemFlags,
     },
 
     /// A store of a pair of registers.
@@ -581,12 +592,14 @@ pub enum Inst {
         rt: Reg,
         rt2: Reg,
         mem: PairAMode,
+        flags: MemFlags,
     },
     /// A load of a pair of registers.
     LoadP64 {
         rt: Writable<Reg>,
         rt2: Writable<Reg>,
         mem: PairAMode,
+        flags: MemFlags,
     },
 
     /// A MOV instruction. These are encoded as ORR's (AluRRR form) but we
@@ -782,31 +795,37 @@ pub enum Inst {
     FpuLoad32 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// Floating-point store, single-precision (32 bit).
     FpuStore32 {
         rd: Reg,
         mem: AMode,
+        flags: MemFlags,
     },
     /// Floating-point load, double-precision (64 bit).
     FpuLoad64 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// Floating-point store, double-precision (64 bit).
     FpuStore64 {
         rd: Reg,
         mem: AMode,
+        flags: MemFlags,
     },
     /// Floating-point/vector load, 128 bit.
     FpuLoad128 {
         rd: Writable<Reg>,
         mem: AMode,
+        flags: MemFlags,
     },
     /// Floating-point/vector store, 128 bit.
     FpuStore128 {
         rd: Reg,
         mem: AMode,
+        flags: MemFlags,
     },
 
     LoadFpuConst64 {
@@ -1411,24 +1430,48 @@ impl Inst {
     }
 
     /// Generic constructor for a load (zero-extending where appropriate).
-    pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type) -> Inst {
+    pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst {
         match ty {
-            B1 | B8 | I8 => Inst::ULoad8 { rd: into_reg, mem },
+            B1 | B8 | I8 => Inst::ULoad8 {
-            B16 | I16 => Inst::ULoad16 { rd: into_reg, mem },
+                rd: into_reg,
-            B32 | I32 | R32 => Inst::ULoad32 { rd: into_reg, mem },
+                mem,
-            B64 | I64 | R64 => Inst::ULoad64 { rd: into_reg, mem },
+                flags,
-            F32 => Inst::FpuLoad32 { rd: into_reg, mem },
+            },
-            F64 => Inst::FpuLoad64 { rd: into_reg, mem },
+            B16 | I16 => Inst::ULoad16 {
+                rd: into_reg,
+                mem,
+                flags,
+            },
+            B32 | I32 | R32 => Inst::ULoad32 {
+                rd: into_reg,
+                mem,
+                flags,
+            },
+            B64 | I64 | R64 => Inst::ULoad64 {
+                rd: into_reg,
+                mem,
+                flags,
+            },
+            F32 => Inst::FpuLoad32 {
+                rd: into_reg,
+                mem,
+                flags,
+            },
+            F64 => Inst::FpuLoad64 {
+                rd: into_reg,
+                mem,
+                flags,
+            },
             _ => {
                 if ty.is_vector() {
                     let bits = ty_bits(ty);
                     let rd = into_reg;
 
                     if bits == 128 {
-                        Inst::FpuLoad128 { rd, mem }
+                        Inst::FpuLoad128 { rd, mem, flags }
                     } else {
                         assert_eq!(bits, 64);
-                        Inst::FpuLoad64 { rd, mem }
+                        Inst::FpuLoad64 { rd, mem, flags }
                     }
                 } else {
                     unimplemented!("gen_load({})", ty);
@@ -1438,24 +1481,48 @@ impl Inst {
     }
 
     /// Generic constructor for a store.
-    pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type) -> Inst {
+    pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst {
         match ty {
-            B1 | B8 | I8 => Inst::Store8 { rd: from_reg, mem },
+            B1 | B8 | I8 => Inst::Store8 {
-            B16 | I16 => Inst::Store16 { rd: from_reg, mem },
+                rd: from_reg,
-            B32 | I32 | R32 => Inst::Store32 { rd: from_reg, mem },
+                mem,
-            B64 | I64 | R64 => Inst::Store64 { rd: from_reg, mem },
+                flags,
-            F32 => Inst::FpuStore32 { rd: from_reg, mem },
+            },
-            F64 => Inst::FpuStore64 { rd: from_reg, mem },
+            B16 | I16 => Inst::Store16 {
+                rd: from_reg,
+                mem,
+                flags,
+            },
+            B32 | I32 | R32 => Inst::Store32 {
+                rd: from_reg,
+                mem,
+                flags,
+            },
+            B64 | I64 | R64 => Inst::Store64 {
+                rd: from_reg,
+                mem,
+                flags,
+            },
+            F32 => Inst::FpuStore32 {
+                rd: from_reg,
+                mem,
+                flags,
+            },
+            F64 => Inst::FpuStore64 {
+                rd: from_reg,
+                mem,
+                flags,
+            },
             _ => {
                 if ty.is_vector() {
                     let bits = ty_bits(ty);
                     let rd = from_reg;
 
                     if bits == 128 {
-                        Inst::FpuStore128 { rd, mem }
+                        Inst::FpuStore128 { rd, mem, flags }
                     } else {
                         assert_eq!(bits, 64);
-                        Inst::FpuStore64 { rd, mem }
+                        Inst::FpuStore64 { rd, mem, flags }
                     }
                 } else {
                     unimplemented!("gen_store({})", ty);
@@ -2126,6 +2193,7 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rt,
             ref mut rt2,
             ref mut mem,
+            ..
         } => {
             map_use(mapper, rt);
             map_use(mapper, rt2);
@@ -2135,6 +2203,7 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rt,
             ref mut rt2,
             ref mut mem,
+            ..
         } => {
             map_def(mapper, rt);
             map_def(mapper, rt2);
@@ -2979,26 +3048,32 @@ impl Inst {
             &Inst::ULoad8 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::SLoad8 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::ULoad16 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::SLoad16 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::ULoad32 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::SLoad32 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::ULoad64 {
                 rd,
@@ -3035,14 +3110,17 @@ impl Inst {
             &Inst::Store8 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::Store16 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::Store32 {
                 rd,
                 ref mem,
+                ..
             }
             | &Inst::Store64 {
                 rd,
@@ -3070,13 +3148,13 @@ impl Inst {
                 let mem = mem.show_rru(mb_rru);
                 format!("{}{} {}, {}", mem_str, op, rd, mem)
             }
-            &Inst::StoreP64 { rt, rt2, ref mem } => {
+            &Inst::StoreP64 { rt, rt2, ref mem, .. } => {
                 let rt = rt.show_rru(mb_rru);
                 let rt2 = rt2.show_rru(mb_rru);
                 let mem = mem.show_rru(mb_rru);
                 format!("stp {}, {}, {}", rt, rt2, mem)
             }
-            &Inst::LoadP64 { rt, rt2, ref mem } => {
+            &Inst::LoadP64 { rt, rt2, ref mem, .. } => {
                 let rt = rt.to_reg().show_rru(mb_rru);
                 let rt2 = rt2.to_reg().show_rru(mb_rru);
                 let mem = mem.show_rru(mb_rru);

cranelift/codegen/src/isa/aarch64/inst/unwind.rs

@@ -29,6 +29,7 @@ impl UnwindInfoGenerator<Inst> for AArch64UnwindInfo {
                     rt,
                     rt2,
                     mem: PairAMode::PreIndexed(rn, imm7),
+                    ..
                 } if *rt == regs::fp_reg()
                     && *rt2 == regs::link_reg()
                     && *rn == regs::writable_stack_reg()
@@ -60,6 +61,7 @@ impl UnwindInfoGenerator<Inst> for AArch64UnwindInfo {
                     rt,
                     rt2,
                     mem: PairAMode::PreIndexed(rn, imm7),
+                    ..
                 } if rn.to_reg() == regs::stack_reg() && imm7.value % (pair_size as i16) == 0 => {
                     // stp r1, r2, [sp, #(i * #16)]
                     let stack_offset = imm7.value as u32;

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -1130,6 +1130,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 | Opcode::Sload32Complex => true,
                 _ => false,
             };
+            let flags = ctx
+                .memflags(insn)
+                .expect("Load instruction should have memflags");
 
             lower_load(
                 ctx,
@@ -1139,19 +1142,19 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 |ctx, rd, elem_ty, mem| {
                     let is_float = ty_has_float_or_vec_representation(elem_ty);
                     ctx.emit(match (ty_bits(elem_ty), sign_extend, is_float) {
-                        (1, _, _) => Inst::ULoad8 { rd, mem },
+                        (1, _, _) => Inst::ULoad8 { rd, mem, flags },
-                        (8, false, _) => Inst::ULoad8 { rd, mem },
+                        (8, false, _) => Inst::ULoad8 { rd, mem, flags },
-                        (8, true, _) => Inst::SLoad8 { rd, mem },
+                        (8, true, _) => Inst::SLoad8 { rd, mem, flags },
-                        (16, false, _) => Inst::ULoad16 { rd, mem },
+                        (16, false, _) => Inst::ULoad16 { rd, mem, flags },
-                        (16, true, _) => Inst::SLoad16 { rd, mem },
+                        (16, true, _) => Inst::SLoad16 { rd, mem, flags },
-                        (32, false, false) => Inst::ULoad32 { rd, mem },
+                        (32, false, false) => Inst::ULoad32 { rd, mem, flags },
-                        (32, true, false) => Inst::SLoad32 { rd, mem },
+                        (32, true, false) => Inst::SLoad32 { rd, mem, flags },
-                        (32, _, true) => Inst::FpuLoad32 { rd, mem },
+                        (32, _, true) => Inst::FpuLoad32 { rd, mem, flags },
-                        (64, _, false) => Inst::ULoad64 { rd, mem },
+                        (64, _, false) => Inst::ULoad64 { rd, mem, flags },
                         // Note that we treat some of the vector loads as scalar floating-point loads,
                         // which is correct in a little endian environment.
-                        (64, _, true) => Inst::FpuLoad64 { rd, mem },
+                        (64, _, true) => Inst::FpuLoad64 { rd, mem, flags },
-                        (128, _, _) => Inst::FpuLoad128 { rd, mem },
+                        (128, _, _) => Inst::FpuLoad128 { rd, mem, flags },
                         _ => panic!("Unsupported size in load"),
                     });
 
@@ -1200,18 +1203,21 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 _ => unreachable!(),
             };
             let is_float = ty_has_float_or_vec_representation(elem_ty);
+            let flags = ctx
+                .memflags(insn)
+                .expect("Store instruction should have memflags");
 
             let mem = lower_address(ctx, elem_ty, &inputs[1..], off);
             let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
 
             ctx.emit(match (ty_bits(elem_ty), is_float) {
-                (1, _) | (8, _) => Inst::Store8 { rd, mem },
+                (1, _) | (8, _) => Inst::Store8 { rd, mem, flags },
-                (16, _) => Inst::Store16 { rd, mem },
+                (16, _) => Inst::Store16 { rd, mem, flags },
-                (32, false) => Inst::Store32 { rd, mem },
+                (32, false) => Inst::Store32 { rd, mem, flags },
-                (32, true) => Inst::FpuStore32 { rd, mem },
+                (32, true) => Inst::FpuStore32 { rd, mem, flags },
-                (64, false) => Inst::Store64 { rd, mem },
+                (64, false) => Inst::Store64 { rd, mem, flags },
-                (64, true) => Inst::FpuStore64 { rd, mem },
+                (64, true) => Inst::FpuStore64 { rd, mem, flags },
-                (128, _) => Inst::FpuStore128 { rd, mem },
+                (128, _) => Inst::FpuStore128 { rd, mem, flags },
                 _ => panic!("Unsupported size in store"),
             });
         }

cranelift/codegen/src/isa/x64/abi.rs

@@ -1,7 +1,7 @@
 //! Implementation of the standard x64 ABI.
 
 use crate::ir::types::*;
-use crate::ir::{self, types, TrapCode, Type};
+use crate::ir::{self, types, MemFlags, TrapCode, Type};
 use crate::isa;
 use crate::isa::{x64::inst::*, CallConv};
 use crate::machinst::abi_impl::*;
@@ -618,6 +618,7 @@ impl From<StackAMode> for SyntheticAmode {
                 SyntheticAmode::Real(Amode::ImmReg {
                     simm32,
                     base: regs::rbp(),
+                    flags: MemFlags::trusted(),
                 })
             }
             StackAMode::NominalSPOffset(off, _ty) => {
@@ -634,6 +635,7 @@ impl From<StackAMode> for SyntheticAmode {
                 SyntheticAmode::Real(Amode::ImmReg {
                     simm32,
                     base: regs::rsp(),
+                    flags: MemFlags::trusted(),
                 })
             }
         }

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -3,6 +3,7 @@
 use super::regs::{self, show_ireg_sized};
 use super::EmitState;
 use crate::ir::condcodes::{FloatCC, IntCC};
+use crate::ir::MemFlags;
 use crate::isa::x64::inst::Inst;
 use crate::machinst::*;
 use regalloc::{
@@ -14,10 +15,14 @@ use std::string::String;
 
 /// A possible addressing mode (amode) that can be used in instructions.
 /// These denote a 64-bit value only.
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub enum Amode {
     /// Immediate sign-extended and a Register.
-    ImmReg { simm32: u32, base: Reg },
+    ImmReg {
+        simm32: u32,
+        base: Reg,
+        flags: MemFlags,
+    },
 
     /// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift)
     ImmRegRegShift {
@@ -25,6 +30,7 @@ pub enum Amode {
         base: Reg,
         index: Reg,
         shift: u8, /* 0 .. 3 only */
+        flags: MemFlags,
     },
 
     /// sign-extend-32-to-64(Immediate) + RIP (instruction pointer).
@@ -35,7 +41,11 @@ pub enum Amode {
 impl Amode {
     pub(crate) fn imm_reg(simm32: u32, base: Reg) -> Self {
         debug_assert!(base.get_class() == RegClass::I64);
-        Self::ImmReg { simm32, base }
+        Self::ImmReg {
+            simm32,
+            base,
+            flags: MemFlags::trusted(),
+        }
     }
 
     pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self {
@@ -47,6 +57,7 @@ impl Amode {
             base,
             index,
             shift,
+            flags: MemFlags::trusted(),
         }
     }
 
@@ -54,6 +65,30 @@ impl Amode {
         Self::RipRelative { target }
     }
 
+    pub(crate) fn with_flags(&self, flags: MemFlags) -> Self {
+        match self {
+            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {
+                simm32,
+                base,
+                flags,
+            },
+            &Self::ImmRegRegShift {
+                simm32,
+                base,
+                index,
+                shift,
+                ..
+            } => Self::ImmRegRegShift {
+                simm32,
+                base,
+                index,
+                shift,
+                flags,
+            },
+            _ => panic!("Amode {:?} cannot take memflags", self),
+        }
+    }
+
     /// Add the regs mentioned by `self` to `collector`.
     pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
         match self {
@@ -69,12 +104,24 @@ impl Amode {
             }
         }
     }
+
+    pub(crate) fn get_flags(&self) -> MemFlags {
+        match self {
+            Amode::ImmReg { flags, .. } => *flags,
+            Amode::ImmRegRegShift { flags, .. } => *flags,
+            Amode::RipRelative { .. } => MemFlags::trusted(),
+        }
+    }
+
+    pub(crate) fn can_trap(&self) -> bool {
+        !self.get_flags().notrap()
+    }
 }
 
 impl PrettyPrint for Amode {
     fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
         match self {
-            Amode::ImmReg { simm32, base } => {
+            Amode::ImmReg { simm32, base, .. } => {
                 format!("{}({})", *simm32 as i32, base.show_rru(mb_rru))
             }
             Amode::ImmRegRegShift {
@@ -82,6 +129,7 @@ impl PrettyPrint for Amode {
                 base,
                 index,
                 shift,
+                ..
             } => format!(
                 "{}({},{},{})",
                 *simm32 as i32,

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -194,14 +194,14 @@ fn emit_std_enc_mem(
     // expression.  But `enc_g` can be derived from a register of any class.
 
     let srcloc = state.cur_srcloc();
-    if srcloc != SourceLoc::default() {
+    if srcloc != SourceLoc::default() && mem_e.can_trap() {
         sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
     }
 
     prefixes.emit(sink);
 
     match mem_e {
-        Amode::ImmReg { simm32, base } => {
+        Amode::ImmReg { simm32, base, .. } => {
             // First, the REX byte.
             let enc_e = int_reg_enc(*base);
             rex.emit_two_op(sink, enc_g, enc_e);
@@ -260,6 +260,7 @@ fn emit_std_enc_mem(
             base: reg_base,
             index: reg_index,
             shift,
+            ..
         } => {
             let enc_base = int_reg_enc(*reg_base);
             let enc_index = int_reg_enc(*reg_index);

cranelift/codegen/src/isa/x64/inst/unwind.rs

@@ -61,7 +61,7 @@ impl UnwindInfoGenerator<Inst> for X64UnwindInfo {
                 }
                 Inst::MovRM {
                     src,
-                    dst: SyntheticAmode::Real(Amode::ImmReg { simm32, base }),
+                    dst: SyntheticAmode::Real(Amode::ImmReg { simm32, base, .. }),
                     ..
                 } if *base == regs::rsp() => {
                     // `mov reg, imm(rsp)`

cranelift/codegen/src/isa/x64/lower.rs

@@ -579,6 +579,10 @@ fn matches_small_constant_shift<C: LowerCtx<I = Inst>>(
 ///
 /// Note: the 32-bit offset in Cranelift has to be sign-extended, which maps x86's behavior.
 fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i32) -> Amode {
+    let flags = ctx
+        .memflags(spec.insn)
+        .expect("Instruction with amode should have memflags");
+
     // We now either have an add that we must materialize, or some other input; as well as the
     // final offset.
     if let Some(add) = matches_input(ctx, spec, Opcode::Iadd) {
@@ -632,7 +636,7 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i
                         let final_offset = (offset as i64).wrapping_add(uext_cst as i64);
                         if low32_will_sign_extend_to_64(final_offset as u64) {
                             let base = put_input_in_reg(ctx, add_inputs[1 - i]);
-                            return Amode::imm_reg(final_offset as u32, base);
+                            return Amode::imm_reg(final_offset as u32, base).with_flags(flags);
                         }
                     }
                 }
@@ -642,7 +646,7 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i
                     let final_offset = (offset as i64).wrapping_add(cst as i64);
                     if low32_will_sign_extend_to_64(final_offset as u64) {
                         let base = put_input_in_reg(ctx, add_inputs[1 - i]);
-                        return Amode::imm_reg(final_offset as u32, base);
+                        return Amode::imm_reg(final_offset as u32, base).with_flags(flags);
                     }
                 }
             }
@@ -654,11 +658,11 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i
             )
         };
 
-        return Amode::imm_reg_reg_shift(offset as u32, base, index, shift);
+        return Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags);
     }
 
     let input = put_input_in_reg(ctx, spec);
-    Amode::imm_reg(offset as u32, input)
+    Amode::imm_reg(offset as u32, input).with_flags(flags)
 }
 
 //=============================================================================
@@ -3060,7 +3064,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     let base = put_input_in_reg(ctx, inputs[0]);
                     let index = put_input_in_reg(ctx, inputs[1]);
                     let shift = 0;
-                    Amode::imm_reg_reg_shift(offset as u32, base, index, shift)
+                    let flags = ctx.memflags(insn).expect("load should have memflags");
+                    Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags)
                 }
 
                 _ => unreachable!(),
@@ -3132,7 +3137,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     let base = put_input_in_reg(ctx, inputs[1]);
                     let index = put_input_in_reg(ctx, inputs[2]);
                     let shift = 0;
-                    Amode::imm_reg_reg_shift(offset as u32, base, index, shift)
+                    let flags = ctx.memflags(insn).expect("store should have memflags");
+                    Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags)
                 }
 
                 _ => unreachable!(),