diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 07ec21645c..9110cedae6 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -209,10 +209,8 @@
        ;; effect of atomically modifying a memory location in a particular way.  Because we have
        ;; no way to explain to the regalloc about earlyclobber registers, this instruction has
        ;; completely fixed operand registers, and we rely on the RA's coalescing to remove copies
-       ;; in the surrounding code to the extent it can.  The sequence is both preceded and
-       ;; followed by a fence which is at least as comprehensive as that of the `Fence`
-       ;; instruction below.  This instruction is sequentially consistent.  The operand
-       ;; conventions are:
+       ;; in the surrounding code to the extent it can. Load- and store-exclusive instructions,
+       ;; with acquire-release semantics, are used to access memory. The operand conventions are:
        ;;
        ;; x25   (rd) address
        ;; x26   (rd) second operand for `op`
@@ -221,28 +219,10 @@
        ;; x28   (wr) scratch reg; value afterwards has no meaning
        (AtomicRMWLoop
         (ty Type) ;; I8, I16, I32 or I64
-        (op AtomicRmwOp))
-
-       ;; An atomic read-modify-write operation. These instructions require the
-       ;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
-       ;; acquire-release semantics.
-       (AtomicRMW
-        (op AtomicRMWOp)
-        (rs Reg)
-        (rt WritableReg)
-        (rn Reg)
-        (ty Type))
-
-       ;; An atomic compare-and-swap operation. This instruction is sequentially consistent.
-       (AtomicCAS
-        (rs WritableReg)
-        (rt Reg)
-        (rn Reg)
-        (ty Type))
+        (op AtomicRMWLoopOp))
 
        ;; Similar to AtomicRMWLoop, a compare-and-swap operation implemented using a load-linked
-       ;; store-conditional loop.
-       ;; This instruction is sequentially consistent.
+       ;; store-conditional loop, with acquire-release semantics.
        ;; Note that the operand conventions, although very similar to AtomicRMWLoop, are different:
        ;;
        ;; x25   (rd) address
@@ -254,6 +234,23 @@
         (ty Type) ;; I8, I16, I32 or I64
         )
 
+       ;; An atomic read-modify-write operation. These instructions require the
+       ;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
+       ;; acquire-release semantics.
+       (AtomicRMW
+         (op AtomicRMWOp)
+         (rs Reg)
+         (rt WritableReg)
+         (rn Reg)
+         (ty Type))
+
+       ;; An atomic compare-and-swap operation. This instruction is sequentially consistent.
+       (AtomicCAS
+         (rs WritableReg)
+         (rt Reg)
+         (rn Reg)
+         (ty Type))
+
        ;; Read `access_ty` bits from address `rt`, either 8, 16, 32 or 64-bits, and put
        ;; it in `rn`, optionally zero-extending to fill a word or double word result.
        ;; This instruction is sequentially consistent.
@@ -1261,8 +1258,30 @@
     (Smin)
     (Umax)
     (Umin)
+    (Swp)
 ))
 
+;; Atomic read-modify-write operations, with acquire-release semantics,
+;; implemented with a loop.
+(type AtomicRMWLoopOp
+  (enum
+    (Add)
+    (Sub)
+    (And)
+    (Nand)
+    (Eor)
+    (Orr)
+    (Smax)
+    (Smin)
+    (Umax)
+    (Umin)
+    (Xchg)
+))
+
+;; Extractors for target features ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(decl use_lse () Inst)
+(extern extractor use_lse use_lse)
+
 ;; Extractor helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (decl move_wide_const_from_u64 (MoveWideConst) u64)
@@ -1304,6 +1323,9 @@
 (decl integral_ty (Type) Type)
 (extern extractor integral_ty integral_ty)
 
+(decl valid_atomic_transaction (Type) Type)
+(extern extractor valid_atomic_transaction valid_atomic_transaction)
+
 ;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
 (decl imm12_from_value (Imm12) Value)
 (extractor
@@ -1345,6 +1367,26 @@
 (decl writable_zero_reg () WritableReg)
 (extern constructor writable_zero_reg writable_zero_reg)
 
+;; Helpers for getting a particular real register
+(decl xreg (u8) Reg)
+(extern constructor xreg xreg)
+
+(decl writable_xreg (u8) WritableReg)
+(extern constructor writable_xreg writable_xreg)
+
+;; Helper for emitting `MInst.Mov64` instructions.
+(decl mov64_to_real (u8 Reg) Reg)
+(rule (mov64_to_real num src)
+      (let ((dst WritableReg (writable_xreg num))
+            (_ Unit (emit (MInst.Mov (operand_size $I64) dst src))))
+        dst))
+
+(decl mov64_from_real (u8) Reg)
+(rule (mov64_from_real num)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.Mov (operand_size $I64) dst (xreg num)))))
+        dst))
+
 ;; Helper for emitting `MInst.MovZ` instructions.
 (decl movz (MoveWideConst OperandSize) Reg)
 (rule (movz imm size)
@@ -2053,3 +2095,30 @@
 (decl cmeq0 (Reg VectorSize) Reg)
 (rule (cmeq0 rn size)
       (vec_misc (VecMisc2.Cmeq0) rn size))
+
+;; Helper for emitting `MInst.AtomicRMW` instructions.
+(decl lse_atomic_rmw (AtomicRMWOp Value Reg Type) Reg)
+(rule (lse_atomic_rmw op p r_arg2 ty)
+      (let (
+          (r_addr Reg p)
+          (dst WritableReg (temp_writable_reg ty))
+          (_ Unit (emit (MInst.AtomicRMW op r_arg2 dst r_addr ty)))
+        )
+        dst))
+
+;; Helper for emitting `MInst.AtomicRMWLoop` instructions.
+;; - Make sure that both args are in virtual regs, since in effect
+;; we have to do a parallel copy to get them safely to the AtomicRMW input
+;; regs, and that's not guaranteed safe if either is in a real reg.
+;; - Move the args to the preordained AtomicRMW input regs
+;; - And finally, copy the preordained AtomicRMW output reg to its destination.
+(decl atomic_rmw_loop (AtomicRMWLoopOp Value Value Type) Reg)
+(rule (atomic_rmw_loop op p arg2 ty)
+      (let (
+          (v_addr Reg (ensure_in_vreg p $I64))
+          (v_arg2 Reg (ensure_in_vreg arg2 $I64))
+          (r_addr Reg (mov64_to_real 25 v_addr))
+          (r_arg2 Reg (mov64_to_real 26 v_arg2))
+          (_ Unit (emit (MInst.AtomicRMWLoop ty op)))
+        )
+        (mov64_from_real 27)))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs
index 2a25c6702a..69f20f5044 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/args.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs
@@ -583,6 +583,13 @@ impl OperandSize {
         }
     }
 
+    pub fn bits(&self) -> u8 {
+        match self {
+            OperandSize::Size32 => 32,
+            OperandSize::Size64 => 64,
+        }
+    }
+
     /// Convert from an integer type into the smallest size that fits.
     pub fn from_ty(ty: Type) -> OperandSize {
         debug_assert!(!ty.is_vector());
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index b8c441aadb..559c9b0c3a 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -7,6 +7,7 @@ use crate::ir::constant::ConstantData;
 use crate::ir::types::*;
 use crate::ir::{LibCall, MemFlags, TrapCode};
 use crate::isa::aarch64::inst::*;
+use crate::isa::aarch64::lower::is_valid_atomic_transaction_ty;
 use crate::machinst::{ty_bits, Reg, RegClass, Writable};
 use core::convert::TryFrom;
 
@@ -505,7 +506,7 @@ fn enc_dmb_ish() -> u32 {
     0xD5033BBF
 }
 
-fn enc_ldal(ty: Type, op: AtomicRMWOp, rs: Reg, rt: Writable<Reg>, rn: Reg) -> u32 {
+fn enc_acq_rel(ty: Type, op: AtomicRMWOp, rs: Reg, rt: Writable<Reg>, rn: Reg) -> u32 {
     assert!(machreg_to_gpr(rt.to_reg()) != 31);
     let sz = match ty {
         I64 => 0b11,
@@ -514,6 +515,10 @@ fn enc_ldal(ty: Type, op: AtomicRMWOp, rs: Reg, rt: Writable<Reg>, rn: Reg) -> u
         I8 => 0b00,
         _ => unreachable!(),
     };
+    let bit15 = match op {
+        AtomicRMWOp::Swp => 0b1,
+        _ => 0b0,
+    };
     let op = match op {
         AtomicRMWOp::Add => 0b000,
         AtomicRMWOp::Clr => 0b001,
@@ -523,10 +528,12 @@ fn enc_ldal(ty: Type, op: AtomicRMWOp, rs: Reg, rt: Writable<Reg>, rn: Reg) -> u
         AtomicRMWOp::Smin => 0b101,
         AtomicRMWOp::Umax => 0b110,
         AtomicRMWOp::Umin => 0b111,
+        AtomicRMWOp::Swp => 0b000,
     };
     0b00_111_000_111_00000_0_000_00_00000_00000
         | (sz << 30)
         | (machreg_to_gpr(rs) << 16)
+        | bit15 << 15
         | (op << 12)
         | (machreg_to_gpr(rn) << 5)
         | machreg_to_gpr(rt.to_reg())
@@ -1371,15 +1378,18 @@ impl MachInstEmit for Inst {
                 sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond));
             }
             &Inst::AtomicRMW { ty, op, rs, rt, rn } => {
+                assert!(is_valid_atomic_transaction_ty(ty));
                 let rs = allocs.next(rs);
                 let rt = allocs.next_writable(rt);
                 let rn = allocs.next(rn);
-                sink.put4(enc_ldal(ty, op, rs, rt, rn));
+                sink.put4(enc_acq_rel(ty, op, rs, rt, rn));
             }
             &Inst::AtomicRMWLoop { ty, op } => {
+                assert!(is_valid_atomic_transaction_ty(ty));
                 /* Emit this:
                      again:
                       ldaxr{,b,h}  x/w27, [x25]
+                      // maybe sign extend
                       op          x28, x27, x26 // op is add,sub,and,orr,eor
                       stlxr{,b,h}  w24, x/w28, [x25]
                       cbnz        x24, again
@@ -1414,10 +1424,31 @@ impl MachInstEmit for Inst {
                 }
                 sink.put4(enc_ldaxr(ty, x27wr, x25)); // ldaxr x27, [x25]
                 let size = OperandSize::from_ty(ty);
+                let sign_ext = match op {
+                    AtomicRMWLoopOp::Smin | AtomicRMWLoopOp::Smax => match ty {
+                        I16 => Some((ExtendOp::SXTH, 16)),
+                        I8 => Some((ExtendOp::SXTB, 8)),
+                        _ => None,
+                    },
+                    _ => None,
+                };
+
+                // sxt{b|h} the loaded result if necessary.
+                if sign_ext.is_some() {
+                    let (_, from_bits) = sign_ext.unwrap();
+                    Inst::Extend {
+                        rd: x27wr,
+                        rn: x27,
+                        signed: true,
+                        from_bits,
+                        to_bits: size.bits(),
+                    }
+                    .emit(&[], sink, emit_info, state);
+                }
 
                 match op {
-                    AtomicRmwOp::Xchg => {} // do nothing
-                    AtomicRmwOp::Nand => {
+                    AtomicRMWLoopOp::Xchg => {} // do nothing
+                    AtomicRMWLoopOp::Nand => {
                         // and x28, x27, x26
                         // mvn x28, x28
 
@@ -1439,29 +1470,42 @@ impl MachInstEmit for Inst {
                         }
                         .emit(&[], sink, emit_info, state);
                     }
-                    AtomicRmwOp::Umin
-                    | AtomicRmwOp::Umax
-                    | AtomicRmwOp::Smin
-                    | AtomicRmwOp::Smax => {
-                        // cmp x27, x26
+                    AtomicRMWLoopOp::Umin
+                    | AtomicRMWLoopOp::Umax
+                    | AtomicRMWLoopOp::Smin
+                    | AtomicRMWLoopOp::Smax => {
+                        // cmp x27, x26 {?sxt}
                         // csel.op x28, x27, x26
 
                         let cond = match op {
-                            AtomicRmwOp::Umin => Cond::Lo,
-                            AtomicRmwOp::Umax => Cond::Hi,
-                            AtomicRmwOp::Smin => Cond::Lt,
-                            AtomicRmwOp::Smax => Cond::Gt,
+                            AtomicRMWLoopOp::Umin => Cond::Lo,
+                            AtomicRMWLoopOp::Umax => Cond::Hi,
+                            AtomicRMWLoopOp::Smin => Cond::Lt,
+                            AtomicRMWLoopOp::Smax => Cond::Gt,
                             _ => unreachable!(),
                         };
 
-                        Inst::AluRRR {
-                            alu_op: ALUOp::SubS,
-                            size,
-                            rd: writable_zero_reg(),
-                            rn: x27,
-                            rm: x26,
+                        if sign_ext.is_some() {
+                            let (extendop, _) = sign_ext.unwrap();
+                            Inst::AluRRRExtend {
+                                alu_op: ALUOp::SubS,
+                                size,
+                                rd: writable_zero_reg(),
+                                rn: x27,
+                                rm: x26,
+                                extendop,
+                            }
+                            .emit(&[], sink, emit_info, state);
+                        } else {
+                            Inst::AluRRR {
+                                alu_op: ALUOp::SubS,
+                                size,
+                                rd: writable_zero_reg(),
+                                rn: x27,
+                                rm: x26,
+                            }
+                            .emit(&[], sink, emit_info, state);
                         }
-                        .emit(&[], sink, emit_info, state);
 
                         Inst::CSel {
                             cond,
@@ -1474,17 +1518,17 @@ impl MachInstEmit for Inst {
                     _ => {
                         // add/sub/and/orr/eor x28, x27, x26
                         let alu_op = match op {
-                            AtomicRmwOp::Add => ALUOp::Add,
-                            AtomicRmwOp::Sub => ALUOp::Sub,
-                            AtomicRmwOp::And => ALUOp::And,
-                            AtomicRmwOp::Or => ALUOp::Orr,
-                            AtomicRmwOp::Xor => ALUOp::Eor,
-                            AtomicRmwOp::Nand
-                            | AtomicRmwOp::Umin
-                            | AtomicRmwOp::Umax
-                            | AtomicRmwOp::Smin
-                            | AtomicRmwOp::Smax
-                            | AtomicRmwOp::Xchg => unreachable!(),
+                            AtomicRMWLoopOp::Add => ALUOp::Add,
+                            AtomicRMWLoopOp::Sub => ALUOp::Sub,
+                            AtomicRMWLoopOp::And => ALUOp::And,
+                            AtomicRMWLoopOp::Orr => ALUOp::Orr,
+                            AtomicRMWLoopOp::Eor => ALUOp::Eor,
+                            AtomicRMWLoopOp::Nand
+                            | AtomicRMWLoopOp::Umin
+                            | AtomicRMWLoopOp::Umax
+                            | AtomicRMWLoopOp::Smin
+                            | AtomicRMWLoopOp::Smax
+                            | AtomicRMWLoopOp::Xchg => unreachable!(),
                         };
 
                         Inst::AluRRR {
@@ -1502,7 +1546,7 @@ impl MachInstEmit for Inst {
                 if srcloc != SourceLoc::default() {
                     sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
                 }
-                if op == AtomicRmwOp::Xchg {
+                if op == AtomicRMWLoopOp::Xchg {
                     sink.put4(enc_stlxr(ty, x24wr, x26, x25)); // stlxr w24, x26, [x25]
                 } else {
                     sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25]
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
index b11d7b3eb5..9d6437ba31 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
@@ -6205,10 +6205,18 @@ fn test_aarch64_binemit() {
         "frintn d23, d24",
     ));
 
+    insns.push((
+        Inst::AtomicRMWLoop {
+            ty: I8,
+            op: AtomicRMWLoopOp::Sub,
+        },
+        "3BFF5F087C031A4B3CFF1808B8FFFFB5",
+        "1: ldaxrb w27, [x25]; sub w28, w27, w26; stlxrb w24, w28, [x25]; cbnz w24, 1b",
+    ));
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I16,
-            op: inst_common::AtomicRmwOp::Xor,
+            op: AtomicRMWLoopOp::Eor,
         },
         "3BFF5F487C031A4A3CFF1848B8FFFFB5",
         "1: ldaxrh w27, [x25]; eor w28, w27, w26; stlxrh w24, w28, [x25]; cbnz w24, 1b",
@@ -6216,7 +6224,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I8,
-            op: inst_common::AtomicRmwOp::Add,
+            op: AtomicRMWLoopOp::Add,
         },
         "3BFF5F087C031A0B3CFF1808B8FFFFB5",
         "1: ldaxrb w27, [x25]; add w28, w27, w26; stlxrb w24, w28, [x25]; cbnz w24, 1b",
@@ -6224,7 +6232,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I32,
-            op: inst_common::AtomicRmwOp::Or,
+            op: AtomicRMWLoopOp::Orr,
         },
         "3BFF5F887C031A2A3CFF1888B8FFFFB5",
         "1: ldaxr w27, [x25]; orr w28, w27, w26; stlxr w24, w28, [x25]; cbnz w24, 1b",
@@ -6232,7 +6240,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I64,
-            op: inst_common::AtomicRmwOp::And,
+            op: AtomicRMWLoopOp::And,
         },
         "3BFF5FC87C031A8A3CFF18C8B8FFFFB5",
         "1: ldaxr x27, [x25]; and x28, x27, x26; stlxr w24, x28, [x25]; cbnz w24, 1b",
@@ -6240,7 +6248,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I8,
-            op: inst_common::AtomicRmwOp::Xchg,
+            op: AtomicRMWLoopOp::Xchg,
         },
         "3BFF5F083AFF1808D8FFFFB5",
         "1: ldaxrb w27, [x25]; stlxrb w24, w26, [x25]; cbnz w24, 1b",
@@ -6248,15 +6256,23 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I16,
-            op: inst_common::AtomicRmwOp::Nand,
+            op: AtomicRMWLoopOp::Nand,
         },
         "3BFF5F487C031A0AFC033C2A3CFF184898FFFFB5",
         "1: ldaxrh w27, [x25]; and w28, w27, w26; mvn w28, w28; stlxrh w24, w28, [x25]; cbnz w24, 1b",
     ));
+    insns.push((
+        Inst::AtomicRMWLoop {
+            ty: I16,
+            op: AtomicRMWLoopOp::Smin,
+        },
+        "3BFF5F487B3F00137FA33A6B7CB39A9A3CFF184878FFFFB5",
+        "1: ldaxrh w27, [x25]; sxth w27, w27; cmp w27, w26, sxth; csel w28, w27, w26, lt; stlxrh w24, w28, [x25]; cbnz w24, 1b",
+    ));
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I32,
-            op: inst_common::AtomicRmwOp::Smin,
+            op: AtomicRMWLoopOp::Smin,
         },
         "3BFF5F887F031A6B7CB39A9A3CFF188898FFFFB5",
         "1: ldaxr w27, [x25]; cmp w27, w26; csel w28, w27, w26, lt; stlxr w24, w28, [x25]; cbnz w24, 1b",
@@ -6264,7 +6280,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I64,
-            op: inst_common::AtomicRmwOp::Smax,
+            op: AtomicRMWLoopOp::Smax,
         },
         "3BFF5FC87F031AEB7CC39A9A3CFF18C898FFFFB5",
         "1: ldaxr x27, [x25]; cmp x27, x26; csel x28, x27, x26, gt; stlxr w24, x28, [x25]; cbnz w24, 1b",
@@ -6272,7 +6288,15 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I8,
-            op: inst_common::AtomicRmwOp::Umin,
+            op: AtomicRMWLoopOp::Smax,
+        },
+        "3BFF5F087B1F00137F833A6B7CC39A9A3CFF180878FFFFB5",
+        "1: ldaxrb w27, [x25]; sxtb w27, w27; cmp w27, w26, sxtb; csel w28, w27, w26, gt; stlxrb w24, w28, [x25]; cbnz w24, 1b",
+    ));
+    insns.push((
+        Inst::AtomicRMWLoop {
+            ty: I8,
+            op: AtomicRMWLoopOp::Umin,
         },
         "3BFF5F087F031A6B7C339A9A3CFF180898FFFFB5",
         "1: ldaxrb w27, [x25]; cmp w27, w26; csel w28, w27, w26, lo; stlxrb w24, w28, [x25]; cbnz w24, 1b",
@@ -6280,7 +6304,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::AtomicRMWLoop {
             ty: I16,
-            op: inst_common::AtomicRmwOp::Umax,
+            op: AtomicRMWLoopOp::Umax,
         },
         "3BFF5F487F031A6B7C839A9A3CFF184898FFFFB5",
         "1: ldaxrh w27, [x25]; cmp w27, w26; csel w28, w27, w26, hi; stlxrh w24, w28, [x25]; cbnz w24, 1b",
@@ -6638,6 +6662,50 @@ fn test_aarch64_binemit() {
         "7A73F9F8",
         "lduminal x25, x26, [x27]",
     ));
+    insns.push((
+        Inst::AtomicRMW {
+            ty: I8,
+            op: AtomicRMWOp::Swp,
+            rs: xreg(28),
+            rt: writable_xreg(29),
+            rn: xreg(30),
+        },
+        "DD83FC38",
+        "swpalb w28, fp, [lr]",
+    ));
+    insns.push((
+        Inst::AtomicRMW {
+            ty: I16,
+            op: AtomicRMWOp::Swp,
+            rs: xreg(0),
+            rt: writable_xreg(1),
+            rn: xreg(2),
+        },
+        "4180E078",
+        "swpalh w0, w1, [x2]",
+    ));
+    insns.push((
+        Inst::AtomicRMW {
+            ty: I32,
+            op: AtomicRMWOp::Swp,
+            rs: xreg(3),
+            rt: writable_xreg(4),
+            rn: xreg(5),
+        },
+        "A480E3B8",
+        "swpal w3, w4, [x5]",
+    ));
+    insns.push((
+        Inst::AtomicRMW {
+            ty: I64,
+            op: AtomicRMWOp::Swp,
+            rs: xreg(6),
+            rt: writable_xreg(7),
+            rn: xreg(8),
+        },
+        "0781E6F8",
+        "swpal x6, x7, [x8]",
+    ));
 
     insns.push((
         Inst::AtomicCAS {
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 2c5d82b4b6..1f82e2c0fd 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -39,9 +39,9 @@ mod emit_tests;
 // Instructions (top level): definition
 
 pub use crate::isa::aarch64::lower::isle::generated_code::{
-    ALUOp, ALUOp3, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp,
-    IntToFpuOp, MInst as Inst, MoveWideOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp,
-    VecRRLongOp, VecRRNarrowOp, VecRRPairLongOp, VecRRRLongOp, VecShiftImmOp,
+    ALUOp, ALUOp3, AtomicRMWLoopOp, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode,
+    FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2,
+    VecPairOp, VecRRLongOp, VecRRNarrowOp, VecRRPairLongOp, VecRRRLongOp, VecShiftImmOp,
 };
 
 /// A floating-point unit (FPU) operation with two args, a register and an immediate.
@@ -676,12 +676,14 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
         &Inst::CCmpImm { rn, .. } => {
             collector.reg_use(rn);
         }
-        &Inst::AtomicRMWLoop { .. } => {
+        &Inst::AtomicRMWLoop { op, .. } => {
             collector.reg_use(xreg(25));
             collector.reg_use(xreg(26));
             collector.reg_def(writable_xreg(24));
             collector.reg_def(writable_xreg(27));
-            collector.reg_def(writable_xreg(28));
+            if op != AtomicRMWLoopOp::Xchg {
+                collector.reg_def(writable_xreg(28));
+            }
         }
         &Inst::AtomicRMW { rs, rt, rn, .. } => {
             collector.reg_use(rs);
@@ -1538,6 +1540,7 @@ impl Inst {
                     AtomicRMWOp::Umax => "ldumaxal",
                     AtomicRMWOp::Smin => "ldsminal",
                     AtomicRMWOp::Umin => "lduminal",
+                    AtomicRMWOp::Swp => "swpal",
                 };
 
                 let size = OperandSize::from_ty(ty);
@@ -1569,28 +1572,39 @@ impl Inst {
                 loop_str.push_str(&format!("ldaxr{} {}, [{}]; ", ty_suffix, r_tmp, r_addr));
 
                 let op_str = match op {
-                    inst_common::AtomicRmwOp::Add => "add",
-                    inst_common::AtomicRmwOp::Sub => "sub",
-                    inst_common::AtomicRmwOp::Xor => "eor",
-                    inst_common::AtomicRmwOp::Or => "orr",
-                    inst_common::AtomicRmwOp::And => "and",
+                    AtomicRMWLoopOp::Add => "add",
+                    AtomicRMWLoopOp::Sub => "sub",
+                    AtomicRMWLoopOp::Eor => "eor",
+                    AtomicRMWLoopOp::Orr => "orr",
+                    AtomicRMWLoopOp::And => "and",
                     _ => "",
                 };
 
                 if op_str.is_empty() {
                     match op {
-                        inst_common::AtomicRmwOp::Xchg => r_dst = r_arg2,
-                        inst_common::AtomicRmwOp::Nand => {
+                        AtomicRMWLoopOp::Xchg => r_dst = r_arg2,
+                        AtomicRMWLoopOp::Nand => {
                             loop_str.push_str(&format!("and {}, {}, {}; ", r_dst, r_tmp, r_arg2));
                             loop_str.push_str(&format!("mvn {}, {}; ", r_dst, r_dst));
                         }
                         _ => {
-                            loop_str.push_str(&format!("cmp {}, {}; ", r_tmp, r_arg2));
+                            if (op == AtomicRMWLoopOp::Smin || op == AtomicRMWLoopOp::Smax)
+                                && (ty == I8 || ty == I16)
+                            {
+                                loop_str
+                                    .push_str(&format!("sxt{} {}, {}; ", ty_suffix, r_tmp, r_tmp));
+                                loop_str.push_str(&format!(
+                                    "cmp {}, {}, sxt{}; ",
+                                    r_tmp, r_arg2, ty_suffix
+                                ));
+                            } else {
+                                loop_str.push_str(&format!("cmp {}, {}; ", r_tmp, r_arg2));
+                            }
                             let cond = match op {
-                                inst_common::AtomicRmwOp::Smin => "lt",
-                                inst_common::AtomicRmwOp::Smax => "gt",
-                                inst_common::AtomicRmwOp::Umin => "lo",
-                                inst_common::AtomicRmwOp::Umax => "hi",
+                                AtomicRMWLoopOp::Smin => "lt",
+                                AtomicRMWLoopOp::Smax => "gt",
+                                AtomicRMWLoopOp::Umin => "lo",
+                                AtomicRMWLoopOp::Umax => "hi",
                                 _ => unreachable!(),
                             };
                             loop_str.push_str(&format!(
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 59aba9b7bb..aee7c7b08f 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -1196,3 +1196,77 @@
       (let ((rn Reg y)
             (vec_size VectorSize (vector_size ty)))
           (value_reg (int_cmp_zero_swap cond rn vec_size))))
+
+;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Add) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Add) addr src ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Xor) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Eor) addr src ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Or) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Set) addr src ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Smax) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Smax) addr src ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Smin) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Smin) addr src ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Umax) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Umax) addr src ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Umin) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Umin) addr src ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.Sub) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Add) addr (sub ty (zero_reg) src) ty))
+(rule (lower (and (use_lse)
+                  (has_type (valid_atomic_transaction ty)
+                      (atomic_rmw flags (AtomicRmwOp.And) addr src))))
+      (lse_atomic_rmw (AtomicRMWOp.Clr) addr (eon ty src (zero_reg)) ty))
+
+
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Add) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Add) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Sub) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Sub) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.And) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.And) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Nand) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Nand) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Or) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Orr) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Xor) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Eor) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Smin) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Smin) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Smax) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Smax) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Umin) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Umin) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Umax) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Umax) addr src ty))
+(rule (lower (has_type (valid_atomic_transaction ty)
+             (atomic_rmw flags (AtomicRmwOp.Xchg) addr src)))
+      (atomic_rmw_loop (AtomicRMWLoopOp.Xchg) addr src ty))
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index 73497dda54..356cc28980 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -5,8 +5,8 @@ pub mod generated_code;
 
 // Types that the generated ISLE code uses via `use super::*`.
 use super::{
-    writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, AtomicRmwOp, BranchTarget,
-    CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FloatCC, Imm12, ImmLogic, ImmShift,
+    writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo,
+    CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FloatCC, Imm12, ImmLogic, ImmShift,
     Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MoveWideConst, MoveWideOp, NarrowValueMode,
     Opcode, OperandSize, PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VecMisc2, VectorSize,
     NZCV,
@@ -17,10 +17,11 @@ use crate::settings::Flags;
 use crate::{
     binemit::CodeOffset,
     ir::{
-        immediates::*, types::*, ExternalName, Inst, InstructionData, MemFlags, TrapCode, Value,
-        ValueList,
+        immediates::*, types::*, AtomicRmwOp, ExternalName, Inst, InstructionData, MemFlags,
+        TrapCode, Value, ValueList,
     },
     isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
+    isa::aarch64::lower::{is_valid_atomic_transaction_ty, writable_xreg, xreg},
     isa::unwind::UnwindInst,
     machinst::{ty_bits, InsnOutput, LowerCtx},
 };
@@ -66,6 +67,14 @@ where
 {
     isle_prelude_methods!();
 
+    fn use_lse(&mut self, _: Inst) -> Option<()> {
+        if self.isa_flags.use_lse() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
     fn move_wide_const_from_u64(&mut self, n: u64) -> Option<MoveWideConst> {
         MoveWideConst::maybe_from_u64(n)
     }
@@ -114,6 +123,14 @@ where
         }
     }
 
+    fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> {
+        if is_valid_atomic_transaction_ty(ty) {
+            Some(ty)
+        } else {
+            None
+        }
+    }
+
     /// This is the fallback case for loading a 64-bit integral constant into a
     /// register.
     ///
@@ -194,6 +211,14 @@ where
         zero_reg()
     }
 
+    fn xreg(&mut self, index: u8) -> Reg {
+        xreg(index)
+    }
+
+    fn writable_xreg(&mut self, index: u8) -> WritableReg {
+        writable_xreg(index)
+    }
+
     fn extended_value_from_value(&mut self, val: Value) -> Option<ExtendedValue> {
         let (val, extend) =
             super::get_as_extended_value(self.lower_ctx, val, NarrowValueMode::None)?;
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
index fa0e304414..8d393724bb 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
@@ -1,4 +1,4 @@
 src/clif.isle 443b34b797fc8ace
-src/prelude.isle d8a93eb727abd7f4
-src/isa/aarch64/inst.isle 77984cc33a05be7
-src/isa/aarch64/lower.isle 71c7e603b0e4bdef
+src/prelude.isle a7915a6b88310eb5
+src/isa/aarch64/inst.isle a2c0ae729bfa24a8
+src/isa/aarch64/lower.isle 15641ca7f0ac061a
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs
index 45b02aed03..35e2fe47fd 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs
@@ -35,6 +35,7 @@ pub trait Context {
     fn invalid_reg(&mut self) -> Reg;
     fn put_in_reg(&mut self, arg0: Value) -> Reg;
     fn put_in_regs(&mut self, arg0: Value) -> ValueRegs;
+    fn ensure_in_vreg(&mut self, arg0: Reg, arg1: Type) -> Reg;
     fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg;
     fn u8_as_u64(&mut self, arg0: u8) -> u64;
     fn u16_as_u64(&mut self, arg0: u16) -> u64;
@@ -90,6 +91,7 @@ pub trait Context {
         arg0: GlobalValue,
     ) -> Option<(ExternalName, RelocDistance, i64)>;
     fn reloc_distance_near(&mut self, arg0: RelocDistance) -> Option<()>;
+    fn use_lse(&mut self, arg0: Inst) -> Option<()>;
     fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option<MoveWideConst>;
     fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option<MoveWideConst>;
     fn imm_logic_from_u64(&mut self, arg0: u64, arg1: Type) -> Option<ImmLogic>;
@@ -103,6 +105,7 @@ pub trait Context {
     fn imm12_from_negated_u64(&mut self, arg0: u64) -> Option<Imm12>;
     fn lshl_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option<ShiftOpAndAmt>;
     fn integral_ty(&mut self, arg0: Type) -> Option<Type>;
+    fn valid_atomic_transaction(&mut self, arg0: Type) -> Option<Type>;
     fn extended_value_from_value(&mut self, arg0: Value) -> Option<ExtendedValue>;
     fn put_extended_in_reg(&mut self, arg0: &ExtendedValue) -> Reg;
     fn get_extended_op(&mut self, arg0: &ExtendedValue) -> ExtendOp;
@@ -111,6 +114,8 @@ pub trait Context {
     fn cond_br_cond(&mut self, arg0: &Cond) -> CondBrKind;
     fn zero_reg(&mut self) -> Reg;
     fn writable_zero_reg(&mut self) -> WritableReg;
+    fn xreg(&mut self, arg0: u8) -> Reg;
+    fn writable_xreg(&mut self, arg0: u8) -> WritableReg;
     fn load_constant64_full(&mut self, arg0: u64) -> Reg;
     fn sinkable_atomic_load(&mut self, arg0: Value) -> Option<SinkableAtomicLoad>;
     fn sink_atomic_load(&mut self, arg0: &SinkableAtomicLoad) -> Reg;
@@ -132,14 +137,14 @@ pub trait Context {
     fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 412.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
     Inst2 { inst1: MInst, inst2: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 434.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 439.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -147,7 +152,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 445.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 450.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -343,7 +348,10 @@ pub enum MInst {
     },
     AtomicRMWLoop {
         ty: Type,
-        op: AtomicRmwOp,
+        op: AtomicRMWLoopOp,
+    },
+    AtomicCASLoop {
+        ty: Type,
     },
     AtomicRMW {
         op: AtomicRMWOp,
@@ -358,9 +366,6 @@ pub enum MInst {
         rn: Reg,
         ty: Type,
     },
-    AtomicCASLoop {
-        ty: Type,
-    },
     LoadAcquire {
         access_ty: Type,
         rt: WritableReg,
@@ -736,7 +741,7 @@ pub enum MInst {
     },
 }
 
-/// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 779.
+/// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 776.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum ALUOp {
     Add,
@@ -764,14 +769,14 @@ pub enum ALUOp {
     SbcS,
 }
 
-/// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 817.
+/// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 814.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum ALUOp3 {
     MAdd,
     MSub,
 }
 
-/// Internal type MoveWideOp: defined at src/isa/aarch64/inst.isle line 825.
+/// Internal type MoveWideOp: defined at src/isa/aarch64/inst.isle line 822.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum MoveWideOp {
     MovZ,
@@ -779,7 +784,7 @@ pub enum MoveWideOp {
     MovK,
 }
 
-/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 863.
+/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 860.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum BitOp {
     RBit,
@@ -787,7 +792,7 @@ pub enum BitOp {
     Cls,
 }
 
-/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 930.
+/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 927.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum FPUOp1 {
     Abs,
@@ -797,7 +802,7 @@ pub enum FPUOp1 {
     Cvt64To32,
 }
 
-/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 940.
+/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 937.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum FPUOp2 {
     Add,
@@ -808,14 +813,14 @@ pub enum FPUOp2 {
     Min,
 }
 
-/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 951.
+/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 948.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum FPUOp3 {
     MAdd32,
     MAdd64,
 }
 
-/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 958.
+/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 955.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum FpuToIntOp {
     F32ToU32,
@@ -828,7 +833,7 @@ pub enum FpuToIntOp {
     F64ToI64,
 }
 
-/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 971.
+/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 968.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum IntToFpuOp {
     U32ToF32,
@@ -841,7 +846,7 @@ pub enum IntToFpuOp {
     I64ToF64,
 }
 
-/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 985.
+/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 982.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum FpuRoundMode {
     Minus32,
@@ -854,7 +859,7 @@ pub enum FpuRoundMode {
     Nearest64,
 }
 
-/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 998.
+/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 995.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecExtendOp {
     Sxtl8,
@@ -865,7 +870,7 @@ pub enum VecExtendOp {
     Uxtl32,
 }
 
-/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1015.
+/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1012.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecALUOp {
     Sqadd,
@@ -907,7 +912,7 @@ pub enum VecALUOp {
     Sqrdmulh,
 }
 
-/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1094.
+/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1091.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecMisc2 {
     Not,
@@ -938,7 +943,7 @@ pub enum VecMisc2 {
     Fcmlt0,
 }
 
-/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1151.
+/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1148.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecRRLongOp {
     Fcvtl16,
@@ -948,7 +953,7 @@ pub enum VecRRLongOp {
     Shll32,
 }
 
-/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1166.
+/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1163.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecRRNarrowOp {
     Xtn16,
@@ -967,7 +972,7 @@ pub enum VecRRNarrowOp {
     Fcvtn64,
 }
 
-/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1198.
+/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1195.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecRRRLongOp {
     Smull8,
@@ -981,13 +986,13 @@ pub enum VecRRRLongOp {
     Umlal32,
 }
 
-/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1215.
+/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1212.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecPairOp {
     Addp,
 }
 
-/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1223.
+/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1220.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecRRPairLongOp {
     Saddlp8,
@@ -996,14 +1001,14 @@ pub enum VecRRPairLongOp {
     Uaddlp16,
 }
 
-/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1234.
+/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1231.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecLanesOp {
     Addv,
     Uminv,
 }
 
-/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1243.
+/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1240.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum VecShiftImmOp {
     Shl,
@@ -1011,7 +1016,7 @@ pub enum VecShiftImmOp {
     Sshr,
 }
 
-/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1254.
+/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1251.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum AtomicRMWOp {
     Add,
@@ -1022,6 +1027,23 @@ pub enum AtomicRMWOp {
     Smin,
     Umax,
     Umin,
+    Swp,
+}
+
+/// Internal type AtomicRMWLoopOp: defined at src/isa/aarch64/inst.isle line 1266.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum AtomicRMWLoopOp {
+    Add,
+    Sub,
+    And,
+    Nand,
+    Eor,
+    Orr,
+    Smax,
+    Smin,
+    Umax,
+    Umin,
+    Xchg,
 }
 
 // Generated as internal constructor for term output_reg.
@@ -1054,7 +1076,7 @@ pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg>
 // Generated as internal constructor for term lo_reg.
 pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
     let pattern0_0 = arg0;
-    // Rule at src/prelude.isle line 145.
+    // Rule at src/prelude.isle line 150.
     let expr0_0 = C::put_in_regs(ctx, pattern0_0);
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -1071,7 +1093,7 @@ pub fn constructor_side_effect<C: Context>(
         &SideEffectNoResult::Inst {
             inst: ref pattern1_0,
         } => {
-            // Rule at src/prelude.isle line 415.
+            // Rule at src/prelude.isle line 420.
             let expr0_0 = C::emit(ctx, pattern1_0);
             let expr1_0 = C::output_none(ctx);
             return Some(expr1_0);
@@ -1080,7 +1102,7 @@ pub fn constructor_side_effect<C: Context>(
             inst1: ref pattern1_0,
             inst2: ref pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 418.
+            // Rule at src/prelude.isle line 423.
             let expr0_0 = C::emit(ctx, pattern1_0);
             let expr1_0 = C::emit(ctx, pattern1_1);
             let expr2_0 = C::output_none(ctx);
@@ -1107,7 +1129,7 @@ pub fn constructor_side_effect_concat<C: Context>(
             inst: ref pattern3_0,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 424.
+            // Rule at src/prelude.isle line 429.
             let expr0_0 = SideEffectNoResult::Inst2 {
                 inst1: pattern1_0.clone(),
                 inst2: pattern3_0.clone(),
@@ -1129,7 +1151,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 461.
+        // Rule at src/prelude.isle line 466.
         return Some(pattern1_1);
     }
     return None;
@@ -1146,7 +1168,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 466.
+            // Rule at src/prelude.isle line 471.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1156,7 +1178,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 468.
+            // Rule at src/prelude.isle line 473.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1185,7 +1207,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 475.
+            // Rule at src/prelude.isle line 480.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -1215,7 +1237,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 500.
+                    // Rule at src/prelude.isle line 505.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1226,7 +1248,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 506.
+                    // Rule at src/prelude.isle line 511.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1239,7 +1261,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 518.
+                    // Rule at src/prelude.isle line 523.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1260,7 +1282,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 494.
+                // Rule at src/prelude.isle line 499.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1280,7 +1302,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 535.
+    // Rule at src/prelude.isle line 540.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -1291,12 +1313,12 @@ pub fn constructor_with_flags_reg<C: Context>(
 pub fn constructor_operand_size<C: Context>(ctx: &mut C, arg0: Type) -> Option<OperandSize> {
     let pattern0_0 = arg0;
     if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
-        // Rule at src/isa/aarch64/inst.isle line 881.
+        // Rule at src/isa/aarch64/inst.isle line 878.
         let expr0_0 = OperandSize::Size32;
         return Some(expr0_0);
     }
     if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) {
-        // Rule at src/isa/aarch64/inst.isle line 882.
+        // Rule at src/isa/aarch64/inst.isle line 879.
         let expr0_0 = OperandSize::Size64;
         return Some(expr0_0);
     }
@@ -1309,28 +1331,28 @@ pub fn constructor_vector_size<C: Context>(ctx: &mut C, arg0: Type) -> Option<Ve
     if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) {
         if pattern1_0 == 8 {
             if pattern1_1 == 16 {
-                // Rule at src/isa/aarch64/inst.isle line 924.
+                // Rule at src/isa/aarch64/inst.isle line 921.
                 let expr0_0 = VectorSize::Size8x16;
                 return Some(expr0_0);
             }
         }
         if pattern1_0 == 16 {
             if pattern1_1 == 8 {
-                // Rule at src/isa/aarch64/inst.isle line 925.
+                // Rule at src/isa/aarch64/inst.isle line 922.
                 let expr0_0 = VectorSize::Size16x8;
                 return Some(expr0_0);
             }
         }
         if pattern1_0 == 32 {
             if pattern1_1 == 4 {
-                // Rule at src/isa/aarch64/inst.isle line 926.
+                // Rule at src/isa/aarch64/inst.isle line 923.
                 let expr0_0 = VectorSize::Size32x4;
                 return Some(expr0_0);
             }
         }
         if pattern1_0 == 64 {
             if pattern1_1 == 2 {
-                // Rule at src/isa/aarch64/inst.isle line 927.
+                // Rule at src/isa/aarch64/inst.isle line 924.
                 let expr0_0 = VectorSize::Size64x2;
                 return Some(expr0_0);
             }
@@ -1339,6 +1361,43 @@ pub fn constructor_vector_size<C: Context>(ctx: &mut C, arg0: Type) -> Option<Ve
     return None;
 }
 
+// Generated as internal constructor for term mov64_to_real.
+pub fn constructor_mov64_to_real<C: Context>(ctx: &mut C, arg0: u8, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/aarch64/inst.isle line 1379.
+    let expr0_0 = C::writable_xreg(ctx, pattern0_0);
+    let expr1_0: Type = I64;
+    let expr2_0 = constructor_operand_size(ctx, expr1_0)?;
+    let expr3_0 = MInst::Mov {
+        size: expr2_0,
+        rd: expr0_0,
+        rm: pattern1_0,
+    };
+    let expr4_0 = C::emit(ctx, &expr3_0);
+    let expr5_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr5_0);
+}
+
+// Generated as internal constructor for term mov64_from_real.
+pub fn constructor_mov64_from_real<C: Context>(ctx: &mut C, arg0: u8) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/aarch64/inst.isle line 1385.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0: Type = I64;
+    let expr3_0 = constructor_operand_size(ctx, expr2_0)?;
+    let expr4_0 = C::xreg(ctx, pattern0_0);
+    let expr5_0 = MInst::Mov {
+        size: expr3_0,
+        rd: expr1_0,
+        rm: expr4_0,
+    };
+    let expr6_0 = C::emit(ctx, &expr5_0);
+    let expr7_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr7_0);
+}
+
 // Generated as internal constructor for term movz.
 pub fn constructor_movz<C: Context>(
     ctx: &mut C,
@@ -1347,7 +1406,7 @@ pub fn constructor_movz<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1350.
+    // Rule at src/isa/aarch64/inst.isle line 1392.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MoveWideOp::MovZ;
@@ -1370,7 +1429,7 @@ pub fn constructor_movn<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1357.
+    // Rule at src/isa/aarch64/inst.isle line 1399.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MoveWideOp::MovN;
@@ -1397,7 +1456,7 @@ pub fn constructor_alu_rr_imm_logic<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1364.
+    // Rule at src/isa/aarch64/inst.isle line 1406.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1425,7 +1484,7 @@ pub fn constructor_alu_rr_imm_shift<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1371.
+    // Rule at src/isa/aarch64/inst.isle line 1413.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1453,7 +1512,7 @@ pub fn constructor_alu_rrr<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1378.
+    // Rule at src/isa/aarch64/inst.isle line 1420.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1481,7 +1540,7 @@ pub fn constructor_vec_rrr<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1385.
+    // Rule at src/isa/aarch64/inst.isle line 1427.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRR {
@@ -1506,7 +1565,7 @@ pub fn constructor_vec_lanes<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1392.
+    // Rule at src/isa/aarch64/inst.isle line 1434.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecLanes {
@@ -1524,7 +1583,7 @@ pub fn constructor_vec_lanes<C: Context>(
 pub fn constructor_vec_dup<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1399.
+    // Rule at src/isa/aarch64/inst.isle line 1441.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecDup {
@@ -1549,7 +1608,7 @@ pub fn constructor_alu_rr_imm12<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1406.
+    // Rule at src/isa/aarch64/inst.isle line 1448.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1579,7 +1638,7 @@ pub fn constructor_alu_rrr_shift<C: Context>(
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
     let pattern4_0 = arg4;
-    // Rule at src/isa/aarch64/inst.isle line 1413.
+    // Rule at src/isa/aarch64/inst.isle line 1455.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1610,7 +1669,7 @@ pub fn constructor_alu_rrr_extend<C: Context>(
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
     let pattern4_0 = arg4;
-    // Rule at src/isa/aarch64/inst.isle line 1420.
+    // Rule at src/isa/aarch64/inst.isle line 1462.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1639,7 +1698,7 @@ pub fn constructor_alu_rr_extend_reg<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1428.
+    // Rule at src/isa/aarch64/inst.isle line 1470.
     let expr0_0 = C::put_extended_in_reg(ctx, pattern3_0);
     let expr1_0 = C::get_extended_op(ctx, pattern3_0);
     let expr2_0 =
@@ -1661,7 +1720,7 @@ pub fn constructor_alu_rrrr<C: Context>(
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
     let pattern4_0 = arg4;
-    // Rule at src/isa/aarch64/inst.isle line 1435.
+    // Rule at src/isa/aarch64/inst.isle line 1477.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1688,7 +1747,7 @@ pub fn constructor_bit_rr<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1442.
+    // Rule at src/isa/aarch64/inst.isle line 1484.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = constructor_operand_size(ctx, pattern1_0)?;
@@ -1713,7 +1772,7 @@ pub fn constructor_add_with_flags_paired<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1449.
+    // Rule at src/isa/aarch64/inst.isle line 1491.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::AddS;
@@ -1743,7 +1802,7 @@ pub fn constructor_adc_paired<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1457.
+    // Rule at src/isa/aarch64/inst.isle line 1499.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::Adc;
@@ -1773,7 +1832,7 @@ pub fn constructor_sub_with_flags_paired<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1465.
+    // Rule at src/isa/aarch64/inst.isle line 1507.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::SubS;
@@ -1801,7 +1860,7 @@ pub fn constructor_cmp64_imm<C: Context>(
 ) -> Option<ProducesFlags> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1472.
+    // Rule at src/isa/aarch64/inst.isle line 1514.
     let expr0_0 = ALUOp::SubS;
     let expr1_0 = OperandSize::Size64;
     let expr2_0 = C::writable_zero_reg(ctx);
@@ -1826,7 +1885,7 @@ pub fn constructor_sbc_paired<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1479.
+    // Rule at src/isa/aarch64/inst.isle line 1521.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::Sbc;
@@ -1856,7 +1915,7 @@ pub fn constructor_vec_misc<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1487.
+    // Rule at src/isa/aarch64/inst.isle line 1529.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecMisc {
@@ -1882,7 +1941,7 @@ pub fn constructor_vec_rrr_long<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1494.
+    // Rule at src/isa/aarch64/inst.isle line 1536.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRRLong {
@@ -1911,7 +1970,7 @@ pub fn constructor_vec_rrrr_long<C: Context>(
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
     let pattern4_0 = arg4;
-    // Rule at src/isa/aarch64/inst.isle line 1504.
+    // Rule at src/isa/aarch64/inst.isle line 1546.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::FpuMove128 {
@@ -1941,7 +2000,7 @@ pub fn constructor_vec_rr_narrow<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1512.
+    // Rule at src/isa/aarch64/inst.isle line 1554.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRNarrow {
@@ -1965,7 +2024,7 @@ pub fn constructor_vec_rr_long<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1519.
+    // Rule at src/isa/aarch64/inst.isle line 1561.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRLong {
@@ -1987,7 +2046,7 @@ pub fn constructor_mov_to_fpu<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1526.
+    // Rule at src/isa/aarch64/inst.isle line 1568.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::MovToFpu {
@@ -2012,7 +2071,7 @@ pub fn constructor_mov_to_vec<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1533.
+    // Rule at src/isa/aarch64/inst.isle line 1575.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::FpuMove128 {
@@ -2041,7 +2100,7 @@ pub fn constructor_mov_from_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1541.
+    // Rule at src/isa/aarch64/inst.isle line 1583.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::MovFromVec {
@@ -2067,7 +2126,7 @@ pub fn constructor_mov_from_vec_signed<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1548.
+    // Rule at src/isa/aarch64/inst.isle line 1590.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::MovFromVecSigned {
@@ -2094,7 +2153,7 @@ pub fn constructor_extend<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1555.
+    // Rule at src/isa/aarch64/inst.isle line 1597.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::Extend {
@@ -2113,7 +2172,7 @@ pub fn constructor_extend<C: Context>(
 pub fn constructor_load_acquire<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1562.
+    // Rule at src/isa/aarch64/inst.isle line 1604.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::LoadAcquire {
@@ -2136,7 +2195,7 @@ pub fn constructor_tst_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1572.
+    // Rule at src/isa/aarch64/inst.isle line 1614.
     let expr0_0 = ALUOp::AndS;
     let expr1_0 = constructor_operand_size(ctx, pattern0_0)?;
     let expr2_0 = C::writable_zero_reg(ctx);
@@ -2161,7 +2220,7 @@ pub fn constructor_csel<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1586.
+    // Rule at src/isa/aarch64/inst.isle line 1628.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::CSel {
@@ -2183,7 +2242,7 @@ pub fn constructor_add<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1595.
+    // Rule at src/isa/aarch64/inst.isle line 1637.
     let expr0_0 = ALUOp::Add;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2199,7 +2258,7 @@ pub fn constructor_add_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1598.
+    // Rule at src/isa/aarch64/inst.isle line 1640.
     let expr0_0 = ALUOp::Add;
     let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2215,7 +2274,7 @@ pub fn constructor_add_extend<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1601.
+    // Rule at src/isa/aarch64/inst.isle line 1643.
     let expr0_0 = ALUOp::Add;
     let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2233,7 +2292,7 @@ pub fn constructor_add_shift<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1604.
+    // Rule at src/isa/aarch64/inst.isle line 1646.
     let expr0_0 = ALUOp::Add;
     let expr1_0 = constructor_alu_rrr_shift(
         ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -2251,7 +2310,7 @@ pub fn constructor_add_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1607.
+    // Rule at src/isa/aarch64/inst.isle line 1649.
     let expr0_0 = VecALUOp::Add;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2262,7 +2321,7 @@ pub fn constructor_sub<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1612.
+    // Rule at src/isa/aarch64/inst.isle line 1654.
     let expr0_0 = ALUOp::Sub;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2278,7 +2337,7 @@ pub fn constructor_sub_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1615.
+    // Rule at src/isa/aarch64/inst.isle line 1657.
     let expr0_0 = ALUOp::Sub;
     let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2294,7 +2353,7 @@ pub fn constructor_sub_extend<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1618.
+    // Rule at src/isa/aarch64/inst.isle line 1660.
     let expr0_0 = ALUOp::Sub;
     let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2312,7 +2371,7 @@ pub fn constructor_sub_shift<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1621.
+    // Rule at src/isa/aarch64/inst.isle line 1663.
     let expr0_0 = ALUOp::Sub;
     let expr1_0 = constructor_alu_rrr_shift(
         ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -2330,7 +2389,7 @@ pub fn constructor_sub_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1624.
+    // Rule at src/isa/aarch64/inst.isle line 1666.
     let expr0_0 = VecALUOp::Sub;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2348,7 +2407,7 @@ pub fn constructor_madd<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1629.
+    // Rule at src/isa/aarch64/inst.isle line 1671.
     let expr0_0 = ALUOp3::MAdd;
     let expr1_0 = constructor_alu_rrrr(
         ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -2368,7 +2427,7 @@ pub fn constructor_msub<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1634.
+    // Rule at src/isa/aarch64/inst.isle line 1676.
     let expr0_0 = ALUOp3::MSub;
     let expr1_0 = constructor_alu_rrrr(
         ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -2386,7 +2445,7 @@ pub fn constructor_uqadd<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1638.
+    // Rule at src/isa/aarch64/inst.isle line 1680.
     let expr0_0 = VecALUOp::Uqadd;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2402,7 +2461,7 @@ pub fn constructor_sqadd<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1642.
+    // Rule at src/isa/aarch64/inst.isle line 1684.
     let expr0_0 = VecALUOp::Sqadd;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2418,7 +2477,7 @@ pub fn constructor_uqsub<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1646.
+    // Rule at src/isa/aarch64/inst.isle line 1688.
     let expr0_0 = VecALUOp::Uqsub;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2434,7 +2493,7 @@ pub fn constructor_sqsub<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1650.
+    // Rule at src/isa/aarch64/inst.isle line 1692.
     let expr0_0 = VecALUOp::Sqsub;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2445,7 +2504,7 @@ pub fn constructor_umulh<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: R
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1654.
+    // Rule at src/isa/aarch64/inst.isle line 1696.
     let expr0_0 = ALUOp::UMulH;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2456,7 +2515,7 @@ pub fn constructor_smulh<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: R
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1658.
+    // Rule at src/isa/aarch64/inst.isle line 1700.
     let expr0_0 = ALUOp::SMulH;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2472,7 +2531,7 @@ pub fn constructor_mul<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1662.
+    // Rule at src/isa/aarch64/inst.isle line 1704.
     let expr0_0 = VecALUOp::Mul;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2482,7 +2541,7 @@ pub fn constructor_mul<C: Context>(
 pub fn constructor_neg<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1666.
+    // Rule at src/isa/aarch64/inst.isle line 1708.
     let expr0_0 = VecMisc2::Neg;
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -2492,7 +2551,7 @@ pub fn constructor_neg<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) ->
 pub fn constructor_rev64<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1670.
+    // Rule at src/isa/aarch64/inst.isle line 1712.
     let expr0_0 = VecMisc2::Rev64;
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -2502,7 +2561,7 @@ pub fn constructor_rev64<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize)
 pub fn constructor_xtn64<C: Context>(ctx: &mut C, arg0: Reg, arg1: bool) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1674.
+    // Rule at src/isa/aarch64/inst.isle line 1716.
     let expr0_0 = VecRRNarrowOp::Xtn64;
     let expr1_0 = constructor_vec_rr_narrow(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -2518,7 +2577,7 @@ pub fn constructor_addp<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1678.
+    // Rule at src/isa/aarch64/inst.isle line 1720.
     let expr0_0 = VecALUOp::Addp;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2528,7 +2587,7 @@ pub fn constructor_addp<C: Context>(
 pub fn constructor_addv<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1682.
+    // Rule at src/isa/aarch64/inst.isle line 1724.
     let expr0_0 = VecLanesOp::Addv;
     let expr1_0 = constructor_vec_lanes(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -2538,7 +2597,7 @@ pub fn constructor_addv<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -
 pub fn constructor_shll32<C: Context>(ctx: &mut C, arg0: Reg, arg1: bool) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1686.
+    // Rule at src/isa/aarch64/inst.isle line 1728.
     let expr0_0 = VecRRLongOp::Shll32;
     let expr1_0 = constructor_vec_rr_long(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -2556,7 +2615,7 @@ pub fn constructor_umlal32<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1690.
+    // Rule at src/isa/aarch64/inst.isle line 1732.
     let expr0_0 = VecRRRLongOp::Umlal32;
     let expr1_0 = constructor_vec_rrrr_long(
         ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -2574,7 +2633,7 @@ pub fn constructor_smull8<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1694.
+    // Rule at src/isa/aarch64/inst.isle line 1736.
     let expr0_0 = VecRRRLongOp::Smull8;
     let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2590,7 +2649,7 @@ pub fn constructor_umull8<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1698.
+    // Rule at src/isa/aarch64/inst.isle line 1740.
     let expr0_0 = VecRRRLongOp::Umull8;
     let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2606,7 +2665,7 @@ pub fn constructor_smull16<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1702.
+    // Rule at src/isa/aarch64/inst.isle line 1744.
     let expr0_0 = VecRRRLongOp::Smull16;
     let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2622,7 +2681,7 @@ pub fn constructor_umull16<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1706.
+    // Rule at src/isa/aarch64/inst.isle line 1748.
     let expr0_0 = VecRRRLongOp::Umull16;
     let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2638,7 +2697,7 @@ pub fn constructor_smull32<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1710.
+    // Rule at src/isa/aarch64/inst.isle line 1752.
     let expr0_0 = VecRRRLongOp::Smull32;
     let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2654,7 +2713,7 @@ pub fn constructor_umull32<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1714.
+    // Rule at src/isa/aarch64/inst.isle line 1756.
     let expr0_0 = VecRRRLongOp::Umull32;
     let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2665,7 +2724,7 @@ pub fn constructor_asr<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1718.
+    // Rule at src/isa/aarch64/inst.isle line 1760.
     let expr0_0 = ALUOp::Asr;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2681,7 +2740,7 @@ pub fn constructor_asr_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1721.
+    // Rule at src/isa/aarch64/inst.isle line 1763.
     let expr0_0 = ALUOp::Asr;
     let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2692,7 +2751,7 @@ pub fn constructor_lsr<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1725.
+    // Rule at src/isa/aarch64/inst.isle line 1767.
     let expr0_0 = ALUOp::Lsr;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2708,7 +2767,7 @@ pub fn constructor_lsr_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1728.
+    // Rule at src/isa/aarch64/inst.isle line 1770.
     let expr0_0 = ALUOp::Lsr;
     let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2719,7 +2778,7 @@ pub fn constructor_lsl<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1732.
+    // Rule at src/isa/aarch64/inst.isle line 1774.
     let expr0_0 = ALUOp::Lsl;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2735,7 +2794,7 @@ pub fn constructor_lsl_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1735.
+    // Rule at src/isa/aarch64/inst.isle line 1777.
     let expr0_0 = ALUOp::Lsl;
     let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2751,7 +2810,7 @@ pub fn constructor_a64_udiv<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1739.
+    // Rule at src/isa/aarch64/inst.isle line 1781.
     let expr0_0 = ALUOp::UDiv;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2767,7 +2826,7 @@ pub fn constructor_a64_sdiv<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1743.
+    // Rule at src/isa/aarch64/inst.isle line 1785.
     let expr0_0 = ALUOp::SDiv;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2777,7 +2836,7 @@ pub fn constructor_a64_sdiv<C: Context>(
 pub fn constructor_not<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1747.
+    // Rule at src/isa/aarch64/inst.isle line 1789.
     let expr0_0 = VecMisc2::Not;
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -2793,7 +2852,7 @@ pub fn constructor_orr_not<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1752.
+    // Rule at src/isa/aarch64/inst.isle line 1794.
     let expr0_0 = ALUOp::OrrNot;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2811,7 +2870,7 @@ pub fn constructor_orr_not_shift<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1755.
+    // Rule at src/isa/aarch64/inst.isle line 1797.
     let expr0_0 = ALUOp::OrrNot;
     let expr1_0 = constructor_alu_rrr_shift(
         ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -2824,7 +2883,7 @@ pub fn constructor_orr<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1760.
+    // Rule at src/isa/aarch64/inst.isle line 1802.
     let expr0_0 = ALUOp::Orr;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2840,7 +2899,7 @@ pub fn constructor_orr_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1763.
+    // Rule at src/isa/aarch64/inst.isle line 1805.
     let expr0_0 = ALUOp::Orr;
     let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2856,7 +2915,7 @@ pub fn constructor_orr_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1766.
+    // Rule at src/isa/aarch64/inst.isle line 1808.
     let expr0_0 = VecALUOp::Orr;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2872,7 +2931,7 @@ pub fn constructor_and_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1771.
+    // Rule at src/isa/aarch64/inst.isle line 1813.
     let expr0_0 = ALUOp::And;
     let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2888,7 +2947,7 @@ pub fn constructor_and_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1774.
+    // Rule at src/isa/aarch64/inst.isle line 1816.
     let expr0_0 = VecALUOp::And;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2904,7 +2963,7 @@ pub fn constructor_eor_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1778.
+    // Rule at src/isa/aarch64/inst.isle line 1820.
     let expr0_0 = VecALUOp::Eor;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2920,7 +2979,7 @@ pub fn constructor_bic_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1782.
+    // Rule at src/isa/aarch64/inst.isle line 1824.
     let expr0_0 = VecALUOp::Bic;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2936,7 +2995,7 @@ pub fn constructor_sshl<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1786.
+    // Rule at src/isa/aarch64/inst.isle line 1828.
     let expr0_0 = VecALUOp::Sshl;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2952,7 +3011,7 @@ pub fn constructor_ushl<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1790.
+    // Rule at src/isa/aarch64/inst.isle line 1832.
     let expr0_0 = VecALUOp::Ushl;
     let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2968,7 +3027,7 @@ pub fn constructor_a64_rotr<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1795.
+    // Rule at src/isa/aarch64/inst.isle line 1837.
     let expr0_0 = ALUOp::RotR;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2984,7 +3043,7 @@ pub fn constructor_a64_rotr_imm<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1798.
+    // Rule at src/isa/aarch64/inst.isle line 1840.
     let expr0_0 = ALUOp::RotR;
     let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -2994,7 +3053,7 @@ pub fn constructor_a64_rotr_imm<C: Context>(
 pub fn constructor_rbit<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1803.
+    // Rule at src/isa/aarch64/inst.isle line 1845.
     let expr0_0 = BitOp::RBit;
     let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -3004,7 +3063,7 @@ pub fn constructor_rbit<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Optio
 pub fn constructor_a64_clz<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1808.
+    // Rule at src/isa/aarch64/inst.isle line 1850.
     let expr0_0 = BitOp::Clz;
     let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -3014,7 +3073,7 @@ pub fn constructor_a64_clz<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Op
 pub fn constructor_a64_cls<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1813.
+    // Rule at src/isa/aarch64/inst.isle line 1855.
     let expr0_0 = BitOp::Cls;
     let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -3025,7 +3084,7 @@ pub fn constructor_eon<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1818.
+    // Rule at src/isa/aarch64/inst.isle line 1860.
     let expr0_0 = ALUOp::EorNot;
     let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -3035,7 +3094,7 @@ pub fn constructor_eon<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg
 pub fn constructor_vec_cnt<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1823.
+    // Rule at src/isa/aarch64/inst.isle line 1865.
     let expr0_0 = VecMisc2::Cnt;
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -3052,7 +3111,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
         };
         if let Some(pattern3_0) = closure3() {
             if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) {
-                // Rule at src/isa/aarch64/inst.isle line 1838.
+                // Rule at src/isa/aarch64/inst.isle line 1880.
                 let expr0_0: Type = I64;
                 let expr1_0 = C::zero_reg(ctx);
                 let expr2_0 = constructor_orr_imm(ctx, expr0_0, expr1_0, pattern4_0)?;
@@ -3060,18 +3119,18 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
             }
         }
         if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) {
-            // Rule at src/isa/aarch64/inst.isle line 1830.
+            // Rule at src/isa/aarch64/inst.isle line 1872.
             let expr0_0 = OperandSize::Size64;
             let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?;
             return Some(expr1_0);
         }
         if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) {
-            // Rule at src/isa/aarch64/inst.isle line 1834.
+            // Rule at src/isa/aarch64/inst.isle line 1876.
             let expr0_0 = OperandSize::Size64;
             let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?;
             return Some(expr1_0);
         }
-        // Rule at src/isa/aarch64/inst.isle line 1845.
+        // Rule at src/isa/aarch64/inst.isle line 1887.
         let expr0_0 = C::load_constant64_full(ctx, pattern2_0);
         return Some(expr0_0);
     }
@@ -3083,17 +3142,17 @@ pub fn constructor_put_in_reg_sext32<C: Context>(ctx: &mut C, arg0: Value) -> Op
     let pattern0_0 = arg0;
     let pattern1_0 = C::value_type(ctx, pattern0_0);
     if pattern1_0 == I32 {
-        // Rule at src/isa/aarch64/inst.isle line 1856.
+        // Rule at src/isa/aarch64/inst.isle line 1898.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if pattern1_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1857.
+        // Rule at src/isa/aarch64/inst.isle line 1899.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1852.
+        // Rule at src/isa/aarch64/inst.isle line 1894.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         let expr1_0: bool = true;
         let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -3109,17 +3168,17 @@ pub fn constructor_put_in_reg_zext32<C: Context>(ctx: &mut C, arg0: Value) -> Op
     let pattern0_0 = arg0;
     let pattern1_0 = C::value_type(ctx, pattern0_0);
     if pattern1_0 == I32 {
-        // Rule at src/isa/aarch64/inst.isle line 1865.
+        // Rule at src/isa/aarch64/inst.isle line 1907.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if pattern1_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1866.
+        // Rule at src/isa/aarch64/inst.isle line 1908.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1861.
+        // Rule at src/isa/aarch64/inst.isle line 1903.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         let expr1_0: bool = false;
         let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -3135,12 +3194,12 @@ pub fn constructor_put_in_reg_sext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
     let pattern0_0 = arg0;
     let pattern1_0 = C::value_type(ctx, pattern0_0);
     if pattern1_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1874.
+        // Rule at src/isa/aarch64/inst.isle line 1916.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1870.
+        // Rule at src/isa/aarch64/inst.isle line 1912.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         let expr1_0: bool = true;
         let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -3156,12 +3215,12 @@ pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
     let pattern0_0 = arg0;
     let pattern1_0 = C::value_type(ctx, pattern0_0);
     if pattern1_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1882.
+        // Rule at src/isa/aarch64/inst.isle line 1924.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1878.
+        // Rule at src/isa/aarch64/inst.isle line 1920.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         let expr1_0: bool = false;
         let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -3175,7 +3234,7 @@ pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
 // Generated as internal constructor for term trap_if_zero_divisor.
 pub fn constructor_trap_if_zero_divisor<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Reg> {
     let pattern0_0 = arg0;
-    // Rule at src/isa/aarch64/inst.isle line 1887.
+    // Rule at src/isa/aarch64/inst.isle line 1929.
     let expr0_0 = C::cond_br_zero(ctx, pattern0_0);
     let expr1_0 = C::trap_code_division_by_zero(ctx);
     let expr2_0 = MInst::TrapIf {
@@ -3190,12 +3249,12 @@ pub fn constructor_trap_if_zero_divisor<C: Context>(ctx: &mut C, arg0: Reg) -> O
 pub fn constructor_size_from_ty<C: Context>(ctx: &mut C, arg0: Type) -> Option<OperandSize> {
     let pattern0_0 = arg0;
     if pattern0_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1893.
+        // Rule at src/isa/aarch64/inst.isle line 1935.
         let expr0_0 = OperandSize::Size64;
         return Some(expr0_0);
     }
     if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1892.
+        // Rule at src/isa/aarch64/inst.isle line 1934.
         let expr0_0 = OperandSize::Size32;
         return Some(expr0_0);
     }
@@ -3212,7 +3271,7 @@ pub fn constructor_trap_if_div_overflow<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1899.
+    // Rule at src/isa/aarch64/inst.isle line 1941.
     let expr0_0 = ALUOp::AddS;
     let expr1_0 = constructor_operand_size(ctx, pattern0_0)?;
     let expr2_0 = C::writable_zero_reg(ctx);
@@ -3281,7 +3340,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
                             C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0)
                         {
                             let pattern9_0 = arg3;
-                            // Rule at src/isa/aarch64/inst.isle line 1944.
+                            // Rule at src/isa/aarch64/inst.isle line 1986.
                             let expr0_0 = C::put_in_reg(ctx, pattern9_0);
                             let expr1_0 = constructor_alu_rr_imm_logic(
                                 ctx, pattern0_0, pattern1_0, expr0_0, pattern8_0,
@@ -3313,7 +3372,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
                                         C::lshl_from_imm64(ctx, pattern10_1, pattern12_0)
                                     {
                                         let pattern14_0 = arg3;
-                                        // Rule at src/isa/aarch64/inst.isle line 1950.
+                                        // Rule at src/isa/aarch64/inst.isle line 1992.
                                         let expr0_0 = C::put_in_reg(ctx, pattern14_0);
                                         let expr1_0 = C::put_in_reg(ctx, pattern7_0);
                                         let expr2_0 = constructor_alu_rrr_shift(
@@ -3351,7 +3410,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
                         if let Some(pattern9_0) =
                             C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0)
                         {
-                            // Rule at src/isa/aarch64/inst.isle line 1942.
+                            // Rule at src/isa/aarch64/inst.isle line 1984.
                             let expr0_0 = C::put_in_reg(ctx, pattern2_0);
                             let expr1_0 = constructor_alu_rr_imm_logic(
                                 ctx, pattern0_0, pattern1_0, expr0_0, pattern9_0,
@@ -3382,7 +3441,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
                                     if let Some(pattern14_0) =
                                         C::lshl_from_imm64(ctx, pattern11_1, pattern13_0)
                                     {
-                                        // Rule at src/isa/aarch64/inst.isle line 1948.
+                                        // Rule at src/isa/aarch64/inst.isle line 1990.
                                         let expr0_0 = C::put_in_reg(ctx, pattern2_0);
                                         let expr1_0 = C::put_in_reg(ctx, pattern8_0);
                                         let expr2_0 = constructor_alu_rrr_shift(
@@ -3404,7 +3463,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
             _ => {}
         }
     }
-    // Rule at src/isa/aarch64/inst.isle line 1938.
+    // Rule at src/isa/aarch64/inst.isle line 1980.
     let expr0_0 = C::put_in_reg(ctx, pattern2_0);
     let expr1_0 = C::put_in_reg(ctx, pattern3_0);
     let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, pattern1_0, expr0_0, expr1_0)?;
@@ -3438,7 +3497,7 @@ pub fn constructor_alu_rs_imm_logic<C: Context>(
                         if let Some(pattern9_0) =
                             C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0)
                         {
-                            // Rule at src/isa/aarch64/inst.isle line 1958.
+                            // Rule at src/isa/aarch64/inst.isle line 2000.
                             let expr0_0 = C::put_in_reg(ctx, pattern2_0);
                             let expr1_0 = constructor_alu_rr_imm_logic(
                                 ctx, pattern0_0, pattern1_0, expr0_0, pattern9_0,
@@ -3469,7 +3528,7 @@ pub fn constructor_alu_rs_imm_logic<C: Context>(
                                     if let Some(pattern14_0) =
                                         C::lshl_from_imm64(ctx, pattern11_1, pattern13_0)
                                     {
-                                        // Rule at src/isa/aarch64/inst.isle line 1960.
+                                        // Rule at src/isa/aarch64/inst.isle line 2002.
                                         let expr0_0 = C::put_in_reg(ctx, pattern2_0);
                                         let expr1_0 = C::put_in_reg(ctx, pattern8_0);
                                         let expr2_0 = constructor_alu_rrr_shift(
@@ -3491,7 +3550,7 @@ pub fn constructor_alu_rs_imm_logic<C: Context>(
             _ => {}
         }
     }
-    // Rule at src/isa/aarch64/inst.isle line 1956.
+    // Rule at src/isa/aarch64/inst.isle line 1998.
     let expr0_0 = C::put_in_reg(ctx, pattern2_0);
     let expr1_0 = C::put_in_reg(ctx, pattern3_0);
     let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, pattern1_0, expr0_0, expr1_0)?;
@@ -3510,7 +3569,7 @@ pub fn constructor_i128_alu_bitop<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1968.
+    // Rule at src/isa/aarch64/inst.isle line 2010.
     let expr0_0 = C::put_in_regs(ctx, pattern2_0);
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -3537,7 +3596,7 @@ pub fn constructor_float_cmp_zero<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 2008.
+    // Rule at src/isa/aarch64/inst.isle line 2050.
     let expr0_0 = C::float_cc_cmp_zero_to_vec_misc_op(ctx, pattern0_0);
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -3553,7 +3612,7 @@ pub fn constructor_float_cmp_zero_swap<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 2013.
+    // Rule at src/isa/aarch64/inst.isle line 2055.
     let expr0_0 = C::float_cc_cmp_zero_to_vec_misc_op_swap(ctx, pattern0_0);
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -3563,7 +3622,7 @@ pub fn constructor_float_cmp_zero_swap<C: Context>(
 pub fn constructor_fcmeq0<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 2018.
+    // Rule at src/isa/aarch64/inst.isle line 2060.
     let expr0_0 = VecMisc2::Fcmeq0;
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
@@ -3579,7 +3638,7 @@ pub fn constructor_int_cmp_zero<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 2044.
+    // Rule at src/isa/aarch64/inst.isle line 2086.
     let expr0_0 = C::int_cc_cmp_zero_to_vec_misc_op(ctx, pattern0_0);
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -3595,7 +3654,7 @@ pub fn constructor_int_cmp_zero_swap<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 2049.
+    // Rule at src/isa/aarch64/inst.isle line 2091.
     let expr0_0 = C::int_cc_cmp_zero_to_vec_misc_op_swap(ctx, pattern0_0);
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?;
     return Some(expr1_0);
@@ -3605,15 +3664,191 @@ pub fn constructor_int_cmp_zero_swap<C: Context>(
 pub fn constructor_cmeq0<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 2054.
+    // Rule at src/isa/aarch64/inst.isle line 2096.
     let expr0_0 = VecMisc2::Cmeq0;
     let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?;
     return Some(expr1_0);
 }
 
+// Generated as internal constructor for term lse_atomic_rmw.
+pub fn constructor_lse_atomic_rmw<C: Context>(
+    ctx: &mut C,
+    arg0: &AtomicRMWOp,
+    arg1: Value,
+    arg2: Reg,
+    arg3: Type,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/aarch64/inst.isle line 2101.
+    let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+    let expr1_0 = C::temp_writable_reg(ctx, pattern3_0);
+    let expr2_0 = MInst::AtomicRMW {
+        op: pattern0_0.clone(),
+        rs: pattern2_0,
+        rt: expr1_0,
+        rn: expr0_0,
+        ty: pattern3_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term atomic_rmw_loop.
+pub fn constructor_atomic_rmw_loop<C: Context>(
+    ctx: &mut C,
+    arg0: &AtomicRMWLoopOp,
+    arg1: Value,
+    arg2: Value,
+    arg3: Type,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/aarch64/inst.isle line 2116.
+    let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+    let expr1_0: Type = I64;
+    let expr2_0 = C::ensure_in_vreg(ctx, expr0_0, expr1_0);
+    let expr3_0 = C::put_in_reg(ctx, pattern2_0);
+    let expr4_0: Type = I64;
+    let expr5_0 = C::ensure_in_vreg(ctx, expr3_0, expr4_0);
+    let expr6_0: u8 = 25;
+    let expr7_0 = constructor_mov64_to_real(ctx, expr6_0, expr2_0)?;
+    let expr8_0: u8 = 26;
+    let expr9_0 = constructor_mov64_to_real(ctx, expr8_0, expr5_0)?;
+    let expr10_0 = MInst::AtomicRMWLoop {
+        ty: pattern3_0,
+        op: pattern0_0.clone(),
+    };
+    let expr11_0 = C::emit(ctx, &expr10_0);
+    let expr12_0: u8 = 27;
+    let expr13_0 = constructor_mov64_from_real(ctx, expr12_0)?;
+    return Some(expr13_0);
+}
+
 // Generated as internal constructor for term lower.
 pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutput> {
     let pattern0_0 = arg0;
+    if let Some(()) = C::use_lse(ctx, pattern0_0) {
+        if let Some(pattern2_0) = C::first_result(ctx, pattern0_0) {
+            let pattern3_0 = C::value_type(ctx, pattern2_0);
+            if let Some(pattern4_0) = C::valid_atomic_transaction(ctx, pattern3_0) {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::AtomicRmw {
+                    opcode: ref pattern6_0,
+                    args: ref pattern6_1,
+                    flags: pattern6_2,
+                    op: ref pattern6_3,
+                } = &pattern5_0
+                {
+                    if let &Opcode::AtomicRmw = pattern6_0 {
+                        let (pattern8_0, pattern8_1) = C::unpack_value_array_2(ctx, pattern6_1);
+                        match pattern6_3 {
+                            &AtomicRmwOp::Add => {
+                                // Rule at src/isa/aarch64/lower.isle line 1202.
+                                let expr0_0 = AtomicRMWOp::Add;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr1_0, pattern4_0,
+                                )?;
+                                let expr3_0 = constructor_output_reg(ctx, expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                            &AtomicRmwOp::And => {
+                                // Rule at src/isa/aarch64/lower.isle line 1234.
+                                let expr0_0 = AtomicRMWOp::Clr;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = C::zero_reg(ctx);
+                                let expr3_0 = constructor_eon(ctx, pattern4_0, expr1_0, expr2_0)?;
+                                let expr4_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr3_0, pattern4_0,
+                                )?;
+                                let expr5_0 = constructor_output_reg(ctx, expr4_0)?;
+                                return Some(expr5_0);
+                            }
+                            &AtomicRmwOp::Or => {
+                                // Rule at src/isa/aarch64/lower.isle line 1210.
+                                let expr0_0 = AtomicRMWOp::Set;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr1_0, pattern4_0,
+                                )?;
+                                let expr3_0 = constructor_output_reg(ctx, expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                            &AtomicRmwOp::Smax => {
+                                // Rule at src/isa/aarch64/lower.isle line 1214.
+                                let expr0_0 = AtomicRMWOp::Smax;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr1_0, pattern4_0,
+                                )?;
+                                let expr3_0 = constructor_output_reg(ctx, expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                            &AtomicRmwOp::Smin => {
+                                // Rule at src/isa/aarch64/lower.isle line 1218.
+                                let expr0_0 = AtomicRMWOp::Smin;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr1_0, pattern4_0,
+                                )?;
+                                let expr3_0 = constructor_output_reg(ctx, expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                            &AtomicRmwOp::Sub => {
+                                // Rule at src/isa/aarch64/lower.isle line 1230.
+                                let expr0_0 = AtomicRMWOp::Add;
+                                let expr1_0 = C::zero_reg(ctx);
+                                let expr2_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr3_0 = constructor_sub(ctx, pattern4_0, expr1_0, expr2_0)?;
+                                let expr4_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr3_0, pattern4_0,
+                                )?;
+                                let expr5_0 = constructor_output_reg(ctx, expr4_0)?;
+                                return Some(expr5_0);
+                            }
+                            &AtomicRmwOp::Umax => {
+                                // Rule at src/isa/aarch64/lower.isle line 1222.
+                                let expr0_0 = AtomicRMWOp::Umax;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr1_0, pattern4_0,
+                                )?;
+                                let expr3_0 = constructor_output_reg(ctx, expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                            &AtomicRmwOp::Umin => {
+                                // Rule at src/isa/aarch64/lower.isle line 1226.
+                                let expr0_0 = AtomicRMWOp::Umin;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr1_0, pattern4_0,
+                                )?;
+                                let expr3_0 = constructor_output_reg(ctx, expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                            &AtomicRmwOp::Xor => {
+                                // Rule at src/isa/aarch64/lower.isle line 1206.
+                                let expr0_0 = AtomicRMWOp::Eor;
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 = constructor_lse_atomic_rmw(
+                                    ctx, &expr0_0, pattern8_0, expr1_0, pattern4_0,
+                                )?;
+                                let expr3_0 = constructor_output_reg(ctx, expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+        }
+    }
     if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) {
         let pattern2_0 = C::value_type(ctx, pattern1_0);
         if pattern2_0 == I8 {
@@ -6683,6 +6918,122 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
                 }
             }
         }
+        if let Some(pattern3_0) = C::valid_atomic_transaction(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::AtomicRmw {
+                opcode: ref pattern5_0,
+                args: ref pattern5_1,
+                flags: pattern5_2,
+                op: ref pattern5_3,
+            } = &pattern4_0
+            {
+                if let &Opcode::AtomicRmw = pattern5_0 {
+                    let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1);
+                    match pattern5_3 {
+                        &AtomicRmwOp::Add => {
+                            // Rule at src/isa/aarch64/lower.isle line 1240.
+                            let expr0_0 = AtomicRMWLoopOp::Add;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::And => {
+                            // Rule at src/isa/aarch64/lower.isle line 1246.
+                            let expr0_0 = AtomicRMWLoopOp::And;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Nand => {
+                            // Rule at src/isa/aarch64/lower.isle line 1249.
+                            let expr0_0 = AtomicRMWLoopOp::Nand;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Or => {
+                            // Rule at src/isa/aarch64/lower.isle line 1252.
+                            let expr0_0 = AtomicRMWLoopOp::Orr;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Smax => {
+                            // Rule at src/isa/aarch64/lower.isle line 1261.
+                            let expr0_0 = AtomicRMWLoopOp::Smax;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Smin => {
+                            // Rule at src/isa/aarch64/lower.isle line 1258.
+                            let expr0_0 = AtomicRMWLoopOp::Smin;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Sub => {
+                            // Rule at src/isa/aarch64/lower.isle line 1243.
+                            let expr0_0 = AtomicRMWLoopOp::Sub;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Umax => {
+                            // Rule at src/isa/aarch64/lower.isle line 1267.
+                            let expr0_0 = AtomicRMWLoopOp::Umax;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Umin => {
+                            // Rule at src/isa/aarch64/lower.isle line 1264.
+                            let expr0_0 = AtomicRMWLoopOp::Umin;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Xchg => {
+                            // Rule at src/isa/aarch64/lower.isle line 1270.
+                            let expr0_0 = AtomicRMWLoopOp::Xchg;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &AtomicRmwOp::Xor => {
+                            // Rule at src/isa/aarch64/lower.isle line 1255.
+                            let expr0_0 = AtomicRMWLoopOp::Eor;
+                            let expr1_0 = constructor_atomic_rmw_loop(
+                                ctx, &expr0_0, pattern7_0, pattern7_1, pattern3_0,
+                            )?;
+                            let expr2_0 = constructor_output_reg(ctx, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        _ => {}
+                    }
+                }
+            }
+        }
     }
     return None;
 }
diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
index fdb277c381..773b5d6a03 100644
--- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
@@ -237,48 +237,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             ctx.emit(inst);
         }
 
-        Opcode::AtomicRmw => {
-            let r_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let mut r_addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let mut r_arg2 = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let ty_access = ty.unwrap();
-            assert!(is_valid_atomic_transaction_ty(ty_access));
-
-            let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
-            let lse_op = match op {
-                AtomicRmwOp::Add => Some(AtomicRMWOp::Add),
-                AtomicRmwOp::And => Some(AtomicRMWOp::Clr),
-                AtomicRmwOp::Xor => Some(AtomicRMWOp::Eor),
-                AtomicRmwOp::Or => Some(AtomicRMWOp::Set),
-                AtomicRmwOp::Smax => Some(AtomicRMWOp::Smax),
-                AtomicRmwOp::Umax => Some(AtomicRMWOp::Umax),
-                AtomicRmwOp::Smin => Some(AtomicRMWOp::Smin),
-                AtomicRmwOp::Umin => Some(AtomicRMWOp::Umin),
-                _ => None,
-            };
-            if isa_flags.use_lse() && lse_op.is_some() {
-                ctx.emit(Inst::AtomicRMW {
-                    op: lse_op.unwrap(),
-                    rs: r_arg2,
-                    rt: r_dst,
-                    rn: r_addr,
-                    ty: ty_access,
-                });
-            } else {
-                // Make sure that both args are in virtual regs, since in effect
-                // we have to do a parallel copy to get them safely to the AtomicRMW input
-                // regs, and that's not guaranteed safe if either is in a real reg.
-                r_addr = ctx.ensure_in_vreg(r_addr, I64);
-                r_arg2 = ctx.ensure_in_vreg(r_arg2, I64);
-                // Move the args to the preordained AtomicRMW input regs
-                ctx.emit(Inst::gen_move(Writable::from_reg(xreg(25)), r_addr, I64));
-                ctx.emit(Inst::gen_move(Writable::from_reg(xreg(26)), r_arg2, I64));
-                ctx.emit(Inst::AtomicRMWLoop { ty: ty_access, op });
-                // And finally, copy the preordained AtomicRMW output reg to its destination.
-                ctx.emit(Inst::gen_move(r_dst, xreg(27), I64));
-                // Also, x24 and x28 are trashed.  `fn aarch64_get_regs` must mention that.
-            }
-        }
+        Opcode::AtomicRmw => implemented_in_isle(ctx),
 
         Opcode::AtomicCas => {
             let r_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
index f7ca087f03..6ff5e690e6 100644
--- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
+++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
@@ -1,4 +1,4 @@
 src/clif.isle 443b34b797fc8ace
-src/prelude.isle d8a93eb727abd7f4
+src/prelude.isle a7915a6b88310eb5
 src/isa/s390x/inst.isle 8218bd9e8556446b
 src/isa/s390x/lower.isle 6a8de81f8dc4e568
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs
index 2ea073acac..0128380d77 100644
--- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs
+++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs
@@ -35,6 +35,7 @@ pub trait Context {
     fn invalid_reg(&mut self) -> Reg;
     fn put_in_reg(&mut self, arg0: Value) -> Reg;
     fn put_in_regs(&mut self, arg0: Value) -> ValueRegs;
+    fn ensure_in_vreg(&mut self, arg0: Reg, arg1: Type) -> Reg;
     fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg;
     fn u8_as_u64(&mut self, arg0: u8) -> u64;
     fn u16_as_u64(&mut self, arg0: u16) -> u64;
@@ -155,14 +156,14 @@ pub trait Context {
     fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 412.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
     Inst2 { inst1: MInst, inst2: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 434.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 439.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -170,7 +171,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 445.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 450.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -943,7 +944,7 @@ pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg>
 // Generated as internal constructor for term lo_reg.
 pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
     let pattern0_0 = arg0;
-    // Rule at src/prelude.isle line 145.
+    // Rule at src/prelude.isle line 150.
     let expr0_0 = C::put_in_regs(ctx, pattern0_0);
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -960,7 +961,7 @@ pub fn constructor_side_effect<C: Context>(
         &SideEffectNoResult::Inst {
             inst: ref pattern1_0,
         } => {
-            // Rule at src/prelude.isle line 415.
+            // Rule at src/prelude.isle line 420.
             let expr0_0 = C::emit(ctx, pattern1_0);
             let expr1_0 = C::output_none(ctx);
             return Some(expr1_0);
@@ -969,7 +970,7 @@ pub fn constructor_side_effect<C: Context>(
             inst1: ref pattern1_0,
             inst2: ref pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 418.
+            // Rule at src/prelude.isle line 423.
             let expr0_0 = C::emit(ctx, pattern1_0);
             let expr1_0 = C::emit(ctx, pattern1_1);
             let expr2_0 = C::output_none(ctx);
@@ -996,7 +997,7 @@ pub fn constructor_side_effect_concat<C: Context>(
             inst: ref pattern3_0,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 424.
+            // Rule at src/prelude.isle line 429.
             let expr0_0 = SideEffectNoResult::Inst2 {
                 inst1: pattern1_0.clone(),
                 inst2: pattern3_0.clone(),
@@ -1018,7 +1019,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 461.
+        // Rule at src/prelude.isle line 466.
         return Some(pattern1_1);
     }
     return None;
@@ -1035,7 +1036,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 466.
+            // Rule at src/prelude.isle line 471.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1045,7 +1046,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 468.
+            // Rule at src/prelude.isle line 473.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1074,7 +1075,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 475.
+            // Rule at src/prelude.isle line 480.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -1104,7 +1105,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 500.
+                    // Rule at src/prelude.isle line 505.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1115,7 +1116,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 506.
+                    // Rule at src/prelude.isle line 511.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1128,7 +1129,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 518.
+                    // Rule at src/prelude.isle line 523.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1149,7 +1150,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 494.
+                // Rule at src/prelude.isle line 499.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1169,7 +1170,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 535.
+    // Rule at src/prelude.isle line 540.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
index 02add8a713..e1d37d192c 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
+++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
@@ -1,4 +1,4 @@
 src/clif.isle 443b34b797fc8ace
-src/prelude.isle d8a93eb727abd7f4
+src/prelude.isle a7915a6b88310eb5
 src/isa/x64/inst.isle 6dcba190988a695
 src/isa/x64/lower.isle b95161bdf07b9365
diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs
index e9e98cdfdc..36168e5b65 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs
@@ -35,6 +35,7 @@ pub trait Context {
     fn invalid_reg(&mut self) -> Reg;
     fn put_in_reg(&mut self, arg0: Value) -> Reg;
     fn put_in_regs(&mut self, arg0: Value) -> ValueRegs;
+    fn ensure_in_vreg(&mut self, arg0: Reg, arg1: Type) -> Reg;
     fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg;
     fn u8_as_u64(&mut self, arg0: u8) -> u64;
     fn u16_as_u64(&mut self, arg0: u16) -> u64;
@@ -163,14 +164,14 @@ pub trait Context {
     fn popcount_low_mask(&mut self) -> VCodeConstant;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 412.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
     Inst2 { inst1: MInst, inst2: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 434.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 439.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -178,7 +179,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 445.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 450.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -562,7 +563,7 @@ pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg>
 // Generated as internal constructor for term lo_reg.
 pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
     let pattern0_0 = arg0;
-    // Rule at src/prelude.isle line 145.
+    // Rule at src/prelude.isle line 150.
     let expr0_0 = C::put_in_regs(ctx, pattern0_0);
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -579,7 +580,7 @@ pub fn constructor_side_effect<C: Context>(
         &SideEffectNoResult::Inst {
             inst: ref pattern1_0,
         } => {
-            // Rule at src/prelude.isle line 415.
+            // Rule at src/prelude.isle line 420.
             let expr0_0 = C::emit(ctx, pattern1_0);
             let expr1_0 = C::output_none(ctx);
             return Some(expr1_0);
@@ -588,7 +589,7 @@ pub fn constructor_side_effect<C: Context>(
             inst1: ref pattern1_0,
             inst2: ref pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 418.
+            // Rule at src/prelude.isle line 423.
             let expr0_0 = C::emit(ctx, pattern1_0);
             let expr1_0 = C::emit(ctx, pattern1_1);
             let expr2_0 = C::output_none(ctx);
@@ -615,7 +616,7 @@ pub fn constructor_side_effect_concat<C: Context>(
             inst: ref pattern3_0,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 424.
+            // Rule at src/prelude.isle line 429.
             let expr0_0 = SideEffectNoResult::Inst2 {
                 inst1: pattern1_0.clone(),
                 inst2: pattern3_0.clone(),
@@ -637,7 +638,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 461.
+        // Rule at src/prelude.isle line 466.
         return Some(pattern1_1);
     }
     return None;
@@ -654,7 +655,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 466.
+            // Rule at src/prelude.isle line 471.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -664,7 +665,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 468.
+            // Rule at src/prelude.isle line 473.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -693,7 +694,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 475.
+            // Rule at src/prelude.isle line 480.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -723,7 +724,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 500.
+                    // Rule at src/prelude.isle line 505.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -734,7 +735,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 506.
+                    // Rule at src/prelude.isle line 511.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -747,7 +748,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 518.
+                    // Rule at src/prelude.isle line 523.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -768,7 +769,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 494.
+                // Rule at src/prelude.isle line 499.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -788,7 +789,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 535.
+    // Rule at src/prelude.isle line 540.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index 615d0a66f7..d69deb76dc 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -120,6 +120,11 @@ macro_rules! isle_prelude_methods {
             self.lower_ctx.put_value_in_regs(val)
         }
 
+        #[inline]
+        fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg {
+            self.lower_ctx.ensure_in_vreg(reg, ty)
+        }
+
         #[inline]
         fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg {
             regs.regs()[i]
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index d5d758b917..dbfd25ca7c 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -129,6 +129,11 @@
 (decl put_in_regs (Value) ValueRegs)
 (extern constructor put_in_regs put_in_regs)
 
+;; If the given reg is a real register, cause the value in reg to be in a virtual
+;; reg, by copying it into a new virtual reg.
+(decl ensure_in_vreg (Reg Type) Reg)
+(extern constructor ensure_in_vreg ensure_in_vreg)
+
 ;; Get the `n`th register inside a `ValueRegs`.
 (decl value_regs_get (ValueRegs usize) Reg)
 (extern constructor value_regs_get value_regs_get)
diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw-lse.clif b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw-lse.clif
index 96cbe97cd9..0e33718593 100644
--- a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw-lse.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw-lse.clif
@@ -41,6 +41,50 @@ block0(v0: i64, v1: i8):
 ;   ldaddalb w1, w4, [x0]
 ;   ret
 
+function %atomic_rmw_sub_i64(i64, i64) {
+block0(v0: i64, v1: i64):
+    v2 = atomic_rmw.i64 sub v0, v1
+    return
+}
+
+; block0:
+;   sub x4, xzr, x1
+;   ldaddal x4, x6, [x0]
+;   ret
+
+function %atomic_rmw_sub_i32(i64, i32) {
+block0(v0: i64, v1: i32):
+    v2 = atomic_rmw.i32 sub v0, v1
+    return
+}
+
+; block0:
+;   sub w4, wzr, w1
+;   ldaddal w4, w6, [x0]
+;   ret
+
+function %atomic_rmw_sub_i16(i64, i16) {
+block0(v0: i64, v1: i16):
+    v2 = atomic_rmw.i16 sub v0, v1
+    return
+}
+
+; block0:
+;   sub w4, wzr, w1
+;   ldaddalh w4, w6, [x0]
+;   ret
+
+function %atomic_rmw_sub_i8(i64, i8) {
+block0(v0: i64, v1: i8):
+    v2 = atomic_rmw.i8 sub v0, v1
+    return
+}
+
+; block0:
+;   sub w4, wzr, w1
+;   ldaddalb w4, w6, [x0]
+;   ret
+
 function %atomic_rmw_and_i64(i64, i64) {
 block0(v0: i64, v1: i64):
     v2 = atomic_rmw.i64 and v0, v1
@@ -48,7 +92,8 @@ block0(v0: i64, v1: i64):
 }
 
 ; block0:
-;   ldclral x1, x4, [x0]
+;   eon x4, x1, xzr
+;   ldclral x4, x6, [x0]
 ;   ret
 
 function %atomic_rmw_and_i32(i64, i32) {
@@ -58,7 +103,8 @@ block0(v0: i64, v1: i32):
 }
 
 ; block0:
-;   ldclral w1, w4, [x0]
+;   eon w4, w1, wzr
+;   ldclral w4, w6, [x0]
 ;   ret
 
 function %atomic_rmw_and_i16(i64, i16) {
@@ -68,7 +114,8 @@ block0(v0: i64, v1: i16):
 }
 
 ; block0:
-;   ldclralh w1, w4, [x0]
+;   eon w4, w1, wzr
+;   ldclralh w4, w6, [x0]
 ;   ret
 
 function %atomic_rmw_and_i8(i64, i8) {
@@ -78,7 +125,8 @@ block0(v0: i64, v1: i8):
 }
 
 ; block0:
-;   ldclralb w1, w4, [x0]
+;   eon w4, w1, wzr
+;   ldclralb w4, w6, [x0]
 ;   ret
 
 function %atomic_rmw_nand_i64(i64, i64) {
diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif
index 1229c08560..0b017ad3df 100644
--- a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif
@@ -89,6 +89,94 @@ block0(v0: i64, v1: i8):
 ;   ldp fp, lr, [sp], #16
 ;   ret
 
+function %atomic_rmw_sub_i64(i64, i64) {
+block0(v0: i64, v1: i64):
+    v2 = atomic_rmw.i64 sub v0, v1
+    return
+}
+
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   str x28, [sp, #-16]!
+;   stp x26, x27, [sp, #-16]!
+;   stp x24, x25, [sp, #-16]!
+; block0:
+;   mov x25, x0
+;   mov x4, x1
+;   mov x26, x4
+;   1: ldaxr x27, [x25]; sub x28, x27, x26; stlxr w24, x28, [x25]; cbnz w24, 1b
+;   ldp x24, x25, [sp], #16
+;   ldp x26, x27, [sp], #16
+;   ldr x28, [sp], #16
+;   ldp fp, lr, [sp], #16
+;   ret
+
+function %atomic_rmw_sub_i32(i64, i32) {
+block0(v0: i64, v1: i32):
+    v2 = atomic_rmw.i32 sub v0, v1
+    return
+}
+
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   str x28, [sp, #-16]!
+;   stp x26, x27, [sp, #-16]!
+;   stp x24, x25, [sp, #-16]!
+; block0:
+;   mov x25, x0
+;   mov x4, x1
+;   mov x26, x4
+;   1: ldaxr w27, [x25]; sub w28, w27, w26; stlxr w24, w28, [x25]; cbnz w24, 1b
+;   ldp x24, x25, [sp], #16
+;   ldp x26, x27, [sp], #16
+;   ldr x28, [sp], #16
+;   ldp fp, lr, [sp], #16
+;   ret
+
+function %atomic_rmw_sub_i16(i64, i16) {
+block0(v0: i64, v1: i16):
+    v2 = atomic_rmw.i16 sub v0, v1
+    return
+}
+
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   str x28, [sp, #-16]!
+;   stp x26, x27, [sp, #-16]!
+;   stp x24, x25, [sp, #-16]!
+; block0:
+;   mov x25, x0
+;   mov x4, x1
+;   mov x26, x4
+;   1: ldaxrh w27, [x25]; sub w28, w27, w26; stlxrh w24, w28, [x25]; cbnz w24, 1b
+;   ldp x24, x25, [sp], #16
+;   ldp x26, x27, [sp], #16
+;   ldr x28, [sp], #16
+;   ldp fp, lr, [sp], #16
+;   ret
+
+function %atomic_rmw_sub_i8(i64, i8) {
+block0(v0: i64, v1: i8):
+    v2 = atomic_rmw.i8 sub v0, v1
+    return
+}
+
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   str x28, [sp, #-16]!
+;   stp x26, x27, [sp, #-16]!
+;   stp x24, x25, [sp, #-16]!
+; block0:
+;   mov x25, x0
+;   mov x4, x1
+;   mov x26, x4
+;   1: ldaxrb w27, [x25]; sub w28, w27, w26; stlxrb w24, w28, [x25]; cbnz w24, 1b
+;   ldp x24, x25, [sp], #16
+;   ldp x26, x27, [sp], #16
+;   ldr x28, [sp], #16
+;   ldp fp, lr, [sp], #16
+;   ret
+
 function %atomic_rmw_and_i64(i64, i64) {
 block0(v0: i64, v1: i64):
     v2 = atomic_rmw.i64 and v0, v1
@@ -500,7 +588,7 @@ block0(v0: i64, v1: i16):
 ;   mov x25, x0
 ;   mov x4, x1
 ;   mov x26, x4
-;   1: ldaxrh w27, [x25]; cmp w27, w26; csel w28, w27, w26, gt; stlxrh w24, w28, [x25]; cbnz w24, 1b
+;   1: ldaxrh w27, [x25]; sxth w27, w27; cmp w27, w26, sxth; csel w28, w27, w26, gt; stlxrh w24, w28, [x25]; cbnz w24, 1b
 ;   ldp x24, x25, [sp], #16
 ;   ldp x26, x27, [sp], #16
 ;   ldr x28, [sp], #16
@@ -522,7 +610,7 @@ block0(v0: i64, v1: i8):
 ;   mov x25, x0
 ;   mov x4, x1
 ;   mov x26, x4
-;   1: ldaxrb w27, [x25]; cmp w27, w26; csel w28, w27, w26, gt; stlxrb w24, w28, [x25]; cbnz w24, 1b
+;   1: ldaxrb w27, [x25]; sxtb w27, w27; cmp w27, w26, sxtb; csel w28, w27, w26, gt; stlxrb w24, w28, [x25]; cbnz w24, 1b
 ;   ldp x24, x25, [sp], #16
 ;   ldp x26, x27, [sp], #16
 ;   ldr x28, [sp], #16
@@ -676,7 +764,7 @@ block0(v0: i64, v1: i16):
 ;   mov x25, x0
 ;   mov x4, x1
 ;   mov x26, x4
-;   1: ldaxrh w27, [x25]; cmp w27, w26; csel w28, w27, w26, lt; stlxrh w24, w28, [x25]; cbnz w24, 1b
+;   1: ldaxrh w27, [x25]; sxth w27, w27; cmp w27, w26, sxth; csel w28, w27, w26, lt; stlxrh w24, w28, [x25]; cbnz w24, 1b
 ;   ldp x24, x25, [sp], #16
 ;   ldp x26, x27, [sp], #16
 ;   ldr x28, [sp], #16
@@ -698,7 +786,7 @@ block0(v0: i64, v1: i8):
 ;   mov x25, x0
 ;   mov x4, x1
 ;   mov x26, x4
-;   1: ldaxrb w27, [x25]; cmp w27, w26; csel w28, w27, w26, lt; stlxrb w24, w28, [x25]; cbnz w24, 1b
+;   1: ldaxrb w27, [x25]; sxtb w27, w27; cmp w27, w26, sxtb; csel w28, w27, w26, lt; stlxrb w24, w28, [x25]; cbnz w24, 1b
 ;   ldp x24, x25, [sp], #16
 ;   ldp x26, x27, [sp], #16
 ;   ldr x28, [sp], #16
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-little.clif b/cranelift/filetests/filetests/runtests/atomic-cas-little.clif
index d80c6dac57..c910befb2b 100644
--- a/cranelift/filetests/filetests/runtests/atomic-cas-little.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-cas-little.clif
@@ -25,7 +25,7 @@ function %atomic_cas_i32(i32, i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32, v2: i32):
-    v3 = stack_addr.i32 ss0
+    v3 = stack_addr.i64 ss0
     store.i32 little v0, v3
 
     v4 = atomic_cas.i32 little v3, v1, v2
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif b/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
new file mode 100644
index 0000000000..4fb63be0fa
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
@@ -0,0 +1,46 @@
+test run
+target s390x
+
+; We can't test that these instructions are right regarding atomicity, but we can
+; test if they perform their operation correctly
+
+function %atomic_cas_big_i16(i32, i64, i16, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16, v3: i16):
+    v4 = stack_addr.i64 ss0
+    store.i32 big v0, v4
+
+    v5 = iadd.i64 v4, v1
+    v6 = atomic_cas.i16 big v5, v2, v3
+
+    v7 = load.i32 big v4
+    return v7
+}
+; run: %atomic_cas_big_i16(0x12345678, 0, 0x1234, 0xabcd) == 0xabcd5678
+; run: %atomic_cas_big_i16(0x12345678, 0, 0x4321, 0xabcd) == 0x12345678
+; run: %atomic_cas_big_i16(0x12345678, 2, 0x5678, 0xabcd) == 0x1234abcd
+; run: %atomic_cas_big_i16(0x12345678, 2, 0x8765, 0xabcd) == 0x12345678
+
+function %atomic_cas_big_i8(i32, i64, i8, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8, v3: i8):
+    v4 = stack_addr.i64 ss0
+    store.i32 big v0, v4
+
+    v5 = iadd.i64 v4, v1
+    v6 = atomic_cas.i8 big v5, v2, v3
+
+    v7 = load.i32 big v4
+    return v7
+}
+; run: %atomic_cas_big_i8(0x12345678, 0, 0x12, 0xab) == 0xab345678
+; run: %atomic_cas_big_i8(0x12345678, 0, 0x21, 0xab) == 0x12345678
+; run: %atomic_cas_big_i8(0x12345678, 1, 0x34, 0xab) == 0x12ab5678
+; run: %atomic_cas_big_i8(0x12345678, 1, 0x43, 0xab) == 0x12345678
+; run: %atomic_cas_big_i8(0x12345678, 2, 0x56, 0xab) == 0x1234ab78
+; run: %atomic_cas_big_i8(0x12345678, 2, 0x65, 0xab) == 0x12345678
+; run: %atomic_cas_big_i8(0x12345678, 3, 0x78, 0xab) == 0x123456ab
+; run: %atomic_cas_big_i8(0x12345678, 3, 0x87, 0xab) == 0x12345678
+
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-subword.clif b/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
similarity index 53%
rename from cranelift/filetests/filetests/runtests/atomic-cas-subword.clif
rename to cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
index 02aeaafecd..cc508ae4b8 100644
--- a/cranelift/filetests/filetests/runtests/atomic-cas-subword.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
@@ -1,27 +1,12 @@
 test run
 target s390x
+target aarch64
+target aarch64 has_lse
+target x86_64
 
 ; We can't test that these instructions are right regarding atomicity, but we can
 ; test if they perform their operation correctly
 
-function %atomic_cas_big_i16(i32, i64, i16, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16, v3: i16):
-    v4 = stack_addr.i64 ss0
-    store.i32 big v0, v4
-
-    v5 = iadd.i64 v4, v1
-    v6 = atomic_cas.i16 big v5, v2, v3
-
-    v7 = load.i32 big v4
-    return v7
-}
-; run: %atomic_cas_big_i16(0x12345678, 0, 0x1234, 0xabcd) == 0xabcd5678
-; run: %atomic_cas_big_i16(0x12345678, 0, 0x4321, 0xabcd) == 0x12345678
-; run: %atomic_cas_big_i16(0x12345678, 2, 0x5678, 0xabcd) == 0x1234abcd
-; run: %atomic_cas_big_i16(0x12345678, 2, 0x8765, 0xabcd) == 0x12345678
-
 function %atomic_cas_little_i16(i32, i64, i16, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -40,28 +25,6 @@ block0(v0: i32, v1: i64, v2: i16, v3: i16):
 ; run: %atomic_cas_little_i16(0x12345678, 0, 0x5678, 0xabcd) == 0x1234abcd
 ; run: %atomic_cas_little_i16(0x12345678, 0, 0x8765, 0xabcd) == 0x12345678
 
-function %atomic_cas_big_i8(i32, i64, i8, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8, v3: i8):
-    v4 = stack_addr.i64 ss0
-    store.i32 big v0, v4
-
-    v5 = iadd.i64 v4, v1
-    v6 = atomic_cas.i8 big v5, v2, v3
-
-    v7 = load.i32 big v4
-    return v7
-}
-; run: %atomic_cas_big_i8(0x12345678, 0, 0x12, 0xab) == 0xab345678
-; run: %atomic_cas_big_i8(0x12345678, 0, 0x21, 0xab) == 0x12345678
-; run: %atomic_cas_big_i8(0x12345678, 1, 0x34, 0xab) == 0x12ab5678
-; run: %atomic_cas_big_i8(0x12345678, 1, 0x43, 0xab) == 0x12345678
-; run: %atomic_cas_big_i8(0x12345678, 2, 0x56, 0xab) == 0x1234ab78
-; run: %atomic_cas_big_i8(0x12345678, 2, 0x65, 0xab) == 0x12345678
-; run: %atomic_cas_big_i8(0x12345678, 3, 0x78, 0xab) == 0x123456ab
-; run: %atomic_cas_big_i8(0x12345678, 3, 0x87, 0xab) == 0x12345678
-
 function %atomic_cas_little_i8(i32, i64, i8, i8) -> i32 {
     ss0 = explicit_slot 4
 
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas.clif b/cranelift/filetests/filetests/runtests/atomic-cas.clif
index 7e95568d5f..9c0783b9bb 100644
--- a/cranelift/filetests/filetests/runtests/atomic-cas.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-cas.clif
@@ -30,7 +30,7 @@ function %atomic_cas_i32(i32, i32, i32) -> i32 {
 block0(v0: i32, v1: i32, v2: i32):
     stack_store.i32 v0, ss0
 
-    v3 = stack_addr.i32 ss0
+    v3 = stack_addr.i64 ss0
     v4 = atomic_cas.i32 v3, v1, v2
 
     v5 = stack_load.i32 ss0
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw-little.clif b/cranelift/filetests/filetests/runtests/atomic-rmw-little.clif
index cd6731cc4f..0ce931876a 100644
--- a/cranelift/filetests/filetests/runtests/atomic-rmw-little.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-rmw-little.clif
@@ -1,5 +1,8 @@
 test run
 target s390x
+target aarch64
+target aarch64 has_lse
+target x86_64
 
 ; We can't test that these instructions are right regarding atomicity, but we can
 ; test if they perform their operation correctly
@@ -26,7 +29,7 @@ function %atomic_rmw_add_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little add v2, v1
@@ -64,7 +67,7 @@ function %atomic_rmw_sub_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little sub v2, v1
@@ -102,7 +105,7 @@ function %atomic_rmw_and_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little and v2, v1
@@ -141,7 +144,7 @@ function %atomic_rmw_or_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little or v2, v1
@@ -180,7 +183,7 @@ function %atomic_rmw_xor_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little xor v2, v1
@@ -218,7 +221,7 @@ function %atomic_rmw_nand_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little nand v2, v1
@@ -257,7 +260,7 @@ function %atomic_rmw_umin_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little umin v2, v1
@@ -297,7 +300,7 @@ function %atomic_rmw_umax_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little umax v2, v1
@@ -337,7 +340,7 @@ function %atomic_rmw_smin_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little smin v2, v1
@@ -377,7 +380,7 @@ function %atomic_rmw_smax_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little smax v2, v1
@@ -415,7 +418,7 @@ function %atomic_rmw_xchg_i32(i32, i32) -> i32 {
     ss0 = explicit_slot 4
 
 block0(v0: i32, v1: i32):
-    v2 = stack_addr.i32 ss0
+    v2 = stack_addr.i64 ss0
     store.i32 little v0, v2
 
     v3 = atomic_rmw.i32 little xchg v2, v1
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
new file mode 100644
index 0000000000..b6218d4d35
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
@@ -0,0 +1,460 @@
+test run
+target s390x
+
+; We can't test that these instructions are right regarding atomicity, but we can
+; test if they perform their operation correctly
+
+function %atomic_rmw_add_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big add v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_add_little_i16(0x12345678, 0, 0x1111) == 0x23455678
+; run: %atomic_rmw_add_little_i16(0x12345678, 0, 0xffff) == 0x12335678
+; run: %atomic_rmw_add_little_i16(0x12345678, 2, 0x1111) == 0x12346789
+; run: %atomic_rmw_add_little_i16(0x12345678, 2, 0xffff) == 0x12345677
+
+function %atomic_rmw_add_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big add v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_add_big_i8(0x12345678, 0, 0x11) == 0x23345678
+; run: %atomic_rmw_add_big_i8(0x12345678, 0, 0xff) == 0x11345678
+; run: %atomic_rmw_add_big_i8(0x12345678, 1, 0x11) == 0x12455678
+; run: %atomic_rmw_add_big_i8(0x12345678, 1, 0xff) == 0x12335678
+; run: %atomic_rmw_add_big_i8(0x12345678, 2, 0x11) == 0x12346778
+; run: %atomic_rmw_add_big_i8(0x12345678, 2, 0xff) == 0x12345578
+; run: %atomic_rmw_add_big_i8(0x12345678, 3, 0x11) == 0x12345689
+; run: %atomic_rmw_add_big_i8(0x12345678, 3, 0xff) == 0x12345677
+
+function %atomic_rmw_sub_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big sub v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_sub_big_i16(0x12345678, 0, 0x1111) == 0x01235678
+; run: %atomic_rmw_sub_big_i16(0x12345678, 0, 0xffff) == 0x12355678
+; run: %atomic_rmw_sub_big_i16(0x12345678, 2, 0x1111) == 0x12344567
+; run: %atomic_rmw_sub_big_i16(0x12345678, 2, 0xffff) == 0x12345679
+
+
+function %atomic_rmw_sub_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big sub v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_sub_big_i8(0x12345678, 0, 0x11) == 0x01345678
+; run: %atomic_rmw_sub_big_i8(0x12345678, 0, 0xff) == 0x13345678
+; run: %atomic_rmw_sub_big_i8(0x12345678, 1, 0x11) == 0x12235678
+; run: %atomic_rmw_sub_big_i8(0x12345678, 1, 0xff) == 0x12355678
+; run: %atomic_rmw_sub_big_i8(0x12345678, 2, 0x11) == 0x12344578
+; run: %atomic_rmw_sub_big_i8(0x12345678, 2, 0xff) == 0x12345778
+; run: %atomic_rmw_sub_big_i8(0x12345678, 3, 0x11) == 0x12345667
+; run: %atomic_rmw_sub_big_i8(0x12345678, 3, 0xff) == 0x12345679
+
+
+function %atomic_rmw_and_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big and v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_and_big_i16(0x12345678, 0, 0xf000) == 0x10005678
+; run: %atomic_rmw_and_big_i16(0x12345678, 0, 0x000f) == 0x00045678
+; run: %atomic_rmw_and_big_i16(0x12345678, 2, 0xf000) == 0x12345000
+; run: %atomic_rmw_and_big_i16(0x12345678, 2, 0x000f) == 0x12340008
+
+
+function %atomic_rmw_and_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big and v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_and_big_i8(0x12345678, 0, 0xf0) == 0x10345678
+; run: %atomic_rmw_and_big_i8(0x12345678, 0, 0x0f) == 0x02345678
+; run: %atomic_rmw_and_big_i8(0x12345678, 1, 0xf0) == 0x12305678
+; run: %atomic_rmw_and_big_i8(0x12345678, 1, 0x0f) == 0x12045678
+; run: %atomic_rmw_and_big_i8(0x12345678, 2, 0xf0) == 0x12345078
+; run: %atomic_rmw_and_big_i8(0x12345678, 2, 0x0f) == 0x12340678
+; run: %atomic_rmw_and_big_i8(0x12345678, 3, 0xf0) == 0x12345670
+; run: %atomic_rmw_and_big_i8(0x12345678, 3, 0x0f) == 0x12345608
+
+
+function %atomic_rmw_or_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big or v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_or_big_i16(0x12345678, 0, 0xf000) == 0xf2345678
+; run: %atomic_rmw_or_big_i16(0x12345678, 0, 0x000f) == 0x123f5678
+; run: %atomic_rmw_or_big_i16(0x12345678, 2, 0xf000) == 0x1234f678
+; run: %atomic_rmw_or_big_i16(0x12345678, 2, 0x000f) == 0x1234567f
+
+
+function %atomic_rmw_or_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big or v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_or_big_i8(0x12345678, 0, 0xf0) == 0xf2345678
+; run: %atomic_rmw_or_big_i8(0x12345678, 0, 0x0f) == 0x1f345678
+; run: %atomic_rmw_or_big_i8(0x12345678, 1, 0xf0) == 0x12f45678
+; run: %atomic_rmw_or_big_i8(0x12345678, 1, 0x0f) == 0x123f5678
+; run: %atomic_rmw_or_big_i8(0x12345678, 2, 0xf0) == 0x1234f678
+; run: %atomic_rmw_or_big_i8(0x12345678, 2, 0x0f) == 0x12345f78
+; run: %atomic_rmw_or_big_i8(0x12345678, 3, 0xf0) == 0x123456f8
+; run: %atomic_rmw_or_big_i8(0x12345678, 3, 0x0f) == 0x1234567f
+
+
+function %atomic_rmw_xor_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big xor v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_xor_big_i16(0x12345678, 0, 0xf000) == 0xe2345678
+; run: %atomic_rmw_xor_big_i16(0x12345678, 0, 0x000f) == 0x123b5678
+; run: %atomic_rmw_xor_big_i16(0x12345678, 2, 0xf000) == 0x1234a678
+; run: %atomic_rmw_xor_big_i16(0x12345678, 2, 0x000f) == 0x12345677
+
+
+function %atomic_rmw_xor_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big xor v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_xor_big_i8(0x12345678, 0, 0xf0) == 0xe2345678
+; run: %atomic_rmw_xor_big_i8(0x12345678, 0, 0x0f) == 0x1d345678
+; run: %atomic_rmw_xor_big_i8(0x12345678, 1, 0xf0) == 0x12c45678
+; run: %atomic_rmw_xor_big_i8(0x12345678, 1, 0x0f) == 0x123b5678
+; run: %atomic_rmw_xor_big_i8(0x12345678, 2, 0xf0) == 0x1234a678
+; run: %atomic_rmw_xor_big_i8(0x12345678, 2, 0x0f) == 0x12345978
+; run: %atomic_rmw_xor_big_i8(0x12345678, 3, 0xf0) == 0x12345688
+; run: %atomic_rmw_xor_big_i8(0x12345678, 3, 0x0f) == 0x12345677
+
+function %atomic_rmw_nand_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big nand v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_nand_big_i16(0x12345678, 0, 0xf000) == 0xefff5678
+; run: %atomic_rmw_nand_big_i16(0x12345678, 0, 0x000f) == 0xfffb5678
+; run: %atomic_rmw_nand_big_i16(0x12345678, 2, 0xf000) == 0x1234afff
+; run: %atomic_rmw_nand_big_i16(0x12345678, 2, 0x000f) == 0x1234fff7
+
+function %atomic_rmw_nand_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big nand v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_nand_big_i8(0x12345678, 0, 0xf0) == 0xef345678
+; run: %atomic_rmw_nand_big_i8(0x12345678, 0, 0x0f) == 0xfd345678
+; run: %atomic_rmw_nand_big_i8(0x12345678, 1, 0xf0) == 0x12cf5678
+; run: %atomic_rmw_nand_big_i8(0x12345678, 1, 0x0f) == 0x12fb5678
+; run: %atomic_rmw_nand_big_i8(0x12345678, 2, 0xf0) == 0x1234af78
+; run: %atomic_rmw_nand_big_i8(0x12345678, 2, 0x0f) == 0x1234f978
+; run: %atomic_rmw_nand_big_i8(0x12345678, 3, 0xf0) == 0x1234568f
+; run: %atomic_rmw_nand_big_i8(0x12345678, 3, 0x0f) == 0x123456f7
+
+
+function %atomic_rmw_umin_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big umin v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_umin_big_i16(0x12345678, 0, 0x1111) == 0x11115678
+; run: %atomic_rmw_umin_big_i16(0x12345678, 0, 0xffff) == 0x12345678
+; run: %atomic_rmw_umin_big_i16(0x12345678, 2, 0x1111) == 0x12341111
+; run: %atomic_rmw_umin_big_i16(0x12345678, 2, 0xffff) == 0x12345678
+
+function %atomic_rmw_umin_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big umin v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_umin_big_i8(0x12345678, 0, 0x11) == 0x11345678
+; run: %atomic_rmw_umin_big_i8(0x12345678, 0, 0xff) == 0x12345678
+; run: %atomic_rmw_umin_big_i8(0x12345678, 1, 0x11) == 0x12115678
+; run: %atomic_rmw_umin_big_i8(0x12345678, 1, 0xff) == 0x12345678
+; run: %atomic_rmw_umin_big_i8(0x12345678, 2, 0x11) == 0x12341178
+; run: %atomic_rmw_umin_big_i8(0x12345678, 2, 0xff) == 0x12345678
+; run: %atomic_rmw_umin_big_i8(0x12345678, 3, 0x11) == 0x12345611
+; run: %atomic_rmw_umin_big_i8(0x12345678, 3, 0xff) == 0x12345678
+
+
+function %atomic_rmw_umax_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big umax v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_umax_big_i16(0x12345678, 0, 0x1111) == 0x12345678
+; run: %atomic_rmw_umax_big_i16(0x12345678, 0, 0xffff) == 0xffff5678
+; run: %atomic_rmw_umax_big_i16(0x12345678, 2, 0x1111) == 0x12345678
+; run: %atomic_rmw_umax_big_i16(0x12345678, 2, 0xffff) == 0x1234ffff
+
+function %atomic_rmw_umax_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big umax v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_umax_big_i8(0x12345678, 0, 0x11) == 0x12345678
+; run: %atomic_rmw_umax_big_i8(0x12345678, 0, 0xff) == 0xff345678
+; run: %atomic_rmw_umax_big_i8(0x12345678, 1, 0x11) == 0x12345678
+; run: %atomic_rmw_umax_big_i8(0x12345678, 1, 0xff) == 0x12ff5678
+; run: %atomic_rmw_umax_big_i8(0x12345678, 2, 0x11) == 0x12345678
+; run: %atomic_rmw_umax_big_i8(0x12345678, 2, 0xff) == 0x1234ff78
+; run: %atomic_rmw_umax_big_i8(0x12345678, 3, 0x11) == 0x12345678
+; run: %atomic_rmw_umax_big_i8(0x12345678, 3, 0xff) == 0x123456ff
+
+
+function %atomic_rmw_smin_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big smin v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_smin_big_i16(0x12345678, 0, 0x1111) == 0x11115678
+; run: %atomic_rmw_smin_big_i16(0x12345678, 0, 0xffff) == 0xffff5678
+; run: %atomic_rmw_smin_big_i16(0x12345678, 2, 0x1111) == 0x12341111
+; run: %atomic_rmw_smin_big_i16(0x12345678, 2, 0xffff) == 0x1234ffff
+
+function %atomic_rmw_smin_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big smin v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_smin_big_i8(0x12345678, 0, 0x11) == 0x11345678
+; run: %atomic_rmw_smin_big_i8(0x12345678, 0, 0xff) == 0xff345678
+; run: %atomic_rmw_smin_big_i8(0x12345678, 1, 0x11) == 0x12115678
+; run: %atomic_rmw_smin_big_i8(0x12345678, 1, 0xff) == 0x12ff5678
+; run: %atomic_rmw_smin_big_i8(0x12345678, 2, 0x11) == 0x12341178
+; run: %atomic_rmw_smin_big_i8(0x12345678, 2, 0xff) == 0x1234ff78
+; run: %atomic_rmw_smin_big_i8(0x12345678, 3, 0x11) == 0x12345611
+; run: %atomic_rmw_smin_big_i8(0x12345678, 3, 0xff) == 0x123456ff
+
+
+function %atomic_rmw_smax_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big smax v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_smax_big_i16(0x12345678, 0, 0xffff) == 0x12345678
+; run: %atomic_rmw_smax_big_i16(0x12345678, 0, 0x7fff) == 0x7fff5678
+; run: %atomic_rmw_smax_big_i16(0x12345678, 2, 0xffff) == 0x12345678
+; run: %atomic_rmw_smax_big_i16(0x12345678, 2, 0x7fff) == 0x12347fff
+
+
+function %atomic_rmw_smax_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big smax v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_smax_big_i8(0x12345678, 0, 0xff) == 0x12345678
+; run: %atomic_rmw_smax_big_i8(0x12345678, 0, 0x7f) == 0x7f345678
+; run: %atomic_rmw_smax_big_i8(0x12345678, 1, 0xff) == 0x12345678
+; run: %atomic_rmw_smax_big_i8(0x12345678, 1, 0x7f) == 0x127f5678
+; run: %atomic_rmw_smax_big_i8(0x12345678, 2, 0xff) == 0x12345678
+; run: %atomic_rmw_smax_big_i8(0x12345678, 2, 0x7f) == 0x12347f78
+; run: %atomic_rmw_smax_big_i8(0x12345678, 3, 0xff) == 0x12345678
+; run: %atomic_rmw_smax_big_i8(0x12345678, 3, 0x7f) == 0x1234567f
+
+
+function %atomic_rmw_xchg_big_i16(i32, i64, i16) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i16):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i16 big xchg v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0x1111) == 0x11115678
+; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0xffff) == 0xffff5678
+; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0x1111) == 0x12341111
+; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0xffff) == 0x1234ffff
+
+
+function %atomic_rmw_xchg_big_i8(i32, i64, i8) -> i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i8):
+    v3 = stack_addr.i64 ss0
+    store.i32 big v0, v3
+
+    v4 = iadd.i64 v3, v1
+    v5 = atomic_rmw.i8 big xchg v4, v2
+
+    v6 = load.i32 big v3
+    return v6
+}
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 0, 0x11) == 0x11345678
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 0, 0xff) == 0xff345678
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 1, 0x11) == 0x12115678
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 1, 0xff) == 0x12ff5678
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 2, 0x11) == 0x12341178
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 2, 0xff) == 0x1234ff78
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 3, 0x11) == 0x12345611
+; run: %atomic_rmw_xchg_big_i8(0x12345678, 3, 0xff) == 0x123456ff
+
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw-subword.clif b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
similarity index 51%
rename from cranelift/filetests/filetests/runtests/atomic-rmw-subword.clif
rename to cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
index efc3068b3f..7e597ecf1d 100644
--- a/cranelift/filetests/filetests/runtests/atomic-rmw-subword.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
@@ -1,27 +1,12 @@
 test run
 target s390x
+target aarch64
+target aarch64 has_lse
+target x86_64
 
 ; We can't test that these instructions are right regarding atomicity, but we can
 ; test if they perform their operation correctly
 
-function %atomic_rmw_add_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big add v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_add_little_i16(0x12345678, 0, 0x1111) == 0x23455678
-; run: %atomic_rmw_add_little_i16(0x12345678, 0, 0xffff) == 0x12335678
-; run: %atomic_rmw_add_little_i16(0x12345678, 2, 0x1111) == 0x12346789
-; run: %atomic_rmw_add_little_i16(0x12345678, 2, 0xffff) == 0x12345677
-
 function %atomic_rmw_add_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -40,28 +25,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_add_little_i16(0x12345678, 0, 0x1111) == 0x12346789
 ; run: %atomic_rmw_add_little_i16(0x12345678, 0, 0xffff) == 0x12345677
 
-function %atomic_rmw_add_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big add v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_add_big_i8(0x12345678, 0, 0x11) == 0x23345678
-; run: %atomic_rmw_add_big_i8(0x12345678, 0, 0xff) == 0x11345678
-; run: %atomic_rmw_add_big_i8(0x12345678, 1, 0x11) == 0x12455678
-; run: %atomic_rmw_add_big_i8(0x12345678, 1, 0xff) == 0x12335678
-; run: %atomic_rmw_add_big_i8(0x12345678, 2, 0x11) == 0x12346778
-; run: %atomic_rmw_add_big_i8(0x12345678, 2, 0xff) == 0x12345578
-; run: %atomic_rmw_add_big_i8(0x12345678, 3, 0x11) == 0x12345689
-; run: %atomic_rmw_add_big_i8(0x12345678, 3, 0xff) == 0x12345677
-
 function %atomic_rmw_add_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -84,26 +47,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_add_little_i8(0x12345678, 0, 0x11) == 0x12345689
 ; run: %atomic_rmw_add_little_i8(0x12345678, 0, 0xff) == 0x12345677
 
-
-
-function %atomic_rmw_sub_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big sub v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_sub_big_i16(0x12345678, 0, 0x1111) == 0x01235678
-; run: %atomic_rmw_sub_big_i16(0x12345678, 0, 0xffff) == 0x12355678
-; run: %atomic_rmw_sub_big_i16(0x12345678, 2, 0x1111) == 0x12344567
-; run: %atomic_rmw_sub_big_i16(0x12345678, 2, 0xffff) == 0x12345679
-
 function %atomic_rmw_sub_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -122,28 +65,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_sub_little_i16(0x12345678, 0, 0x1111) == 0x12344567
 ; run: %atomic_rmw_sub_little_i16(0x12345678, 0, 0xffff) == 0x12345679
 
-function %atomic_rmw_sub_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big sub v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_sub_big_i8(0x12345678, 0, 0x11) == 0x01345678
-; run: %atomic_rmw_sub_big_i8(0x12345678, 0, 0xff) == 0x13345678
-; run: %atomic_rmw_sub_big_i8(0x12345678, 1, 0x11) == 0x12235678
-; run: %atomic_rmw_sub_big_i8(0x12345678, 1, 0xff) == 0x12355678
-; run: %atomic_rmw_sub_big_i8(0x12345678, 2, 0x11) == 0x12344578
-; run: %atomic_rmw_sub_big_i8(0x12345678, 2, 0xff) == 0x12345778
-; run: %atomic_rmw_sub_big_i8(0x12345678, 3, 0x11) == 0x12345667
-; run: %atomic_rmw_sub_big_i8(0x12345678, 3, 0xff) == 0x12345679
-
 function %atomic_rmw_sub_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -166,26 +87,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_sub_little_i8(0x12345678, 0, 0x11) == 0x12345667
 ; run: %atomic_rmw_sub_little_i8(0x12345678, 0, 0xff) == 0x12345679
 
-
-
-function %atomic_rmw_and_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big and v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_and_big_i16(0x12345678, 0, 0xf000) == 0x10005678
-; run: %atomic_rmw_and_big_i16(0x12345678, 0, 0x000f) == 0x00045678
-; run: %atomic_rmw_and_big_i16(0x12345678, 2, 0xf000) == 0x12345000
-; run: %atomic_rmw_and_big_i16(0x12345678, 2, 0x000f) == 0x12340008
-
 function %atomic_rmw_and_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -204,28 +105,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_and_little_i16(0x12345678, 0, 0xf000) == 0x12345000
 ; run: %atomic_rmw_and_little_i16(0x12345678, 0, 0x000f) == 0x12340008
 
-function %atomic_rmw_and_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big and v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_and_big_i8(0x12345678, 0, 0xf0) == 0x10345678
-; run: %atomic_rmw_and_big_i8(0x12345678, 0, 0x0f) == 0x02345678
-; run: %atomic_rmw_and_big_i8(0x12345678, 1, 0xf0) == 0x12305678
-; run: %atomic_rmw_and_big_i8(0x12345678, 1, 0x0f) == 0x12045678
-; run: %atomic_rmw_and_big_i8(0x12345678, 2, 0xf0) == 0x12345078
-; run: %atomic_rmw_and_big_i8(0x12345678, 2, 0x0f) == 0x12340678
-; run: %atomic_rmw_and_big_i8(0x12345678, 3, 0xf0) == 0x12345670
-; run: %atomic_rmw_and_big_i8(0x12345678, 3, 0x0f) == 0x12345608
-
 function %atomic_rmw_and_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -249,25 +128,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_and_little_i8(0x12345678, 0, 0x0f) == 0x12345608
 
 
-
-function %atomic_rmw_or_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big or v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_or_big_i16(0x12345678, 0, 0xf000) == 0xf2345678
-; run: %atomic_rmw_or_big_i16(0x12345678, 0, 0x000f) == 0x123f5678
-; run: %atomic_rmw_or_big_i16(0x12345678, 2, 0xf000) == 0x1234f678
-; run: %atomic_rmw_or_big_i16(0x12345678, 2, 0x000f) == 0x1234567f
-
 function %atomic_rmw_or_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -286,28 +146,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_or_little_i16(0x12345678, 0, 0xf000) == 0x1234f678
 ; run: %atomic_rmw_or_little_i16(0x12345678, 0, 0x000f) == 0x1234567f
 
-function %atomic_rmw_or_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big or v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_or_big_i8(0x12345678, 0, 0xf0) == 0xf2345678
-; run: %atomic_rmw_or_big_i8(0x12345678, 0, 0x0f) == 0x1f345678
-; run: %atomic_rmw_or_big_i8(0x12345678, 1, 0xf0) == 0x12f45678
-; run: %atomic_rmw_or_big_i8(0x12345678, 1, 0x0f) == 0x123f5678
-; run: %atomic_rmw_or_big_i8(0x12345678, 2, 0xf0) == 0x1234f678
-; run: %atomic_rmw_or_big_i8(0x12345678, 2, 0x0f) == 0x12345f78
-; run: %atomic_rmw_or_big_i8(0x12345678, 3, 0xf0) == 0x123456f8
-; run: %atomic_rmw_or_big_i8(0x12345678, 3, 0x0f) == 0x1234567f
-
 function %atomic_rmw_or_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -330,27 +168,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_or_little_i8(0x12345678, 0, 0xf0) == 0x123456f8
 ; run: %atomic_rmw_or_little_i8(0x12345678, 0, 0x0f) == 0x1234567f
 
-
-
-
-function %atomic_rmw_xor_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big xor v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_xor_big_i16(0x12345678, 0, 0xf000) == 0xe2345678
-; run: %atomic_rmw_xor_big_i16(0x12345678, 0, 0x000f) == 0x123b5678
-; run: %atomic_rmw_xor_big_i16(0x12345678, 2, 0xf000) == 0x1234a678
-; run: %atomic_rmw_xor_big_i16(0x12345678, 2, 0x000f) == 0x12345677
-
 function %atomic_rmw_xor_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -369,28 +186,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_xor_little_i16(0x12345678, 0, 0xf000) == 0x1234a678
 ; run: %atomic_rmw_xor_little_i16(0x12345678, 0, 0x000f) == 0x12345677
 
-function %atomic_rmw_xor_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big xor v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_xor_big_i8(0x12345678, 0, 0xf0) == 0xe2345678
-; run: %atomic_rmw_xor_big_i8(0x12345678, 0, 0x0f) == 0x1d345678
-; run: %atomic_rmw_xor_big_i8(0x12345678, 1, 0xf0) == 0x12c45678
-; run: %atomic_rmw_xor_big_i8(0x12345678, 1, 0x0f) == 0x123b5678
-; run: %atomic_rmw_xor_big_i8(0x12345678, 2, 0xf0) == 0x1234a678
-; run: %atomic_rmw_xor_big_i8(0x12345678, 2, 0x0f) == 0x12345978
-; run: %atomic_rmw_xor_big_i8(0x12345678, 3, 0xf0) == 0x12345688
-; run: %atomic_rmw_xor_big_i8(0x12345678, 3, 0x0f) == 0x12345677
-
 function %atomic_rmw_xor_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -414,25 +209,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_xor_little_i8(0x12345678, 0, 0x0f) == 0x12345677
 
 
-
-function %atomic_rmw_nand_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big nand v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_nand_big_i16(0x12345678, 0, 0xf000) == 0xefff5678
-; run: %atomic_rmw_nand_big_i16(0x12345678, 0, 0x000f) == 0xfffb5678
-; run: %atomic_rmw_nand_big_i16(0x12345678, 2, 0xf000) == 0x1234afff
-; run: %atomic_rmw_nand_big_i16(0x12345678, 2, 0x000f) == 0x1234fff7
-
 function %atomic_rmw_nand_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -451,28 +227,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_nand_little_i16(0x12345678, 0, 0xf000) == 0x1234afff
 ; run: %atomic_rmw_nand_little_i16(0x12345678, 0, 0x000f) == 0x1234fff7
 
-function %atomic_rmw_nand_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big nand v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_nand_big_i8(0x12345678, 0, 0xf0) == 0xef345678
-; run: %atomic_rmw_nand_big_i8(0x12345678, 0, 0x0f) == 0xfd345678
-; run: %atomic_rmw_nand_big_i8(0x12345678, 1, 0xf0) == 0x12cf5678
-; run: %atomic_rmw_nand_big_i8(0x12345678, 1, 0x0f) == 0x12fb5678
-; run: %atomic_rmw_nand_big_i8(0x12345678, 2, 0xf0) == 0x1234af78
-; run: %atomic_rmw_nand_big_i8(0x12345678, 2, 0x0f) == 0x1234f978
-; run: %atomic_rmw_nand_big_i8(0x12345678, 3, 0xf0) == 0x1234568f
-; run: %atomic_rmw_nand_big_i8(0x12345678, 3, 0x0f) == 0x123456f7
-
 function %atomic_rmw_nand_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -496,25 +250,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_nand_little_i8(0x12345678, 0, 0x0f) == 0x123456f7
 
 
-
-function %atomic_rmw_umin_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big umin v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_umin_big_i16(0x12345678, 0, 0x1111) == 0x11115678
-; run: %atomic_rmw_umin_big_i16(0x12345678, 0, 0xffff) == 0x12345678
-; run: %atomic_rmw_umin_big_i16(0x12345678, 2, 0x1111) == 0x12341111
-; run: %atomic_rmw_umin_big_i16(0x12345678, 2, 0xffff) == 0x12345678
-
 function %atomic_rmw_umin_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -533,28 +268,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_umin_little_i16(0x12345678, 0, 0x1111) == 0x12341111
 ; run: %atomic_rmw_umin_little_i16(0x12345678, 0, 0xffff) == 0x12345678
 
-function %atomic_rmw_umin_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big umin v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_umin_big_i8(0x12345678, 0, 0x11) == 0x11345678
-; run: %atomic_rmw_umin_big_i8(0x12345678, 0, 0xff) == 0x12345678
-; run: %atomic_rmw_umin_big_i8(0x12345678, 1, 0x11) == 0x12115678
-; run: %atomic_rmw_umin_big_i8(0x12345678, 1, 0xff) == 0x12345678
-; run: %atomic_rmw_umin_big_i8(0x12345678, 2, 0x11) == 0x12341178
-; run: %atomic_rmw_umin_big_i8(0x12345678, 2, 0xff) == 0x12345678
-; run: %atomic_rmw_umin_big_i8(0x12345678, 3, 0x11) == 0x12345611
-; run: %atomic_rmw_umin_big_i8(0x12345678, 3, 0xff) == 0x12345678
-
 function %atomic_rmw_umin_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -578,25 +291,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_umin_little_i8(0x12345678, 0, 0xff) == 0x12345678
 
 
-
-function %atomic_rmw_umax_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big umax v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_umax_big_i16(0x12345678, 0, 0x1111) == 0x12345678
-; run: %atomic_rmw_umax_big_i16(0x12345678, 0, 0xffff) == 0xffff5678
-; run: %atomic_rmw_umax_big_i16(0x12345678, 2, 0x1111) == 0x12345678
-; run: %atomic_rmw_umax_big_i16(0x12345678, 2, 0xffff) == 0x1234ffff
-
 function %atomic_rmw_umax_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -615,28 +309,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_umax_little_i16(0x12345678, 0, 0x1111) == 0x12345678
 ; run: %atomic_rmw_umax_little_i16(0x12345678, 0, 0xffff) == 0x1234ffff
 
-function %atomic_rmw_umax_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big umax v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_umax_big_i8(0x12345678, 0, 0x11) == 0x12345678
-; run: %atomic_rmw_umax_big_i8(0x12345678, 0, 0xff) == 0xff345678
-; run: %atomic_rmw_umax_big_i8(0x12345678, 1, 0x11) == 0x12345678
-; run: %atomic_rmw_umax_big_i8(0x12345678, 1, 0xff) == 0x12ff5678
-; run: %atomic_rmw_umax_big_i8(0x12345678, 2, 0x11) == 0x12345678
-; run: %atomic_rmw_umax_big_i8(0x12345678, 2, 0xff) == 0x1234ff78
-; run: %atomic_rmw_umax_big_i8(0x12345678, 3, 0x11) == 0x12345678
-; run: %atomic_rmw_umax_big_i8(0x12345678, 3, 0xff) == 0x123456ff
-
 function %atomic_rmw_umax_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
 
@@ -660,25 +332,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_umax_little_i8(0x12345678, 0, 0xff) == 0x123456ff
 
 
-
-function %atomic_rmw_smin_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big smin v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_smin_big_i16(0x12345678, 0, 0x1111) == 0x11115678
-; run: %atomic_rmw_smin_big_i16(0x12345678, 0, 0xffff) == 0xffff5678
-; run: %atomic_rmw_smin_big_i16(0x12345678, 2, 0x1111) == 0x12341111
-; run: %atomic_rmw_smin_big_i16(0x12345678, 2, 0xffff) == 0x1234ffff
-
 function %atomic_rmw_smin_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -697,27 +350,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_smin_little_i16(0x12345678, 0, 0x1111) == 0x12341111
 ; run: %atomic_rmw_smin_little_i16(0x12345678, 0, 0xffff) == 0x1234ffff
 
-function %atomic_rmw_smin_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big smin v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_smin_big_i8(0x12345678, 0, 0x11) == 0x11345678
-; run: %atomic_rmw_smin_big_i8(0x12345678, 0, 0xff) == 0xff345678
-; run: %atomic_rmw_smin_big_i8(0x12345678, 1, 0x11) == 0x12115678
-; run: %atomic_rmw_smin_big_i8(0x12345678, 1, 0xff) == 0x12ff5678
-; run: %atomic_rmw_smin_big_i8(0x12345678, 2, 0x11) == 0x12341178
-; run: %atomic_rmw_smin_big_i8(0x12345678, 2, 0xff) == 0x1234ff78
-; run: %atomic_rmw_smin_big_i8(0x12345678, 3, 0x11) == 0x12345611
-; run: %atomic_rmw_smin_big_i8(0x12345678, 3, 0xff) == 0x123456ff
 
 function %atomic_rmw_smin_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
@@ -742,25 +374,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_smin_little_i8(0x12345678, 0, 0xff) == 0x123456ff
 
 
-
-function %atomic_rmw_smax_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big smax v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_smax_big_i16(0x12345678, 0, 0xffff) == 0x12345678
-; run: %atomic_rmw_smax_big_i16(0x12345678, 0, 0x7fff) == 0x7fff5678
-; run: %atomic_rmw_smax_big_i16(0x12345678, 2, 0xffff) == 0x12345678
-; run: %atomic_rmw_smax_big_i16(0x12345678, 2, 0x7fff) == 0x12347fff
-
 function %atomic_rmw_smax_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -779,27 +392,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_smax_little_i16(0x12345678, 0, 0xffff) == 0x12345678
 ; run: %atomic_rmw_smax_little_i16(0x12345678, 0, 0x7fff) == 0x12347fff
 
-function %atomic_rmw_smax_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big smax v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_smax_big_i8(0x12345678, 0, 0xff) == 0x12345678
-; run: %atomic_rmw_smax_big_i8(0x12345678, 0, 0x7f) == 0x7f345678
-; run: %atomic_rmw_smax_big_i8(0x12345678, 1, 0xff) == 0x12345678
-; run: %atomic_rmw_smax_big_i8(0x12345678, 1, 0x7f) == 0x127f5678
-; run: %atomic_rmw_smax_big_i8(0x12345678, 2, 0xff) == 0x12345678
-; run: %atomic_rmw_smax_big_i8(0x12345678, 2, 0x7f) == 0x12347f78
-; run: %atomic_rmw_smax_big_i8(0x12345678, 3, 0xff) == 0x12345678
-; run: %atomic_rmw_smax_big_i8(0x12345678, 3, 0x7f) == 0x1234567f
 
 function %atomic_rmw_smax_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
@@ -824,25 +416,6 @@ block0(v0: i32, v1: i64, v2: i8):
 ; run: %atomic_rmw_smax_little_i8(0x12345678, 0, 0x7f) == 0x1234567f
 
 
-
-function %atomic_rmw_xchg_big_i16(i32, i64, i16) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i16):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i16 big xchg v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0x1111) == 0x11115678
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0xffff) == 0xffff5678
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0x1111) == 0x12341111
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0xffff) == 0x1234ffff
-
 function %atomic_rmw_xchg_little_i16(i32, i64, i16) -> i32 {
     ss0 = explicit_slot 4
 
@@ -861,27 +434,6 @@ block0(v0: i32, v1: i64, v2: i16):
 ; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0x1111) == 0x12341111
 ; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0xffff) == 0x1234ffff
 
-function %atomic_rmw_xchg_big_i8(i32, i64, i8) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i64, v2: i8):
-    v3 = stack_addr.i64 ss0
-    store.i32 big v0, v3
-
-    v4 = iadd.i64 v3, v1
-    v5 = atomic_rmw.i8 big xchg v4, v2
-
-    v6 = load.i32 big v3
-    return v6
-}
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 0, 0x11) == 0x11345678
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 0, 0xff) == 0xff345678
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 1, 0x11) == 0x12115678
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 1, 0xff) == 0x12ff5678
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 2, 0x11) == 0x12341178
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 2, 0xff) == 0x1234ff78
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 3, 0x11) == 0x12345611
-; run: %atomic_rmw_xchg_big_i8(0x12345678, 3, 0xff) == 0x123456ff
 
 function %atomic_rmw_xchg_little_i8(i32, i64, i8) -> i32 {
     ss0 = explicit_slot 4
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw.clif b/cranelift/filetests/filetests/runtests/atomic-rmw.clif
deleted file mode 100644
index 26c466c5e8..0000000000
--- a/cranelift/filetests/filetests/runtests/atomic-rmw.clif
+++ /dev/null
@@ -1,432 +0,0 @@
-test run
-target aarch64
-target aarch64 has_lse
-target x86_64
-target s390x
-
-; We can't test that these instructions are right regarding atomicity, but we can
-; test if they perform their operation correctly
-
-function %atomic_rmw_add_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 add v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_add_i64(0, 0) == 0
-; run: %atomic_rmw_add_i64(1, 0) == 1
-; run: %atomic_rmw_add_i64(0, 1) == 1
-; run: %atomic_rmw_add_i64(1, 1) == 2
-; run: %atomic_rmw_add_i64(0xC0FFEEEE_C0FFEEEE, 0x1DCB1111_1DCB1111) == 0xDECAFFFF_DECAFFFF
-
-function %atomic_rmw_add_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 add v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_add_i32(0, 0) == 0
-; run: %atomic_rmw_add_i32(1, 0) == 1
-; run: %atomic_rmw_add_i32(0, 1) == 1
-; run: %atomic_rmw_add_i32(1, 1) == 2
-; run: %atomic_rmw_add_i32(0xC0FFEEEE, 0x1DCB1111) == 0xDECAFFFF
-
-
-
-function %atomic_rmw_sub_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 sub v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_sub_i64(0, 0) == 0
-; run: %atomic_rmw_sub_i64(1, 0) == 1
-; run: %atomic_rmw_sub_i64(0, 1) == -1
-; run: %atomic_rmw_sub_i64(1, 1) == 0
-; run: %atomic_rmw_sub_i64(0xDECAFFFF_DECAFFFF, 0x1DCB1111_1DCB1111) == 0xC0FFEEEE_C0FFEEEE
-
-function %atomic_rmw_sub_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 sub v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_sub_i32(0, 0) == 0
-; run: %atomic_rmw_sub_i32(1, 0) == 1
-; run: %atomic_rmw_sub_i32(0, 1) == -1
-; run: %atomic_rmw_sub_i32(1, 1) == 0
-; run: %atomic_rmw_sub_i32(0xDECAFFFF, 0x1DCB1111) == 0xC0FFEEEE
-
-
-
-function %atomic_rmw_and_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 and v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_and_i64(0, 0) == 0
-; run: %atomic_rmw_and_i64(1, 0) == 0
-; run: %atomic_rmw_and_i64(0, 1) == 0
-; run: %atomic_rmw_and_i64(1, 1) == 1
-; run: %atomic_rmw_and_i64(0xF1FFFEFE_FEEEFFFF, 0xCEFFEFEF_DFDBFFFF) == 0xC0FFEEEE_DECAFFFF
-
-function %atomic_rmw_and_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 and v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-
-; run: %atomic_rmw_and_i64(0, 0) == 0
-; run: %atomic_rmw_and_i64(1, 0) == 0
-; run: %atomic_rmw_and_i64(0, 1) == 0
-; run: %atomic_rmw_and_i64(1, 1) == 1
-; run: %atomic_rmw_and_i64(0xF1FFFEFE, 0xCEFFEFEF) == 0xC0FFEEEE
-
-
-
-function %atomic_rmw_or_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 or v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_or_i64(0, 0) == 0
-; run: %atomic_rmw_or_i64(1, 0) == 1
-; run: %atomic_rmw_or_i64(0, 1) == 1
-; run: %atomic_rmw_or_i64(1, 1) == 1
-; run: %atomic_rmw_or_i64(0x80AAAAAA_8A8AAAAA, 0x40554444_54405555) == 0xC0FFEEEE_DECAFFFF
-
-function %atomic_rmw_or_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 or v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-
-; run: %atomic_rmw_or_i32(0, 0) == 0
-; run: %atomic_rmw_or_i32(1, 0) == 1
-; run: %atomic_rmw_or_i32(0, 1) == 1
-; run: %atomic_rmw_or_i32(1, 1) == 1
-; run: %atomic_rmw_or_i32(0x80AAAAAA, 0x40554444) == 0xC0FFEEEE
-
-
-
-function %atomic_rmw_xor_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 xor v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_xor_i64(0, 0) == 0
-; run: %atomic_rmw_xor_i64(1, 0) == 1
-; run: %atomic_rmw_xor_i64(0, 1) == 1
-; run: %atomic_rmw_xor_i64(1, 1) == 0
-; run: %atomic_rmw_xor_i64(0x8FA50A64_9440A07D, 0x4F5AE48A_4A8A5F82) == 0xC0FFEEEE_DECAFFFF
-
-function %atomic_rmw_xor_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 xor v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_xor_i32(0, 0) == 0
-; run: %atomic_rmw_xor_i32(1, 0) == 1
-; run: %atomic_rmw_xor_i32(0, 1) == 1
-; run: %atomic_rmw_xor_i32(1, 1) == 0
-; run: %atomic_rmw_xor_i32(0x8FA50A64, 0x4F5AE48A) == 0xC0FFEEEE
-
-
-
-function %atomic_rmw_nand_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 nand v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_nand_i64(0, 0) == -1
-; run: %atomic_rmw_nand_i64(1, 0) == -1
-; run: %atomic_rmw_nand_i64(0, 1) == -1
-; run: %atomic_rmw_nand_i64(1, 1) == -2
-; run: %atomic_rmw_nand_i64(0xC0FFEEEE_DECAFFFF, 0x7DCB5691_7DCB5691) == 0xBF34B97F_A335A96E
-
-function %atomic_rmw_nand_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 nand v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_nand_i32(0, 0) == -1
-; run: %atomic_rmw_nand_i32(1, 0) == -1
-; run: %atomic_rmw_nand_i32(0, 1) == -1
-; run: %atomic_rmw_nand_i32(1, 1) == -2
-; run: %atomic_rmw_nand_i32(0xC0FFEEEE, 0x7DCB5691) == 0xBF34B97F
-
-
-
-function %atomic_rmw_umin_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 umin v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_umin_i64(0, 0) == 0
-; run: %atomic_rmw_umin_i64(1, 0) == 0
-; run: %atomic_rmw_umin_i64(0, 1) == 0
-; run: %atomic_rmw_umin_i64(1, 1) == 1
-; run: %atomic_rmw_umin_i64(-1, 1) == 1
-; run: %atomic_rmw_umin_i64(-1, -3) == -3
-
-function %atomic_rmw_umin_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 umin v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_umin_i32(0, 0) == 0
-; run: %atomic_rmw_umin_i32(1, 0) == 0
-; run: %atomic_rmw_umin_i32(0, 1) == 0
-; run: %atomic_rmw_umin_i32(1, 1) == 1
-; run: %atomic_rmw_umin_i32(-1, 1) == 1
-; run: %atomic_rmw_umin_i32(-1, -3) == -3
-
-
-
-function %atomic_rmw_umax_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 umax v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_umax_i64(0, 0) == 0
-; run: %atomic_rmw_umax_i64(1, 0) == 1
-; run: %atomic_rmw_umax_i64(0, 1) == 1
-; run: %atomic_rmw_umax_i64(1, 1) == 1
-; run: %atomic_rmw_umax_i64(-1, 1) == -1
-; run: %atomic_rmw_umax_i64(-1, -3) == -1
-
-function %atomic_rmw_umax_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 umax v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_umax_i32(0, 0) == 0
-; run: %atomic_rmw_umax_i32(1, 0) == 1
-; run: %atomic_rmw_umax_i32(0, 1) == 1
-; run: %atomic_rmw_umax_i32(1, 1) == 1
-; run: %atomic_rmw_umax_i32(-1, 1) == -1
-; run: %atomic_rmw_umax_i32(-1, -3) == -1
-
-
-
-function %atomic_rmw_smin_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 smin v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_smin_i64(0, 0) == 0
-; run: %atomic_rmw_smin_i64(1, 0) == 0
-; run: %atomic_rmw_smin_i64(0, 1) == 0
-; run: %atomic_rmw_smin_i64(1, 1) == 1
-; run: %atomic_rmw_smin_i64(-1, 1) == -1
-; run: %atomic_rmw_smin_i64(-1, -3) == -3
-
-function %atomic_rmw_smin_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 smin v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_smin_i32(0, 0) == 0
-; run: %atomic_rmw_smin_i32(1, 0) == 0
-; run: %atomic_rmw_smin_i32(0, 1) == 0
-; run: %atomic_rmw_smin_i32(1, 1) == 1
-; run: %atomic_rmw_smin_i32(-1, -1) == -1
-; run: %atomic_rmw_smin_i32(-1, -3) == -3
-
-
-
-function %atomic_rmw_smax_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 smax v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_smax_i64(0, 0) == 0
-; run: %atomic_rmw_smax_i64(1, 0) == 1
-; run: %atomic_rmw_smax_i64(0, 1) == 1
-; run: %atomic_rmw_smax_i64(1, 1) == 1
-; run: %atomic_rmw_smax_i64(-1, 1) == 1
-; run: %atomic_rmw_smax_i64(-1, -3) == -1
-
-function %atomic_rmw_smax_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 smax v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_smax_i32(0, 0) == 0
-; run: %atomic_rmw_smax_i32(1, 0) == 1
-; run: %atomic_rmw_smax_i32(0, 1) == 1
-; run: %atomic_rmw_smax_i32(1, 1) == 1
-; run: %atomic_rmw_smax_i32(-1, 1) == 1
-; run: %atomic_rmw_smax_i32(-1, -3) == -1
-
-
-
-function %atomic_rmw_xchg_i64(i64, i64) -> i64 {
-    ss0 = explicit_slot 8
-
-block0(v0: i64, v1: i64):
-    stack_store.i64 v0, ss0
-
-    v2 = stack_addr.i64 ss0
-    v3 = atomic_rmw.i64 xchg v2, v1
-
-    v4 = stack_load.i64 ss0
-    return v4
-}
-; run: %atomic_rmw_xchg_i64(0, 0) == 0
-; run: %atomic_rmw_xchg_i64(1, 0) == 0
-; run: %atomic_rmw_xchg_i64(0, 1) == 1
-; run: %atomic_rmw_xchg_i64(0, 0xC0FFEEEE_DECAFFFF) == 0xC0FFEEEE_DECAFFFF
-
-function %atomic_rmw_xchg_i32(i32, i32) -> i32 {
-    ss0 = explicit_slot 4
-
-block0(v0: i32, v1: i32):
-    stack_store.i32 v0, ss0
-
-    v2 = stack_addr.i32 ss0
-    v3 = atomic_rmw.i32 xchg v2, v1
-
-    v4 = stack_load.i32 ss0
-    return v4
-}
-; run: %atomic_rmw_xchg_i32(0, 0) == 0
-; run: %atomic_rmw_xchg_i32(1, 0) == 0
-; run: %atomic_rmw_xchg_i32(0, 1) == 1
-; run: %atomic_rmw_xchg_i32(0, 0xC0FFEEEE) == 0xC0FFEEEE