[AArch64] Port AtomicLoad and AtomicStore to ISLE (#4301)

Copyright (c) 2022, Arm Limited.

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -1618,6 +1618,11 @@
             (_ Unit (emit (MInst.LoadAcquire ty dst addr))))
         dst))
 
+;; Helper for emitting `MInst.StoreRelease` instructions.
+(decl store_release (Type Reg Reg) SideEffectNoResult)
+(rule (store_release ty src addr)
+      (SideEffectNoResult.Inst (MInst.StoreRelease ty src addr)))
+
 ;; Helper for generating a `tst` instruction.
 ;;
 ;; Produces a `ProducesFlags` rather than a register or emitted instruction

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -6,7 +6,6 @@ use crate::binemit::{CodeOffset, Reloc, StackMap};
 use crate::ir::types::*;
 use crate::ir::{LibCall, MemFlags, TrapCode};
 use crate::isa::aarch64::inst::*;
-use crate::isa::aarch64::lower::is_valid_atomic_transaction_ty;
 use crate::machinst::{ty_bits, Reg, RegClass, Writable};
 use core::convert::TryFrom;
 
@@ -1374,14 +1373,12 @@ impl MachInstEmit for Inst {
                 sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond));
             }
             &Inst::AtomicRMW { ty, op, rs, rt, rn } => {
-                assert!(is_valid_atomic_transaction_ty(ty));
                 let rs = allocs.next(rs);
                 let rt = allocs.next_writable(rt);
                 let rn = allocs.next(rn);
                 sink.put4(enc_acq_rel(ty, op, rs, rt, rn));
             }
             &Inst::AtomicRMWLoop { ty, op } => {
-                assert!(is_valid_atomic_transaction_ty(ty));
                 /* Emit this:
                      again:
                       ldaxr{,b,h}  x/w27, [x25]

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -1271,7 +1271,20 @@
       (let ((use_allocated_encoding bool (is_not_baldrdash_call_conv)))
          (side_effect (udf use_allocated_encoding trap_code))))
 
-;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(rule (lower (has_type (valid_atomic_transaction ty) (atomic_load flags addr)))
+      (load_acquire ty addr))
+
+
+;;;; Rules for `AtomicStore` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(rule (lower (atomic_store flags
+                src @ (value_type (valid_atomic_transaction ty))
+                addr))
+      (side_effect (store_release ty src addr)))
+
+
+;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule 1 (lower (and (use_lse)
                   (has_type (valid_atomic_transaction ty)

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -1477,30 +1477,6 @@ pub(crate) fn materialize_bool_result<C: LowerCtx<I = Inst>>(
     }
 }
 
-/// This is target-word-size dependent.  And it excludes booleans and reftypes.
-pub(crate) fn is_valid_atomic_transaction_ty(ty: Type) -> bool {
-    match ty {
-        I8 | I16 | I32 | I64 => true,
-        _ => false,
-    }
-}
-
-pub(crate) fn emit_atomic_load<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    rt: Writable<Reg>,
-    insn: IRInst,
-) -> Inst {
-    assert!(ctx.data(insn).opcode() == Opcode::AtomicLoad);
-    let inputs = insn_inputs(ctx, insn);
-    let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-    let access_ty = ctx.output_ty(insn, 0);
-    assert!(is_valid_atomic_transaction_ty(access_ty));
-    // We're ignoring the result type of the load because the LoadAcquire will
-    // explicitly zero extend to the nearest word, and also zero the high half
-    // of an X register.
-    Inst::LoadAcquire { access_ty, rt, rn }
-}
-
 fn load_op_to_ty(op: Opcode) -> Option<Type> {
     match op {
         Opcode::Sload8 | Opcode::Uload8 => Some(I8),

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -21,7 +21,7 @@ use crate::{
         TrapCode, Value, ValueList,
     },
     isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
-    isa::aarch64::lower::{is_valid_atomic_transaction_ty, writable_xreg, xreg},
+    isa::aarch64::lower::{writable_xreg, xreg},
     isa::unwind::UnwindInst,
     machinst::{ty_bits, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData},
 };
@@ -123,11 +123,11 @@ where
         }
     }
 
+    /// This is target-word-size dependent.  And it excludes booleans and reftypes.
     fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> {
-        if is_valid_atomic_transaction_ty(ty) {
+        match ty {
-            Some(ty)
+            I8 | I16 | I32 | I64 => Some(ty),
-        } else {
+            _ => None,
-            None
         }
     }
 

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -241,19 +241,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::AtomicCas => implemented_in_isle(ctx),
 
-        Opcode::AtomicLoad => {
+        Opcode::AtomicLoad => implemented_in_isle(ctx),
-            let rt = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let inst = emit_atomic_load(ctx, rt, insn);
-            ctx.emit(inst);
-        }
 
-        Opcode::AtomicStore => {
+        Opcode::AtomicStore => implemented_in_isle(ctx),
-            let rt = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let access_ty = ctx.input_ty(insn, 0);
-            assert!(is_valid_atomic_transaction_ty(access_ty));
-            ctx.emit(Inst::StoreRelease { access_ty, rt, rn });
-        }
 
         Opcode::Fence => {
             ctx.emit(Inst::Fence {});