x64: port some atomics to ISLE (#4212)

* x64: port `fence` to ISLE
* x64: port `atomic_load` to ISLE
* x64: port `atomic_store` to ISLE

cranelift/codegen/src/isa/x64/inst.isle

@@ -988,6 +988,10 @@
 (decl amode_offset (Amode u32) Amode)
 (extern constructor amode_offset amode_offset)
 
+;; Return a zero offset as an `Offset32`.
+(decl zero_offset () Offset32)
+(extern constructor zero_offset zero_offset)
+
 ;; Shift kinds.
 
 (type ShiftKind extern
@@ -2866,6 +2870,12 @@
 (rule (x64_xor_mem ty addr val)
       (alu_rm ty (AluRmiROpcode.Xor) addr val))
 
+;;;; Atomics ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl x64_mfence () SideEffectNoResult)
+(rule (x64_mfence)
+      (SideEffectNoResult.Inst (MInst.Fence (FenceKind.MFence))))
+
 ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (convert Gpr InstOutput output_gpr)

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -524,15 +524,6 @@ impl Inst {
         }
     }
 
-    /// A convenience function to be able to use a RegMem as the source of a move.
-    pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable<Reg>) -> Inst {
-        src.assert_regclass_is(RegClass::Int);
-        match src {
-            RegMem::Reg { reg } => Self::mov_r_r(OperandSize::Size64, reg, dst),
-            RegMem::Mem { addr } => Self::mov64_m_r(addr, dst),
-        }
-    }
-
     pub(crate) fn mov_r_m(size: OperandSize, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
         debug_assert!(src.class() == RegClass::Int);
         Inst::MovRM {

cranelift/codegen/src/isa/x64/lower.isle

@@ -2781,3 +2781,34 @@
       (let ((_ RegMemImm (sink_load sink)))
         (side_effect
          (x64_xor_mem ty (to_amode flags addr offset) src2))))
+
+;; Rules for `fence` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (fence))
+      (side_effect (x64_mfence)))
+
+;; Rules for `atomic_load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; This is a normal load. The x86-TSO memory model provides sufficient
+;; sequencing to satisfy the CLIF synchronisation requirements for `AtomicLoad`
+;; without the need for any fence instructions.
+;;
+;; As described in the `atomic_load` documentation, this lowering is only valid
+;; for I8, I16, I32, and I64. The sub-64-bit types are zero extended, as with a
+;; normal load.
+(rule (lower (has_type $I64 (atomic_load flags address)))
+      (x64_mov (to_amode flags address (zero_offset))))
+(rule (lower (has_type (and (fits_in_32 ty) (ty_int _)) (atomic_load flags address)))
+      (x64_movzx (ext_mode (ty_bits_u16 ty) 64) (to_amode flags address (zero_offset))))
+
+;; Rules for `atomic_store` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; This is a normal store followed by an `mfence` instruction. As described in
+;; the `atomic_load` documentation, this lowering is only valid for I8, I16,
+;; I32, and I64.
+(rule (lower (atomic_store flags
+                           value @ (value_type (and (fits_in_64 ty) (ty_int _)))
+                           address))
+      (side_effect (side_effect_concat
+       (x64_movrm ty (to_amode flags address (zero_offset)) value)
+       (x64_mfence))))

cranelift/codegen/src/isa/x64/lower.rs

@@ -2234,46 +2234,15 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::AtomicLoad => {
-            // This is a normal load.  The x86-TSO memory model provides sufficient sequencing
-            // to satisfy the CLIF synchronisation requirements for `AtomicLoad` without the
-            // need for any fence instructions.
-            let data = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let addr = lower_to_amode(ctx, inputs[0], 0);
-            let ty_access = ty.unwrap();
-            assert!(is_valid_atomic_transaction_ty(ty_access));
-
-            let rm = RegMem::mem(addr);
-            if ty_access == types::I64 {
-                ctx.emit(Inst::mov64_rm_r(rm, data));
-            } else {
-                let ext_mode = ExtMode::new(ty_access.bits(), 64).unwrap_or_else(|| {
-                    panic!(
-                        "invalid extension during AtomicLoad: {} -> {}",
-                        ty_access.bits(),
-                        64
-                    )
-                });
-                ctx.emit(Inst::movzx_rm_r(ext_mode, rm, data));
-            }
+            implemented_in_isle(ctx);
         }
 
         Opcode::AtomicStore => {
-            // This is a normal store, followed by an `mfence` instruction.
-            let data = put_input_in_reg(ctx, inputs[0]);
-            let addr = lower_to_amode(ctx, inputs[1], 0);
-            let ty_access = ctx.input_ty(insn, 0);
-            assert!(is_valid_atomic_transaction_ty(ty_access));
-
-            ctx.emit(Inst::store(ty_access, data, addr));
-            ctx.emit(Inst::Fence {
-                kind: FenceKind::MFence,
-            });
+            implemented_in_isle(ctx);
         }
 
         Opcode::Fence => {
-            ctx.emit(Inst::Fence {
-                kind: FenceKind::MFence,
-            });
+            implemented_in_isle(ctx);
         }
 
         Opcode::FuncAddr => {

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -542,6 +542,11 @@ where
     fn amode_offset(&mut self, addr: &Amode, offset: u32) -> Amode {
         addr.offset(offset)
     }
+
+    #[inline]
+    fn zero_offset(&mut self) -> Offset32 {
+        Offset32::new(0)
+    }
 }
 
 // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we

cranelift/codegen/src/machinst/isle.rs

@@ -300,6 +300,11 @@ macro_rules! isle_prelude_methods {
             }
         }
 
+        #[inline]
+        fn ty_int(&mut self, ty: Type) -> Option<Type> {
+            ty.is_int().then(|| ty)
+        }
+
         #[inline]
         fn ty_scalar_float(&mut self, ty: Type) -> Option<Type> {
             match ty {

cranelift/codegen/src/prelude.isle

@@ -318,6 +318,10 @@
 (decl ty_int_bool_128 (Type) Type)
 (extern extractor ty_int_bool_128 ty_int_bool_128)
 
+;; An extractor that only matches integers.
+(decl ty_int (Type) Type)
+(extern extractor ty_int ty_int)
+
 ;; An extractor that only matches scalar floating-point types--F32 or F64.
 (decl ty_scalar_float (Type) Type)
 (extern extractor ty_scalar_float ty_scalar_float)