diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 0d7015ade6..cab888bc26 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -988,6 +988,10 @@ (decl amode_offset (Amode u32) Amode) (extern constructor amode_offset amode_offset) +;; Return a zero offset as an `Offset32`. +(decl zero_offset () Offset32) +(extern constructor zero_offset zero_offset) + ;; Shift kinds. (type ShiftKind extern @@ -2866,6 +2870,12 @@ (rule (x64_xor_mem ty addr val) (alu_rm ty (AluRmiROpcode.Xor) addr val)) +;;;; Atomics ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl x64_mfence () SideEffectNoResult) +(rule (x64_mfence) + (SideEffectNoResult.Inst (MInst.Fence (FenceKind.MFence)))) + ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (convert Gpr InstOutput output_gpr) diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 435643f1d6..c022d16b2d 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -524,15 +524,6 @@ impl Inst { } } - /// A convenience function to be able to use a RegMem as the source of a move. - pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable) -> Inst { - src.assert_regclass_is(RegClass::Int); - match src { - RegMem::Reg { reg } => Self::mov_r_r(OperandSize::Size64, reg, dst), - RegMem::Mem { addr } => Self::mov64_m_r(addr, dst), - } - } - pub(crate) fn mov_r_m(size: OperandSize, src: Reg, dst: impl Into) -> Inst { debug_assert!(src.class() == RegClass::Int); Inst::MovRM { diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index af12600da8..16e5689ec0 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -2781,3 +2781,34 @@ (let ((_ RegMemImm (sink_load sink))) (side_effect (x64_xor_mem ty (to_amode flags addr offset) src2)))) + +;; Rules for `fence` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (fence)) + (side_effect (x64_mfence))) + +;; Rules for `atomic_load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; This is a normal load. The x86-TSO memory model provides sufficient +;; sequencing to satisfy the CLIF synchronisation requirements for `AtomicLoad` +;; without the need for any fence instructions. +;; +;; As described in the `atomic_load` documentation, this lowering is only valid +;; for I8, I16, I32, and I64. The sub-64-bit types are zero extended, as with a +;; normal load. +(rule (lower (has_type $I64 (atomic_load flags address))) + (x64_mov (to_amode flags address (zero_offset)))) +(rule (lower (has_type (and (fits_in_32 ty) (ty_int _)) (atomic_load flags address))) + (x64_movzx (ext_mode (ty_bits_u16 ty) 64) (to_amode flags address (zero_offset)))) + +;; Rules for `atomic_store` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; This is a normal store followed by an `mfence` instruction. As described in +;; the `atomic_load` documentation, this lowering is only valid for I8, I16, +;; I32, and I64. +(rule (lower (atomic_store flags + value @ (value_type (and (fits_in_64 ty) (ty_int _))) + address)) + (side_effect (side_effect_concat + (x64_movrm ty (to_amode flags address (zero_offset)) value) + (x64_mfence)))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 68dff7c772..594ffc94f0 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2234,46 +2234,15 @@ fn lower_insn_to_regs>( } Opcode::AtomicLoad => { - // This is a normal load. The x86-TSO memory model provides sufficient sequencing - // to satisfy the CLIF synchronisation requirements for `AtomicLoad` without the - // need for any fence instructions. - let data = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let addr = lower_to_amode(ctx, inputs[0], 0); - let ty_access = ty.unwrap(); - assert!(is_valid_atomic_transaction_ty(ty_access)); - - let rm = RegMem::mem(addr); - if ty_access == types::I64 { - ctx.emit(Inst::mov64_rm_r(rm, data)); - } else { - let ext_mode = ExtMode::new(ty_access.bits(), 64).unwrap_or_else(|| { - panic!( - "invalid extension during AtomicLoad: {} -> {}", - ty_access.bits(), - 64 - ) - }); - ctx.emit(Inst::movzx_rm_r(ext_mode, rm, data)); - } + implemented_in_isle(ctx); } Opcode::AtomicStore => { - // This is a normal store, followed by an `mfence` instruction. - let data = put_input_in_reg(ctx, inputs[0]); - let addr = lower_to_amode(ctx, inputs[1], 0); - let ty_access = ctx.input_ty(insn, 0); - assert!(is_valid_atomic_transaction_ty(ty_access)); - - ctx.emit(Inst::store(ty_access, data, addr)); - ctx.emit(Inst::Fence { - kind: FenceKind::MFence, - }); + implemented_in_isle(ctx); } Opcode::Fence => { - ctx.emit(Inst::Fence { - kind: FenceKind::MFence, - }); + implemented_in_isle(ctx); } Opcode::FuncAddr => { diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index b7e0c9a3ce..2c428ec292 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -542,6 +542,11 @@ where fn amode_offset(&mut self, addr: &Amode, offset: u32) -> Amode { addr.offset(offset) } + + #[inline] + fn zero_offset(&mut self) -> Offset32 { + Offset32::new(0) + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index d02b53193b..3e26be3d4f 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -300,6 +300,11 @@ macro_rules! isle_prelude_methods { } } + #[inline] + fn ty_int(&mut self, ty: Type) -> Option { + ty.is_int().then(|| ty) + } + #[inline] fn ty_scalar_float(&mut self, ty: Type) -> Option { match ty { diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 75b586d186..1f9641a31d 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -318,6 +318,10 @@ (decl ty_int_bool_128 (Type) Type) (extern extractor ty_int_bool_128 ty_int_bool_128) +;; An extractor that only matches integers. +(decl ty_int (Type) Type) +(extern extractor ty_int ty_int) + ;; An extractor that only matches scalar floating-point types--F32 or F64. (decl ty_scalar_float (Type) Type) (extern extractor ty_scalar_float ty_scalar_float)