diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 8345d6eab4..e57223a8f9 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -1618,6 +1618,11 @@ (_ Unit (emit (MInst.LoadAcquire ty dst addr)))) dst)) +;; Helper for emitting `MInst.StoreRelease` instructions. +(decl store_release (Type Reg Reg) SideEffectNoResult) +(rule (store_release ty src addr) + (SideEffectNoResult.Inst (MInst.StoreRelease ty src addr))) + ;; Helper for generating a `tst` instruction. ;; ;; Produces a `ProducesFlags` rather than a register or emitted instruction diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index d5cbc4d649..a4aadb67a1 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -6,7 +6,6 @@ use crate::binemit::{CodeOffset, Reloc, StackMap}; use crate::ir::types::*; use crate::ir::{LibCall, MemFlags, TrapCode}; use crate::isa::aarch64::inst::*; -use crate::isa::aarch64::lower::is_valid_atomic_transaction_ty; use crate::machinst::{ty_bits, Reg, RegClass, Writable}; use core::convert::TryFrom; @@ -1374,14 +1373,12 @@ impl MachInstEmit for Inst { sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond)); } &Inst::AtomicRMW { ty, op, rs, rt, rn } => { - assert!(is_valid_atomic_transaction_ty(ty)); let rs = allocs.next(rs); let rt = allocs.next_writable(rt); let rn = allocs.next(rn); sink.put4(enc_acq_rel(ty, op, rs, rt, rn)); } &Inst::AtomicRMWLoop { ty, op } => { - assert!(is_valid_atomic_transaction_ty(ty)); /* Emit this: again: ldaxr{,b,h} x/w27, [x25] diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 006dd3ecf1..8c159e674f 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -1271,7 +1271,20 @@ (let ((use_allocated_encoding bool (is_not_baldrdash_call_conv))) (side_effect (udf use_allocated_encoding trap_code)))) -;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(rule (lower (has_type (valid_atomic_transaction ty) (atomic_load flags addr))) + (load_acquire ty addr)) + + +;;;; Rules for `AtomicStore` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(rule (lower (atomic_store flags + src @ (value_type (valid_atomic_transaction ty)) + addr)) + (side_effect (store_release ty src addr))) + + +;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule 1 (lower (and (use_lse) (has_type (valid_atomic_transaction ty) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index a69d7ee24b..a7dfcc0a5c 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -1477,30 +1477,6 @@ pub(crate) fn materialize_bool_result>( } } -/// This is target-word-size dependent. And it excludes booleans and reftypes. -pub(crate) fn is_valid_atomic_transaction_ty(ty: Type) -> bool { - match ty { - I8 | I16 | I32 | I64 => true, - _ => false, - } -} - -pub(crate) fn emit_atomic_load>( - ctx: &mut C, - rt: Writable, - insn: IRInst, -) -> Inst { - assert!(ctx.data(insn).opcode() == Opcode::AtomicLoad); - let inputs = insn_inputs(ctx, insn); - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - let access_ty = ctx.output_ty(insn, 0); - assert!(is_valid_atomic_transaction_ty(access_ty)); - // We're ignoring the result type of the load because the LoadAcquire will - // explicitly zero extend to the nearest word, and also zero the high half - // of an X register. - Inst::LoadAcquire { access_ty, rt, rn } -} - fn load_op_to_ty(op: Opcode) -> Option { match op { Opcode::Sload8 | Opcode::Uload8 => Some(I8), diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index fbe3a90dea..e6d51db796 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -21,7 +21,7 @@ use crate::{ TrapCode, Value, ValueList, }, isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm}, - isa::aarch64::lower::{is_valid_atomic_transaction_ty, writable_xreg, xreg}, + isa::aarch64::lower::{writable_xreg, xreg}, isa::unwind::UnwindInst, machinst::{ty_bits, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData}, }; @@ -123,11 +123,11 @@ where } } + /// This is target-word-size dependent. And it excludes booleans and reftypes. fn valid_atomic_transaction(&mut self, ty: Type) -> Option { - if is_valid_atomic_transaction_ty(ty) { - Some(ty) - } else { - None + match ty { + I8 | I16 | I32 | I64 => Some(ty), + _ => None, } } diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 0c95ad12f6..fd0c1006e6 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -241,19 +241,9 @@ pub(crate) fn lower_insn_to_regs>( Opcode::AtomicCas => implemented_in_isle(ctx), - Opcode::AtomicLoad => { - let rt = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let inst = emit_atomic_load(ctx, rt, insn); - ctx.emit(inst); - } + Opcode::AtomicLoad => implemented_in_isle(ctx), - Opcode::AtomicStore => { - let rt = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - let access_ty = ctx.input_ty(insn, 0); - assert!(is_valid_atomic_transaction_ty(access_ty)); - ctx.emit(Inst::StoreRelease { access_ty, rt, rn }); - } + Opcode::AtomicStore => implemented_in_isle(ctx), Opcode::Fence => { ctx.emit(Inst::Fence {});