T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cranelift AArch64: Fix the atomic memory operations (#4831)

Browse Source 
Previously the implementations of the various atomic memory IR operations
ignored the memory operation flags that were passed.

Copyright (c) 2022, Arm Limited.

Co-authored-by: Chris Fallin <chris@cfallin.org>
This commit is contained in:
Anton Kirilov
2022-09-02 17:35:21 +01:00
committed by GitHub
parent d2e19b8d74
commit 48bf078c83
6 changed files with 194 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
cranelift/codegen/src/isa/aarch64/inst.isle
View File

@@ -251,6 +251,7 @@
(AtomicRMWLoop (AtomicRMWLoop
(ty Type) ;; I8, I16, I32 or I64 (ty Type) ;; I8, I16, I32 or I64
(op AtomicRMWLoopOp) (op AtomicRMWLoopOp)
(flags MemFlags)
(addr Reg) (addr Reg)
(operand Reg) (operand Reg)
(oldval WritableReg) (oldval WritableReg)
@@ -268,6 +269,7 @@
;; x24 (wr) scratch reg; value afterwards has no meaning ;; x24 (wr) scratch reg; value afterwards has no meaning
(AtomicCASLoop (AtomicCASLoop
(ty Type) ;; I8, I16, I32 or I64 (ty Type) ;; I8, I16, I32 or I64
(flags MemFlags)
(addr Reg) (addr Reg)
(expected Reg) (expected Reg)
(replacement Reg) (replacement Reg)
@@ -282,7 +284,8 @@
(rs Reg) (rs Reg)
(rt WritableReg) (rt WritableReg)
(rn Reg) (rn Reg)
(ty Type)) (ty Type)
(flags MemFlags))
;; An atomic compare-and-swap operation. These instructions require the ;; An atomic compare-and-swap operation. These instructions require the
;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have ;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
@@ -294,7 +297,8 @@
(rs Reg) (rs Reg)
(rt Reg) (rt Reg)
(rn Reg) (rn Reg)
(ty Type)) (ty Type)
(flags MemFlags))
;; Read `access_ty` bits from address `rt`, either 8, 16, 32 or 64-bits, and put ;; Read `access_ty` bits from address `rt`, either 8, 16, 32 or 64-bits, and put
;; it in `rn`, optionally zero-extending to fill a word or double word result. ;; it in `rn`, optionally zero-extending to fill a word or double word result.
@@ -302,14 +306,16 @@
(LoadAcquire (LoadAcquire
(access_ty Type) ;; I8, I16, I32 or I64 (access_ty Type) ;; I8, I16, I32 or I64
(rt WritableReg) (rt WritableReg)
(rn Reg)) (rn Reg)
(flags MemFlags))
;; Write the lowest `ty` bits of `rt` to address `rn`. ;; Write the lowest `ty` bits of `rt` to address `rn`.
;; This instruction is sequentially consistent. ;; This instruction is sequentially consistent.
(StoreRelease (StoreRelease
(access_ty Type) ;; I8, I16, I32 or I64 (access_ty Type) ;; I8, I16, I32 or I64
(rt Reg) (rt Reg)
(rn Reg)) (rn Reg)
(flags MemFlags))
;; A memory fence. This must provide ordering to ensure that, at a minimum, neither loads ;; A memory fence. This must provide ordering to ensure that, at a minimum, neither loads
;; nor stores may move forwards or backwards across the fence. Currently emitted as "dmb ;; nor stores may move forwards or backwards across the fence. Currently emitted as "dmb
@@ -2124,16 +2130,16 @@
dst)) dst))
;; Helper for emitting `MInst.LoadAcquire` instructions. ;; Helper for emitting `MInst.LoadAcquire` instructions.
(decl load_acquire (Type Reg) Reg) (decl load_acquire (Type MemFlags Reg) Reg)
(rule (load_acquire ty addr) (rule (load_acquire ty flags addr)
(let ((dst WritableReg (temp_writable_reg $I64)) (let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadAcquire ty dst addr)))) (_ Unit (emit (MInst.LoadAcquire ty dst addr flags))))
dst)) dst))
;; Helper for emitting `MInst.StoreRelease` instructions. ;; Helper for emitting `MInst.StoreRelease` instructions.
(decl store_release (Type Reg Reg) SideEffectNoResult) (decl store_release (Type MemFlags Reg Reg) SideEffectNoResult)
(rule (store_release ty src addr) (rule (store_release ty flags src addr)
(SideEffectNoResult.Inst (MInst.StoreRelease ty src addr))) (SideEffectNoResult.Inst (MInst.StoreRelease ty src addr flags)))
;; Helper for generating a `tst` instruction. ;; Helper for generating a `tst` instruction.
;; ;;
@@ -2694,21 +2700,10 @@
) )
x)) x))
;; An atomic load that can be sunk into another operation. (decl sink_atomic_load (Inst) Reg)
(type SinkableAtomicLoad extern (enum)) (rule (sink_atomic_load x @ (atomic_load _ addr))
(let ((_ Unit (sink_inst x)))
;; Extract a `SinkableAtomicLoad` that works with `Reg` from a value (put_in_reg addr)))
;; operand.
(decl sinkable_atomic_load (SinkableAtomicLoad) Value)
(extern extractor sinkable_atomic_load sinkable_atomic_load)
;; Sink a `SinkableAtomicLoad` into a `Reg`.
;;
;; This is a side-effectful operation that notifies the context that the
;; instruction that produced the `SinkableAtomicLoad` has been sunk into another
;; instruction, and no longer needs to be lowered.
(decl sink_atomic_load (SinkableAtomicLoad) Reg)
(extern constructor sink_atomic_load sink_atomic_load)
;; Helper for generating either an `AluRRR`, `AluRRRShift`, or `AluRRImmLogic` ;; Helper for generating either an `AluRRR`, `AluRRRShift`, or `AluRRImmLogic`
;; instruction depending on the input. Note that this requires that the `ALUOp` ;; instruction depending on the input. Note that this requires that the `ALUOp`
@@ -2890,21 +2885,21 @@
(vec_misc (VecMisc2.Cmeq0) rn size)) (vec_misc (VecMisc2.Cmeq0) rn size))
;; Helper for emitting `MInst.AtomicRMW` instructions. ;; Helper for emitting `MInst.AtomicRMW` instructions.
(decl lse_atomic_rmw (AtomicRMWOp Value Reg Type) Reg) (decl lse_atomic_rmw (AtomicRMWOp Value Reg Type MemFlags) Reg)
(rule (lse_atomic_rmw op p r_arg2 ty) (rule (lse_atomic_rmw op p r_arg2 ty flags)
(let ( (let (
(r_addr Reg p) (r_addr Reg p)
(dst WritableReg (temp_writable_reg ty)) (dst WritableReg (temp_writable_reg ty))
(_ Unit (emit (MInst.AtomicRMW op r_arg2 dst r_addr ty))) (_ Unit (emit (MInst.AtomicRMW op r_arg2 dst r_addr ty flags)))
) )
dst)) dst))
;; Helper for emitting `MInst.AtomicCAS` instructions. ;; Helper for emitting `MInst.AtomicCAS` instructions.
(decl lse_atomic_cas (Reg Reg Reg Type) Reg) (decl lse_atomic_cas (Reg Reg Reg Type MemFlags) Reg)
(rule (lse_atomic_cas addr expect replace ty) (rule (lse_atomic_cas addr expect replace ty flags)
(let ( (let (
(dst WritableReg (temp_writable_reg ty)) (dst WritableReg (temp_writable_reg ty))
(_ Unit (emit (MInst.AtomicCAS dst expect replace addr ty))) (_ Unit (emit (MInst.AtomicCAS dst expect replace addr ty flags)))
) )
dst)) dst))
@@ -2914,12 +2909,12 @@
;; regs, and that's not guaranteed safe if either is in a real reg. ;; regs, and that's not guaranteed safe if either is in a real reg.
;; - Move the args to the preordained AtomicRMW input regs ;; - Move the args to the preordained AtomicRMW input regs
;; - And finally, copy the preordained AtomicRMW output reg to its destination. ;; - And finally, copy the preordained AtomicRMW output reg to its destination.
(decl atomic_rmw_loop (AtomicRMWLoopOp Reg Reg Type) Reg) (decl atomic_rmw_loop (AtomicRMWLoopOp Reg Reg Type MemFlags) Reg)
(rule (atomic_rmw_loop op addr operand ty) (rule (atomic_rmw_loop op addr operand ty flags)
(let ((dst WritableReg (temp_writable_reg $I64)) (let ((dst WritableReg (temp_writable_reg $I64))
(scratch1 WritableReg (temp_writable_reg $I64)) (scratch1 WritableReg (temp_writable_reg $I64))
(scratch2 WritableReg (temp_writable_reg $I64)) (scratch2 WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AtomicRMWLoop ty op addr operand dst scratch1 scratch2)))) (_ Unit (emit (MInst.AtomicRMWLoop ty op flags addr operand dst scratch1 scratch2))))
dst)) dst))
;; Helper for emitting `MInst.AtomicCASLoop` instructions. ;; Helper for emitting `MInst.AtomicCASLoop` instructions.
@@ -2928,11 +2923,11 @@
;; about zero-extending narrow (I8/I16/I32) values here. ;; about zero-extending narrow (I8/I16/I32) values here.
;; Make sure that all three args are in virtual regs. See corresponding comment ;; Make sure that all three args are in virtual regs. See corresponding comment
;; for `atomic_rmw_loop` above. ;; for `atomic_rmw_loop` above.
(decl atomic_cas_loop (Reg Reg Reg Type) Reg) (decl atomic_cas_loop (Reg Reg Reg Type MemFlags) Reg)
(rule (atomic_cas_loop addr expect replace ty) (rule (atomic_cas_loop addr expect replace ty flags)
(let ((dst WritableReg (temp_writable_reg $I64)) (let ((dst WritableReg (temp_writable_reg $I64))
(scratch WritableReg (temp_writable_reg $I64)) (scratch WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AtomicCASLoop ty addr expect replace dst scratch)))) (_ Unit (emit (MInst.AtomicCASLoop ty flags addr expect replace dst scratch))))
dst)) dst))
;; Helper for emitting `MInst.MovPReg` instructions. ;; Helper for emitting `MInst.MovPReg` instructions.

72
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -1424,13 +1424,26 @@ impl MachInstEmit for Inst {
let rn = allocs.next(rn); let rn = allocs.next(rn);
sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond)); sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond));
} }
&Inst::AtomicRMW { ty, op, rs, rt, rn } => { &Inst::AtomicRMW {
ty,
op,
rs,
rt,
rn,
flags,
} => {
let rs = allocs.next(rs); let rs = allocs.next(rs);
let rt = allocs.next_writable(rt); let rt = allocs.next_writable(rt);
let rn = allocs.next(rn); let rn = allocs.next(rn);
let srcloc = state.cur_srcloc();
if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds);
}
sink.put4(enc_acq_rel(ty, op, rs, rt, rn)); sink.put4(enc_acq_rel(ty, op, rs, rt, rn));
} }
&Inst::AtomicRMWLoop { ty, op, .. } => { &Inst::AtomicRMWLoop { ty, op, flags, .. } => {
/* Emit this: /* Emit this:
again: again:
ldaxr{,b,h} x/w27, [x25] ldaxr{,b,h} x/w27, [x25]
@@ -1463,10 +1476,12 @@ impl MachInstEmit for Inst {
// again: // again:
sink.bind_label(again_label); sink.bind_label(again_label);
let srcloc = state.cur_srcloc(); let srcloc = state.cur_srcloc();
if !srcloc.is_default() { if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds); sink.add_trap(TrapCode::HeapOutOfBounds);
} }
sink.put4(enc_ldaxr(ty, x27wr, x25)); // ldaxr x27, [x25] sink.put4(enc_ldaxr(ty, x27wr, x25)); // ldaxr x27, [x25]
let size = OperandSize::from_ty(ty); let size = OperandSize::from_ty(ty);
let sign_ext = match op { let sign_ext = match op {
@@ -1588,7 +1603,7 @@ impl MachInstEmit for Inst {
} }
let srcloc = state.cur_srcloc(); let srcloc = state.cur_srcloc();
if !srcloc.is_default() { if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds); sink.add_trap(TrapCode::HeapOutOfBounds);
} }
if op == AtomicRMWLoopOp::Xchg { if op == AtomicRMWLoopOp::Xchg {
@@ -1608,7 +1623,14 @@ impl MachInstEmit for Inst {
)); ));
sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19); sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
} }
&Inst::AtomicCAS { rd, rs, rt, rn, ty } => { &Inst::AtomicCAS {
rd,
rs,
rt,
rn,
ty,
flags,
} => {
let rd = allocs.next_writable(rd); let rd = allocs.next_writable(rd);
let rs = allocs.next(rs); let rs = allocs.next(rs);
debug_assert_eq!(rd.to_reg(), rs); debug_assert_eq!(rd.to_reg(), rs);
@@ -1622,9 +1644,14 @@ impl MachInstEmit for Inst {
_ => panic!("Unsupported type: {}", ty), _ => panic!("Unsupported type: {}", ty),
}; };
let srcloc = state.cur_srcloc();
if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds);
}
sink.put4(enc_cas(size, rd, rt, rn)); sink.put4(enc_cas(size, rd, rt, rn));
} }
&Inst::AtomicCASLoop { ty, .. } => { &Inst::AtomicCASLoop { ty, flags, .. } => {
/* Emit this: /* Emit this:
again: again:
ldaxr{,b,h} x/w27, [x25] ldaxr{,b,h} x/w27, [x25]
@@ -1651,10 +1678,12 @@ impl MachInstEmit for Inst {
// again: // again:
sink.bind_label(again_label); sink.bind_label(again_label);
let srcloc = state.cur_srcloc(); let srcloc = state.cur_srcloc();
if !srcloc.is_default() { if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds); sink.add_trap(TrapCode::HeapOutOfBounds);
} }
// ldaxr x27, [x25] // ldaxr x27, [x25]
sink.put4(enc_ldaxr(ty, x27wr, x25)); sink.put4(enc_ldaxr(ty, x27wr, x25));
@@ -1679,9 +1708,10 @@ impl MachInstEmit for Inst {
sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19); sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19);
let srcloc = state.cur_srcloc(); let srcloc = state.cur_srcloc();
if !srcloc.is_default() { if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds); sink.add_trap(TrapCode::HeapOutOfBounds);
} }
sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25] sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25]
// cbnz w24, again. // cbnz w24, again.
@@ -1698,14 +1728,36 @@ impl MachInstEmit for Inst {
// out: // out:
sink.bind_label(out_label); sink.bind_label(out_label);
} }
&Inst::LoadAcquire { access_ty, rt, rn } => { &Inst::LoadAcquire {
access_ty,
rt,
rn,
flags,
} => {
let rn = allocs.next(rn); let rn = allocs.next(rn);
let rt = allocs.next_writable(rt); let rt = allocs.next_writable(rt);
let srcloc = state.cur_srcloc();
if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds);
}
sink.put4(enc_ldar(access_ty, rt, rn)); sink.put4(enc_ldar(access_ty, rt, rn));
} }
&Inst::StoreRelease { access_ty, rt, rn } => { &Inst::StoreRelease {
access_ty,
rt,
rn,
flags,
} => {
let rn = allocs.next(rn); let rn = allocs.next(rn);
let rt = allocs.next(rt); let rt = allocs.next(rt);
let srcloc = state.cur_srcloc();
if !srcloc.is_default() && !flags.notrap() {
sink.add_trap(TrapCode::HeapOutOfBounds);
}
sink.put4(enc_stlr(access_ty, rt, rn)); sink.put4(enc_stlr(access_ty, rt, rn));
} }
&Inst::Fence {} => { &Inst::Fence {} => {

65
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -6926,6 +6926,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I8, ty: I8,
op: AtomicRMWLoopOp::Sub, op: AtomicRMWLoopOp::Sub,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -6939,6 +6940,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I16, ty: I16,
op: AtomicRMWLoopOp::Eor, op: AtomicRMWLoopOp::Eor,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -6952,6 +6954,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I8, ty: I8,
op: AtomicRMWLoopOp::Add, op: AtomicRMWLoopOp::Add,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -6965,6 +6968,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I32, ty: I32,
op: AtomicRMWLoopOp::Orr, op: AtomicRMWLoopOp::Orr,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -6978,6 +6982,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I64, ty: I64,
op: AtomicRMWLoopOp::And, op: AtomicRMWLoopOp::And,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -6991,6 +6996,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I8, ty: I8,
op: AtomicRMWLoopOp::Xchg, op: AtomicRMWLoopOp::Xchg,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7004,6 +7010,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I16, ty: I16,
op: AtomicRMWLoopOp::Nand, op: AtomicRMWLoopOp::Nand,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7017,6 +7024,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I16, ty: I16,
op: AtomicRMWLoopOp::Smin, op: AtomicRMWLoopOp::Smin,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7030,6 +7038,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I32, ty: I32,
op: AtomicRMWLoopOp::Smin, op: AtomicRMWLoopOp::Smin,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7043,6 +7052,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I64, ty: I64,
op: AtomicRMWLoopOp::Smax, op: AtomicRMWLoopOp::Smax,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7056,6 +7066,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I8, ty: I8,
op: AtomicRMWLoopOp::Smax, op: AtomicRMWLoopOp::Smax,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7069,6 +7080,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I8, ty: I8,
op: AtomicRMWLoopOp::Umin, op: AtomicRMWLoopOp::Umin,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7082,6 +7094,7 @@ fn test_aarch64_binemit() {
Inst::AtomicRMWLoop { Inst::AtomicRMWLoop {
ty: I16, ty: I16,
op: AtomicRMWLoopOp::Umax, op: AtomicRMWLoopOp::Umax,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
operand: xreg(26), operand: xreg(26),
oldval: writable_xreg(27), oldval: writable_xreg(27),
@@ -7099,6 +7112,7 @@ fn test_aarch64_binemit() {
rs: xreg(1), rs: xreg(1),
rt: writable_xreg(2), rt: writable_xreg(2),
rn: xreg(3), rn: xreg(3),
flags: MemFlags::trusted(),
}, },
"6200E138", "6200E138",
"ldaddalb w1, w2, [x3]", "ldaddalb w1, w2, [x3]",
@@ -7110,6 +7124,7 @@ fn test_aarch64_binemit() {
rs: xreg(4), rs: xreg(4),
rt: writable_xreg(5), rt: writable_xreg(5),
rn: xreg(6), rn: xreg(6),
flags: MemFlags::trusted(),
}, },
"C500E478", "C500E478",
"ldaddalh w4, w5, [x6]", "ldaddalh w4, w5, [x6]",
@@ -7121,6 +7136,7 @@ fn test_aarch64_binemit() {
rs: xreg(7), rs: xreg(7),
rt: writable_xreg(8), rt: writable_xreg(8),
rn: xreg(9), rn: xreg(9),
flags: MemFlags::trusted(),
}, },
"2801E7B8", "2801E7B8",
"ldaddal w7, w8, [x9]", "ldaddal w7, w8, [x9]",
@@ -7132,6 +7148,7 @@ fn test_aarch64_binemit() {
rs: xreg(10), rs: xreg(10),
rt: writable_xreg(11), rt: writable_xreg(11),
rn: xreg(12), rn: xreg(12),
flags: MemFlags::trusted(),
}, },
"8B01EAF8", "8B01EAF8",
"ldaddal x10, x11, [x12]", "ldaddal x10, x11, [x12]",
@@ -7143,6 +7160,7 @@ fn test_aarch64_binemit() {
rs: xreg(13), rs: xreg(13),
rt: writable_xreg(14), rt: writable_xreg(14),
rn: xreg(15), rn: xreg(15),
flags: MemFlags::trusted(),
}, },
"EE11ED38", "EE11ED38",
"ldclralb w13, w14, [x15]", "ldclralb w13, w14, [x15]",
@@ -7154,6 +7172,7 @@ fn test_aarch64_binemit() {
rs: xreg(16), rs: xreg(16),
rt: writable_xreg(17), rt: writable_xreg(17),
rn: xreg(18), rn: xreg(18),
flags: MemFlags::trusted(),
}, },
"5112F078", "5112F078",
"ldclralh w16, w17, [x18]", "ldclralh w16, w17, [x18]",
@@ -7165,6 +7184,7 @@ fn test_aarch64_binemit() {
rs: xreg(19), rs: xreg(19),
rt: writable_xreg(20), rt: writable_xreg(20),
rn: xreg(21), rn: xreg(21),
flags: MemFlags::trusted(),
}, },
"B412F3B8", "B412F3B8",
"ldclral w19, w20, [x21]", "ldclral w19, w20, [x21]",
@@ -7176,6 +7196,7 @@ fn test_aarch64_binemit() {
rs: xreg(22), rs: xreg(22),
rt: writable_xreg(23), rt: writable_xreg(23),
rn: xreg(24), rn: xreg(24),
flags: MemFlags::trusted(),
}, },
"1713F6F8", "1713F6F8",
"ldclral x22, x23, [x24]", "ldclral x22, x23, [x24]",
@@ -7187,6 +7208,7 @@ fn test_aarch64_binemit() {
rs: xreg(25), rs: xreg(25),
rt: writable_xreg(26), rt: writable_xreg(26),
rn: xreg(27), rn: xreg(27),
flags: MemFlags::trusted(),
}, },
"7A23F938", "7A23F938",
"ldeoralb w25, w26, [x27]", "ldeoralb w25, w26, [x27]",
@@ -7198,6 +7220,7 @@ fn test_aarch64_binemit() {
rs: xreg(28), rs: xreg(28),
rt: writable_xreg(29), rt: writable_xreg(29),
rn: xreg(30), rn: xreg(30),
flags: MemFlags::trusted(),
}, },
"DD23FC78", "DD23FC78",
"ldeoralh w28, fp, [lr]", "ldeoralh w28, fp, [lr]",
@@ -7209,6 +7232,7 @@ fn test_aarch64_binemit() {
rs: xreg(29), rs: xreg(29),
rt: writable_xreg(28), rt: writable_xreg(28),
rn: xreg(27), rn: xreg(27),
flags: MemFlags::trusted(),
}, },
"7C23FDB8", "7C23FDB8",
"ldeoral fp, w28, [x27]", "ldeoral fp, w28, [x27]",
@@ -7220,6 +7244,7 @@ fn test_aarch64_binemit() {
rs: xreg(26), rs: xreg(26),
rt: writable_xreg(25), rt: writable_xreg(25),
rn: xreg(24), rn: xreg(24),
flags: MemFlags::trusted(),
}, },
"1923FAF8", "1923FAF8",
"ldeoral x26, x25, [x24]", "ldeoral x26, x25, [x24]",
@@ -7231,6 +7256,7 @@ fn test_aarch64_binemit() {
rs: xreg(23), rs: xreg(23),
rt: writable_xreg(22), rt: writable_xreg(22),
rn: xreg(21), rn: xreg(21),
flags: MemFlags::trusted(),
}, },
"B632F738", "B632F738",
"ldsetalb w23, w22, [x21]", "ldsetalb w23, w22, [x21]",
@@ -7242,6 +7268,7 @@ fn test_aarch64_binemit() {
rs: xreg(20), rs: xreg(20),
rt: writable_xreg(19), rt: writable_xreg(19),
rn: xreg(18), rn: xreg(18),
flags: MemFlags::trusted(),
}, },
"5332F478", "5332F478",
"ldsetalh w20, w19, [x18]", "ldsetalh w20, w19, [x18]",
@@ -7253,6 +7280,7 @@ fn test_aarch64_binemit() {
rs: xreg(17), rs: xreg(17),
rt: writable_xreg(16), rt: writable_xreg(16),
rn: xreg(15), rn: xreg(15),
flags: MemFlags::trusted(),
}, },
"F031F1B8", "F031F1B8",
"ldsetal w17, w16, [x15]", "ldsetal w17, w16, [x15]",
@@ -7264,6 +7292,7 @@ fn test_aarch64_binemit() {
rs: xreg(14), rs: xreg(14),
rt: writable_xreg(13), rt: writable_xreg(13),
rn: xreg(12), rn: xreg(12),
flags: MemFlags::trusted(),
}, },
"8D31EEF8", "8D31EEF8",
"ldsetal x14, x13, [x12]", "ldsetal x14, x13, [x12]",
@@ -7275,6 +7304,7 @@ fn test_aarch64_binemit() {
rs: xreg(11), rs: xreg(11),
rt: writable_xreg(10), rt: writable_xreg(10),
rn: xreg(9), rn: xreg(9),
flags: MemFlags::trusted(),
}, },
"2A41EB38", "2A41EB38",
"ldsmaxalb w11, w10, [x9]", "ldsmaxalb w11, w10, [x9]",
@@ -7286,6 +7316,7 @@ fn test_aarch64_binemit() {
rs: xreg(8), rs: xreg(8),
rt: writable_xreg(7), rt: writable_xreg(7),
rn: xreg(6), rn: xreg(6),
flags: MemFlags::trusted(),
}, },
"C740E878", "C740E878",
"ldsmaxalh w8, w7, [x6]", "ldsmaxalh w8, w7, [x6]",
@@ -7297,6 +7328,7 @@ fn test_aarch64_binemit() {
rs: xreg(5), rs: xreg(5),
rt: writable_xreg(4), rt: writable_xreg(4),
rn: xreg(3), rn: xreg(3),
flags: MemFlags::trusted(),
}, },
"6440E5B8", "6440E5B8",
"ldsmaxal w5, w4, [x3]", "ldsmaxal w5, w4, [x3]",
@@ -7308,6 +7340,7 @@ fn test_aarch64_binemit() {
rs: xreg(2), rs: xreg(2),
rt: writable_xreg(1), rt: writable_xreg(1),
rn: xreg(0), rn: xreg(0),
flags: MemFlags::trusted(),
}, },
"0140E2F8", "0140E2F8",
"ldsmaxal x2, x1, [x0]", "ldsmaxal x2, x1, [x0]",
@@ -7319,6 +7352,7 @@ fn test_aarch64_binemit() {
rs: xreg(1), rs: xreg(1),
rt: writable_xreg(2), rt: writable_xreg(2),
rn: xreg(3), rn: xreg(3),
flags: MemFlags::trusted(),
}, },
"6250E138", "6250E138",
"ldsminalb w1, w2, [x3]", "ldsminalb w1, w2, [x3]",
@@ -7330,6 +7364,7 @@ fn test_aarch64_binemit() {
rs: xreg(4), rs: xreg(4),
rt: writable_xreg(5), rt: writable_xreg(5),
rn: xreg(6), rn: xreg(6),
flags: MemFlags::trusted(),
}, },
"C550E478", "C550E478",
"ldsminalh w4, w5, [x6]", "ldsminalh w4, w5, [x6]",
@@ -7341,6 +7376,7 @@ fn test_aarch64_binemit() {
rs: xreg(7), rs: xreg(7),
rt: writable_xreg(8), rt: writable_xreg(8),
rn: xreg(9), rn: xreg(9),
flags: MemFlags::trusted(),
}, },
"2851E7B8", "2851E7B8",
"ldsminal w7, w8, [x9]", "ldsminal w7, w8, [x9]",
@@ -7352,6 +7388,7 @@ fn test_aarch64_binemit() {
rs: xreg(10), rs: xreg(10),
rt: writable_xreg(11), rt: writable_xreg(11),
rn: xreg(12), rn: xreg(12),
flags: MemFlags::trusted(),
}, },
"8B51EAF8", "8B51EAF8",
"ldsminal x10, x11, [x12]", "ldsminal x10, x11, [x12]",
@@ -7363,6 +7400,7 @@ fn test_aarch64_binemit() {
rs: xreg(13), rs: xreg(13),
rt: writable_xreg(14), rt: writable_xreg(14),
rn: xreg(15), rn: xreg(15),
flags: MemFlags::trusted(),
}, },
"EE61ED38", "EE61ED38",
"ldumaxalb w13, w14, [x15]", "ldumaxalb w13, w14, [x15]",
@@ -7374,6 +7412,7 @@ fn test_aarch64_binemit() {
rs: xreg(16), rs: xreg(16),
rt: writable_xreg(17), rt: writable_xreg(17),
rn: xreg(18), rn: xreg(18),
flags: MemFlags::trusted(),
}, },
"5162F078", "5162F078",
"ldumaxalh w16, w17, [x18]", "ldumaxalh w16, w17, [x18]",
@@ -7385,6 +7424,7 @@ fn test_aarch64_binemit() {
rs: xreg(19), rs: xreg(19),
rt: writable_xreg(20), rt: writable_xreg(20),
rn: xreg(21), rn: xreg(21),
flags: MemFlags::trusted(),
}, },
"B462F3B8", "B462F3B8",
"ldumaxal w19, w20, [x21]", "ldumaxal w19, w20, [x21]",
@@ -7396,6 +7436,7 @@ fn test_aarch64_binemit() {
rs: xreg(22), rs: xreg(22),
rt: writable_xreg(23), rt: writable_xreg(23),
rn: xreg(24), rn: xreg(24),
flags: MemFlags::trusted(),
}, },
"1763F6F8", "1763F6F8",
"ldumaxal x22, x23, [x24]", "ldumaxal x22, x23, [x24]",
@@ -7407,6 +7448,7 @@ fn test_aarch64_binemit() {
rs: xreg(16), rs: xreg(16),
rt: writable_xreg(17), rt: writable_xreg(17),
rn: xreg(18), rn: xreg(18),
flags: MemFlags::trusted(),
}, },
"5172F038", "5172F038",
"lduminalb w16, w17, [x18]", "lduminalb w16, w17, [x18]",
@@ -7418,6 +7460,7 @@ fn test_aarch64_binemit() {
rs: xreg(19), rs: xreg(19),
rt: writable_xreg(20), rt: writable_xreg(20),
rn: xreg(21), rn: xreg(21),
flags: MemFlags::trusted(),
}, },
"B472F378", "B472F378",
"lduminalh w19, w20, [x21]", "lduminalh w19, w20, [x21]",
@@ -7429,6 +7472,7 @@ fn test_aarch64_binemit() {
rs: xreg(22), rs: xreg(22),
rt: writable_xreg(23), rt: writable_xreg(23),
rn: xreg(24), rn: xreg(24),
flags: MemFlags::trusted(),
}, },
"1773F6B8", "1773F6B8",
"lduminal w22, w23, [x24]", "lduminal w22, w23, [x24]",
@@ -7440,6 +7484,7 @@ fn test_aarch64_binemit() {
rs: xreg(25), rs: xreg(25),
rt: writable_xreg(26), rt: writable_xreg(26),
rn: xreg(27), rn: xreg(27),
flags: MemFlags::trusted(),
}, },
"7A73F9F8", "7A73F9F8",
"lduminal x25, x26, [x27]", "lduminal x25, x26, [x27]",
@@ -7451,6 +7496,7 @@ fn test_aarch64_binemit() {
rs: xreg(28), rs: xreg(28),
rt: writable_xreg(29), rt: writable_xreg(29),
rn: xreg(30), rn: xreg(30),
flags: MemFlags::trusted(),
}, },
"DD83FC38", "DD83FC38",
"swpalb w28, fp, [lr]", "swpalb w28, fp, [lr]",
@@ -7462,6 +7508,7 @@ fn test_aarch64_binemit() {
rs: xreg(0), rs: xreg(0),
rt: writable_xreg(1), rt: writable_xreg(1),
rn: xreg(2), rn: xreg(2),
flags: MemFlags::trusted(),
}, },
"4180E078", "4180E078",
"swpalh w0, w1, [x2]", "swpalh w0, w1, [x2]",
@@ -7473,6 +7520,7 @@ fn test_aarch64_binemit() {
rs: xreg(3), rs: xreg(3),
rt: writable_xreg(4), rt: writable_xreg(4),
rn: xreg(5), rn: xreg(5),
flags: MemFlags::trusted(),
}, },
"A480E3B8", "A480E3B8",
"swpal w3, w4, [x5]", "swpal w3, w4, [x5]",
@@ -7484,6 +7532,7 @@ fn test_aarch64_binemit() {
rs: xreg(6), rs: xreg(6),
rt: writable_xreg(7), rt: writable_xreg(7),
rn: xreg(8), rn: xreg(8),
flags: MemFlags::trusted(),
}, },
"0781E6F8", "0781E6F8",
"swpal x6, x7, [x8]", "swpal x6, x7, [x8]",
@@ -7496,6 +7545,7 @@ fn test_aarch64_binemit() {
rt: xreg(20), rt: xreg(20),
rn: xreg(10), rn: xreg(10),
ty: I8, ty: I8,
flags: MemFlags::trusted(),
}, },
"54FDFC08", "54FDFC08",
"casalb w28, w28, w20, [x10]", "casalb w28, w28, w20, [x10]",
@@ -7507,6 +7557,7 @@ fn test_aarch64_binemit() {
rt: xreg(19), rt: xreg(19),
rn: xreg(23), rn: xreg(23),
ty: I16, ty: I16,
flags: MemFlags::trusted(),
}, },
"F3FEE248", "F3FEE248",
"casalh w2, w2, w19, [x23]", "casalh w2, w2, w19, [x23]",
@@ -7518,6 +7569,7 @@ fn test_aarch64_binemit() {
rt: zero_reg(), rt: zero_reg(),
rn: stack_reg(), rn: stack_reg(),
ty: I32, ty: I32,
flags: MemFlags::trusted(),
}, },
"FFFFE088", "FFFFE088",
"casal w0, w0, wzr, [sp]", "casal w0, w0, wzr, [sp]",
@@ -7529,6 +7581,7 @@ fn test_aarch64_binemit() {
rt: xreg(15), rt: xreg(15),
rn: xreg(27), rn: xreg(27),
ty: I64, ty: I64,
flags: MemFlags::trusted(),
}, },
"6FFFE7C8", "6FFFE7C8",
"casal x7, x7, x15, [x27]", "casal x7, x7, x15, [x27]",
@@ -7536,6 +7589,7 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::AtomicCASLoop { Inst::AtomicCASLoop {
ty: I8, ty: I8,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
expected: xreg(26), expected: xreg(26),
replacement: xreg(28), replacement: xreg(28),
@@ -7549,6 +7603,7 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::AtomicCASLoop { Inst::AtomicCASLoop {
ty: I16, ty: I16,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
expected: xreg(26), expected: xreg(26),
replacement: xreg(28), replacement: xreg(28),
@@ -7562,6 +7617,7 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::AtomicCASLoop { Inst::AtomicCASLoop {
ty: I32, ty: I32,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
expected: xreg(26), expected: xreg(26),
replacement: xreg(28), replacement: xreg(28),
@@ -7575,6 +7631,7 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::AtomicCASLoop { Inst::AtomicCASLoop {
ty: I64, ty: I64,
flags: MemFlags::trusted(),
addr: xreg(25), addr: xreg(25),
expected: xreg(26), expected: xreg(26),
replacement: xreg(28), replacement: xreg(28),
@@ -7590,6 +7647,7 @@ fn test_aarch64_binemit() {
access_ty: I8, access_ty: I8,
rt: writable_xreg(7), rt: writable_xreg(7),
rn: xreg(28), rn: xreg(28),
flags: MemFlags::trusted(),
}, },
"87FFDF08", "87FFDF08",
"ldarb w7, [x28]", "ldarb w7, [x28]",
@@ -7600,6 +7658,7 @@ fn test_aarch64_binemit() {
access_ty: I16, access_ty: I16,
rt: writable_xreg(2), rt: writable_xreg(2),
rn: xreg(3), rn: xreg(3),
flags: MemFlags::trusted(),
}, },
"62FCDF48", "62FCDF48",
"ldarh w2, [x3]", "ldarh w2, [x3]",
@@ -7610,6 +7669,7 @@ fn test_aarch64_binemit() {
access_ty: I32, access_ty: I32,
rt: writable_xreg(15), rt: writable_xreg(15),
rn: xreg(0), rn: xreg(0),
flags: MemFlags::trusted(),
}, },
"0FFCDF88", "0FFCDF88",
"ldar w15, [x0]", "ldar w15, [x0]",
@@ -7620,6 +7680,7 @@ fn test_aarch64_binemit() {
access_ty: I64, access_ty: I64,
rt: writable_xreg(28), rt: writable_xreg(28),
rn: xreg(7), rn: xreg(7),
flags: MemFlags::trusted(),
}, },
"FCFCDFC8", "FCFCDFC8",
"ldar x28, [x7]", "ldar x28, [x7]",
@@ -7630,6 +7691,7 @@ fn test_aarch64_binemit() {
access_ty: I8, access_ty: I8,
rt: xreg(7), rt: xreg(7),
rn: xreg(28), rn: xreg(28),
flags: MemFlags::trusted(),
}, },
"87FF9F08", "87FF9F08",
"stlrb w7, [x28]", "stlrb w7, [x28]",
@@ -7640,6 +7702,7 @@ fn test_aarch64_binemit() {
access_ty: I16, access_ty: I16,
rt: xreg(2), rt: xreg(2),
rn: xreg(3), rn: xreg(3),
flags: MemFlags::trusted(),
}, },
"62FC9F48", "62FC9F48",
"stlrh w2, [x3]", "stlrh w2, [x3]",
@@ -7650,6 +7713,7 @@ fn test_aarch64_binemit() {
access_ty: I32, access_ty: I32,
rt: xreg(15), rt: xreg(15),
rn: xreg(0), rn: xreg(0),
flags: MemFlags::trusted(),
}, },
"0FFC9F88", "0FFC9F88",
"stlr w15, [x0]", "stlr w15, [x0]",
@@ -7660,6 +7724,7 @@ fn test_aarch64_binemit() {
access_ty: I64, access_ty: I64,
rt: xreg(28), rt: xreg(28),
rn: xreg(7), rn: xreg(7),
flags: MemFlags::trusted(),
}, },
"FCFC9FC8", "FCFC9FC8",
"stlr x28, [x7]", "stlr x28, [x7]",

10
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -1604,7 +1604,9 @@ impl Inst {
let cond = cond.pretty_print(0, allocs); let cond = cond.pretty_print(0, allocs);
format!("ccmp {}, {}, {}, {}", rn, imm, nzcv, cond) format!("ccmp {}, {}, {}, {}", rn, imm, nzcv, cond)
} }
&Inst::AtomicRMW { rs, rt, rn, ty, op } => { &Inst::AtomicRMW {
rs, rt, rn, ty, op, ..
} => {
let op = match op { let op = match op {
AtomicRMWOp::Add => "ldaddal", AtomicRMWOp::Add => "ldaddal",
AtomicRMWOp::Clr => "ldclral", AtomicRMWOp::Clr => "ldclral",
@@ -1637,6 +1639,7 @@ impl Inst {
oldval, oldval,
scratch1, scratch1,
scratch2, scratch2,
..
} => { } => {
let op = match op { let op = match op {
AtomicRMWLoopOp::Add => "add", AtomicRMWLoopOp::Add => "add",
@@ -1667,7 +1670,9 @@ impl Inst {
scratch2, scratch2,
) )
} }
&Inst::AtomicCAS { rd, rs, rt, rn, ty } => { &Inst::AtomicCAS {
rd, rs, rt, rn, ty, ..
} => {
let op = match ty { let op = match ty {
I8 => "casalb", I8 => "casalb",
I16 => "casalh", I16 => "casalh",
@@ -1689,6 +1694,7 @@ impl Inst {
replacement, replacement,
oldval, oldval,
scratch, scratch,
..
} => { } => {
let addr = pretty_print_ireg(addr, OperandSize::Size64, allocs); let addr = pretty_print_ireg(addr, OperandSize::Size64, allocs);
let expected = pretty_print_ireg(expected, OperandSize::Size64, allocs); let expected = pretty_print_ireg(expected, OperandSize::Size64, allocs);

53
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -926,8 +926,9 @@
;; Atomic loads will also automatically zero their upper bits so the `uextend` ;; Atomic loads will also automatically zero their upper bits so the `uextend`
;; instruction can effectively get skipped here. ;; instruction can effectively get skipped here.
(rule (lower (has_type (fits_in_64 out) (rule (lower (has_type (fits_in_64 out)
(uextend (and (value_type in) (sinkable_atomic_load addr))))) (uextend x @ (and (value_type in) (atomic_load flags _)))))
(load_acquire in (sink_atomic_load addr))) (if-let mem_op (is_sinkable_inst x))
(load_acquire in flags (sink_atomic_load mem_op)))
;; Conversion to 128-bit needs a zero-extension of the lower bits and the upper ;; Conversion to 128-bit needs a zero-extension of the lower bits and the upper
;; bits are all zero. ;; bits are all zero.
@@ -1780,98 +1781,98 @@
;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (valid_atomic_transaction ty) (atomic_load flags addr))) (rule (lower (has_type (valid_atomic_transaction ty) (atomic_load flags addr)))
(load_acquire ty addr)) (load_acquire ty flags addr))
;;;; Rules for `AtomicStore` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Rules for `AtomicStore` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (atomic_store flags (rule (lower (atomic_store flags
src @ (value_type (valid_atomic_transaction ty)) src @ (value_type (valid_atomic_transaction ty))
addr)) addr))
(side_effect (store_release ty src addr))) (side_effect (store_release ty flags src addr)))
;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Add) addr src)))) (atomic_rmw flags (AtomicRmwOp.Add) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Add) addr src ty)) (lse_atomic_rmw (AtomicRMWOp.Add) addr src ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Xor) addr src)))) (atomic_rmw flags (AtomicRmwOp.Xor) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Eor) addr src ty)) (lse_atomic_rmw (AtomicRMWOp.Eor) addr src ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Or) addr src)))) (atomic_rmw flags (AtomicRmwOp.Or) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Set) addr src ty)) (lse_atomic_rmw (AtomicRMWOp.Set) addr src ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Smax) addr src)))) (atomic_rmw flags (AtomicRmwOp.Smax) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Smax) addr src ty)) (lse_atomic_rmw (AtomicRMWOp.Smax) addr src ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Smin) addr src)))) (atomic_rmw flags (AtomicRmwOp.Smin) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Smin) addr src ty)) (lse_atomic_rmw (AtomicRMWOp.Smin) addr src ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Umax) addr src)))) (atomic_rmw flags (AtomicRmwOp.Umax) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Umax) addr src ty)) (lse_atomic_rmw (AtomicRMWOp.Umax) addr src ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Umin) addr src)))) (atomic_rmw flags (AtomicRmwOp.Umin) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Umin) addr src ty)) (lse_atomic_rmw (AtomicRMWOp.Umin) addr src ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Sub) addr src)))) (atomic_rmw flags (AtomicRmwOp.Sub) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Add) addr (sub ty (zero_reg) src) ty)) (lse_atomic_rmw (AtomicRMWOp.Add) addr (sub ty (zero_reg) src) ty flags))
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.And) addr src)))) (atomic_rmw flags (AtomicRmwOp.And) addr src))))
(lse_atomic_rmw (AtomicRMWOp.Clr) addr (eon ty src (zero_reg)) ty)) (lse_atomic_rmw (AtomicRMWOp.Clr) addr (eon ty src (zero_reg)) ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Add) addr src))) (atomic_rmw flags (AtomicRmwOp.Add) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Add) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Add) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Sub) addr src))) (atomic_rmw flags (AtomicRmwOp.Sub) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Sub) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Sub) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.And) addr src))) (atomic_rmw flags (AtomicRmwOp.And) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.And) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.And) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Nand) addr src))) (atomic_rmw flags (AtomicRmwOp.Nand) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Nand) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Nand) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Or) addr src))) (atomic_rmw flags (AtomicRmwOp.Or) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Orr) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Orr) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Xor) addr src))) (atomic_rmw flags (AtomicRmwOp.Xor) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Eor) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Eor) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Smin) addr src))) (atomic_rmw flags (AtomicRmwOp.Smin) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Smin) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Smin) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Smax) addr src))) (atomic_rmw flags (AtomicRmwOp.Smax) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Smax) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Smax) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Umin) addr src))) (atomic_rmw flags (AtomicRmwOp.Umin) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Umin) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Umin) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Umax) addr src))) (atomic_rmw flags (AtomicRmwOp.Umax) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Umax) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Umax) addr src ty flags))
(rule (lower (has_type (valid_atomic_transaction ty) (rule (lower (has_type (valid_atomic_transaction ty)
(atomic_rmw flags (AtomicRmwOp.Xchg) addr src))) (atomic_rmw flags (AtomicRmwOp.Xchg) addr src)))
(atomic_rmw_loop (AtomicRMWLoopOp.Xchg) addr src ty)) (atomic_rmw_loop (AtomicRMWLoopOp.Xchg) addr src ty flags))
;;;; Rules for `AtomicCAS` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Rules for `AtomicCAS` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule 1 (lower (and (use_lse) (rule 1 (lower (and (use_lse)
(has_type (valid_atomic_transaction ty) (has_type (valid_atomic_transaction ty)
(atomic_cas flags addr src1 src2)))) (atomic_cas flags addr src1 src2))))
(lse_atomic_cas addr src1 src2 ty)) (lse_atomic_cas addr src1 src2 ty flags))
(rule (lower (and (has_type (valid_atomic_transaction ty) (rule (lower (and (has_type (valid_atomic_transaction ty)
(atomic_cas flags addr src1 src2)))) (atomic_cas flags addr src1 src2))))
(atomic_cas_loop addr src1 src2 ty)) (atomic_cas_loop addr src1 src2 ty flags))
;;;; Rules for 'fvdemote' ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Rules for 'fvdemote' ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (fvdemote x)) (rule (lower (fvdemote x))

24
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -68,11 +68,6 @@ pub struct ExtendedValue {
extend: ExtendOp, extend: ExtendOp,
} }
pub struct SinkableAtomicLoad {
atomic_load: Inst,
atomic_addr: Value,
}
impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
isle_prelude_method_helpers!(AArch64Caller); isle_prelude_method_helpers!(AArch64Caller);
} }
@@ -366,25 +361,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
} }
} }
fn sinkable_atomic_load(&mut self, val: Value) -> Option<SinkableAtomicLoad> {
let input = self.lower_ctx.get_value_as_source_or_const(val);
if let InputSourceInst::UniqueUse(atomic_load, 0) = input.inst {
if self.lower_ctx.data(atomic_load).opcode() == Opcode::AtomicLoad {
let atomic_addr = self.lower_ctx.input_as_value(atomic_load, 0);
return Some(SinkableAtomicLoad {
atomic_load,
atomic_addr,
});
}
}
None
}
fn sink_atomic_load(&mut self, load: &SinkableAtomicLoad) -> Reg {
self.lower_ctx.sink_inst(load.atomic_load);
self.put_in_reg(load.atomic_addr)
}
fn shift_mask(&mut self, ty: Type) -> ImmLogic { fn shift_mask(&mut self, ty: Type) -> ImmLogic {
debug_assert!(ty.lane_bits().is_power_of_two()); debug_assert!(ty.lane_bits().is_power_of_two());