diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 9e252a44ef..458dfe4789 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -2843,6 +2843,9 @@ (decl preg_stack () PReg) (extern constructor preg_stack preg_stack) +(decl preg_gpr_0 () PReg) +(extern constructor preg_gpr_0 preg_gpr_0) + ;; Copy the physical stack register into a virtual register. (decl sp () Reg) (rule (sp) @@ -3414,7 +3417,7 @@ ;; be written if the memory location still holds the old value in %r0. ;; The result should be passed to "casloop_result" or (in the case of ;; subword loops) to "casloop_rotate_result". -(decl casloop_emit (VecMInstBuilder Type MemFlags Reg Reg) Reg) +(decl casloop_emit (VecMInstBuilder Type MemFlags Reg Reg) PReg) (rule (casloop_emit ib ty flags aligned_addr val) (let (;; Construct a memory argument for the aligned word. (aligned_mem MemArg (memarg_reg_plus_off aligned_addr 0 0 flags)) @@ -3424,17 +3427,23 @@ ;; Emit initial load followed by compare-and-swap loop. (_ Unit (emit_load (ty_ext32 ty) (casloop_val_reg) aligned_mem)) (_ Unit (emit_loop ib (intcc_as_cond (IntCC.NotEqual))))) - result)) + + ;; push_atomic_cas above returns its destination register argument, + ;; cas_loop_val_reg, as its result. As cas_loop_val_reg is a writable + ;; version of `gpr 0`, we return that directly here as a physical + ;; register to avoid accidentally using it with a non-preg move + ;; instruction. + (preg_gpr_0))) ;; Compute the previous memory value after a (fullword) compare-and-swap loop. ;; In the big-endian case, the value is already correct, but may need to be ;; copied out of the hard register. In the little-endian case, we need to ;; byte-swap since the compare-and-swap instruction is always big-endian. -(decl casloop_result (Type MemFlags Reg) Reg) +(decl casloop_result (Type MemFlags PReg) Reg) (rule 1 (casloop_result (ty_32_or_64 ty) (bigendian) result) - (copy_reg ty result)) + (mov_preg result)) (rule (casloop_result (ty_32_or_64 ty) (littleendian) result) - (bswap_reg ty result)) + (bswap_reg ty (preg_to_reg result))) ;; Emit a fullword compare-and-swap loop, returning the previous memory value. (decl casloop (VecMInstBuilder Type MemFlags Reg Reg) Reg) diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index 4b5eb35ff4..23ea1e7369 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -2350,7 +2350,7 @@ impl Inst { } &Inst::MovPReg { rd, rm } => { let rm: Reg = rm.into(); - debug_assert!([regs::gpr(15)].contains(&rm)); + debug_assert!([regs::gpr(0), regs::gpr(14), regs::gpr(15)].contains(&rm)); let rd = allocs.next_writable(rd); Inst::Mov64 { rd, rm }.emit(&[], sink, emit_info, state); } diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index 722915e9b5..88a40a928b 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -620,7 +620,7 @@ fn s390x_get_operands VReg>(inst: &Inst, collector: &mut OperandC collector.reg_use(rm); } &Inst::MovPReg { rd, rm } => { - debug_assert!([regs::gpr(14), regs::gpr(15)].contains(&rm.into())); + debug_assert!([regs::gpr(0), regs::gpr(14), regs::gpr(15)].contains(&rm.into())); debug_assert!(rd.to_reg().is_virtual()); collector.reg_def(rd); } diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 5a1813e3ce..df615b36c7 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -938,6 +938,11 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { stack_reg().to_real_reg().unwrap().into() } + #[inline] + fn preg_gpr_0(&mut self) -> PReg { + gpr(0).to_real_reg().unwrap().into() + } + #[inline] fn writable_regpair(&mut self, hi: WritableReg, lo: WritableReg) -> WritableRegPair { WritableRegPair { hi, lo } diff --git a/cranelift/filetests/filetests/isa/s390x/issue-5425.clif b/cranelift/filetests/filetests/isa/s390x/issue-5425.clif new file mode 100644 index 0000000000..28a21b3b85 --- /dev/null +++ b/cranelift/filetests/filetests/isa/s390x/issue-5425.clif @@ -0,0 +1,16 @@ +test compile +set regalloc_checker=1 +target s390x + +function %a() system_v { + fn0 = %callee_f64(i64) -> i32 + +block0: + v1 = iconst.i64 0 + v2 = call fn0(v1) ; v1 = 0 + + v21 = iconst.i64 0 + v22 = iconst.i32 2 + v23 = atomic_rmw.i32 xchg v21, v22 ; v21 = 0, v22 = 2 + trap user0 +}