s390x: Add support for atomic operations (part 1)
This adds full back-end support for the Fence, AtomicLoad and AtomicStore operations, and partial support for the AtomicCas and AtomicRmw operations. The missing pieces include sub-word operations, operations on little-endian memory requiring byte-swapping, and some of the subtypes of AtomicRmw -- everything that cannot be implemented without a compare-and-swap loop. This will be done in a follow-up patch. This patch already suffices to make the test suite green again after a recent change that now requires atomic operations when accessing the heap.
This commit is contained in:
Ulrich Weigand
@@ -181,6 +181,56 @@ pub fn mem_emit(
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mem_rs_emit(
|
||||
rd: Reg,
|
||||
rn: Reg,
|
||||
mem: &MemArg,
|
||||
opcode_rs: Option<u16>,
|
||||
opcode_rsy: Option<u16>,
|
||||
add_trap: bool,
|
||||
sink: &mut MachBuffer<Inst>,
|
||||
emit_info: &EmitInfo,
|
||||
state: &mut EmitState,
|
||||
) {
|
||||
let (mem_insts, mem) = mem_finalize(
|
||||
mem,
|
||||
state,
|
||||
opcode_rs.is_some(),
|
||||
opcode_rsy.is_some(),
|
||||
false,
|
||||
false,
|
||||
);
|
||||
for inst in mem_insts.into_iter() {
|
||||
inst.emit(sink, emit_info, state);
|
||||
}
|
||||
|
||||
if add_trap && mem.can_trap() {
|
||||
let srcloc = state.cur_srcloc();
|
||||
if srcloc != SourceLoc::default() {
|
||||
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
|
||||
}
|
||||
}
|
||||
|
||||
match &mem {
|
||||
&MemArg::BXD12 {
|
||||
base, index, disp, ..
|
||||
} => {
|
||||
assert!(index == zero_reg());
|
||||
put(sink, &enc_rs(opcode_rs.unwrap(), rd, rn, base, disp.bits()));
|
||||
}
|
||||
&MemArg::BXD20 {
|
||||
base, index, disp, ..
|
||||
} => {
|
||||
assert!(index == zero_reg());
|
||||
put(
|
||||
sink,
|
||||
&enc_rsy(opcode_rsy.unwrap(), rd, rn, base, disp.bits()),
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mem_imm8_emit(
|
||||
imm: u8,
|
||||
mem: &MemArg,
|
||||
@@ -1301,6 +1351,53 @@ impl MachInstEmit for Inst {
|
||||
);
|
||||
}
|
||||
|
||||
&Inst::AtomicRmw {
|
||||
alu_op,
|
||||
rd,
|
||||
rn,
|
||||
ref mem,
|
||||
} => {
|
||||
let opcode = match alu_op {
|
||||
ALUOp::Add32 => 0xebf8, // LAA
|
||||
ALUOp::Add64 => 0xebe8, // LAAG
|
||||
ALUOp::And32 => 0xebf4, // LAN
|
||||
ALUOp::And64 => 0xebe4, // LANG
|
||||
ALUOp::Orr32 => 0xebf6, // LAO
|
||||
ALUOp::Orr64 => 0xebe6, // LAOG
|
||||
ALUOp::Xor32 => 0xebf7, // LAX
|
||||
ALUOp::Xor64 => 0xebe7, // LAXG
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let rd = rd.to_reg();
|
||||
mem_rs_emit(
|
||||
rd,
|
||||
rn,
|
||||
mem,
|
||||
None,
|
||||
Some(opcode),
|
||||
true,
|
||||
sink,
|
||||
emit_info,
|
||||
state,
|
||||
);
|
||||
}
|
||||
&Inst::AtomicCas32 { rd, rn, ref mem } | &Inst::AtomicCas64 { rd, rn, ref mem } => {
|
||||
let (opcode_rs, opcode_rsy) = match self {
|
||||
&Inst::AtomicCas32 { .. } => (Some(0xba), Some(0xeb14)), // CS(Y)
|
||||
&Inst::AtomicCas64 { .. } => (None, Some(0xeb30)), // CSG
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let rd = rd.to_reg();
|
||||
mem_rs_emit(
|
||||
rd, rn, mem, opcode_rs, opcode_rsy, true, sink, emit_info, state,
|
||||
);
|
||||
}
|
||||
&Inst::Fence => {
|
||||
put(sink, &enc_e(0x07e0));
|
||||
}
|
||||
|
||||
&Inst::Load32 { rd, ref mem }
|
||||
| &Inst::Load32ZExt8 { rd, ref mem }
|
||||
| &Inst::Load32SExt8 { rd, ref mem }
|
||||
|
||||
@@ -2204,6 +2204,656 @@ fn test_s390x_binemit() {
|
||||
"srag %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080F8",
|
||||
"laa %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FF8",
|
||||
"laa %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080F8",
|
||||
"laa %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FF8",
|
||||
"laa %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080E8",
|
||||
"laag %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FE8",
|
||||
"laag %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080E8",
|
||||
"laag %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Add64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FE8",
|
||||
"laag %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080F4",
|
||||
"lan %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FF4",
|
||||
"lan %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080F4",
|
||||
"lan %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FF4",
|
||||
"lan %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080E4",
|
||||
"lang %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FE4",
|
||||
"lang %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080E4",
|
||||
"lang %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::And64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FE4",
|
||||
"lang %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080F6",
|
||||
"lao %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FF6",
|
||||
"lao %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080F6",
|
||||
"lao %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FF6",
|
||||
"lao %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080E6",
|
||||
"laog %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FE6",
|
||||
"laog %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080E6",
|
||||
"laog %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Orr64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FE6",
|
||||
"laog %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080F7",
|
||||
"lax %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FF7",
|
||||
"lax %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080F7",
|
||||
"lax %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor32,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FF7",
|
||||
"lax %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45000080E7",
|
||||
"laxg %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7FE7",
|
||||
"laxg %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB45600080E7",
|
||||
"laxg %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicRmw {
|
||||
alu_op: ALUOp::Xor64,
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7FE7",
|
||||
"laxg %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD12 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: UImm12::maybe_from_u64(0).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"BA450000",
|
||||
"cs %r4, %r5, 0",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD12 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: UImm12::maybe_from_u64(4095).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"BA450FFF",
|
||||
"cs %r4, %r5, 4095",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB4500008014",
|
||||
"csy %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7F14",
|
||||
"csy %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD12 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: UImm12::maybe_from_u64(0).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"BA456000",
|
||||
"cs %r4, %r5, 0(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD12 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: UImm12::maybe_from_u64(4095).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"BA456FFF",
|
||||
"cs %r4, %r5, 4095(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB4560008014",
|
||||
"csy %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas32 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7F14",
|
||||
"csy %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas64 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB4500008030",
|
||||
"csg %r4, %r5, -524288",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas64 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: zero_reg(),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB450FFF7F30",
|
||||
"csg %r4, %r5, 524287",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas64 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(-524288).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB4560008030",
|
||||
"csg %r4, %r5, -524288(%r6)",
|
||||
));
|
||||
insns.push((
|
||||
Inst::AtomicCas64 {
|
||||
rd: writable_gpr(4),
|
||||
rn: gpr(5),
|
||||
mem: MemArg::BXD20 {
|
||||
base: gpr(6),
|
||||
index: zero_reg(),
|
||||
disp: SImm20::maybe_from_i64(524287).unwrap(),
|
||||
flags: MemFlags::trusted(),
|
||||
},
|
||||
},
|
||||
"EB456FFF7F30",
|
||||
"csg %r4, %r5, 524287(%r6)",
|
||||
));
|
||||
insns.push((Inst::Fence, "07E0", "bcr 14, 0"));
|
||||
|
||||
insns.push((
|
||||
Inst::Load32 {
|
||||
rd: writable_gpr(1),
|
||||
|
||||
@@ -404,6 +404,30 @@ pub enum Inst {
|
||||
trap_code: TrapCode,
|
||||
},
|
||||
|
||||
/// An atomic read-modify-write operation with a memory in-/out operand,
|
||||
/// a register destination, and a register source.
|
||||
/// a memory source.
|
||||
AtomicRmw {
|
||||
alu_op: ALUOp,
|
||||
rd: Writable<Reg>,
|
||||
rn: Reg,
|
||||
mem: MemArg,
|
||||
},
|
||||
/// A 32-bit atomic compare-and-swap operation.
|
||||
AtomicCas32 {
|
||||
rd: Writable<Reg>,
|
||||
rn: Reg,
|
||||
mem: MemArg,
|
||||
},
|
||||
/// A 64-bit atomic compare-and-swap operation.
|
||||
AtomicCas64 {
|
||||
rd: Writable<Reg>,
|
||||
rn: Reg,
|
||||
mem: MemArg,
|
||||
},
|
||||
/// A memory fence operation.
|
||||
Fence,
|
||||
|
||||
/// A 32-bit load.
|
||||
Load32 {
|
||||
rd: Writable<Reg>,
|
||||
@@ -1190,6 +1214,24 @@ fn s390x_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
|
||||
&Inst::CmpTrapRUImm16 { rn, .. } => {
|
||||
collector.add_use(rn);
|
||||
}
|
||||
&Inst::AtomicRmw {
|
||||
rd, rn, ref mem, ..
|
||||
} => {
|
||||
collector.add_def(rd);
|
||||
collector.add_use(rn);
|
||||
memarg_regs(mem, collector);
|
||||
}
|
||||
&Inst::AtomicCas32 {
|
||||
rd, rn, ref mem, ..
|
||||
}
|
||||
| &Inst::AtomicCas64 {
|
||||
rd, rn, ref mem, ..
|
||||
} => {
|
||||
collector.add_mod(rd);
|
||||
collector.add_use(rn);
|
||||
memarg_regs(mem, collector);
|
||||
}
|
||||
&Inst::Fence => {}
|
||||
&Inst::Load32 { rd, ref mem, .. }
|
||||
| &Inst::Load32ZExt8 { rd, ref mem, .. }
|
||||
| &Inst::Load32SExt8 { rd, ref mem, .. }
|
||||
@@ -1589,6 +1631,38 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
|
||||
map_use(mapper, rn);
|
||||
}
|
||||
|
||||
&mut Inst::AtomicRmw {
|
||||
ref mut rd,
|
||||
ref mut rn,
|
||||
ref mut mem,
|
||||
..
|
||||
} => {
|
||||
map_def(mapper, rd);
|
||||
map_use(mapper, rn);
|
||||
map_mem(mapper, mem);
|
||||
}
|
||||
&mut Inst::AtomicCas32 {
|
||||
ref mut rd,
|
||||
ref mut rn,
|
||||
ref mut mem,
|
||||
..
|
||||
} => {
|
||||
map_mod(mapper, rd);
|
||||
map_use(mapper, rn);
|
||||
map_mem(mapper, mem);
|
||||
}
|
||||
&mut Inst::AtomicCas64 {
|
||||
ref mut rd,
|
||||
ref mut rn,
|
||||
ref mut mem,
|
||||
..
|
||||
} => {
|
||||
map_mod(mapper, rd);
|
||||
map_use(mapper, rn);
|
||||
map_mem(mapper, mem);
|
||||
}
|
||||
&mut Inst::Fence => {}
|
||||
|
||||
&mut Inst::Load32 {
|
||||
ref mut rd,
|
||||
ref mut mem,
|
||||
@@ -2735,6 +2809,61 @@ impl Inst {
|
||||
let cond = cond.show_rru(mb_rru);
|
||||
format!("{}{} {}, {}", op, cond, rn, imm)
|
||||
}
|
||||
&Inst::AtomicRmw {
|
||||
alu_op,
|
||||
rd,
|
||||
rn,
|
||||
ref mem,
|
||||
} => {
|
||||
let op = match alu_op {
|
||||
ALUOp::Add32 => "laa",
|
||||
ALUOp::Add64 => "laag",
|
||||
ALUOp::And32 => "lan",
|
||||
ALUOp::And64 => "lang",
|
||||
ALUOp::Orr32 => "lao",
|
||||
ALUOp::Orr64 => "laog",
|
||||
ALUOp::Xor32 => "lax",
|
||||
ALUOp::Xor64 => "laxg",
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let (mem_str, mem) =
|
||||
mem_finalize_for_show(mem, mb_rru, state, false, true, false, false);
|
||||
|
||||
let rd = rd.to_reg().show_rru(mb_rru);
|
||||
let rn = rn.show_rru(mb_rru);
|
||||
let mem = mem.show_rru(mb_rru);
|
||||
format!("{}{} {}, {}, {}", mem_str, op, rd, rn, mem)
|
||||
}
|
||||
&Inst::AtomicCas32 { rd, rn, ref mem } | &Inst::AtomicCas64 { rd, rn, ref mem } => {
|
||||
let (opcode_rs, opcode_rsy) = match self {
|
||||
&Inst::AtomicCas32 { .. } => (Some("cs"), Some("csy")),
|
||||
&Inst::AtomicCas64 { .. } => (None, Some("csg")),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let (mem_str, mem) = mem_finalize_for_show(
|
||||
mem,
|
||||
mb_rru,
|
||||
state,
|
||||
opcode_rs.is_some(),
|
||||
opcode_rsy.is_some(),
|
||||
false,
|
||||
false,
|
||||
);
|
||||
|
||||
let op = match &mem {
|
||||
&MemArg::BXD12 { .. } => opcode_rs,
|
||||
&MemArg::BXD20 { .. } => opcode_rsy,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let rd = rd.to_reg().show_rru(mb_rru);
|
||||
let rn = rn.show_rru(mb_rru);
|
||||
let mem = mem.show_rru(mb_rru);
|
||||
format!("{}{} {}, {}, {}", mem_str, op.unwrap(), rd, rn, mem)
|
||||
}
|
||||
&Inst::Fence => "bcr 14, 0".to_string(),
|
||||
&Inst::Load32 { rd, ref mem }
|
||||
| &Inst::Load32ZExt8 { rd, ref mem }
|
||||
| &Inst::Load32SExt8 { rd, ref mem }
|
||||
|
||||
@@ -33,6 +33,13 @@ fn ty_is_float(ty: Type) -> bool {
|
||||
!ty_is_int(ty)
|
||||
}
|
||||
|
||||
fn is_valid_atomic_transaction_ty(ty: Type) -> bool {
|
||||
match ty {
|
||||
types::I8 | types::I16 | types::I32 | types::I64 => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn choose_32_64<T: Copy>(ty: Type, op32: T, op64: T) -> T {
|
||||
let bits = ty_bits(ty);
|
||||
if bits <= 32 {
|
||||
@@ -2500,13 +2507,159 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|
||||
// N.B.: the Ret itself is generated by the ABI.
|
||||
}
|
||||
|
||||
Opcode::AtomicRmw
|
||||
| Opcode::AtomicCas
|
||||
| Opcode::AtomicLoad
|
||||
| Opcode::AtomicStore
|
||||
| Opcode::Fence => {
|
||||
// TODO
|
||||
panic!("Atomic operations not implemented");
|
||||
Opcode::AtomicRmw => {
|
||||
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
|
||||
let addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
|
||||
let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
|
||||
let flags = ctx.memflags(insn).unwrap();
|
||||
let endianness = flags.endianness(Endianness::Big);
|
||||
let ty = ty.unwrap();
|
||||
assert!(is_valid_atomic_transaction_ty(ty));
|
||||
if endianness == Endianness::Little {
|
||||
panic!("Little-endian atomic operations not implemented");
|
||||
}
|
||||
if ty_bits(ty) < 32 {
|
||||
panic!("Sub-word atomic operations not implemented");
|
||||
}
|
||||
let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
|
||||
let (alu_op, rn) = match op {
|
||||
AtomicRmwOp::And => (choose_32_64(ty, ALUOp::And32, ALUOp::And64), rn),
|
||||
AtomicRmwOp::Or => (choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64), rn),
|
||||
AtomicRmwOp::Xor => (choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64), rn),
|
||||
AtomicRmwOp::Add => (choose_32_64(ty, ALUOp::Add32, ALUOp::Add64), rn),
|
||||
AtomicRmwOp::Sub => {
|
||||
let tmp_ty = choose_32_64(ty, types::I32, types::I64);
|
||||
let tmp = ctx.alloc_tmp(tmp_ty).only_reg().unwrap();
|
||||
let neg_op = choose_32_64(ty, UnaryOp::Neg32, UnaryOp::Neg64);
|
||||
ctx.emit(Inst::UnaryRR {
|
||||
op: neg_op,
|
||||
rd: tmp,
|
||||
rn,
|
||||
});
|
||||
(choose_32_64(ty, ALUOp::Add32, ALUOp::Add64), tmp.to_reg())
|
||||
}
|
||||
_ => panic!("AtomicRmw operation type {:?} not implemented", op),
|
||||
};
|
||||
let mem = MemArg::reg(addr, flags);
|
||||
ctx.emit(Inst::AtomicRmw {
|
||||
alu_op,
|
||||
rd,
|
||||
rn,
|
||||
mem,
|
||||
});
|
||||
}
|
||||
Opcode::AtomicCas => {
|
||||
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
|
||||
let addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
|
||||
let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
|
||||
let rn = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
|
||||
let flags = ctx.memflags(insn).unwrap();
|
||||
let endianness = flags.endianness(Endianness::Big);
|
||||
let ty = ty.unwrap();
|
||||
assert!(is_valid_atomic_transaction_ty(ty));
|
||||
if endianness == Endianness::Little {
|
||||
panic!("Little-endian atomic operations not implemented");
|
||||
}
|
||||
if ty_bits(ty) < 32 {
|
||||
panic!("Sub-word atomic operations not implemented");
|
||||
}
|
||||
let mem = MemArg::reg(addr, flags);
|
||||
ctx.emit(Inst::gen_move(rd, rm, ty));
|
||||
if ty_bits(ty) == 32 {
|
||||
ctx.emit(Inst::AtomicCas32 { rd, rn, mem });
|
||||
} else {
|
||||
ctx.emit(Inst::AtomicCas64 { rd, rn, mem });
|
||||
}
|
||||
}
|
||||
Opcode::AtomicLoad => {
|
||||
let flags = ctx.memflags(insn).unwrap();
|
||||
let endianness = flags.endianness(Endianness::Big);
|
||||
let ty = ty.unwrap();
|
||||
assert!(is_valid_atomic_transaction_ty(ty));
|
||||
|
||||
let mem = lower_address(ctx, &inputs[..], 0, flags);
|
||||
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
|
||||
|
||||
if endianness == Endianness::Big {
|
||||
ctx.emit(match ty_bits(ty) {
|
||||
8 => Inst::Load32ZExt8 { rd, mem },
|
||||
16 => Inst::Load32ZExt16 { rd, mem },
|
||||
32 => Inst::Load32 { rd, mem },
|
||||
64 => Inst::Load64 { rd, mem },
|
||||
_ => panic!("Unsupported size in load"),
|
||||
});
|
||||
} else {
|
||||
ctx.emit(match ty_bits(ty) {
|
||||
8 => Inst::Load32ZExt8 { rd, mem },
|
||||
16 => Inst::LoadRev16 { rd, mem },
|
||||
32 => Inst::LoadRev32 { rd, mem },
|
||||
64 => Inst::LoadRev64 { rd, mem },
|
||||
_ => panic!("Unsupported size in load"),
|
||||
});
|
||||
}
|
||||
}
|
||||
Opcode::AtomicStore => {
|
||||
let flags = ctx.memflags(insn).unwrap();
|
||||
let endianness = flags.endianness(Endianness::Big);
|
||||
let ty = ctx.input_ty(insn, 0);
|
||||
assert!(is_valid_atomic_transaction_ty(ty));
|
||||
|
||||
let mem = lower_address(ctx, &inputs[1..], 0, flags);
|
||||
|
||||
if ty_bits(ty) <= 16 {
|
||||
if let Some(imm) = input_matches_const(ctx, inputs[0]) {
|
||||
ctx.emit(match (endianness, ty_bits(ty)) {
|
||||
(_, 8) => Inst::StoreImm8 {
|
||||
imm: imm as u8,
|
||||
mem,
|
||||
},
|
||||
(Endianness::Big, 16) => Inst::StoreImm16 {
|
||||
imm: imm as i16,
|
||||
mem,
|
||||
},
|
||||
(Endianness::Little, 16) => Inst::StoreImm16 {
|
||||
imm: (imm as i16).swap_bytes(),
|
||||
mem,
|
||||
},
|
||||
_ => panic!("Unsupported size in store"),
|
||||
});
|
||||
} else {
|
||||
let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
|
||||
ctx.emit(match (endianness, ty_bits(ty)) {
|
||||
(_, 8) => Inst::Store8 { rd, mem },
|
||||
(Endianness::Big, 16) => Inst::Store16 { rd, mem },
|
||||
(Endianness::Little, 16) => Inst::StoreRev16 { rd, mem },
|
||||
_ => panic!("Unsupported size in store"),
|
||||
});
|
||||
}
|
||||
} else if endianness == Endianness::Big {
|
||||
if let Some(imm) = input_matches_simm16(ctx, inputs[0]) {
|
||||
ctx.emit(match ty_bits(ty) {
|
||||
32 => Inst::StoreImm32SExt16 { imm, mem },
|
||||
64 => Inst::StoreImm64SExt16 { imm, mem },
|
||||
_ => panic!("Unsupported size in store"),
|
||||
});
|
||||
} else {
|
||||
let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
|
||||
ctx.emit(match ty_bits(ty) {
|
||||
32 => Inst::Store32 { rd, mem },
|
||||
64 => Inst::Store64 { rd, mem },
|
||||
_ => panic!("Unsupported size in store"),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
|
||||
ctx.emit(match ty_bits(ty) {
|
||||
32 => Inst::StoreRev32 { rd, mem },
|
||||
64 => Inst::StoreRev64 { rd, mem },
|
||||
_ => panic!("Unsupported size in store"),
|
||||
});
|
||||
}
|
||||
|
||||
ctx.emit(Inst::Fence);
|
||||
}
|
||||
Opcode::Fence => {
|
||||
ctx.emit(Inst::Fence);
|
||||
}
|
||||
|
||||
Opcode::RawBitcast
|
||||
|
||||
Reference in New Issue
Block a user