T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s390x: Add support for all remaining atomic operations (#3746)

Browse Source 
This adds support for all atomic operations that were unimplemented
so far in the s390x back end:
- atomic_rmw operations xchg, nand, smin, smax, umin, umax
- $I8 and $I16 versions of atomic_rmw and atomic_cas
- little endian versions of atomic_rmw and atomic_cas

All of these have to be implemented by a compare-and-swap loop;
and for the $I8 and $I16 versions the actual atomic instruction
needs to operate on the surrounding aligned 32-bit word.

Since we cannot emit new control flow during ISLE instruction
selection, these compare-and-swap loops are emitted as a single
meta-instruction to be expanded at emit time.

However, since there is a large number of different versions of
the loop required to implement all the above operations, I've
implemented a facility to allow specifying the loop bodies
from within ISLE after all, by creating a vector of MInst
structures that will be emitted as part of the meta-instruction.

There are still restrictions, in particular instructions that
are part of the loop body may not modify any virtual register.
But even so, this approach looks preferable to doing everything
in emit.rs.

A few instructions needed in those compare-and-swap loop bodies
were added as well, in particular the RxSBG family of instructions
as well as the LOAD REVERSED in-register byte-swap instructions.

This patch also adds filetest runtests to verify the semantics
of all operations, in particular the subword and little-endian
variants (those are currently only executed on s390x).
This commit is contained in:
Ulrich Weigand
2022-02-08 22:48:44 +01:00
committed by GitHub
parent 5cd97c054d
commit 9c5c872b3b
21 changed files with 6413 additions and 891 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
cranelift/codegen/src/isa/s390x/inst/emit.rs
View File

@@ -426,6 +426,28 @@ fn enc_rie_d(opcode: u16, r1: Reg, r3: Reg, i2: u16) -> [u8; 6] {
enc
}
/// RIEf-type instructions.
///
/// 47 39 35 31 23 15 7
/// opcode1 r1 r2 i3 i4 i5 opcode2
/// 40 36 32 24 16 8 0
///
fn enc_rie_f(opcode: u16, r1: Reg, r2: Reg, i3: u8, i4: u8, i5: u8) -> [u8; 6] {
let mut enc: [u8; 6] = [0; 6];
let opcode1 = ((opcode >> 8) & 0xff) as u8;
let opcode2 = (opcode & 0xff) as u8;
let r1 = machreg_to_gpr(r1) & 0x0f;
let r2 = machreg_to_gpr(r2) & 0x0f;
enc[0] = opcode1;
enc[1] = r1 << 4 | r2;
enc[2] = i3;
enc[3] = i4;
enc[4] = i5;
enc[5] = opcode2;
enc
}
/// RIEg-type instructions.
///
/// 47 39 35 31 15 7
@@ -1188,6 +1210,60 @@ impl MachInstEmit for Inst {
);
}
&Inst::RxSBG {
op,
rd,
rn,
start_bit,
end_bit,
rotate_amt,
} => {
let opcode = match op {
RxSBGOp::Insert => 0xec59, // RISBGN
RxSBGOp::And => 0xec54, // RNSBG
RxSBGOp::Or => 0xec56, // ROSBG
RxSBGOp::Xor => 0xec57, // RXSBG
};
put(
sink,
&enc_rie_f(
opcode,
rd.to_reg(),
rn,
start_bit,
end_bit,
(rotate_amt as u8) & 63,
),
);
}
&Inst::RxSBGTest {
op,
rd,
rn,
start_bit,
end_bit,
rotate_amt,
} => {
let opcode = match op {
RxSBGOp::And => 0xec54, // RNSBG
RxSBGOp::Or => 0xec56, // ROSBG
RxSBGOp::Xor => 0xec57, // RXSBG
_ => unreachable!(),
};
put(
sink,
&enc_rie_f(
opcode,
rd,
rn,
start_bit | 0x80,
end_bit,
(rotate_amt as u8) & 63,
),
);
}
&Inst::UnaryRR { op, rd, rn } => {
match op {
UnaryOp::Abs32 => {
@@ -1222,6 +1298,14 @@ impl MachInstEmit for Inst {
let opcode = 0xb9e1; // POPCNT
put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 8, 0));
}
UnaryOp::BSwap32 => {
let opcode = 0xb91f; // LRVR
put(sink, &enc_rre(opcode, rd.to_reg(), rn));
}
UnaryOp::BSwap64 => {
let opcode = 0xb90f; // LRVRG
put(sink, &enc_rre(opcode, rd.to_reg(), rn));
}
}
}
@@ -1406,6 +1490,39 @@ impl MachInstEmit for Inst {
state,
);
}
&Inst::Loop { ref body, cond } => {
// This sequence is *one* instruction in the vcode, and is expanded only here at
// emission time, because it requires branching to internal labels.
let loop_label = sink.get_label();
let done_label = sink.get_label();
// Emit label at the start of the loop.
sink.bind_label(loop_label);
for inst in (&body).into_iter() {
match &inst {
// Replace a CondBreak with a branch to done_label.
&Inst::CondBreak { cond } => {
let inst = Inst::OneWayCondBr {
target: done_label,
cond: *cond,
};
inst.emit(sink, emit_info, state);
}
_ => inst.emit(sink, emit_info, state),
};
}
let inst = Inst::OneWayCondBr {
target: loop_label,
cond,
};
inst.emit(sink, emit_info, state);
// Emit label at the end of the loop.
sink.bind_label(done_label);
}
&Inst::CondBreak { .. } => unreachable!(), // Only valid inside a Loop.
&Inst::AtomicCas32 { rd, rn, ref mem } | &Inst::AtomicCas64 { rd, rn, ref mem } => {
let (opcode_rs, opcode_rsy) = match self {
&Inst::AtomicCas32 { .. } => (Some(0xba), Some(0xeb14)), // CS(Y)

131
cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
View File

@@ -1478,6 +1478,24 @@ fn test_s390x_binemit() {
"B9E1801A",
"popcnt %r1, %r10, 8",
));
insns.push((
Inst::UnaryRR {
op: UnaryOp::BSwap32,
rd: writable_gpr(1),
rn: gpr(10),
},
"B91F001A",
"lrvr %r1, %r10",
));
insns.push((
Inst::UnaryRR {
op: UnaryOp::BSwap64,
rd: writable_gpr(1),
rn: gpr(10),
},
"B90F001A",
"lrvgr %r1, %r10",
));
insns.push((
Inst::CmpRR {
@@ -2410,6 +2428,91 @@ fn test_s390x_binemit() {
"srag %r4, %r5, 63(%r6)",
));
insns.push((
Inst::RxSBG {
op: RxSBGOp::Insert,
rd: writable_gpr(4),
rn: gpr(5),
start_bit: 8,
end_bit: 32,
rotate_amt: -16,
},
"EC4508203059",
"risbgn %r4, %r5, 8, 32, 48",
));
insns.push((
Inst::RxSBG {
op: RxSBGOp::And,
rd: writable_gpr(4),
rn: gpr(5),
start_bit: 8,
end_bit: 32,
rotate_amt: 63,
},
"EC4508203F54",
"rnsbg %r4, %r5, 8, 32, 63",
));
insns.push((
Inst::RxSBG {
op: RxSBGOp::Or,
rd: writable_gpr(4),
rn: gpr(5),
start_bit: 8,
end_bit: 32,
rotate_amt: 63,
},
"EC4508203F56",
"rosbg %r4, %r5, 8, 32, 63",
));
insns.push((
Inst::RxSBG {
op: RxSBGOp::Xor,
rd: writable_gpr(4),
rn: gpr(5),
start_bit: 8,
end_bit: 32,
rotate_amt: 63,
},
"EC4508203F57",
"rxsbg %r4, %r5, 8, 32, 63",
));
insns.push((
Inst::RxSBGTest {
op: RxSBGOp::And,
rd: gpr(4),
rn: gpr(5),
start_bit: 8,
end_bit: 32,
rotate_amt: 63,
},
"EC4588203F54",
"rnsbg %r4, %r5, 136, 32, 63",
));
insns.push((
Inst::RxSBGTest {
op: RxSBGOp::Or,
rd: gpr(4),
rn: gpr(5),
start_bit: 8,
end_bit: 32,
rotate_amt: 63,
},
"EC4588203F56",
"rosbg %r4, %r5, 136, 32, 63",
));
insns.push((
Inst::RxSBGTest {
op: RxSBGOp::Xor,
rd: gpr(4),
rn: gpr(5),
start_bit: 8,
end_bit: 32,
rotate_amt: 63,
},
"EC4588203F57",
"rxsbg %r4, %r5, 136, 32, 63",
));
insns.push((
Inst::AtomicRmw {
alu_op: ALUOp::Add32,
@@ -6699,6 +6802,34 @@ fn test_s390x_binemit() {
"jno 6 ; trap",
));
insns.push((
Inst::Loop {
body: vec![
Inst::CmpRR {
op: CmpOp::CmpS32,
rn: gpr(2),
rm: gpr(3),
},
Inst::CondBreak {
cond: Cond::from_mask(13),
},
Inst::AtomicCas32 {
rd: writable_gpr(4),
rn: gpr(5),
mem: MemArg::BXD12 {
base: gpr(6),
index: zero_reg(),
disp: UImm12::maybe_from_u64(0).unwrap(),
flags: MemFlags::trusted(),
},
},
],
cond: Cond::from_mask(6),
},
"1923C0D400000008BA456000C064FFFFFFFA",
"0: cr %r2, %r3 ; jgnh 1f ; cs %r4, %r5, 0(%r6) ; jglh 0b ; 1:",
));
insns.push((
Inst::FpuMove32 {
rd: writable_fpr(8),

109
cranelift/codegen/src/isa/s390x/inst/mod.rs
View File

@@ -35,7 +35,7 @@ mod emit_tests;
pub use crate::isa::s390x::lower::isle::generated_code::{
ALUOp, CmpOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst,
ShiftOp, UnaryOp,
RxSBGOp, ShiftOp, UnaryOp,
};
/// Additional information for (direct) Call instructions, left out of line to lower the size of
@@ -93,6 +93,8 @@ impl Inst {
| Inst::AluRUImm16Shifted { .. }
| Inst::AluRUImm32Shifted { .. }
| Inst::ShiftRR { .. }
| Inst::RxSBG { .. }
| Inst::RxSBGTest { .. }
| Inst::SMulWide { .. }
| Inst::UMulWide { .. }
| Inst::SDivMod32 { .. }
@@ -191,6 +193,8 @@ impl Inst {
| Inst::JTSequence { .. }
| Inst::LoadExtNameFar { .. }
| Inst::LoadAddr { .. }
| Inst::Loop { .. }
| Inst::CondBreak { .. }
| Inst::VirtualSPOffsetAdj { .. }
| Inst::ValueLabelMarker { .. }
| Inst::Unwind { .. } => InstructionSet::Base,
@@ -437,6 +441,14 @@ fn s390x_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_use(shift_reg);
}
}
&Inst::RxSBG { rd, rn, .. } => {
collector.add_mod(rd);
collector.add_use(rn);
}
&Inst::RxSBGTest { rd, rn, .. } => {
collector.add_use(rd);
collector.add_use(rn);
}
&Inst::UnaryRR { rd, rn, .. } => {
collector.add_def(rd);
collector.add_use(rn);
@@ -687,6 +699,12 @@ fn s390x_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_def(rd);
memarg_regs(mem, collector);
}
&Inst::Loop { ref body, .. } => {
for inst in body.iter() {
s390x_get_regs(inst, collector);
}
}
&Inst::CondBreak { .. } => {}
&Inst::VirtualSPOffsetAdj { .. } => {}
&Inst::ValueLabelMarker { reg, .. } => {
collector.add_use(reg);
@@ -812,6 +830,22 @@ pub fn s390x_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
mapper.map_use(shift_reg);
}
}
&mut Inst::RxSBG {
ref mut rd,
ref mut rn,
..
} => {
mapper.map_mod(rd);
mapper.map_use(rn);
}
&mut Inst::RxSBGTest {
ref mut rd,
ref mut rn,
..
} => {
mapper.map_use(rd);
mapper.map_use(rn);
}
&mut Inst::UnaryRR {
ref mut rd,
ref mut rn,
@@ -1408,6 +1442,12 @@ pub fn s390x_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
mapper.map_def(rd);
map_mem(mapper, mem);
}
&mut Inst::Loop { ref mut body, .. } => {
for inst in body.iter_mut() {
s390x_map_regs(inst, mapper);
}
}
&mut Inst::CondBreak { .. } => {}
&mut Inst::VirtualSPOffsetAdj { .. } => {}
&mut Inst::ValueLabelMarker { ref mut reg, .. } => {
mapper.map_use(reg);
@@ -1909,6 +1949,58 @@ impl Inst {
};
format!("{} {}, {}, {}{}", op, rd, rn, shift_imm, shift_reg)
}
&Inst::RxSBG {
op,
rd,
rn,
start_bit,
end_bit,
rotate_amt,
} => {
let op = match op {
RxSBGOp::Insert => "risbgn",
RxSBGOp::And => "rnsbg",
RxSBGOp::Or => "rosbg",
RxSBGOp::Xor => "rxsbg",
};
let rd = rd.to_reg().show_rru(mb_rru);
let rn = rn.show_rru(mb_rru);
format!(
"{} {}, {}, {}, {}, {}",
op,
rd,
rn,
start_bit,
end_bit,
(rotate_amt as u8) & 63
)
}
&Inst::RxSBGTest {
op,
rd,
rn,
start_bit,
end_bit,
rotate_amt,
} => {
let op = match op {
RxSBGOp::And => "rnsbg",
RxSBGOp::Or => "rosbg",
RxSBGOp::Xor => "rxsbg",
_ => unreachable!(),
};
let rd = rd.show_rru(mb_rru);
let rn = rn.show_rru(mb_rru);
format!(
"{} {}, {}, {}, {}, {}",
op,
rd,
rn,
start_bit | 0x80,
end_bit,
(rotate_amt as u8) & 63
)
}
&Inst::UnaryRR { op, rd, rn } => {
let (op, extra) = match op {
UnaryOp::Abs32 => ("lpr", ""),
@@ -1919,6 +2011,8 @@ impl Inst {
UnaryOp::Neg64Ext32 => ("lcgfr", ""),
UnaryOp::PopcntByte => ("popcnt", ""),
UnaryOp::PopcntReg => ("popcnt", ", 8"),
UnaryOp::BSwap32 => ("lrvr", ""),
UnaryOp::BSwap64 => ("lrvgr", ""),
};
let rd = rd.to_reg().show_rru(mb_rru);
let rn = rn.show_rru(mb_rru);
@@ -2644,6 +2738,19 @@ impl Inst {
let mem = mem.show_rru(mb_rru);
format!("{}{} {}, {}", mem_str, op, rd, mem)
}
&Inst::Loop { ref body, cond } => {
let body = body
.into_iter()
.map(|inst| inst.show_rru(mb_rru))
.collect::<Vec<_>>()
.join(" ; ");
let cond = cond.show_rru(mb_rru);
format!("0: {} ; jg{} 0b ; 1:", body, cond)
}
&Inst::CondBreak { cond } => {
let cond = cond.show_rru(mb_rru);
format!("jg{} 1f", cond)
}
&Inst::VirtualSPOffsetAdj { offset } => {
state.virtual_sp_offset += offset;
format!("virtual_sp_offset_adjust {}", offset)