T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3717 from uweigand/s390x-branchtarget

Browse Source 
s390x: Refactor branch and jumptable emission
This commit is contained in:
Chris Fallin
2022-01-25 08:55:31 -08:00
committed by GitHub
parent 5fc01bafc7 cee00c6591
commit ce63a113ab
10 changed files with 731 additions and 870 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cranelift/codegen/src/isa/s390x/inst.isle
View File

@@ -589,15 +589,15 @@
;; An unconditional branch.
(Jump
(dest BranchTarget))
(dest MachLabel))
;; A conditional branch. Contains two targets; at emission time, both are emitted, but
;; the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the
;; choice of taken/not_taken (inverting the branch polarity as needed) based on the
;; fallthrough at the time of lowering.
(CondBr
(taken BranchTarget)
(not_taken BranchTarget)
(taken MachLabel)
(not_taken MachLabel)
(cond Cond))
;; A conditional trap execute a `Trap` if the condition is true. This is
@@ -624,7 +624,7 @@
;;
;; See, e.g., the lowering of `trapif` (conditional trap) for an example.
(OneWayCondBr
(target BranchTarget)
(target MachLabel)
(cond Cond))
;; An indirect branch through a register, augmented with set of all
@@ -644,10 +644,8 @@
;; Jump-table sequence, as one compound instruction (see note in lower.rs
;; for rationale).
(JTSequence
(info BoxJTSequenceInfo)
(ridx Reg)
(rtmp1 WritableReg)
(rtmp2 WritableReg))
(targets VecMachLabel))
;; Load an inline symbol reference with RelocDistance::Far.
(LoadExtNameFar
@@ -680,8 +678,8 @@
(type BoxCallInfo (primitive BoxCallInfo))
(type BoxCallIndInfo (primitive BoxCallIndInfo))
(type MachLabel (primitive MachLabel))
(type VecMachLabel (primitive VecMachLabel))
(type BranchTarget (primitive BranchTarget))
(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
(type BoxExternalName (primitive BoxExternalName))
(type ValueLabel (primitive ValueLabel))

54
cranelift/codegen/src/isa/s390x/inst/args.rs
View File

@@ -38,7 +38,7 @@ pub enum MemArg {
},
/// PC-relative Reference to a label.
Label { target: BranchTarget },
Label { target: MachLabel },
/// PC-relative Reference to a near symbol.
Symbol {
@@ -182,47 +182,6 @@ impl Cond {
}
}
/// A branch target. Either unresolved (basic-block index) or resolved (offset
/// from end of current instruction).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BranchTarget {
/// An unresolved reference to a Label, as passed into
/// `lower_branch_group()`.
Label(MachLabel),
/// A fixed PC offset.
ResolvedOffset(i32),
}
impl BranchTarget {
/// Return the target's label, if it is a label-based target.
pub fn as_label(self) -> Option<MachLabel> {
match self {
BranchTarget::Label(l) => Some(l),
_ => None,
}
}
/// Return the target's offset, if specified, or zero if label-based.
pub fn as_ri_offset_or_zero(self) -> u16 {
let off = match self {
BranchTarget::ResolvedOffset(off) => off >> 1,
_ => 0,
};
assert!(off <= 0x7fff);
assert!(off >= -0x8000);
off as u16
}
/// Return the target's offset, if specified, or zero if label-based.
pub fn as_ril_offset_or_zero(self) -> u32 {
let off = match self {
BranchTarget::ResolvedOffset(off) => off >> 1,
_ => 0,
};
off as u32
}
}
impl PrettyPrint for MemArg {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
match self {
@@ -270,7 +229,7 @@ impl PrettyPrint for MemArg {
}
}
}
&MemArg::Label { ref target } => target.show_rru(mb_rru),
&MemArg::Label { target } => target.to_string(),
&MemArg::Symbol {
ref name, offset, ..
} => format!("{} + {}", name, offset),
@@ -306,12 +265,3 @@ impl PrettyPrint for Cond {
s.to_string()
}
}
impl PrettyPrint for BranchTarget {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
match self {
&BranchTarget::Label(label) => format!("label{:?}", label.get()),
&BranchTarget::ResolvedOffset(off) => format!("{}", off),
}
}
}

128
cranelift/codegen/src/isa/s390x/inst/emit.rs
View File

@@ -1,7 +1,6 @@
//! S390x ISA: binary code emission.
use crate::binemit::{Reloc, StackMap};
use crate::ir::condcodes::IntCC;
use crate::ir::MemFlags;
use crate::ir::{SourceLoc, TrapCode};
use crate::isa::s390x::inst::*;
@@ -153,14 +152,9 @@ pub fn mem_emit(
&enc_rxy(opcode_rxy.unwrap(), rd, base, index, disp.bits()),
);
}
&MemArg::Label { ref target } => {
if let Some(l) = target.as_label() {
sink.use_label_at_offset(sink.cur_offset(), l, LabelUse::BranchRIL);
}
put(
sink,
&enc_ril_b(opcode_ril.unwrap(), rd, target.as_ril_offset_or_zero()),
);
&MemArg::Label { target } => {
sink.use_label_at_offset(sink.cur_offset(), target, LabelUse::BranchRIL);
put(sink, &enc_ril_b(opcode_ril.unwrap(), rd, 0));
}
&MemArg::Symbol {
ref name, offset, ..
@@ -1904,60 +1898,43 @@ impl MachInstEmit for Inst {
&Inst::EpiloguePlaceholder => {
// Noop; this is just a placeholder for epilogues.
}
&Inst::Jump { ref dest } => {
&Inst::Jump { dest } => {
let off = sink.cur_offset();
// Indicate that the jump uses a label, if so, so that a fixup can occur later.
if let Some(l) = dest.as_label() {
sink.use_label_at_offset(off, l, LabelUse::BranchRIL);
sink.add_uncond_branch(off, off + 6, l);
}
sink.use_label_at_offset(off, dest, LabelUse::BranchRIL);
sink.add_uncond_branch(off, off + 6, dest);
// Emit the jump itself.
let opcode = 0xc04; // BCRL
put(sink, &enc_ril_c(opcode, 15, dest.as_ril_offset_or_zero()));
put(sink, &enc_ril_c(opcode, 15, 0));
}
&Inst::IndirectBr { rn, .. } => {
let opcode = 0x07; // BCR
put(sink, &enc_rr(opcode, gpr(15), rn));
}
&Inst::CondBr {
ref taken,
ref not_taken,
taken,
not_taken,
cond,
} => {
let opcode = 0xc04; // BCRL
// Conditional part first.
let cond_off = sink.cur_offset();
if let Some(l) = taken.as_label() {
sink.use_label_at_offset(cond_off, l, LabelUse::BranchRIL);
sink.use_label_at_offset(cond_off, taken, LabelUse::BranchRIL);
let inverted = &enc_ril_c(opcode, cond.invert().bits(), 0);
sink.add_cond_branch(cond_off, cond_off + 6, l, inverted);
}
put(
sink,
&enc_ril_c(opcode, cond.bits(), taken.as_ril_offset_or_zero()),
);
sink.add_cond_branch(cond_off, cond_off + 6, taken, inverted);
put(sink, &enc_ril_c(opcode, cond.bits(), 0));
// Unconditional part next.
let uncond_off = sink.cur_offset();
if let Some(l) = not_taken.as_label() {
sink.use_label_at_offset(uncond_off, l, LabelUse::BranchRIL);
sink.add_uncond_branch(uncond_off, uncond_off + 6, l);
sink.use_label_at_offset(uncond_off, not_taken, LabelUse::BranchRIL);
sink.add_uncond_branch(uncond_off, uncond_off + 6, not_taken);
put(sink, &enc_ril_c(opcode, 15, 0));
}
put(
sink,
&enc_ril_c(opcode, 15, not_taken.as_ril_offset_or_zero()),
);
}
&Inst::OneWayCondBr { ref target, cond } => {
&Inst::OneWayCondBr { target, cond } => {
let opcode = 0xc04; // BCRL
if let Some(l) = target.as_label() {
sink.use_label_at_offset(sink.cur_offset(), l, LabelUse::BranchRIL);
}
put(
sink,
&enc_ril_c(opcode, cond.bits(), target.as_ril_offset_or_zero()),
);
sink.use_label_at_offset(sink.cur_offset(), target, LabelUse::BranchRIL);
put(sink, &enc_ril_c(opcode, cond.bits(), 0));
}
&Inst::Nop0 => {}
&Inst::Nop2 => {
@@ -1984,86 +1961,49 @@ impl MachInstEmit for Inst {
let srcloc = state.cur_srcloc();
put_with_trap(sink, &enc_e(0x0000), srcloc, trap_code);
}
&Inst::JTSequence {
ridx,
rtmp1,
rtmp2,
ref info,
..
} => {
&Inst::JTSequence { ridx, ref targets } => {
let table_label = sink.get_label();
// This sequence is *one* instruction in the vcode, and is expanded only here at
// emission time, because we cannot allow the regalloc to insert spills/reloads in
// the middle; we depend on hardcoded PC-rel addressing below.
// Bounds-check index and branch to default.
let inst = Inst::CmpRUImm32 {
op: CmpOp::CmpL64,
rn: ridx,
imm: info.targets.len() as u32,
};
inst.emit(sink, emit_info, state);
let inst = Inst::OneWayCondBr {
target: info.default_target,
cond: Cond::from_intcc(IntCC::UnsignedGreaterThanOrEqual),
};
inst.emit(sink, emit_info, state);
// Set rtmp2 to index scaled by entry size.
let inst = Inst::ShiftRR {
shift_op: ShiftOp::LShL64,
rd: rtmp2,
rn: ridx,
shift_imm: 2,
shift_reg: zero_reg(),
};
inst.emit(sink, emit_info, state);
// Set rtmp1 to address of jump table.
// Set temp register to address of jump table.
let rtmp = writable_spilltmp_reg();
let inst = Inst::LoadAddr {
rd: rtmp1,
rd: rtmp,
mem: MemArg::Label {
target: BranchTarget::Label(table_label),
target: table_label,
},
};
inst.emit(sink, emit_info, state);
// Set rtmp2 to value loaded out of jump table.
let inst = Inst::Load64SExt32 {
rd: rtmp2,
mem: MemArg::reg_plus_reg(rtmp1.to_reg(), rtmp2.to_reg(), MemFlags::trusted()),
};
inst.emit(sink, emit_info, state);
// Set rtmp1 to target address (rtmp1 + rtmp2).
let inst = Inst::AluRRR {
alu_op: ALUOp::Add64,
rd: rtmp1,
rn: rtmp1.to_reg(),
rm: rtmp2.to_reg(),
// Set temp to target address by adding the value of the jump table entry.
let inst = Inst::AluRX {
alu_op: ALUOp::Add64Ext32,
rd: rtmp,
mem: MemArg::reg_plus_reg(rtmp.to_reg(), ridx, MemFlags::trusted()),
};
inst.emit(sink, emit_info, state);
// Branch to computed address. (`targets` here is only used for successor queries
// and is not needed for emission.)
let inst = Inst::IndirectBr {
rn: rtmp1.to_reg(),
rn: rtmp.to_reg(),
targets: vec![],
};
inst.emit(sink, emit_info, state);
// Emit jump table (table of 32-bit offsets).
// The first entry is the default target, which is not emitted
// into the jump table, so we skip it here. It is only in the
// list so MachTerminator will see the potential target.
sink.bind_label(table_label);
let jt_off = sink.cur_offset();
for &target in info.targets.iter() {
for &target in targets.iter().skip(1) {
let word_off = sink.cur_offset();
let off_into_table = word_off - jt_off;
sink.use_label_at_offset(
word_off,
target.as_label().unwrap(),
LabelUse::PCRel32,
);
sink.use_label_at_offset(word_off, target, LabelUse::PCRel32);
sink.put4(off_into_table.swap_bytes());
}

345
cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
View File

@@ -1567,11 +1567,11 @@ fn test_s390x_binemit() {
op: CmpOp::CmpS32,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61D00000020",
"crl %r1, 64",
"C61D00000003",
"crl %r1, label1",
));
insns.push((
Inst::CmpRX {
@@ -1606,11 +1606,11 @@ fn test_s390x_binemit() {
op: CmpOp::CmpS32Ext16,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61500000020",
"chrl %r1, 64",
"C61500000003",
"chrl %r1, label1",
));
insns.push((
Inst::CmpRX {
@@ -1631,11 +1631,11 @@ fn test_s390x_binemit() {
op: CmpOp::CmpS64,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61800000020",
"cgrl %r1, 64",
"C61800000003",
"cgrl %r1, label1",
));
insns.push((
Inst::CmpRX {
@@ -1656,11 +1656,11 @@ fn test_s390x_binemit() {
op: CmpOp::CmpS64Ext16,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61400000020",
"cghrl %r1, 64",
"C61400000003",
"cghrl %r1, label1",
));
insns.push((
Inst::CmpRX {
@@ -1681,11 +1681,11 @@ fn test_s390x_binemit() {
op: CmpOp::CmpS64Ext32,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61C00000020",
"cgfrl %r1, 64",
"C61C00000003",
"cgfrl %r1, label1",
));
insns.push((
Inst::CmpRX {
@@ -1720,22 +1720,22 @@ fn test_s390x_binemit() {
op: CmpOp::CmpL32,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61F00000020",
"clrl %r1, 64",
"C61F00000003",
"clrl %r1, label1",
));
insns.push((
Inst::CmpRX {
op: CmpOp::CmpL32Ext16,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61700000020",
"clhrl %r1, 64",
"C61700000003",
"clhrl %r1, label1",
));
insns.push((
Inst::CmpRX {
@@ -1756,22 +1756,22 @@ fn test_s390x_binemit() {
op: CmpOp::CmpL64,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61A00000020",
"clgrl %r1, 64",
"C61A00000003",
"clgrl %r1, label1",
));
insns.push((
Inst::CmpRX {
op: CmpOp::CmpL64Ext16,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61600000020",
"clghrl %r1, 64",
"C61600000003",
"clghrl %r1, label1",
));
insns.push((
Inst::CmpRX {
@@ -1792,11 +1792,11 @@ fn test_s390x_binemit() {
op: CmpOp::CmpL64Ext32,
rn: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C61E00000020",
"clgfrl %r1, 64",
"C61E00000003",
"clgfrl %r1, label1",
));
insns.push((
@@ -4433,81 +4433,81 @@ fn test_s390x_binemit() {
Inst::Load32 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41D00000020",
"lrl %r1, 64",
"C41D00000003",
"lrl %r1, label1",
));
insns.push((
Inst::Load32SExt16 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41500000020",
"lhrl %r1, 64",
"C41500000003",
"lhrl %r1, label1",
));
insns.push((
Inst::Load32ZExt16 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41200000020",
"llhrl %r1, 64",
"C41200000003",
"llhrl %r1, label1",
));
insns.push((
Inst::Load64 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41800000020",
"lgrl %r1, 64",
"C41800000003",
"lgrl %r1, label1",
));
insns.push((
Inst::Load64SExt16 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41400000020",
"lghrl %r1, 64",
"C41400000003",
"lghrl %r1, label1",
));
insns.push((
Inst::Load64ZExt16 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41600000020",
"llghrl %r1, 64",
"C41600000003",
"llghrl %r1, label1",
));
insns.push((
Inst::Load64SExt32 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41C00000020",
"lgfrl %r1, 64",
"C41C00000003",
"lgfrl %r1, label1",
));
insns.push((
Inst::Load64ZExt32 {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41E00000020",
"llgfrl %r1, 64",
"C41E00000003",
"llgfrl %r1, label1",
));
insns.push((
Inst::LoadRev16 {
@@ -5687,31 +5687,31 @@ fn test_s390x_binemit() {
Inst::Store16 {
rd: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41700000020",
"sthrl %r1, 64",
"C41700000003",
"sthrl %r1, label1",
));
insns.push((
Inst::Store32 {
rd: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41F00000020",
"strl %r1, 64",
"C41F00000003",
"strl %r1, label1",
));
insns.push((
Inst::Store64 {
rd: gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C41B00000020",
"stgrl %r1, 64",
"C41B00000003",
"stgrl %r1, label1",
));
insns.push((
@@ -5932,11 +5932,11 @@ fn test_s390x_binemit() {
Inst::LoadAddr {
rd: writable_gpr(1),
mem: MemArg::Label {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(1),
},
},
"C01000000020",
"larl %r1, 64",
"C01000000003",
"larl %r1, label1",
));
insns.push((
Inst::LoadAddr {
@@ -6396,250 +6396,250 @@ fn test_s390x_binemit() {
insns.push((
Inst::Jump {
dest: BranchTarget::ResolvedOffset(64),
dest: MachLabel::from_block(0),
},
"C0F400000020",
"jg 64",
"C0F400000000",
"jg label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(1),
},
"C01400000020",
"jgo 64",
"C01400000000",
"jgo label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(2),
},
"C02400000020",
"jgh 64",
"C02400000000",
"jgh label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(3),
},
"C03400000020",
"jgnle 64",
"C03400000000",
"jgnle label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(4),
},
"C04400000020",
"jgl 64",
"C04400000000",
"jgl label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(5),
},
"C05400000020",
"jgnhe 64",
"C05400000000",
"jgnhe label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(6),
},
"C06400000020",
"jglh 64",
"C06400000000",
"jglh label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(7),
},
"C07400000020",
"jgne 64",
"C07400000000",
"jgne label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(8),
},
"C08400000020",
"jge 64",
"C08400000000",
"jge label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(9),
},
"C09400000020",
"jgnlh 64",
"C09400000000",
"jgnlh label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(10),
},
"C0A400000020",
"jghe 64",
"C0A400000000",
"jghe label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(11),
},
"C0B400000020",
"jgnl 64",
"C0B400000000",
"jgnl label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(12),
},
"C0C400000020",
"jgle 64",
"C0C400000000",
"jgle label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(13),
},
"C0D400000020",
"jgnh 64",
"C0D400000000",
"jgnh label0",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
target: MachLabel::from_block(0),
cond: Cond::from_mask(14),
},
"C0E400000020",
"jgno 64",
"C0E400000000",
"jgno label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(1),
},
"C01400000020C0F400000040",
"jgo 64 ; jg 128",
"C01400000000C0F4FFFFFFFD",
"jgo label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(2),
},
"C02400000020C0F400000040",
"jgh 64 ; jg 128",
"C02400000000C0F4FFFFFFFD",
"jgh label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(3),
},
"C03400000020C0F400000040",
"jgnle 64 ; jg 128",
"C03400000000C0F4FFFFFFFD",
"jgnle label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(4),
},
"C04400000020C0F400000040",
"jgl 64 ; jg 128",
"C04400000000C0F4FFFFFFFD",
"jgl label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(5),
},
"C05400000020C0F400000040",
"jgnhe 64 ; jg 128",
"C05400000000C0F4FFFFFFFD",
"jgnhe label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(6),
},
"C06400000020C0F400000040",
"jglh 64 ; jg 128",
"C06400000000C0F4FFFFFFFD",
"jglh label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(7),
},
"C07400000020C0F400000040",
"jgne 64 ; jg 128",
"C07400000000C0F4FFFFFFFD",
"jgne label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(8),
},
"C08400000020C0F400000040",
"jge 64 ; jg 128",
"C08400000000C0F4FFFFFFFD",
"jge label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(9),
},
"C09400000020C0F400000040",
"jgnlh 64 ; jg 128",
"C09400000000C0F4FFFFFFFD",
"jgnlh label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(10),
},
"C0A400000020C0F400000040",
"jghe 64 ; jg 128",
"C0A400000000C0F4FFFFFFFD",
"jghe label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(11),
},
"C0B400000020C0F400000040",
"jgnl 64 ; jg 128",
"C0B400000000C0F4FFFFFFFD",
"jgnl label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(12),
},
"C0C400000020C0F400000040",
"jgle 64 ; jg 128",
"C0C400000000C0F4FFFFFFFD",
"jgle label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(13),
},
"C0D400000020C0F400000040",
"jgnh 64 ; jg 128",
"C0D400000000C0F4FFFFFFFD",
"jgnh label0 ; jg label0",
));
insns.push((
Inst::CondBr {
taken: BranchTarget::ResolvedOffset(64),
not_taken: BranchTarget::ResolvedOffset(128),
taken: MachLabel::from_block(0),
not_taken: MachLabel::from_block(0),
cond: Cond::from_mask(14),
},
"C0E400000020C0F400000040",
"jgno 64 ; jg 128",
"C0E400000000C0F4FFFFFFFD",
"jgno label0 ; jg label0",
));
insns.push((
@@ -8128,7 +8128,18 @@ fn test_s390x_binemit() {
assert_eq!(expected_printing, actual_printing);
let mut buffer = MachBuffer::new();
// Label 0 before the instruction.
let label0 = buffer.get_label();
buffer.bind_label(label0);
// Emit the instruction.
insn.emit(&mut buffer, &emit_info, &mut Default::default());
// Label 1 after the instruction.
let label1 = buffer.get_label();
buffer.bind_label(label1);
let buffer = buffer.finish();
let actual_encoding = &buffer.stringify_code_bytes();
assert_eq!(expected_encoding, actual_encoding);

97
cranelift/codegen/src/isa/s390x/inst/mod.rs
View File

@@ -58,15 +58,6 @@ pub struct CallIndInfo {
pub opcode: Opcode,
}
/// Additional information for JTSequence instructions, left out of line to lower the size of the Inst
/// enum.
#[derive(Clone, Debug)]
pub struct JTSequenceInfo {
pub default_target: BranchTarget,
pub targets: Vec<BranchTarget>,
pub targets_for_term: Vec<MachLabel>, // needed for MachTerminator.
}
#[test]
fn inst_size_test() {
// This test will help with unintentionally growing the size
@@ -686,12 +677,8 @@ fn s390x_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
&Inst::Debugtrap => {}
&Inst::Trap { .. } => {}
&Inst::TrapIf { .. } => {}
&Inst::JTSequence {
ridx, rtmp1, rtmp2, ..
} => {
&Inst::JTSequence { ridx, .. } => {
collector.add_use(ridx);
collector.add_def(rtmp1);
collector.add_def(rtmp2);
}
&Inst::LoadExtNameFar { rd, .. } => {
collector.add_def(rd);
@@ -1408,15 +1395,8 @@ pub fn s390x_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
&mut Inst::CondBr { .. } | &mut Inst::OneWayCondBr { .. } => {}
&mut Inst::Debugtrap | &mut Inst::Trap { .. } | &mut Inst::TrapIf { .. } => {}
&mut Inst::Nop0 | &mut Inst::Nop2 => {}
&mut Inst::JTSequence {
ref mut ridx,
ref mut rtmp1,
ref mut rtmp2,
..
} => {
&mut Inst::JTSequence { ref mut ridx, .. } => {
mapper.map_use(ridx);
mapper.map_def(rtmp1);
mapper.map_def(rtmp2);
}
&mut Inst::LoadExtNameFar { ref mut rd, .. } => {
mapper.map_def(rd);
@@ -1471,18 +1451,16 @@ impl MachInst for Inst {
fn is_term<'a>(&'a self) -> MachTerminator<'a> {
match self {
&Inst::Ret { .. } | &Inst::EpiloguePlaceholder => MachTerminator::Ret,
&Inst::Jump { dest } => MachTerminator::Uncond(dest.as_label().unwrap()),
&Inst::Jump { dest } => MachTerminator::Uncond(dest),
&Inst::CondBr {
taken, not_taken, ..
} => MachTerminator::Cond(taken.as_label().unwrap(), not_taken.as_label().unwrap()),
} => MachTerminator::Cond(taken, not_taken),
&Inst::OneWayCondBr { .. } => {
// Explicitly invisible to CFG processing.
MachTerminator::None
}
&Inst::IndirectBr { ref targets, .. } => MachTerminator::Indirect(&targets[..]),
&Inst::JTSequence { ref info, .. } => {
MachTerminator::Indirect(&info.targets_for_term[..])
}
&Inst::JTSequence { ref targets, .. } => MachTerminator::Indirect(&targets[..]),
_ => MachTerminator::None,
}
}
@@ -1589,9 +1567,7 @@ impl MachInst for Inst {
}
fn gen_jump(target: MachLabel) -> Inst {
Inst::Jump {
dest: BranchTarget::Label(target),
}
Inst::Jump { dest: target }
}
fn worst_case_size() -> CodeOffset {
@@ -2592,8 +2568,8 @@ impl Inst {
format!("br {}", link)
}
&Inst::EpiloguePlaceholder => "epilogue placeholder".to_string(),
&Inst::Jump { ref dest } => {
let dest = dest.show_rru(mb_rru);
&Inst::Jump { dest } => {
let dest = dest.to_string();
format!("jg {}", dest)
}
&Inst::IndirectBr { rn, .. } => {
@@ -2601,17 +2577,17 @@ impl Inst {
format!("br {}", rn)
}
&Inst::CondBr {
ref taken,
ref not_taken,
taken,
not_taken,
cond,
} => {
let taken = taken.show_rru(mb_rru);
let not_taken = not_taken.show_rru(mb_rru);
let taken = taken.to_string();
let not_taken = not_taken.to_string();
let cond = cond.show_rru(mb_rru);
format!("jg{} {} ; jg {}", cond, taken, not_taken)
}
&Inst::OneWayCondBr { ref target, cond } => {
let target = target.show_rru(mb_rru);
&Inst::OneWayCondBr { target, cond } => {
let target = target.to_string();
let cond = cond.show_rru(mb_rru);
format!("jg{} {}", cond, target)
}
@@ -2621,42 +2597,25 @@ impl Inst {
let cond = cond.invert().show_rru(mb_rru);
format!("j{} 6 ; trap", cond)
}
&Inst::JTSequence {
ref info,
ridx,
rtmp1,
rtmp2,
..
} => {
&Inst::JTSequence { ridx, ref targets } => {
let ridx = ridx.show_rru(mb_rru);
let rtmp1 = rtmp1.show_rru(mb_rru);
let rtmp2 = rtmp2.show_rru(mb_rru);
let default_target = info.default_target.show_rru(mb_rru);
let rtmp = writable_spilltmp_reg().to_reg().show_rru(mb_rru);
// The first entry is the default target, which is not emitted
// into the jump table, so we skip it here. It is only in the
// list so MachTerminator will see the potential target.
let jt_entries: String = targets
.iter()
.skip(1)
.map(|label| format!(" {}", label.to_string()))
.collect();
format!(
concat!(
"clgfi {}, {} ; ",
"jghe {} ; ",
"sllg {}, {}, 2 ; ",
"larl {}, 18 ; ",
"lgf {}, 0({}, {}) ; ",
"agrk {}, {}, {} ; ",
"larl {}, 14 ; ",
"agf {}, 0({}, {}) ; ",
"br {} ; ",
"jt_entries {:?}"
"jt_entries{}"
),
ridx,
info.targets.len(),
default_target,
rtmp2,
ridx,
rtmp1,
rtmp2,
rtmp2,
rtmp1,
rtmp1,
rtmp1,
rtmp2,
rtmp1,
info.targets
rtmp, rtmp, rtmp, ridx, rtmp, jt_entries,
)
}
&Inst::LoadExtNameFar {

62
cranelift/codegen/src/isa/s390x/lower.rs
View File

@@ -12,7 +12,6 @@ use crate::machinst::*;
use crate::settings::Flags;
use crate::CodegenResult;
use alloc::boxed::Box;
use alloc::vec::Vec;
use core::convert::TryFrom;
use regalloc::{Reg, Writable};
use smallvec::SmallVec;
@@ -1018,8 +1017,8 @@ fn lower_branch<C: LowerCtx<I = Inst>>(
let op1 = ctx.data(branches[1]).opcode();
assert!(op1 == Opcode::Jump);
let taken = BranchTarget::Label(targets[0]);
let not_taken = BranchTarget::Label(targets[1]);
let taken = targets[0];
let not_taken = targets[1];
match op0 {
Opcode::Brz | Opcode::Brnz => {
@@ -1068,9 +1067,7 @@ fn lower_branch<C: LowerCtx<I = Inst>>(
match op {
Opcode::Jump => {
assert!(branches.len() == 1);
ctx.emit(Inst::Jump {
dest: BranchTarget::Label(targets[0]),
});
ctx.emit(Inst::Jump { dest: targets[0] });
}
Opcode::BrTable => {
@@ -1087,19 +1084,35 @@ fn lower_branch<C: LowerCtx<I = Inst>>(
NarrowValueMode::ZeroExtend64,
);
// Temp registers needed by the compound instruction.
let rtmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let rtmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
// Bounds-check index and branch to default.
// This is an internal branch that is not a terminator insn.
// Instead, the default target is listed a potential target
// in the final JTSequence, which is the block terminator.
ctx.emit(Inst::CmpRUImm32 {
op: CmpOp::CmpL64,
rn: ridx,
imm: jt_size as u32,
});
ctx.emit(Inst::OneWayCondBr {
target: targets[0],
cond: Cond::from_intcc(IntCC::UnsignedGreaterThanOrEqual),
});
// Compute index scaled by entry size.
let rtmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
ctx.emit(Inst::ShiftRR {
shift_op: ShiftOp::LShL64,
rd: rtmp,
rn: ridx,
shift_imm: 2,
shift_reg: zero_reg(),
});
// Emit the compound instruction that does:
//
// clgfi %rIdx, <jt-size>
// jghe <default-target>
// sllg %rTmp2, %rIdx, 2
// larl %rTmp1, <jt-base>
// lgf %rTmp2, 0(%rTmp2, %rTmp1)
// agrk %rTmp1, %rTmp1, %rTmp2
// br %rA
// larl %r1, <jt-base>
// agf %r1, 0(%r1, %rTmp)
// br %r1
// [jt entries]
//
// This must be *one* instruction in the vcode because
@@ -1109,22 +1122,9 @@ fn lower_branch<C: LowerCtx<I = Inst>>(
// (The alternative is to introduce a relocation pass
// for inlined jumptables, which is much worse, IMHO.)
let default_target = BranchTarget::Label(targets[0]);
let jt_targets: Vec<BranchTarget> = targets
.iter()
.skip(1)
.map(|bix| BranchTarget::Label(*bix))
.collect();
let targets_for_term: Vec<MachLabel> = targets.to_vec();
ctx.emit(Inst::JTSequence {
ridx,
rtmp1,
rtmp2,
info: Box::new(JTSequenceInfo {
default_target,
targets: jt_targets,
targets_for_term,
}),
ridx: rtmp.to_reg(),
targets: targets.to_vec(),
});
}

5
cranelift/codegen/src/isa/s390x/lower/isle.rs
View File

@@ -5,8 +5,8 @@ pub mod generated_code;
// Types that the generated ISLE code uses via `use super::*`.
use super::{
BranchTarget, CallIndInfo, CallInfo, Cond, Inst as MInst, JTSequenceInfo, MachLabel, MemArg,
MemFlags, Opcode, Reg, UImm16Shifted, UImm32Shifted,
CallIndInfo, CallInfo, Cond, Inst as MInst, MachLabel, MemArg, MemFlags, Opcode, Reg,
UImm16Shifted, UImm32Shifted,
};
use crate::isa::s390x::settings::Flags as IsaFlags;
use crate::machinst::isle::*;
@@ -27,7 +27,6 @@ use std::vec::Vec;
type BoxCallInfo = Box<CallInfo>;
type BoxCallIndInfo = Box<CallIndInfo>;
type VecMachLabel = Vec<MachLabel>;
type BoxJTSequenceInfo = Box<JTSequenceInfo>;
type BoxExternalName = Box<ExternalName>;
/// The main entry point for lowering with ISLE.

2
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle 51d2aef2566c1c96
src/isa/s390x/inst.isle 63cf833b5cfd727d
src/isa/s390x/inst.isle 17b77476355c4509
src/isa/s390x/lower.isle a0e21a567040bc33

884
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

8
cranelift/filetests/filetests/isa/s390x/jumptable.clif
View File

@@ -28,14 +28,20 @@ block5(v5: i64):
return v6
}
; check: clgfi %r2, 3 ; jghe label1 ; sllg %r4, %r2, 2 ; larl %r3, 18 ; lgf %r4, 0(%r4, %r3) ; agrk %r3, %r3, %r4 ; br %r3 ; jt_entries
; check: clgfi %r2, 3
; nextln: jghe label1
; nextln: sllg %r3, %r2, 2
; nextln: larl %r1, 14 ; agf %r1, 0(%r1, %r3) ; br %r1 ; jt_entries label3 label5 label7
; check: Block 3
; check: lghi %r3, 1
; nextln: jg
; check: Block 5
; check: lghi %r3, 2
; nextln: jg
; check: Block 7
; check: lghi %r3, 3
; nextln: jg