T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1962 from cfallin/aarch64-lowering-condbr

Browse Source 
AArch64: avoid branches with explicit offsets at lowering stage.
This commit is contained in:
Chris Fallin
2020-07-02 14:05:40 -07:00
committed by GitHub
parent bffd54c016 b7ecad1d74
commit b700646c93
11 changed files with 267 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -1490,12 +1490,20 @@ impl MachInstEmit for Inst {
}
sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
}
&Inst::OneWayCondBr { target, kind } => {
&Inst::TrapIf { kind, trap_info } => {
// condbr KIND, LABEL
let off = sink.cur_offset();
if let Some(l) = target.as_label() {
sink.use_label_at_offset(off, l, LabelUse::Branch19);
}
sink.put4(enc_conditional_br(target, kind));
let label = sink.get_label();
sink.put4(enc_conditional_br(
BranchTarget::Label(label),
kind.invert(),
));
sink.use_label_at_offset(off, label, LabelUse::Branch19);
// udf
let trap = Inst::Udf { trap_info };
trap.emit(sink, flags, state);
// LABEL:
sink.bind_label(label);
}
&Inst::IndirectBr { rn, .. } => {
sink.put4(enc_br(rn));
@@ -1534,6 +1542,17 @@ impl MachInstEmit for Inst {
// emission time, because we cannot allow the regalloc to insert spills/reloads in
// the middle; we depend on hardcoded PC-rel addressing below.
// Branch to default when condition code from prior comparison indicates.
let br = enc_conditional_br(info.default_target, CondBrKind::Cond(Cond::Hs));
// No need to inform the sink's branch folding logic about this branch, because it
// will not be merged with any other branch, flipped, or elided (it is not preceded
// or succeeded by any other branch). Just emit it with the label use.
let default_br_offset = sink.cur_offset();
if let BranchTarget::Label(l) = info.default_target {
sink.use_label_at_offset(default_br_offset, l, LabelUse::Branch19);
}
sink.put4(br);
// Save index in a tmp (the live range of ridx only goes to start of this
// sequence; rtmp1 or rtmp2 may overwrite it).
let inst = Inst::gen_move(rtmp2, ridx, I64);

169
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -2681,149 +2681,148 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Zero(xreg(8)),
},
"080200B4",
"cbz x8, 64",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::NotZero(xreg(8)),
},
"080200B5",
"cbnz x8, 64",
"480000B40000A0D4",
"cbz x8, 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Eq),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Zero(xreg(8)),
},
"00020054",
"b.eq 64",
"480000B50000A0D4",
"cbnz x8, 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Ne),
},
"01020054",
"b.ne 64",
"400000540000A0D4",
"b.eq 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Hs),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Eq),
},
"02020054",
"b.hs 64",
"410000540000A0D4",
"b.ne 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Lo),
},
"03020054",
"b.lo 64",
"420000540000A0D4",
"b.hs 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Mi),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Hs),
},
"04020054",
"b.mi 64",
"430000540000A0D4",
"b.lo 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Pl),
},
"05020054",
"b.pl 64",
"440000540000A0D4",
"b.mi 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Vs),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Mi),
},
"06020054",
"b.vs 64",
"450000540000A0D4",
"b.pl 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Vc),
},
"07020054",
"b.vc 64",
"460000540000A0D4",
"b.vs 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Hi),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Vs),
},
"08020054",
"b.hi 64",
"470000540000A0D4",
"b.vc 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Ls),
},
"09020054",
"b.ls 64",
"480000540000A0D4",
"b.hi 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Ge),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Hi),
},
"0A020054",
"b.ge 64",
"490000540000A0D4",
"b.ls 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Lt),
},
"0B020054",
"b.lt 64",
"4A0000540000A0D4",
"b.ge 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Gt),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Ge),
},
"0C020054",
"b.gt 64",
"4B0000540000A0D4",
"b.lt 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Le),
},
"0D020054",
"b.le 64",
"4C0000540000A0D4",
"b.gt 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Al),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Gt),
},
"0E020054",
"b.al 64",
"4D0000540000A0D4",
"b.le 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Nv),
},
"0F020054",
"b.nv 64",
"4E0000540000A0D4",
"b.al 8 ; udf",
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Al),
},
"4F0000540000A0D4",
"b.nv 8 ; udf",
));
insns.push((

86
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -342,6 +342,7 @@ pub struct CallIndInfo {
#[derive(Clone, Debug)]
pub struct JTSequenceInfo {
pub targets: Vec<BranchTarget>,
pub default_target: BranchTarget,
pub targets_for_term: Vec<MachLabel>, // needed for MachTerminator.
}
@@ -825,20 +826,17 @@ pub enum Inst {
kind: CondBrKind,
},
/// A one-way conditional branch, invisible to the CFG processing; used *only* as part of
/// straight-line sequences in code to be emitted.
/// A conditional trap: execute a `udf` if the condition is true. This is
/// one VCode instruction because it uses embedded control flow; it is
/// logically a single-in, single-out region, but needs to appear as one
/// unit to the register allocator.
///
/// In more detail:
/// - This branch is lowered to a branch at the machine-code level, but does not end a basic
/// block, and does not create edges in the CFG seen by regalloc.
/// - Thus, it is *only* valid to use as part of a single-in, single-out sequence that is
/// lowered from a single CLIF instruction. For example, certain arithmetic operations may
/// use these branches to handle certain conditions, such as overflows, traps, etc.
///
/// See, e.g., the lowering of `trapif` (conditional trap) for an example.
OneWayCondBr {
target: BranchTarget,
/// The `CondBrKind` gives the conditional-branch condition that will
/// *execute* the embedded `Inst`. (In the emitted code, we use the inverse
/// of this condition in a branch that skips the trap instruction.)
TrapIf {
kind: CondBrKind,
trap_info: (SourceLoc, TrapCode),
},
/// An indirect branch through a register, augmented with set of all
@@ -1354,7 +1352,7 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_defs(&*info.defs);
collector.add_use(info.rn);
}
&Inst::CondBr { ref kind, .. } | &Inst::OneWayCondBr { ref kind, .. } => match kind {
&Inst::CondBr { ref kind, .. } => match kind {
CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => {
collector.add_use(*rt);
}
@@ -1366,6 +1364,12 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
&Inst::Nop0 | Inst::Nop4 => {}
&Inst::Brk => {}
&Inst::Udf { .. } => {}
&Inst::TrapIf { ref kind, .. } => match kind {
CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => {
collector.add_use(*rt);
}
CondBrKind::Cond(_) => {}
},
&Inst::Adr { rd, .. } => {
collector.add_def(rd);
}
@@ -1957,13 +1961,16 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
}
map_use(mapper, &mut info.rn);
}
&mut Inst::CondBr { ref mut kind, .. } | &mut Inst::OneWayCondBr { ref mut kind, .. } => {
&mut Inst::CondBr { ref mut kind, .. } => {
map_br(mapper, kind);
}
&mut Inst::IndirectBr { ref mut rn, .. } => {
map_use(mapper, rn);
}
&mut Inst::Nop0 | &mut Inst::Nop4 | &mut Inst::Brk | &mut Inst::Udf { .. } => {}
&mut Inst::TrapIf { ref mut kind, .. } => {
map_br(mapper, kind);
}
&mut Inst::Adr { ref mut rd, .. } => {
map_def(mapper, rd);
}
@@ -2034,10 +2041,6 @@ impl MachInst for Inst {
&Inst::CondBr {
taken, not_taken, ..
} => MachTerminator::Cond(taken.as_label().unwrap(), not_taken.as_label().unwrap()),
&Inst::OneWayCondBr { .. } => {
// Explicitly invisible to CFG processing.
MachTerminator::None
}
&Inst::IndirectBr { ref targets, .. } => MachTerminator::Indirect(&targets[..]),
&Inst::JTSequence { ref info, .. } => {
MachTerminator::Indirect(&info.targets_for_term[..])
@@ -2887,32 +2890,26 @@ impl ShowWithRRU for Inst {
}
}
}
&Inst::OneWayCondBr {
ref target,
ref kind,
} => {
let target = target.show_rru(mb_rru);
match kind {
&CondBrKind::Zero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbz {}, {}", reg, target)
}
&CondBrKind::NotZero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbnz {}, {}", reg, target)
}
&CondBrKind::Cond(c) => {
let c = c.show_rru(mb_rru);
format!("b.{} {}", c, target)
}
}
}
&Inst::IndirectBr { rn, .. } => {
let rn = rn.show_rru(mb_rru);
format!("br {}", rn)
}
&Inst::Brk => "brk #0".to_string(),
&Inst::Udf { .. } => "udf".to_string(),
&Inst::TrapIf { ref kind, .. } => match kind {
&CondBrKind::Zero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbnz {}, 8 ; udf", reg)
}
&CondBrKind::NotZero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbz {}, 8 ; udf", reg)
}
&CondBrKind::Cond(c) => {
let c = c.invert().show_rru(mb_rru);
format!("b.{} 8 ; udf", c)
}
},
&Inst::Adr { rd, off } => {
let rd = rd.show_rru(mb_rru);
format!("adr {}, pc+{}", rd, off)
@@ -2929,15 +2926,26 @@ impl ShowWithRRU for Inst {
let ridx = ridx.show_rru(mb_rru);
let rtmp1 = rtmp1.show_rru(mb_rru);
let rtmp2 = rtmp2.show_rru(mb_rru);
let default_target = info.default_target.show_rru(mb_rru);
format!(
concat!(
"b.hs {} ; ",
"adr {}, pc+16 ; ",
"ldrsw {}, [{}, {}, LSL 2] ; ",
"add {}, {}, {} ; ",
"br {} ; ",
"jt_entries {:?}"
),
rtmp1, rtmp2, rtmp1, ridx, rtmp1, rtmp1, rtmp2, rtmp1, info.targets
default_target,
rtmp1,
rtmp2,
rtmp1,
ridx,
rtmp1,
rtmp1,
rtmp2,
rtmp1,
info.targets
)
}
&Inst::LoadConst64 { rd, const_data } => {