T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

AArch64: avoid branches with explicit offsets at lowering stage.

Browse Source 
In discussions with @bnjbvr, it came up that generating `OneWayCondBr`s
with explicit, hardcoded PC-offsets as part of lowered instruction
sequences is actually unsafe, because the register allocator *might*
insert a spill or reload into the middle of our sequence. We were
careful about this in some cases but somehow missed that it was a
general restriction. Conceptually, all inter-instruction references
should be via labels at the VCode level; explicit offsets are only ever
known at emission time, and resolved by the `MachBuffer`.

To allow for conditional trap checks without modifying the CFG (as seen
by regalloc) during lowering, this PR instead adds a `TrapIf`
pseudo-instruction that conditionally skips a single embedded trap
instruction. It lowers to the same `condbr label ; trap ; label: ...`
sequence, but without the hardcoded branch-target offset in the lowering
code.
This commit is contained in:
Chris Fallin
2020-07-01 16:28:41 -07:00
parent f2dd1535d5
commit b7ecad1d74
11 changed files with 267 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -631,14 +631,11 @@ impl AArch64ABIBody {
rn: stack_reg(),
rm: stack_limit,
});
insts.push(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(8),
// Here `Hs` == "higher or same" when interpreting the two
// operands as unsigned integers.
kind: CondBrKind::Cond(Cond::Hs),
});
insts.push(Inst::Udf {
insts.push(Inst::TrapIf {
trap_info: (ir::SourceLoc::default(), ir::TrapCode::StackOverflow),
// Here `Lo` == "less than" when interpreting the two
// operands as unsigned integers.
kind: CondBrKind::Cond(Cond::Lo),
});
}
}

29
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -1471,12 +1471,20 @@ impl MachInstEmit for Inst {
}
sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
}
&Inst::OneWayCondBr { target, kind } => {
&Inst::TrapIf { kind, trap_info } => {
// condbr KIND, LABEL
let off = sink.cur_offset();
if let Some(l) = target.as_label() {
sink.use_label_at_offset(off, l, LabelUse::Branch19);
}
sink.put4(enc_conditional_br(target, kind));
let label = sink.get_label();
sink.put4(enc_conditional_br(
BranchTarget::Label(label),
kind.invert(),
));
sink.use_label_at_offset(off, label, LabelUse::Branch19);
// udf
let trap = Inst::Udf { trap_info };
trap.emit(sink, flags, state);
// LABEL:
sink.bind_label(label);
}
&Inst::IndirectBr { rn, .. } => {
sink.put4(enc_br(rn));
@@ -1515,6 +1523,17 @@ impl MachInstEmit for Inst {
// emission time, because we cannot allow the regalloc to insert spills/reloads in
// the middle; we depend on hardcoded PC-rel addressing below.
// Branch to default when condition code from prior comparison indicates.
let br = enc_conditional_br(info.default_target, CondBrKind::Cond(Cond::Hs));
// No need to inform the sink's branch folding logic about this branch, because it
// will not be merged with any other branch, flipped, or elided (it is not preceded
// or succeeded by any other branch). Just emit it with the label use.
let default_br_offset = sink.cur_offset();
if let BranchTarget::Label(l) = info.default_target {
sink.use_label_at_offset(default_br_offset, l, LabelUse::Branch19);
}
sink.put4(br);
// Save index in a tmp (the live range of ridx only goes to start of this
// sequence; rtmp1 or rtmp2 may overwrite it).
let inst = Inst::gen_move(rtmp2, ridx, I64);

169
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -2505,149 +2505,148 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Zero(xreg(8)),
},
"080200B4",
"cbz x8, 64",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::NotZero(xreg(8)),
},
"080200B5",
"cbnz x8, 64",
"480000B40000A0D4",
"cbz x8, 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Eq),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Zero(xreg(8)),
},
"00020054",
"b.eq 64",
"480000B50000A0D4",
"cbnz x8, 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Ne),
},
"01020054",
"b.ne 64",
"400000540000A0D4",
"b.eq 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Hs),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Eq),
},
"02020054",
"b.hs 64",
"410000540000A0D4",
"b.ne 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Lo),
},
"03020054",
"b.lo 64",
"420000540000A0D4",
"b.hs 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Mi),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Hs),
},
"04020054",
"b.mi 64",
"430000540000A0D4",
"b.lo 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Pl),
},
"05020054",
"b.pl 64",
"440000540000A0D4",
"b.mi 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Vs),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Mi),
},
"06020054",
"b.vs 64",
"450000540000A0D4",
"b.pl 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Vc),
},
"07020054",
"b.vc 64",
"460000540000A0D4",
"b.vs 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Hi),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Vs),
},
"08020054",
"b.hi 64",
"470000540000A0D4",
"b.vc 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Ls),
},
"09020054",
"b.ls 64",
"480000540000A0D4",
"b.hi 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Ge),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Hi),
},
"0A020054",
"b.ge 64",
"490000540000A0D4",
"b.ls 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Lt),
},
"0B020054",
"b.lt 64",
"4A0000540000A0D4",
"b.ge 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Gt),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Ge),
},
"0C020054",
"b.gt 64",
"4B0000540000A0D4",
"b.lt 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Le),
},
"0D020054",
"b.le 64",
"4C0000540000A0D4",
"b.gt 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
kind: CondBrKind::Cond(Cond::Al),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Gt),
},
"0E020054",
"b.al 64",
"4D0000540000A0D4",
"b.le 8 ; udf",
));
insns.push((
Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(64),
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Nv),
},
"0F020054",
"b.nv 64",
"4E0000540000A0D4",
"b.al 8 ; udf",
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
kind: CondBrKind::Cond(Cond::Al),
},
"4F0000540000A0D4",
"b.nv 8 ; udf",
));
insns.push((

86
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -334,6 +334,7 @@ pub struct CallIndInfo {
#[derive(Clone, Debug)]
pub struct JTSequenceInfo {
pub targets: Vec<BranchTarget>,
pub default_target: BranchTarget,
pub targets_for_term: Vec<MachLabel>, // needed for MachTerminator.
}
@@ -817,20 +818,17 @@ pub enum Inst {
kind: CondBrKind,
},
/// A one-way conditional branch, invisible to the CFG processing; used *only* as part of
/// straight-line sequences in code to be emitted.
/// A conditional trap: execute a `udf` if the condition is true. This is
/// one VCode instruction because it uses embedded control flow; it is
/// logically a single-in, single-out region, but needs to appear as one
/// unit to the register allocator.
///
/// In more detail:
/// - This branch is lowered to a branch at the machine-code level, but does not end a basic
/// block, and does not create edges in the CFG seen by regalloc.
/// - Thus, it is *only* valid to use as part of a single-in, single-out sequence that is
/// lowered from a single CLIF instruction. For example, certain arithmetic operations may
/// use these branches to handle certain conditions, such as overflows, traps, etc.
///
/// See, e.g., the lowering of `trapif` (conditional trap) for an example.
OneWayCondBr {
target: BranchTarget,
/// The `CondBrKind` gives the conditional-branch condition that will
/// *execute* the embedded `Inst`. (In the emitted code, we use the inverse
/// of this condition in a branch that skips the trap instruction.)
TrapIf {
kind: CondBrKind,
trap_info: (SourceLoc, TrapCode),
},
/// An indirect branch through a register, augmented with set of all
@@ -1346,7 +1344,7 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_defs(&*info.defs);
collector.add_use(info.rn);
}
&Inst::CondBr { ref kind, .. } | &Inst::OneWayCondBr { ref kind, .. } => match kind {
&Inst::CondBr { ref kind, .. } => match kind {
CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => {
collector.add_use(*rt);
}
@@ -1358,6 +1356,12 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
&Inst::Nop0 | Inst::Nop4 => {}
&Inst::Brk => {}
&Inst::Udf { .. } => {}
&Inst::TrapIf { ref kind, .. } => match kind {
CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => {
collector.add_use(*rt);
}
CondBrKind::Cond(_) => {}
},
&Inst::Adr { rd, .. } => {
collector.add_def(rd);
}
@@ -1949,13 +1953,16 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
}
map_use(mapper, &mut info.rn);
}
&mut Inst::CondBr { ref mut kind, .. } | &mut Inst::OneWayCondBr { ref mut kind, .. } => {
&mut Inst::CondBr { ref mut kind, .. } => {
map_br(mapper, kind);
}
&mut Inst::IndirectBr { ref mut rn, .. } => {
map_use(mapper, rn);
}
&mut Inst::Nop0 | &mut Inst::Nop4 | &mut Inst::Brk | &mut Inst::Udf { .. } => {}
&mut Inst::TrapIf { ref mut kind, .. } => {
map_br(mapper, kind);
}
&mut Inst::Adr { ref mut rd, .. } => {
map_def(mapper, rd);
}
@@ -2026,10 +2033,6 @@ impl MachInst for Inst {
&Inst::CondBr {
taken, not_taken, ..
} => MachTerminator::Cond(taken.as_label().unwrap(), not_taken.as_label().unwrap()),
&Inst::OneWayCondBr { .. } => {
// Explicitly invisible to CFG processing.
MachTerminator::None
}
&Inst::IndirectBr { ref targets, .. } => MachTerminator::Indirect(&targets[..]),
&Inst::JTSequence { ref info, .. } => {
MachTerminator::Indirect(&info.targets_for_term[..])
@@ -2880,32 +2883,26 @@ impl ShowWithRRU for Inst {
}
}
}
&Inst::OneWayCondBr {
ref target,
ref kind,
} => {
let target = target.show_rru(mb_rru);
match kind {
&CondBrKind::Zero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbz {}, {}", reg, target)
}
&CondBrKind::NotZero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbnz {}, {}", reg, target)
}
&CondBrKind::Cond(c) => {
let c = c.show_rru(mb_rru);
format!("b.{} {}", c, target)
}
}
}
&Inst::IndirectBr { rn, .. } => {
let rn = rn.show_rru(mb_rru);
format!("br {}", rn)
}
&Inst::Brk => "brk #0".to_string(),
&Inst::Udf { .. } => "udf".to_string(),
&Inst::TrapIf { ref kind, .. } => match kind {
&CondBrKind::Zero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbnz {}, 8 ; udf", reg)
}
&CondBrKind::NotZero(reg) => {
let reg = reg.show_rru(mb_rru);
format!("cbz {}, 8 ; udf", reg)
}
&CondBrKind::Cond(c) => {
let c = c.invert().show_rru(mb_rru);
format!("b.{} 8 ; udf", c)
}
},
&Inst::Adr { rd, off } => {
let rd = rd.show_rru(mb_rru);
format!("adr {}, pc+{}", rd, off)
@@ -2922,15 +2919,26 @@ impl ShowWithRRU for Inst {
let ridx = ridx.show_rru(mb_rru);
let rtmp1 = rtmp1.show_rru(mb_rru);
let rtmp2 = rtmp2.show_rru(mb_rru);
let default_target = info.default_target.show_rru(mb_rru);
format!(
concat!(
"b.hs {} ; ",
"adr {}, pc+16 ; ",
"ldrsw {}, [{}, {}, LSL 2] ; ",
"add {}, {}, {} ; ",
"br {} ; ",
"jt_entries {:?}"
),
rtmp1, rtmp2, rtmp1, ridx, rtmp1, rtmp1, rtmp2, rtmp1, info.targets
default_target,
rtmp1,
rtmp2,
rtmp1,
ridx,
rtmp1,
rtmp1,
rtmp2,
rtmp1,
info.targets
)
}
&Inst::LoadConst64 { rd, const_data } => {

109
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -282,14 +282,11 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// msub rd, rd, rm, rn ; rd = rn - rd * rm
// Check for divide by 0.
let branch_size = 8;
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(branch_size),
kind: CondBrKind::NotZero(rm),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero);
ctx.emit(Inst::Udf { trap_info });
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Zero(rm),
});
ctx.emit(Inst::AluRRRR {
alu_op: ALUOp::MSub64,
@@ -300,17 +297,17 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
});
} else {
if div_op == ALUOp::SDiv64 {
// cbz rm, #20
// cbnz rm, #8
// udf ; divide by zero
// cmn rm, 1
// ccmp rn, 1, #nzcv, eq
// b.vc 12
// b.vc #8
// udf ; signed overflow
// udf ; divide by zero
// Check for divide by 0.
let branch_size = 20;
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(branch_size),
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero);
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Zero(rm),
});
@@ -336,27 +333,22 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
nzcv: NZCV::new(false, false, false, false),
cond: Cond::Eq,
});
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(12),
kind: CondBrKind::Cond(Cond::Vc),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
ctx.emit(Inst::Udf { trap_info });
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Cond(Cond::Vs),
});
} else {
// cbnz rm, #8
// udf ; divide by zero
// Check for divide by 0.
let branch_size = 8;
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(branch_size),
kind: CondBrKind::NotZero(rm),
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero);
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Zero(rm),
});
}
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero);
ctx.emit(Inst::Udf { trap_info });
}
}
@@ -1324,15 +1316,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
cond
};
// Branch around the break instruction with inverted cond. Go straight to lowered
// one-target form; this is logically part of a single-in single-out template lowering.
let cond = cond.invert();
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(8),
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Cond(cond),
});
ctx.emit(Inst::Udf { trap_info })
}
Opcode::Safepoint => {
@@ -1711,12 +1698,11 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else {
ctx.emit(Inst::FpuCmp64 { rn, rm: rn });
}
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(8),
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Ordered)),
});
let trap_info = (ctx.srcloc(insn), TrapCode::BadConversionToInteger);
ctx.emit(Inst::Udf { trap_info });
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Unordered)),
});
let tmp = ctx.alloc_tmp(RegClass::V128, I128);
@@ -1752,12 +1738,11 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(8),
kind: CondBrKind::Cond(lower_fp_condcode(low_cond)),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
ctx.emit(Inst::Udf { trap_info });
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()),
});
// <= high_bound
lower_constant_f32(ctx, tmp, high_bound);
@@ -1765,12 +1750,11 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(8),
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan)),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
ctx.emit(Inst::Udf { trap_info });
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()),
});
} else {
// From float64.
let (low_bound, low_cond, high_bound) = match (signed, out_bits) {
@@ -1795,12 +1779,11 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(8),
kind: CondBrKind::Cond(lower_fp_condcode(low_cond)),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
ctx.emit(Inst::Udf { trap_info });
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()),
});
// <= high_bound
lower_constant_f64(ctx, tmp, high_bound);
@@ -1808,12 +1791,11 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
ctx.emit(Inst::OneWayCondBr {
target: BranchTarget::ResolvedOffset(8),
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan)),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
ctx.emit(Inst::Udf { trap_info });
ctx.emit(Inst::TrapIf {
trap_info,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()),
});
};
// Do the conversion.
@@ -2307,7 +2289,8 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
let rtmp1 = ctx.alloc_tmp(RegClass::I64, I32);
let rtmp2 = ctx.alloc_tmp(RegClass::I64, I32);
// Bounds-check and branch to default.
// Bounds-check, leaving condition codes for JTSequence's
// branch to default target below.
if let Some(imm12) = Imm12::maybe_from_u64(jt_size as u64) {
ctx.emit(Inst::AluRRImm12 {
alu_op: ALUOp::SubS32,
@@ -2324,14 +2307,10 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
rm: rtmp1.to_reg(),
});
}
let default_target = BranchTarget::Label(targets[0]);
ctx.emit(Inst::OneWayCondBr {
target: default_target.clone(),
kind: CondBrKind::Cond(Cond::Hs), // unsigned >=
});
// Emit the compound instruction that does:
//
// b.hs default
// adr rA, jt
// ldrsw rB, [rA, rIndex, UXTW 2]
// add rA, rA, rB
@@ -2350,6 +2329,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
.skip(1)
.map(|bix| BranchTarget::Label(*bix))
.collect();
let default_target = BranchTarget::Label(targets[0]);
let targets_for_term: Vec<MachLabel> = targets.to_vec();
ctx.emit(Inst::JTSequence {
ridx,
@@ -2357,6 +2337,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
rtmp2,
info: Box::new(JTSequenceInfo {
targets: jt_targets,
default_target,
targets_for_term: targets_for_term,
}),
});

11
cranelift/codegen/src/machinst/buffer.rs
View File

@@ -1390,7 +1390,9 @@ mod test {
inst.emit(&mut buf, &flags, &mut state);
buf.bind_label(label(1));
let inst = Inst::Nop4;
let inst = Inst::Udf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
};
inst.emit(&mut buf, &flags, &mut state);
buf.bind_label(label(2));
@@ -1403,14 +1405,13 @@ mod test {
let mut buf2 = MachBuffer::new();
let mut state = Default::default();
let inst = Inst::OneWayCondBr {
kind: CondBrKind::Zero(xreg(0)),
target: BranchTarget::ResolvedOffset(8),
let inst = Inst::TrapIf {
kind: CondBrKind::NotZero(xreg(0)),
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
};
inst.emit(&mut buf2, &flags, &mut state);
let inst = Inst::Nop4;
inst.emit(&mut buf2, &flags, &mut state);
inst.emit(&mut buf2, &flags, &mut state);
let buf2 = buf2.finish();

59
cranelift/filetests/filetests/vcode/aarch64/arithmetic.clif
View File

@@ -76,12 +76,10 @@ block0(v0: i64, v1: i64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: sdiv x2, x0, x1
; nextln: cbz x1, 20
; nextln: cbnz x1, 8 ; udf
; nextln: adds xzr, x1, #1
; nextln: ccmp x0, #1, #nzcv, eq
; nextln: b.vc 12
; nextln: udf
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: mov x0, x2
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -98,12 +96,10 @@ block0(v0: i64):
; nextln: mov fp, sp
; nextln: movz x2, #2
; nextln: sdiv x1, x0, x2
; nextln: cbz x2, 20
; nextln: cbnz x2, 8 ; udf
; nextln: adds xzr, x2, #1
; nextln: ccmp x0, #1, #nzcv, eq
; nextln: b.vc 12
; nextln: udf
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: mov x0, x1
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -118,8 +114,7 @@ block0(v0: i64, v1: i64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: udiv x0, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
@@ -135,8 +130,7 @@ block0(v0: i64):
; nextln: mov fp, sp
; nextln: movz x1, #2
; nextln: udiv x0, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
@@ -150,8 +144,7 @@ block0(v0: i64, v1: i64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: sdiv x2, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: msub x0, x2, x1, x0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -166,8 +159,7 @@ block0(v0: i64, v1: i64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: udiv x2, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: msub x0, x2, x1, x0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -185,12 +177,10 @@ block0(v0: i32, v1: i32):
; nextln: sxtw x3, w0
; nextln: sxtw x2, w1
; nextln: sdiv x0, x3, x2
; nextln: cbz x2, 20
; nextln: cbnz x2, 8 ; udf
; nextln: adds wzr, w2, #1
; nextln: ccmp w3, #1, #nzcv, eq
; nextln: b.vc 12
; nextln: udf
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
@@ -204,16 +194,15 @@ block0(v0: i32):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: sxtw x1, w0
; nextln: movz x0, #2
; nextln: sxtw x2, w0
; nextln: sdiv x0, x1, x2
; nextln: cbz x2, 20
; nextln: sxtw x0, w0
; nextln: movz x1, #2
; nextln: sxtw x2, w1
; nextln: sdiv x1, x0, x2
; nextln: cbnz x2, 8 ; udf
; nextln: adds wzr, w2, #1
; nextln: ccmp w1, #1, #nzcv, eq
; nextln: b.vc 12
; nextln: udf
; nextln: udf
; nextln: ccmp w0, #1, #nzcv, eq
; nextln: b.vc 8 ; udf
; nextln: mov x0, x1
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
@@ -229,8 +218,7 @@ block0(v0: i32, v1: i32):
; nextln: mov w0, w0
; nextln: mov w1, w1
; nextln: udiv x0, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
@@ -248,8 +236,7 @@ block0(v0: i32):
; nextln: mov w0, w0
; nextln: movz x1, #2
; nextln: udiv x0, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
@@ -265,8 +252,7 @@ block0(v0: i32, v1: i32):
; nextln: sxtw x0, w0
; nextln: sxtw x1, w1
; nextln: sdiv x2, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: msub x0, x2, x1, x0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -283,8 +269,7 @@ block0(v0: i32, v1: i32):
; nextln: mov w0, w0
; nextln: mov w1, w1
; nextln: udiv x2, x0, x1
; nextln: cbnz x1, 8
; nextln: udf
; nextln: cbnz x1, 8 ; udf
; nextln: msub x0, x2, x1, x0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16

72
cranelift/filetests/filetests/vcode/aarch64/floating-point.clif
View File

@@ -426,16 +426,13 @@ block0(v0: f32):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp s0, s0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -1
; nextln: fcmp s0, s1
; nextln: b.gt 8
; nextln: udf
; nextln: b.gt 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 4294967300
; nextln: fcmp s0, s1
; nextln: b.mi 8
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzu w0, s0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -450,16 +447,13 @@ block0(v0: f32):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp s0, s0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -2147483600
; nextln: fcmp s0, s1
; nextln: b.ge 8
; nextln: udf
; nextln: b.ge 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 2147483600
; nextln: fcmp s0, s1
; nextln: b.mi
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzs w0, s0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -474,16 +468,13 @@ block0(v0: f32):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp s0, s0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -1
; nextln: fcmp s0, s1
; nextln: b.gt 8
; nextln: udf
; nextln: b.gt 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 18446744000000000000
; nextln: fcmp s0, s1
; nextln: b.mi 8
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzu x0, s0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -498,16 +489,13 @@ block0(v0: f32):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp s0, s0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -9223372000000000000
; nextln: fcmp s0, s1
; nextln: b.ge 8
; nextln: udf
; nextln: b.ge 8 ; udf
; nextln: ldr s1, pc+8 ; b 8 ; data.f32 9223372000000000000
; nextln: fcmp s0, s1
; nextln: b.mi 8
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzs x0, s0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -522,16 +510,13 @@ block0(v0: f64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp d0, d0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -1
; nextln: fcmp d0, d1
; nextln: b.gt 8
; nextln: udf
; nextln: b.gt 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 4294967296
; nextln: fcmp d0, d1
; nextln: b.mi 8
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzu w0, d0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -546,16 +531,13 @@ block0(v0: f64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp d0, d0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -2147483649
; nextln: fcmp d0, d1
; nextln: b.gt 8
; nextln: udf
; nextln: b.gt 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 2147483648
; nextln: fcmp d0, d1
; nextln: b.mi 8
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzs w0, d0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -570,16 +552,13 @@ block0(v0: f64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp d0, d0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -1
; nextln: fcmp d0, d1
; nextln: b.gt 8
; nextln: udf
; nextln: b.gt 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 18446744073709552000
; nextln: fcmp d0, d1
; nextln: b.mi 8
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzu x0, d0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -594,16 +573,13 @@ block0(v0: f64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: fcmp d0, d0
; nextln: b.vc 8
; nextln: udf
; nextln: b.vc 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -9223372036854776000
; nextln: fcmp d0, d1
; nextln: b.ge 8
; nextln: udf
; nextln: b.ge 8 ; udf
; nextln: ldr d1, pc+8 ; b 12 ; data.f64 9223372036854776000
; nextln: fcmp d0, d1
; nextln: b.mi 8
; nextln: udf
; nextln: b.mi 8 ; udf
; nextln: fcvtzs x0, d0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16

3
cranelift/filetests/filetests/vcode/aarch64/jumptable.clif
View File

@@ -29,8 +29,7 @@ block5(v5: i64):
}
; check: subs wzr, w0, #3
; nextln: b.hs
; nextln: adr x1, pc+16 ; ldrsw x2, [x1, x0, LSL 2] ; add x1, x1, x2 ; br x1 ; jt_entries
; nextln: b.hs label1 ; adr x1, pc+16 ; ldrsw x2, [x1, x0, LSL 2] ; add x1, x1, x2 ; br x1 ; jt_entries
; check: movz x1, #1
; nextln: b

27
cranelift/filetests/filetests/vcode/aarch64/stack-limit.clif
View File

@@ -43,8 +43,7 @@ block0(v0: i64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: subs xzr, sp, x0
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: ldr x16
; nextln: blr x16
; nextln: mov sp, fp
@@ -67,8 +66,7 @@ block0(v0: i64):
; nextln: ldur x16, [x0]
; nextln: ldur x16, [x16, #4]
; nextln: subs xzr, sp, x16
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: ldr x16
; nextln: blr x16
; nextln: mov sp, fp
@@ -86,8 +84,7 @@ block0(v0: i64):
; nextln: mov fp, sp
; nextln: add x16, x0, #176
; nextln: subs xzr, sp, x16
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: sub sp, sp, #176
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -102,14 +99,12 @@ block0(v0: i64):
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: subs xzr, sp, x0
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: movz x17, #6784
; nextln: movk x17, #6, LSL #16
; nextln: add x16, x0, x17, UXTX
; nextln: subs xzr, sp, x16
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: ldr x16, 8 ; b 12 ; data 400000
; nextln: sub sp, sp, x16, UXTX
; nextln: mov sp, fp
@@ -132,8 +127,7 @@ block0(v0: i64):
; nextln: ldur x16, [x16, #4]
; nextln: add x16, x16, #32
; nextln: subs xzr, sp, x16
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: sub sp, sp, #32
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
@@ -154,14 +148,12 @@ block0(v0: i64):
; nextln: ldur x16, [x0]
; nextln: ldur x16, [x16, #4]
; nextln: subs xzr, sp, x16
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: movz x17, #6784
; nextln: movk x17, #6, LSL #16
; nextln: add x16, x16, x17, UXTX
; nextln: subs xzr, sp, x16
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: ldr x16, 8 ; b 12 ; data 400000
; nextln: sub sp, sp, x16, UXTX
; nextln: mov sp, fp
@@ -182,8 +174,7 @@ block0(v0: i64):
; nextln: movz x16, #6784 ; movk x16, #6, LSL #16 ; add x16, x0, x16, UXTX ; ldr x16, [x16]
; nextln: add x16, x16, #32
; nextln: subs xzr, sp, x16
; nextln: b.hs 8
; nextln: udf
; nextln: b.hs 8 ; udf
; nextln: sub sp, sp, #32
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16

3
cranelift/filetests/filetests/vcode/aarch64/traps.clif
View File

@@ -17,8 +17,7 @@ block0(v0: i64):
}
; check: subs xzr, x0, #42
; nextln: b.ne 8
; nextln: udf
; nextln: b.ne 8 ; udf
function %h() {
block0: