T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a MachBuffer::defer_trap method (#6011)

Browse Source 
* Add a `MachBuffer::defer_trap` method

This commit adds a new method to `MachBuffer` to defer trap opcodes to
the end of a function in a similar manner to how constants are deferred
to the end of the function. This is useful for backends which frequently
use `TrapIf`-style opcodes. Currently a jump is emitted which skips the
next instruction, a trap, and then execution continues normally. While
there isn't any pressing problem with this construction the trap opcode
is in the middle of the instruction stream as opposed to "off on the
side" despite rarely being taken.

With this method in place all the backends (except riscv64 since I
couldn't figure it out easily enough) have a new lowering of their
`TrapIf` opcode. Now a trap is deferred, which returns a label, and then
that label is jumped to when executing the trap. A fixup is then
recorded in `MachBuffer` to get patched later on during emission, or at
the end of the function. Subsequently all `TrapIf` instructions
translate to a single branch plus a single trap at the end of the
function.

I've additionally further updated some more lowerings in the x64 backend
which were explicitly using traps to instead use `TrapIf` where
applicable to avoid jumping over traps mid-function. Other backends
didn't appear to have many jump-over-the-next-trap patterns.

Lots of tests have had their expectations updated here which should
reflect all the traps being sunk to the end of functions.

* Print trap code on all platforms

* Emit traps before constants

* Preserve source location information for traps

* Fix test expectations

* Attempt to fix s390x

The MachBuffer was registering trap codes with the first byte of the
trap, but the SIGILL handler was expecting it to be registered with the
last byte of the trap. Exploit that SIGILL is always represented with a
2-byte instruction and always march 2-backwards for SIGILL, continuing
to march backwards 1 byte for SIGFPE-generating instructions.

* Back out s390x changes

* Back out more s390x bits

* Review comments
This commit is contained in:
Alex Crichton
2023-03-20 16:24:47 -05:00
committed by GitHub
parent 6a03398faf
commit a3b21031d4
52 changed files with 702 additions and 588 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -3115,20 +3115,15 @@ impl MachInstEmit for Inst {
sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
}
&Inst::TrapIf { kind, trap_code } => {
let label = sink.defer_trap(trap_code, state.take_stack_map());
// condbr KIND, LABEL
let off = sink.cur_offset();
let label = sink.get_label();
sink.put4(enc_conditional_br(
BranchTarget::Label(label),
kind.invert(),
kind,
&mut allocs,
));
sink.use_label_at_offset(off, label, LabelUse::Branch19);
// udf
let trap = Inst::Udf { trap_code };
trap.emit(&[], sink, emit_info, state);
// LABEL:
sink.bind_label(label);
}
&Inst::IndirectBr { rn, .. } => {
let rn = allocs.next(rn);
@@ -3142,15 +3137,11 @@ impl MachInstEmit for Inst {
sink.put4(0xd4200000);
}
&Inst::Udf { trap_code } => {
// "CLIF" in hex, to make the trap recognizable during
// debugging.
let encoding = 0xc11f;
sink.add_trap(trap_code);
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
}
sink.put4(encoding);
sink.put_data(Inst::TRAP_OPCODE);
}
&Inst::Adr { rd, off } => {
let rd = allocs.next_writable(rd);

72
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -5873,144 +5873,144 @@ fn test_aarch64_binemit() {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::NotZero(xreg(8)),
},
"480000B41FC10000",
"cbz x8, 8 ; udf",
"280000B51FC10000",
"cbnz x8, #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Zero(xreg(8)),
},
"480000B51FC10000",
"cbnz x8, 8 ; udf",
"280000B41FC10000",
"cbz x8, #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ne),
},
"400000541FC10000",
"b.eq 8 ; udf",
"210000541FC10000",
"b.ne #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Eq),
},
"410000541FC10000",
"b.ne 8 ; udf",
"200000541FC10000",
"b.eq #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Lo),
},
"420000541FC10000",
"b.hs 8 ; udf",
"230000541FC10000",
"b.lo #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Hs),
},
"430000541FC10000",
"b.lo 8 ; udf",
"220000541FC10000",
"b.hs #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Pl),
},
"440000541FC10000",
"b.mi 8 ; udf",
"250000541FC10000",
"b.pl #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Mi),
},
"450000541FC10000",
"b.pl 8 ; udf",
"240000541FC10000",
"b.mi #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Vc),
},
"460000541FC10000",
"b.vs 8 ; udf",
"270000541FC10000",
"b.vc #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Vs),
},
"470000541FC10000",
"b.vc 8 ; udf",
"260000541FC10000",
"b.vs #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ls),
},
"480000541FC10000",
"b.hi 8 ; udf",
"290000541FC10000",
"b.ls #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Hi),
},
"490000541FC10000",
"b.ls 8 ; udf",
"280000541FC10000",
"b.hi #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Lt),
},
"4A0000541FC10000",
"b.ge 8 ; udf",
"2B0000541FC10000",
"b.lt #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ge),
},
"4B0000541FC10000",
"b.lt 8 ; udf",
"2A0000541FC10000",
"b.ge #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Le),
},
"4C0000541FC10000",
"b.gt 8 ; udf",
"2D0000541FC10000",
"b.le #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Gt),
},
"4D0000541FC10000",
"b.le 8 ; udf",
"2C0000541FC10000",
"b.gt #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Nv),
},
"4E0000541FC10000",
"b.al 8 ; udf",
"2F0000541FC10000",
"b.nv #trap=interrupt",
));
insns.push((
Inst::TrapIf {
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Al),
},
"4F0000541FC10000",
"b.nv 8 ; udf",
"2E0000541FC10000",
"b.al #trap=interrupt",
));
insns.push((

17
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -926,6 +926,10 @@ impl MachInst for Inst {
type ABIMachineSpec = AArch64MachineDeps;
type LabelUse = LabelUse;
// "CLIF" in hex, to make the trap recognizable during
// debugging.
const TRAP_OPCODE: &'static [u8] = &0xc11f_u32.to_le_bytes();
fn get_operands<F: Fn(VReg) -> VReg>(&self, collector: &mut OperandCollector<'_, F>) {
aarch64_get_operands(self, collector);
}
@@ -2519,18 +2523,21 @@ impl Inst {
}
&Inst::Brk => "brk #0".to_string(),
&Inst::Udf { .. } => "udf #0xc11f".to_string(),
&Inst::TrapIf { ref kind, .. } => match kind {
&Inst::TrapIf {
ref kind,
trap_code,
} => match kind {
&CondBrKind::Zero(reg) => {
let reg = pretty_print_reg(reg, allocs);
format!("cbnz {}, 8 ; udf", reg)
format!("cbz {reg}, #trap={trap_code}")
}
&CondBrKind::NotZero(reg) => {
let reg = pretty_print_reg(reg, allocs);
format!("cbz {}, 8 ; udf", reg)
format!("cbnz {reg}, #trap={trap_code}")
}
&CondBrKind::Cond(c) => {
let c = c.invert().pretty_print(0, allocs);
format!("b.{} 8 ; udf", c)
let c = c.pretty_print(0, allocs);
format!("b.{c} #trap={trap_code}")
}
},
&Inst::Adr { rd, off } => {

4
cranelift/codegen/src/isa/riscv64/inst/emit.rs
View File

@@ -1908,9 +1908,7 @@ impl MachInstEmit for Inst {
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
}
// https://github.com/riscv/riscv-isa-manual/issues/850
// all zero will cause invalid opcode.
sink.put4(0);
sink.put_data(Inst::TRAP_OPCODE);
}
&Inst::SelectIf {
if_spectre_guard: _if_spectre_guard, // _if_spectre_guard not use because it is used to not be removed by optimization pass and some other staff.

4
cranelift/codegen/src/isa/riscv64/inst/mod.rs
View File

@@ -642,6 +642,10 @@ impl MachInst for Inst {
type LabelUse = LabelUse;
type ABIMachineSpec = Riscv64MachineDeps;
// https://github.com/riscv/riscv-isa-manual/issues/850
// all zero will cause invalid opcode.
const TRAP_OPCODE: &'static [u8] = &[0; 4];
fn gen_dummy_use(reg: Reg) -> Self {
Inst::DummyUse { reg }
}

1
cranelift/codegen/src/isa/s390x/inst/mod.rs
View File

@@ -998,6 +998,7 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
impl MachInst for Inst {
type ABIMachineSpec = S390xMachineDeps;
type LabelUse = LabelUse;
const TRAP_OPCODE: &'static [u8] = &[0, 0];
fn get_operands<F: Fn(VReg) -> VReg>(&self, collector: &mut OperandCollector<'_, F>) {
s390x_get_operands(self, collector);

87
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -1681,16 +1681,8 @@ pub(crate) fn emit(
}
Inst::TrapIf { cc, trap_code } => {
let else_label = sink.get_label();
// Jump over if the invert of CC is set (i.e. CC is not set).
one_way_jmp(sink, cc.invert(), else_label);
// Trap!
let inst = Inst::trap(*trap_code);
inst.emit(&[], sink, info, state);
sink.bind_label(else_label);
let trap_label = sink.defer_trap(*trap_code, state.take_stack_map());
one_way_jmp(sink, *cc, trap_label);
}
Inst::TrapIfAnd {
@@ -1698,15 +1690,13 @@ pub(crate) fn emit(
cc2,
trap_code,
} => {
let trap_label = sink.defer_trap(*trap_code, state.take_stack_map());
let else_label = sink.get_label();
// Jump over if either condition code is not set.
// Jump to the end if the first condition isn't true, and then if
// the second condition is true go to the trap.
one_way_jmp(sink, cc1.invert(), else_label);
one_way_jmp(sink, cc2.invert(), else_label);
// Trap!
let inst = Inst::trap(*trap_code);
inst.emit(&[], sink, info, state);
one_way_jmp(sink, *cc2, trap_label);
sink.bind_label(else_label);
}
@@ -1716,19 +1706,11 @@ pub(crate) fn emit(
cc2,
trap_code,
} => {
let trap_label = sink.get_label();
let else_label = sink.get_label();
let trap_label = sink.defer_trap(*trap_code, state.take_stack_map());
// trap immediately if cc1 is set, otherwise jump over the trap if cc2 is not.
// Emit two jumps to the same trap if either condition code is true.
one_way_jmp(sink, *cc1, trap_label);
one_way_jmp(sink, cc2.invert(), else_label);
// Trap!
sink.bind_label(trap_label);
let inst = Inst::trap(*trap_code);
inst.emit(&[], sink, info, state);
sink.bind_label(else_label);
one_way_jmp(sink, *cc2, trap_label);
}
Inst::XmmUnaryRmR { op, src, dst } => {
@@ -3056,7 +3038,6 @@ pub(crate) fn emit(
};
let done = sink.get_label();
let not_nan = sink.get_label();
// The truncation.
let inst = Inst::xmm_to_gpr(trunc_op, src, Writable::from_reg(dst), *dst_size);
@@ -3073,9 +3054,10 @@ pub(crate) fn emit(
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), src);
inst.emit(&[], sink, info, state);
one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN
if *is_saturating {
let not_nan = sink.get_label();
one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN
// For NaN, emit 0.
let inst = Inst::alu_rmi_r(
*dst_size,
@@ -3119,9 +3101,7 @@ pub(crate) fn emit(
inst.emit(&[], sink, info, state);
}
} else {
let check_positive = sink.get_label();
let inst = Inst::trap(TrapCode::BadConversionToInteger);
let inst = Inst::trap_if(CC::P, TrapCode::BadConversionToInteger);
inst.emit(&[], sink, info, state);
// Check if INT_MIN was the correct result: determine the smallest floating point
@@ -3130,8 +3110,6 @@ pub(crate) fn emit(
// If the src register is less (or in some cases, less-or-equal) than the threshold,
// trap!
sink.bind_label(not_nan);
let mut no_overflow_cc = CC::NB; // >=
let output_bits = dst_size.to_bits();
match *src_size {
@@ -3168,16 +3146,12 @@ pub(crate) fn emit(
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm), src);
inst.emit(&[], sink, info, state);
// jump over trap if src >= or > threshold
one_way_jmp(sink, no_overflow_cc, check_positive);
let inst = Inst::trap(TrapCode::IntegerOverflow);
// no trap if src >= or > threshold
let inst = Inst::trap_if(no_overflow_cc.invert(), TrapCode::IntegerOverflow);
inst.emit(&[], sink, info, state);
// If positive, it was a real overflow.
sink.bind_label(check_positive);
// Zero out the tmp_xmm register.
let inst = Inst::xmm_rm_r(
SseOpcode::Xorpd,
@@ -3189,9 +3163,8 @@ pub(crate) fn emit(
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm);
inst.emit(&[], sink, info, state);
one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src
let inst = Inst::trap(TrapCode::IntegerOverflow);
// no trap if 0 >= src
let inst = Inst::trap_if(CC::B, TrapCode::IntegerOverflow);
inst.emit(&[], sink, info, state);
}
@@ -3291,11 +3264,10 @@ pub(crate) fn emit(
let handle_large = sink.get_label();
one_way_jmp(sink, CC::NB, handle_large); // jump to handle_large if src >= large_threshold
let not_nan = sink.get_label();
one_way_jmp(sink, CC::NP, not_nan); // jump over trap if not NaN
if *is_saturating {
// Emit 0.
// If not NaN jump over this 0-return, otherwise return 0
let not_nan = sink.get_label();
one_way_jmp(sink, CC::NP, not_nan);
let inst = Inst::alu_rmi_r(
*dst_size,
AluRmiROpcode::Xor,
@@ -3306,14 +3278,13 @@ pub(crate) fn emit(
let inst = Inst::jmp_known(done);
inst.emit(&[], sink, info, state);
sink.bind_label(not_nan);
} else {
// Trap.
let inst = Inst::trap(TrapCode::BadConversionToInteger);
let inst = Inst::trap_if(CC::P, TrapCode::BadConversionToInteger);
inst.emit(&[], sink, info, state);
}
sink.bind_label(not_nan);
// Actual truncation for small inputs: if the result is not positive, then we had an
// overflow.
@@ -3360,10 +3331,10 @@ pub(crate) fn emit(
let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst);
inst.emit(&[], sink, info, state);
let next_is_large = sink.get_label();
one_way_jmp(sink, CC::NL, next_is_large); // if dst >= 0, jump to next_is_large
if *is_saturating {
let next_is_large = sink.get_label();
one_way_jmp(sink, CC::NL, next_is_large); // if dst >= 0, jump to next_is_large
// The input was "large" (>= 2**(width -1)), so the only way to get an integer
// overflow is because the input was too large: saturate to the max value.
let inst = Inst::imm(
@@ -3379,13 +3350,12 @@ pub(crate) fn emit(
let inst = Inst::jmp_known(done);
inst.emit(&[], sink, info, state);
sink.bind_label(next_is_large);
} else {
let inst = Inst::trap(TrapCode::IntegerOverflow);
let inst = Inst::trap_if(CC::L, TrapCode::IntegerOverflow);
inst.emit(&[], sink, info, state);
}
sink.bind_label(next_is_large);
if *dst_size == OperandSize::Size64 {
let inst = Inst::imm(OperandSize::Size64, 1 << 63, Writable::from_reg(tmp_gpr));
inst.emit(&[], sink, info, state);
@@ -3615,8 +3585,7 @@ pub(crate) fn emit(
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
}
sink.put1(0x0f);
sink.put1(0x0b);
sink.put_data(Inst::TRAP_OPCODE);
}
Inst::VirtualSPOffsetAdj { offset } => {

8
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -472,6 +472,10 @@ impl Inst {
Inst::Ud2 { trap_code }
}
pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
Inst::TrapIf { cc, trap_code }
}
pub(crate) fn cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
debug_assert!(size.is_one_of(&[
OperandSize::Size16,
@@ -1675,7 +1679,7 @@ impl PrettyPrint for Inst {
}
Inst::TrapIf { cc, trap_code, .. } => {
format!("j{} ; ud2 {} ;", cc.invert().to_string(), trap_code)
format!("j{cc} #trap={trap_code}")
}
Inst::TrapIfAnd {
@@ -2502,6 +2506,8 @@ impl MachInst for Inst {
}
type LabelUse = LabelUse;
const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
}
/// State carried between emissions of a sequence of instructions.

113
cranelift/codegen/src/machinst/buffer.rs
View File

@@ -229,6 +229,8 @@ pub struct MachBuffer<I: VCodeInst> {
label_aliases: SmallVec<[MachLabel; 16]>,
/// Constants that must be emitted at some point.
pending_constants: SmallVec<[MachLabelConstant; 16]>,
/// Traps that must be emitted at some point.
pending_traps: SmallVec<[MachLabelTrap; 16]>,
/// Fixups that must be performed after all code is emitted.
fixup_records: SmallVec<[MachLabelFixup<I>; 16]>,
/// Current deadline at which all constants are flushed and all code labels
@@ -371,6 +373,7 @@ impl<I: VCodeInst> MachBuffer<I> {
label_offsets: SmallVec::new(),
label_aliases: SmallVec::new(),
pending_constants: SmallVec::new(),
pending_traps: SmallVec::new(),
fixup_records: SmallVec::new(),
island_deadline: UNKNOWN_LABEL_OFFSET,
island_worst_case_size: 0,
@@ -1077,15 +1080,41 @@ impl<I: VCodeInst> MachBuffer<I> {
align,
label
);
let deadline = self.cur_offset().saturating_add(max_distance);
self.island_worst_case_size += data.len() as CodeOffset;
self.island_worst_case_size =
(self.island_worst_case_size + I::LabelUse::ALIGN - 1) & !(I::LabelUse::ALIGN - 1);
self.update_deadline(data.len(), max_distance);
self.pending_constants.push(MachLabelConstant {
label,
align,
data: SmallVec::from(data),
});
}
/// Emit a trap at some point in the future with the specified code and
/// stack map.
///
/// This function returns a [`MachLabel`] which will be the future address
/// of the trap. Jumps should refer to this label, likely by using the
/// [`MachBuffer::use_label_at_offset`] method, to get a relocation
/// patched in once the address of the trap is known.
///
/// This will batch all traps into the end of the function.
pub fn defer_trap(&mut self, code: TrapCode, stack_map: Option<StackMap>) -> MachLabel {
let label = self.get_label();
self.update_deadline(I::TRAP_OPCODE.len(), u32::MAX);
self.pending_traps.push(MachLabelTrap {
label,
code,
stack_map,
loc: self.cur_srcloc.map(|(_start, loc)| loc),
});
label
}
fn update_deadline(&mut self, len: usize, max_distance: CodeOffset) {
trace!("defer: eventually emit {} bytes", len);
let deadline = self.cur_offset().saturating_add(max_distance);
self.island_worst_case_size += len as CodeOffset;
self.island_worst_case_size =
(self.island_worst_case_size + I::LabelUse::ALIGN - 1) & !(I::LabelUse::ALIGN - 1);
if deadline < self.island_deadline {
self.island_deadline = deadline;
}
@@ -1131,8 +1160,48 @@ impl<I: VCodeInst> MachBuffer<I> {
self.island_deadline = UNKNOWN_LABEL_OFFSET;
self.island_worst_case_size = 0;
// First flush out all constants so we have more labels in case fixups
// are applied against these labels.
// End the current location tracking since anything emitted during this
// function shouldn't be attributed to whatever the current source
// location is.
//
// Note that the current source location, if it's set right now, will be
// restored at the end of this island emission.
let cur_loc = self.cur_srcloc.map(|(_, loc)| loc);
if cur_loc.is_some() {
self.end_srcloc();
}
// First flush out all traps/constants so we have more labels in case
// fixups are applied against these labels.
//
// Note that traps are placed first since this typically happens at the
// end of the function and for disassemblers we try to keep all the code
// contiguously together.
for MachLabelTrap {
label,
code,
stack_map,
loc,
} in mem::take(&mut self.pending_traps)
{
// If this trap has source information associated with it then
// emit this information for the trap instruction going out now too.
if let Some(loc) = loc {
self.start_srcloc(loc);
}
self.align_to(I::LabelUse::ALIGN);
self.bind_label(label);
self.add_trap(code);
if let Some(map) = stack_map {
let extent = StackMapExtent::UpcomingBytes(I::TRAP_OPCODE.len() as u32);
self.add_stack_map(extent, map);
}
self.put_data(I::TRAP_OPCODE);
if loc.is_some() {
self.end_srcloc();
}
}
for MachLabelConstant { label, align, data } in mem::take(&mut self.pending_constants) {
self.align_to(align);
self.bind_label(label);
@@ -1209,6 +1278,10 @@ impl<I: VCodeInst> MachBuffer<I> {
}
}
}
if let Some(loc) = cur_loc {
self.start_srcloc(loc);
}
}
/// Emits a "veneer" the `kind` code at `offset` to jump to `label`.
@@ -1256,7 +1329,10 @@ impl<I: VCodeInst> MachBuffer<I> {
}
fn finish_emission_maybe_forcing_veneers(&mut self, force_veneers: bool) {
while !self.pending_constants.is_empty() || !self.fixup_records.is_empty() {
while !self.pending_constants.is_empty()
|| !self.pending_traps.is_empty()
|| !self.fixup_records.is_empty()
{
// `emit_island()` will emit any pending veneers and constants, and
// as a side-effect, will also take care of any fixups with resolved
// labels eagerly.
@@ -1479,6 +1555,19 @@ struct MachLabelConstant {
data: SmallVec<[u8; 16]>,
}
/// A trap that is deferred to the next time an island is emitted for either
/// traps, constants, or fixups.
struct MachLabelTrap {
/// This label will refer to the trap's offset.
label: MachLabel,
/// The code associated with this trap.
code: TrapCode,
/// An optional stack map to associate with this trap.
stack_map: Option<StackMap>,
/// An optional source location to assign for this trap.
loc: Option<RelSourceLoc>,
}
/// A fixup to perform on the buffer once code is emitted. Fixups always refer
/// to labels and patch the code based on label offsets. Hence, they are like
/// relocations, but internal to one buffer.
@@ -1748,20 +1837,20 @@ mod test {
buf.bind_label(label(0));
let inst = Inst::CondBr {
kind: CondBrKind::NotZero(xreg(0)),
kind: CondBrKind::Zero(xreg(0)),
taken: target(1),
not_taken: target(2),
};
inst.emit(&[], &mut buf, &info, &mut state);
buf.bind_label(label(1));
let inst = Inst::Udf {
trap_code: TrapCode::Interrupt,
};
let inst = Inst::Nop4;
inst.emit(&[], &mut buf, &info, &mut state);
buf.bind_label(label(2));
let inst = Inst::Nop4;
let inst = Inst::Udf {
trap_code: TrapCode::Interrupt,
};
inst.emit(&[], &mut buf, &info, &mut state);
buf.bind_label(label(3));

4
cranelift/codegen/src/machinst/mod.rs
View File

@@ -177,6 +177,10 @@ pub trait MachInst: Clone + Debug {
/// A label-use kind: a type that describes the types of label references that
/// can occur in an instruction.
type LabelUse: MachInstLabelUse;
/// Byte representation of a trap opcode which is inserted by `MachBuffer`
/// during its `defer_trap` method.
const TRAP_OPCODE: &'static [u8];
}
/// A descriptor of a label reference (use) in an instruction set.