T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a MachBuffer::defer_trap method (#6011)

Browse Source 
* Add a `MachBuffer::defer_trap` method

This commit adds a new method to `MachBuffer` to defer trap opcodes to
the end of a function in a similar manner to how constants are deferred
to the end of the function. This is useful for backends which frequently
use `TrapIf`-style opcodes. Currently a jump is emitted which skips the
next instruction, a trap, and then execution continues normally. While
there isn't any pressing problem with this construction the trap opcode
is in the middle of the instruction stream as opposed to "off on the
side" despite rarely being taken.

With this method in place all the backends (except riscv64 since I
couldn't figure it out easily enough) have a new lowering of their
`TrapIf` opcode. Now a trap is deferred, which returns a label, and then
that label is jumped to when executing the trap. A fixup is then
recorded in `MachBuffer` to get patched later on during emission, or at
the end of the function. Subsequently all `TrapIf` instructions
translate to a single branch plus a single trap at the end of the
function.

I've additionally further updated some more lowerings in the x64 backend
which were explicitly using traps to instead use `TrapIf` where
applicable to avoid jumping over traps mid-function. Other backends
didn't appear to have many jump-over-the-next-trap patterns.

Lots of tests have had their expectations updated here which should
reflect all the traps being sunk to the end of functions.

* Print trap code on all platforms

* Emit traps before constants

* Preserve source location information for traps

* Fix test expectations

* Attempt to fix s390x

The MachBuffer was registering trap codes with the first byte of the
trap, but the SIGILL handler was expecting it to be registered with the
last byte of the trap. Exploit that SIGILL is always represented with a
2-byte instruction and always march 2-backwards for SIGILL, continuing
to march backwards 1 byte for SIGFPE-generating instructions.

* Back out s390x changes

* Back out more s390x bits

* Review comments
This commit is contained in:
Alex Crichton
2023-03-20 16:24:47 -05:00
committed by GitHub
parent 6a03398faf
commit a3b21031d4
52 changed files with 702 additions and 588 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -1681,16 +1681,8 @@ pub(crate) fn emit(
}
Inst::TrapIf { cc, trap_code } => {
let else_label = sink.get_label();
// Jump over if the invert of CC is set (i.e. CC is not set).
one_way_jmp(sink, cc.invert(), else_label);
// Trap!
let inst = Inst::trap(*trap_code);
inst.emit(&[], sink, info, state);
sink.bind_label(else_label);
let trap_label = sink.defer_trap(*trap_code, state.take_stack_map());
one_way_jmp(sink, *cc, trap_label);
}
Inst::TrapIfAnd {
@@ -1698,15 +1690,13 @@ pub(crate) fn emit(
cc2,
trap_code,
} => {
let trap_label = sink.defer_trap(*trap_code, state.take_stack_map());
let else_label = sink.get_label();
// Jump over if either condition code is not set.
// Jump to the end if the first condition isn't true, and then if
// the second condition is true go to the trap.
one_way_jmp(sink, cc1.invert(), else_label);
one_way_jmp(sink, cc2.invert(), else_label);
// Trap!
let inst = Inst::trap(*trap_code);
inst.emit(&[], sink, info, state);
one_way_jmp(sink, *cc2, trap_label);
sink.bind_label(else_label);
}
@@ -1716,19 +1706,11 @@ pub(crate) fn emit(
cc2,
trap_code,
} => {
let trap_label = sink.get_label();
let else_label = sink.get_label();
let trap_label = sink.defer_trap(*trap_code, state.take_stack_map());
// trap immediately if cc1 is set, otherwise jump over the trap if cc2 is not.
// Emit two jumps to the same trap if either condition code is true.
one_way_jmp(sink, *cc1, trap_label);
one_way_jmp(sink, cc2.invert(), else_label);
// Trap!
sink.bind_label(trap_label);
let inst = Inst::trap(*trap_code);
inst.emit(&[], sink, info, state);
sink.bind_label(else_label);
one_way_jmp(sink, *cc2, trap_label);
}
Inst::XmmUnaryRmR { op, src, dst } => {
@@ -3056,7 +3038,6 @@ pub(crate) fn emit(
};
let done = sink.get_label();
let not_nan = sink.get_label();
// The truncation.
let inst = Inst::xmm_to_gpr(trunc_op, src, Writable::from_reg(dst), *dst_size);
@@ -3073,9 +3054,10 @@ pub(crate) fn emit(
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), src);
inst.emit(&[], sink, info, state);
one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN
if *is_saturating {
let not_nan = sink.get_label();
one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN
// For NaN, emit 0.
let inst = Inst::alu_rmi_r(
*dst_size,
@@ -3119,9 +3101,7 @@ pub(crate) fn emit(
inst.emit(&[], sink, info, state);
}
} else {
let check_positive = sink.get_label();
let inst = Inst::trap(TrapCode::BadConversionToInteger);
let inst = Inst::trap_if(CC::P, TrapCode::BadConversionToInteger);
inst.emit(&[], sink, info, state);
// Check if INT_MIN was the correct result: determine the smallest floating point
@@ -3130,8 +3110,6 @@ pub(crate) fn emit(
// If the src register is less (or in some cases, less-or-equal) than the threshold,
// trap!
sink.bind_label(not_nan);
let mut no_overflow_cc = CC::NB; // >=
let output_bits = dst_size.to_bits();
match *src_size {
@@ -3168,16 +3146,12 @@ pub(crate) fn emit(
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(tmp_xmm), src);
inst.emit(&[], sink, info, state);
// jump over trap if src >= or > threshold
one_way_jmp(sink, no_overflow_cc, check_positive);
let inst = Inst::trap(TrapCode::IntegerOverflow);
// no trap if src >= or > threshold
let inst = Inst::trap_if(no_overflow_cc.invert(), TrapCode::IntegerOverflow);
inst.emit(&[], sink, info, state);
// If positive, it was a real overflow.
sink.bind_label(check_positive);
// Zero out the tmp_xmm register.
let inst = Inst::xmm_rm_r(
SseOpcode::Xorpd,
@@ -3189,9 +3163,8 @@ pub(crate) fn emit(
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm);
inst.emit(&[], sink, info, state);
one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src
let inst = Inst::trap(TrapCode::IntegerOverflow);
// no trap if 0 >= src
let inst = Inst::trap_if(CC::B, TrapCode::IntegerOverflow);
inst.emit(&[], sink, info, state);
}
@@ -3291,11 +3264,10 @@ pub(crate) fn emit(
let handle_large = sink.get_label();
one_way_jmp(sink, CC::NB, handle_large); // jump to handle_large if src >= large_threshold
let not_nan = sink.get_label();
one_way_jmp(sink, CC::NP, not_nan); // jump over trap if not NaN
if *is_saturating {
// Emit 0.
// If not NaN jump over this 0-return, otherwise return 0
let not_nan = sink.get_label();
one_way_jmp(sink, CC::NP, not_nan);
let inst = Inst::alu_rmi_r(
*dst_size,
AluRmiROpcode::Xor,
@@ -3306,14 +3278,13 @@ pub(crate) fn emit(
let inst = Inst::jmp_known(done);
inst.emit(&[], sink, info, state);
sink.bind_label(not_nan);
} else {
// Trap.
let inst = Inst::trap(TrapCode::BadConversionToInteger);
let inst = Inst::trap_if(CC::P, TrapCode::BadConversionToInteger);
inst.emit(&[], sink, info, state);
}
sink.bind_label(not_nan);
// Actual truncation for small inputs: if the result is not positive, then we had an
// overflow.
@@ -3360,10 +3331,10 @@ pub(crate) fn emit(
let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst);
inst.emit(&[], sink, info, state);
let next_is_large = sink.get_label();
one_way_jmp(sink, CC::NL, next_is_large); // if dst >= 0, jump to next_is_large
if *is_saturating {
let next_is_large = sink.get_label();
one_way_jmp(sink, CC::NL, next_is_large); // if dst >= 0, jump to next_is_large
// The input was "large" (>= 2**(width -1)), so the only way to get an integer
// overflow is because the input was too large: saturate to the max value.
let inst = Inst::imm(
@@ -3379,13 +3350,12 @@ pub(crate) fn emit(
let inst = Inst::jmp_known(done);
inst.emit(&[], sink, info, state);
sink.bind_label(next_is_large);
} else {
let inst = Inst::trap(TrapCode::IntegerOverflow);
let inst = Inst::trap_if(CC::L, TrapCode::IntegerOverflow);
inst.emit(&[], sink, info, state);
}
sink.bind_label(next_is_large);
if *dst_size == OperandSize::Size64 {
let inst = Inst::imm(OperandSize::Size64, 1 << 63, Writable::from_reg(tmp_gpr));
inst.emit(&[], sink, info, state);
@@ -3615,8 +3585,7 @@ pub(crate) fn emit(
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
}
sink.put1(0x0f);
sink.put1(0x0b);
sink.put_data(Inst::TRAP_OPCODE);
}
Inst::VirtualSPOffsetAdj { offset } => {

8
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -472,6 +472,10 @@ impl Inst {
Inst::Ud2 { trap_code }
}
pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
Inst::TrapIf { cc, trap_code }
}
pub(crate) fn cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
debug_assert!(size.is_one_of(&[
OperandSize::Size16,
@@ -1675,7 +1679,7 @@ impl PrettyPrint for Inst {
}
Inst::TrapIf { cc, trap_code, .. } => {
format!("j{} ; ud2 {} ;", cc.invert().to_string(), trap_code)
format!("j{cc} #trap={trap_code}")
}
Inst::TrapIfAnd {
@@ -2502,6 +2506,8 @@ impl MachInst for Inst {
}
type LabelUse = LabelUse;
const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
}
/// State carried between emissions of a sequence of instructions.