T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MachInst backends: handle SourceLocs out-of-band, not in Insts.

Browse Source 
In existing MachInst backends, many instructions -- any that can trap or
result in a relocation -- carry `SourceLoc` values in order to propagate
the location-in-original-source to use to describe resulting traps or
relocation errors.

This is quite tedious, and also error-prone: it is likely that the
necessary plumbing will be missed in some cases, and in any case, it's
unnecessarily verbose.

This PR factors out the `SourceLoc` handling so that it is tracked
during emission as part of the `EmitState`, and plumbed through
automatically by the machine-independent framework. Instruction emission
code that directly emits trap or relocation records can query the
current location as necessary. Then we only need to ensure that memory
references and trap instructions, at their (one) emission point rather
than their (many) lowering/generation points, are wired up correctly.

This does have the side-effect that some loads and stores that do not
correspond directly to user code's heap accesses will have unnecessary
but harmless trap metadata. For example, the load that fetches a code
offset from a jump table will have a 'heap out of bounds' trap record
attached to it; but because it is bounds-checked, and will never
actually trap if the lowering is correct, this should be harmless.  The
simplicity improvement here seemed more worthwhile to me than plumbing
through a "corresponds to user-level load/store" bit, because the latter
is a bit complex when we allow for op merging.

Closes #2290: though it does not implement a full "metadata" scheme as
described in that issue, this seems simpler overall.
This commit is contained in:
Chris Fallin
2020-11-10 14:37:11 -08:00
parent 0568f4fb02
commit 4dce51096d
19 changed files with 849 additions and 1714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -1,7 +1,7 @@
//! Implementation of the standard x64 ABI.
use crate::ir::types::*;
use crate::ir::{self, types, SourceLoc, TrapCode, Type};
use crate::ir::{self, types, TrapCode, Type};
use crate::isa;
use crate::isa::{x64::inst::*, CallConv};
use crate::machinst::abi_impl::*;
@@ -252,11 +252,11 @@ impl ABIMachineSpec for X64ABIMachineSpec {
_ if ty.bytes() == 16 => ExtKind::None,
_ => panic!("load_stack({})", ty),
};
Inst::load(ty, mem, into_reg, ext_kind, /* infallible */ None)
Inst::load(ty, mem, into_reg, ext_kind)
}
fn gen_store_stack(mem: StackAMode, from_reg: Reg, ty: Type) -> Self::I {
Inst::store(ty, from_reg, mem, /* infallible */ None)
Inst::store(ty, from_reg, mem)
}
fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self::I {
@@ -274,9 +274,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
let ext_mode = ExtMode::new(from_bits as u16, to_bits as u16)
.expect(&format!("invalid extension: {} -> {}", from_bits, to_bits));
if is_signed {
Inst::movsx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg, None)
Inst::movsx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg)
} else {
Inst::movzx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg, None)
Inst::movzx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg)
}
}
@@ -308,7 +308,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
Inst::TrapIf {
// NBE == "> unsigned"; args above are reversed; this tests limit_reg > rsp.
cc: CC::NBE,
srcloc: SourceLoc::default(),
trap_code: TrapCode::StackOverflow,
},
]
@@ -335,13 +334,13 @@ impl ABIMachineSpec for X64ABIMachineSpec {
assert_eq!(ty, I64);
let simm32 = offset as u32;
let mem = Amode::imm_reg(simm32, base);
Inst::load(ty, mem, into_reg, ExtKind::None, None)
Inst::load(ty, mem, into_reg, ExtKind::None)
}
fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Self::I {
let simm32 = offset as u32;
let mem = Amode::imm_reg(simm32, base);
Inst::store(ty, from_reg, mem, None)
Inst::store(ty, from_reg, mem)
}
fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Self::I; 2]> {
@@ -426,7 +425,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
/* bytes = */ 8,
r_reg.to_reg(),
Amode::imm_reg(cur_offset, regs::rsp()),
None,
));
cur_offset += 8;
}
@@ -461,7 +459,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
insts.push(Inst::mov64_m_r(
Amode::imm_reg(cur_offset, regs::rsp()),
Writable::from_reg(rreg.to_reg()),
None,
));
cur_offset += 8;
}
@@ -486,7 +483,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
insts.push(Inst::mov64_m_r(
Amode::imm_reg(off as u32, regs::rbp()),
Writable::from_reg(regs::r14()),
None,
));
}
@@ -498,7 +494,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
dest: &CallDest,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: ir::Opcode,
tmp: Writable<Reg>,
_callee_conv: isa::CallConv,
@@ -509,7 +504,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
&CallDest::ExtName(ref name, RelocDistance::Near) => {
insts.push((
InstIsSafepoint::Yes,
Inst::call_known(name.clone(), uses, defs, loc, opcode),
Inst::call_known(name.clone(), uses, defs, opcode),
));
}
&CallDest::ExtName(ref name, RelocDistance::Far) => {
@@ -519,18 +514,17 @@ impl ABIMachineSpec for X64ABIMachineSpec {
dst: tmp,
name: Box::new(name.clone()),
offset: 0,
srcloc: loc,
},
));
insts.push((
InstIsSafepoint::Yes,
Inst::call_unknown(RegMem::reg(tmp.to_reg()), uses, defs, loc, opcode),
Inst::call_unknown(RegMem::reg(tmp.to_reg()), uses, defs, opcode),
));
}
&CallDest::Reg(reg) => {
insts.push((
InstIsSafepoint::Yes,
Inst::call_unknown(RegMem::reg(reg), uses, defs, loc, opcode),
Inst::call_unknown(RegMem::reg(reg), uses, defs, opcode),
));
}
}

366
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -1,4 +1,4 @@
use crate::binemit::Reloc;
use crate::binemit::{Addend, Reloc};
use crate::ir::immediates::{Ieee32, Ieee64};
use crate::ir::TrapCode;
use crate::isa::x64::inst::args::*;
@@ -181,6 +181,7 @@ impl LegacyPrefixes {
/// indicate a 64-bit operation.
fn emit_std_enc_mem(
sink: &mut MachBuffer<Inst>,
state: &EmitState,
prefixes: LegacyPrefixes,
opcodes: u32,
mut num_opcodes: usize,
@@ -192,6 +193,11 @@ fn emit_std_enc_mem(
// 64-bit integer registers, because they are part of an address
// expression. But `enc_g` can be derived from a register of any class.
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
prefixes.emit(sink);
match mem_e {
@@ -342,6 +348,7 @@ fn emit_std_enc_enc(
fn emit_std_reg_mem(
sink: &mut MachBuffer<Inst>,
state: &EmitState,
prefixes: LegacyPrefixes,
opcodes: u32,
num_opcodes: usize,
@@ -350,7 +357,16 @@ fn emit_std_reg_mem(
rex: RexFlags,
) {
let enc_g = reg_enc(reg_g);
emit_std_enc_mem(sink, prefixes, opcodes, num_opcodes, enc_g, mem_e, rex);
emit_std_enc_mem(
sink,
state,
prefixes,
opcodes,
num_opcodes,
enc_g,
mem_e,
rex,
);
}
fn emit_std_reg_reg(
@@ -407,6 +423,18 @@ fn one_way_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
sink.put4(0x0);
}
/// Emits a relocation, attaching the current source location as well.
fn emit_reloc(
sink: &mut MachBuffer<Inst>,
state: &EmitState,
kind: Reloc,
name: &ExternalName,
addend: Addend,
) {
let srcloc = state.cur_srcloc();
sink.add_reloc(srcloc, kind, name, addend);
}
/// The top-level emit function.
///
/// Important! Do not add improved (shortened) encoding cases to existing
@@ -505,13 +533,15 @@ pub(crate) fn emit(
}
RegMemImm::Mem { addr } => {
let amode = addr.finalize(state);
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
0x0FAF,
2,
reg_g.to_reg(),
&addr.finalize(state),
&amode,
rex,
);
}
@@ -562,13 +592,15 @@ pub(crate) fn emit(
RegMemImm::Mem { addr } => {
// Here we revert to the "normal" G-E ordering.
let amode = addr.finalize(state);
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
opcode_m,
1,
reg_g.to_reg(),
&addr.finalize(state),
&amode,
rex,
);
}
@@ -616,15 +648,19 @@ pub(crate) fn emit(
*src,
rex_flags,
),
RegMem::Mem { addr: src } => emit_std_reg_mem(
sink,
prefix,
opcode,
num_opcodes,
dst.to_reg(),
&src.finalize(state),
rex_flags,
),
RegMem::Mem { addr: src } => {
let amode = src.finalize(state);
emit_std_reg_mem(
sink,
state,
prefix,
opcode,
num_opcodes,
dst.to_reg(),
&amode,
rex_flags,
);
}
}
}
@@ -660,7 +696,6 @@ pub(crate) fn emit(
size,
signed,
divisor,
loc,
} => {
let (opcode, prefix, rex_flags) = match size {
1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
@@ -670,7 +705,8 @@ pub(crate) fn emit(
_ => unreachable!("{}", size),
};
sink.add_trap(*loc, TrapCode::IntegerDivisionByZero);
let loc = state.cur_srcloc();
sink.add_trap(loc, TrapCode::IntegerDivisionByZero);
let subopcode = if *signed { 7 } else { 6 };
match divisor {
@@ -678,15 +714,10 @@ pub(crate) fn emit(
let src = int_reg_enc(*reg);
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
}
RegMem::Mem { addr: src } => emit_std_enc_mem(
sink,
prefix,
opcode,
1,
subopcode,
&src.finalize(state),
rex_flags,
),
RegMem::Mem { addr: src } => {
let amode = src.finalize(state);
emit_std_enc_mem(sink, state, prefix, opcode, 1, subopcode, &amode, rex_flags);
}
}
}
@@ -704,15 +735,10 @@ pub(crate) fn emit(
let src = int_reg_enc(*reg);
emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags)
}
RegMem::Mem { addr: src } => emit_std_enc_mem(
sink,
prefix,
0xF7,
1,
subopcode,
&src.finalize(state),
rex_flags,
),
RegMem::Mem { addr: src } => {
let amode = src.finalize(state);
emit_std_enc_mem(sink, state, prefix, 0xF7, 1, subopcode, &amode, rex_flags);
}
}
}
@@ -737,7 +763,6 @@ pub(crate) fn emit(
kind,
size,
divisor,
loc,
tmp,
} => {
// Generates the following code sequence:
@@ -773,7 +798,7 @@ pub(crate) fn emit(
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg());
inst.emit(sink, info, state);
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero, *loc);
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero);
inst.emit(sink, info, state);
let (do_op, done_label) = if kind.is_signed() {
@@ -818,7 +843,7 @@ pub(crate) fn emit(
}
// If not equal, jump over the trap.
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerOverflow, *loc);
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerOverflow);
inst.emit(sink, info, state);
(Some(do_op), None)
@@ -847,7 +872,7 @@ pub(crate) fn emit(
inst.emit(sink, info, state);
}
let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg()), *loc);
let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg()));
inst.emit(sink, info, state);
// Lowering takes care of moving the result back into the right register, see comment
@@ -900,12 +925,7 @@ pub(crate) fn emit(
emit_std_reg_reg(sink, LegacyPrefixes::None, 0x89, 1, *src, dst.to_reg(), rex);
}
Inst::MovzxRmR {
ext_mode,
src,
dst,
srcloc,
} => {
Inst::MovzxRmR { ext_mode, src, dst } => {
let (opcodes, num_opcodes, mut rex_flags) = match ext_mode {
ExtMode::BL => {
// MOVZBL is (REX.W==0) 0F B6 /r
@@ -963,13 +983,9 @@ pub(crate) fn emit(
RegMem::Mem { addr: src } => {
let src = &src.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
opcodes,
num_opcodes,
@@ -981,16 +997,12 @@ pub(crate) fn emit(
}
}
Inst::Mov64MR { src, dst, srcloc } => {
Inst::Mov64MR { src, dst } => {
let src = &src.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
0x8B,
1,
@@ -1000,22 +1012,22 @@ pub(crate) fn emit(
)
}
Inst::LoadEffectiveAddress { addr, dst } => emit_std_reg_mem(
sink,
LegacyPrefixes::None,
0x8D,
1,
dst.to_reg(),
&addr.finalize(state),
RexFlags::set_w(),
),
Inst::LoadEffectiveAddress { addr, dst } => {
let amode = addr.finalize(state);
Inst::MovsxRmR {
ext_mode,
src,
dst,
srcloc,
} => {
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
0x8D,
1,
dst.to_reg(),
&amode,
RexFlags::set_w(),
);
}
Inst::MovsxRmR { ext_mode, src, dst } => {
let (opcodes, num_opcodes, mut rex_flags) = match ext_mode {
ExtMode::BL => {
// MOVSBL is (REX.W==0) 0F BE /r
@@ -1065,13 +1077,9 @@ pub(crate) fn emit(
RegMem::Mem { addr: src } => {
let src = &src.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
opcodes,
num_opcodes,
@@ -1083,19 +1091,9 @@ pub(crate) fn emit(
}
}
Inst::MovRM {
size,
src,
dst,
srcloc,
} => {
Inst::MovRM { size, src, dst } => {
let dst = &dst.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
match size {
1 => {
// This is one of the few places where the presence of a
@@ -1109,13 +1107,14 @@ pub(crate) fn emit(
};
// MOV r8, r/m8 is (REX.W==0) 88 /r
emit_std_reg_mem(sink, LegacyPrefixes::None, 0x88, 1, *src, dst, rex)
emit_std_reg_mem(sink, state, LegacyPrefixes::None, 0x88, 1, *src, dst, rex)
}
2 => {
// MOV r16, r/m16 is 66 (REX.W==0) 89 /r
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::_66,
0x89,
1,
@@ -1129,6 +1128,7 @@ pub(crate) fn emit(
// MOV r32, r/m32 is (REX.W==0) 89 /r
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
0x89,
1,
@@ -1142,6 +1142,7 @@ pub(crate) fn emit(
// MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem(
sink,
state,
LegacyPrefixes::None,
0x89,
1,
@@ -1248,7 +1249,16 @@ pub(crate) fn emit(
}
RegMemImm::Mem { addr } => {
let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode_bytes, 2, dst.to_reg(), addr, rex);
emit_std_reg_mem(
sink,
state,
prefix,
opcode_bytes,
2,
dst.to_reg(),
addr,
rex,
);
}
RegMemImm::Imm { .. } => unreachable!(),
}
@@ -1300,7 +1310,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state);
// Whereas here we revert to the "normal" G-E ordering.
let opcode = if *size == 1 { 0x3A } else { 0x3B };
emit_std_reg_mem(sink, prefix, opcode, 1, *reg_g, addr, rex);
emit_std_reg_mem(sink, state, prefix, opcode, 1, *reg_g, addr, rex);
}
RegMemImm::Imm { simm32 } => {
@@ -1358,7 +1368,16 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex_flags);
emit_std_reg_mem(
sink,
state,
prefix,
opcode,
2,
reg_g.to_reg(),
addr,
rex_flags,
);
}
}
}
@@ -1402,6 +1421,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state);
emit_std_enc_mem(
sink,
state,
LegacyPrefixes::None,
0xFF,
1,
@@ -1432,25 +1452,22 @@ pub(crate) fn emit(
sink.put1(0x58 + (enc_dst & 7));
}
Inst::CallKnown {
dest, loc, opcode, ..
} => {
Inst::CallKnown { dest, opcode, .. } => {
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s);
}
sink.put1(0xE8);
// The addend adjusts for the difference between the end of the instruction and the
// beginning of the immediate field.
sink.add_reloc(*loc, Reloc::X86CallPCRel4, &dest, -4);
emit_reloc(sink, state, Reloc::X86CallPCRel4, &dest, -4);
sink.put4(0);
if opcode.is_call() {
sink.add_call_site(*loc, *opcode);
let loc = state.cur_srcloc();
sink.add_call_site(loc, *opcode);
}
}
Inst::CallUnknown {
dest, opcode, loc, ..
} => {
Inst::CallUnknown { dest, opcode, .. } => {
let start_offset = sink.cur_offset();
match dest {
RegMem::Reg { reg } => {
@@ -1470,6 +1487,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state);
emit_std_enc_mem(
sink,
state,
LegacyPrefixes::None,
0xFF,
1,
@@ -1483,7 +1501,8 @@ pub(crate) fn emit(
sink.add_stack_map(StackMapExtent::StartedAtOffset(start_offset), s);
}
if opcode.is_call() {
sink.add_call_site(*loc, *opcode);
let loc = state.cur_srcloc();
sink.add_call_site(loc, *opcode);
}
}
@@ -1566,6 +1585,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state);
emit_std_enc_mem(
sink,
state,
LegacyPrefixes::None,
0xFF,
1,
@@ -1609,7 +1629,7 @@ pub(crate) fn emit(
one_way_jmp(sink, CC::NB, *default_target); // idx unsigned >= jmp table size
// Copy the index (and make sure to clear the high 32-bits lane of tmp2).
let inst = Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(*idx), *tmp2, None);
let inst = Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(*idx), *tmp2);
inst.emit(sink, info, state);
// Load base address of jump table.
@@ -1623,7 +1643,6 @@ pub(crate) fn emit(
ExtMode::LQ,
RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)),
*tmp2,
None,
);
inst.emit(sink, info, state);
@@ -1655,18 +1674,14 @@ pub(crate) fn emit(
}
}
Inst::TrapIf {
cc,
trap_code,
srcloc,
} => {
Inst::TrapIf { cc, trap_code } => {
let else_label = sink.get_label();
// Jump over if the invert of CC is set (i.e. CC is not set).
one_way_jmp(sink, cc.invert(), else_label);
// Trap!
let inst = Inst::trap(*srcloc, *trap_code);
let inst = Inst::trap(*trap_code);
inst.emit(sink, info, state);
sink.bind_label(else_label);
@@ -1676,7 +1691,6 @@ pub(crate) fn emit(
op,
src: src_e,
dst: reg_g,
srcloc,
} => {
let rex = RexFlags::clear_w();
@@ -1715,11 +1729,16 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem(sink, prefix, opcode, num_opcodes, reg_g.to_reg(), addr, rex);
emit_std_reg_mem(
sink,
state,
prefix,
opcode,
num_opcodes,
reg_g.to_reg(),
addr,
rex,
);
}
};
}
@@ -1728,7 +1747,6 @@ pub(crate) fn emit(
op,
src: src_e,
dst: reg_g,
srcloc,
} => {
let rex = RexFlags::clear_w();
let (prefix, opcode, length) = match op {
@@ -1823,12 +1841,17 @@ pub(crate) fn emit(
emit_std_reg_reg(sink, prefix, opcode, length, reg_g.to_reg(), *reg_e, rex);
}
RegMem::Mem { addr } => {
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, length, reg_g.to_reg(), addr, rex);
emit_std_reg_mem(
sink,
state,
prefix,
opcode,
length,
reg_g.to_reg(),
addr,
rex,
);
}
}
}
@@ -1897,7 +1920,7 @@ pub(crate) fn emit(
// and negative zero. These instructions merge the sign bits in that
// case, and are no-ops otherwise.
let op = if *is_min { or_op } else { and_op };
let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst, None);
let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst);
inst.emit(sink, info, state);
let inst = Inst::jmp_known(done);
@@ -1907,13 +1930,13 @@ pub(crate) fn emit(
// read-only operand: perform an addition between the two operands, which has the
// desired NaN propagation effects.
sink.bind_label(propagate_nan);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst, None);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst);
inst.emit(sink, info, state);
one_way_jmp(sink, CC::P, done);
sink.bind_label(do_min_max);
let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst, None);
let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst);
inst.emit(sink, info, state);
sink.bind_label(done);
@@ -1925,7 +1948,6 @@ pub(crate) fn emit(
dst,
imm,
is64,
srcloc,
} => {
let (prefix, opcode, len) = match op {
SseOpcode::Cmpps => (LegacyPrefixes::None, 0x0FC2, 2),
@@ -1964,16 +1986,12 @@ pub(crate) fn emit(
}
}
RegMem::Mem { addr } => {
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
let addr = &addr.finalize(state);
assert!(
!regs_swapped,
"No existing way to encode a mem argument in the ModRM r/m field."
);
emit_std_reg_mem(sink, prefix, opcode, len, dst.to_reg(), addr, rex);
emit_std_reg_mem(sink, state, prefix, opcode, len, dst.to_reg(), addr, rex);
}
}
sink.put1(*imm);
@@ -1981,7 +1999,7 @@ pub(crate) fn emit(
Inst::XmmLoadConst { src, dst, ty } => {
let load_offset = Amode::rip_relative(sink.get_label_for_constant(*src));
let load = Inst::load(*ty, load_offset, *dst, ExtKind::None, None);
let load = Inst::load(*ty, load_offset, *dst, ExtKind::None);
load.emit(sink, info, state);
}
@@ -1990,12 +2008,7 @@ pub(crate) fn emit(
// emitted.
}
Inst::XmmMovRM {
op,
src,
dst,
srcloc,
} => {
Inst::XmmMovRM { op, src, dst } => {
let (prefix, opcode) = match op {
SseOpcode::Movaps => (LegacyPrefixes::None, 0x0F29),
SseOpcode::Movapd => (LegacyPrefixes::_66, 0x0F29),
@@ -2008,11 +2021,16 @@ pub(crate) fn emit(
_ => unimplemented!("Opcode {:?} not implemented", op),
};
let dst = &dst.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, RexFlags::clear_w());
emit_std_reg_mem(
sink,
state,
prefix,
opcode,
2,
*src,
dst,
RexFlags::clear_w(),
);
}
Inst::XmmToGpr {
@@ -2070,7 +2088,7 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex);
emit_std_reg_mem(sink, state, prefix, opcode, 2, reg_g.to_reg(), addr, rex);
}
}
}
@@ -2090,7 +2108,7 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, len, *dst, addr, rex);
emit_std_reg_mem(sink, state, prefix, opcode, len, *dst, addr, rex);
}
}
}
@@ -2185,7 +2203,7 @@ pub(crate) fn emit(
} else {
SseOpcode::Addss
};
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst, None);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst);
inst.emit(sink, info, state);
sink.bind_label(done);
@@ -2199,7 +2217,6 @@ pub(crate) fn emit(
dst,
tmp_gpr,
tmp_xmm,
srcloc,
} => {
// Emits the following common sequence:
//
@@ -2292,12 +2309,8 @@ pub(crate) fn emit(
// If the input was positive, saturate to INT_MAX.
// Zero out tmp_xmm.
let inst = Inst::xmm_rm_r(
SseOpcode::Xorpd,
RegMem::reg(tmp_xmm.to_reg()),
*tmp_xmm,
None,
);
let inst =
Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm);
inst.emit(sink, info, state);
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg());
@@ -2317,7 +2330,7 @@ pub(crate) fn emit(
} else {
let check_positive = sink.get_label();
let inst = Inst::trap(*srcloc, TrapCode::BadConversionToInteger);
let inst = Inst::trap(TrapCode::BadConversionToInteger);
inst.emit(sink, info, state);
// Check if INT_MIN was the correct result: determine the smallest floating point
@@ -2360,7 +2373,7 @@ pub(crate) fn emit(
// jump over trap if src >= or > threshold
one_way_jmp(sink, no_overflow_cc, check_positive);
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow);
let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state);
// If positive, it was a real overflow.
@@ -2368,12 +2381,8 @@ pub(crate) fn emit(
sink.bind_label(check_positive);
// Zero out the tmp_xmm register.
let inst = Inst::xmm_rm_r(
SseOpcode::Xorpd,
RegMem::reg(tmp_xmm.to_reg()),
*tmp_xmm,
None,
);
let inst =
Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm);
inst.emit(sink, info, state);
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg());
@@ -2381,7 +2390,7 @@ pub(crate) fn emit(
one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow);
let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state);
}
@@ -2396,7 +2405,6 @@ pub(crate) fn emit(
dst,
tmp_gpr,
tmp_xmm,
srcloc,
} => {
// The only difference in behavior between saturating and non-saturating is how we
// handle errors. Emits the following sequence:
@@ -2487,7 +2495,7 @@ pub(crate) fn emit(
inst.emit(sink, info, state);
} else {
// Trap.
let inst = Inst::trap(*srcloc, TrapCode::BadConversionToInteger);
let inst = Inst::trap(TrapCode::BadConversionToInteger);
inst.emit(sink, info, state);
}
@@ -2519,7 +2527,7 @@ pub(crate) fn emit(
inst.emit(sink, info, state);
} else {
// Trap.
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow);
let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state);
}
@@ -2527,7 +2535,7 @@ pub(crate) fn emit(
sink.bind_label(handle_large);
let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src, None);
let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src);
inst.emit(sink, info, state);
let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size);
@@ -2556,7 +2564,7 @@ pub(crate) fn emit(
let inst = Inst::jmp_known(done);
inst.emit(sink, info, state);
} else {
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow);
let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state);
}
@@ -2582,18 +2590,13 @@ pub(crate) fn emit(
sink.bind_label(done);
}
Inst::LoadExtName {
dst,
name,
offset,
srcloc,
} => {
Inst::LoadExtName { dst, name, offset } => {
// The full address can be encoded in the register, with a relocation.
// Generates: movabsq $name, %dst
let enc_dst = int_reg_enc(dst.to_reg());
sink.put1(0x48 | ((enc_dst >> 3) & 1));
sink.put1(0xB8 | (enc_dst & 7));
sink.add_reloc(*srcloc, Reloc::Abs8, name, *offset);
emit_reloc(sink, state, Reloc::Abs8, name, *offset);
if info.flags().emit_all_ones_funcaddrs() {
sink.put8(u64::max_value());
} else {
@@ -2601,15 +2604,7 @@ pub(crate) fn emit(
}
}
Inst::LockCmpxchg {
ty,
src,
dst,
srcloc,
} => {
if let Some(srcloc) = srcloc {
sink.add_trap(*srcloc, TrapCode::HeapOutOfBounds);
}
Inst::LockCmpxchg { ty, src, dst } => {
// lock cmpxchg{b,w,l,q} %src, (dst)
// Note that 0xF0 is the Lock prefix.
let (prefix, rex, opcodes) = match *ty {
@@ -2626,10 +2621,11 @@ pub(crate) fn emit(
types::I64 => (LegacyPrefixes::_F0, RexFlags::set_w(), 0x0FB1),
_ => unreachable!(),
};
emit_std_reg_mem(sink, prefix, opcodes, 2, *src, &dst.finalize(state), rex);
let amode = dst.finalize(state);
emit_std_reg_mem(sink, state, prefix, opcodes, 2, *src, &amode, rex);
}
Inst::AtomicRmwSeq { ty, op, srcloc } => {
Inst::AtomicRmwSeq { ty, op } => {
// Emit this:
//
// mov{zbq,zwq,zlq,q} (%r9), %rax // rax = old value
@@ -2657,7 +2653,7 @@ pub(crate) fn emit(
// mov{zbq,zwq,zlq,q} (%r9), %rax
// No need to call `add_trap` here, since the `i1` emit will do that.
let i1 = Inst::load(*ty, amode.clone(), rax_w, ExtKind::ZeroExtend, *srcloc);
let i1 = Inst::load(*ty, amode.clone(), rax_w, ExtKind::ZeroExtend);
i1.emit(sink, info, state);
// again:
@@ -2690,7 +2686,6 @@ pub(crate) fn emit(
ty: *ty,
src: r11,
dst: amode.into(),
srcloc: *srcloc,
};
i4.emit(sink, info, state);
@@ -2712,8 +2707,9 @@ pub(crate) fn emit(
sink.put1(0xcc);
}
Inst::Ud2 { trap_info } => {
sink.add_trap(trap_info.0, trap_info.1);
Inst::Ud2 { trap_code } => {
let cur_srcloc = state.cur_srcloc();
sink.add_trap(cur_srcloc, *trap_code);
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
}

744
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

File diff suppressed because it is too large Load Diff

237
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -71,7 +71,6 @@ pub enum Inst {
size: u8, // 1, 2, 4 or 8
signed: bool,
divisor: RegMem,
loc: SourceLoc,
},
/// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs.
@@ -95,7 +94,6 @@ pub enum Inst {
/// different from the temporary.
divisor: Writable<Reg>,
tmp: Option<Writable<Reg>>,
loc: SourceLoc,
},
/// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo)
@@ -126,16 +124,12 @@ pub enum Inst {
ext_mode: ExtMode,
src: RegMem,
dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
},
/// A plain 64-bit integer load, since MovZX_RM_R can't represent that.
Mov64MR {
src: SyntheticAmode,
dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
},
/// Loads the memory address of addr into dst.
@@ -149,8 +143,6 @@ pub enum Inst {
ext_mode: ExtMode,
src: RegMem,
dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
},
/// Integer stores: mov (b w l q) reg addr.
@@ -158,8 +150,6 @@ pub enum Inst {
size: u8, // 1, 2, 4 or 8.
src: Reg,
dst: SyntheticAmode,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
},
/// Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
@@ -213,7 +203,6 @@ pub enum Inst {
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
},
/// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt,
@@ -226,8 +215,6 @@ pub enum Inst {
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
},
/// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq
@@ -235,8 +222,6 @@ pub enum Inst {
op: SseOpcode,
src: Reg,
dst: SyntheticAmode,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
},
/// XMM (vector) unary op (to move a constant value into an xmm register): movups
@@ -289,7 +274,6 @@ pub enum Inst {
dst: Writable<Reg>,
tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
},
/// Converts a scalar xmm to an unsigned int32/int64.
@@ -305,7 +289,6 @@ pub enum Inst {
dst: Writable<Reg>,
tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
},
/// A sequence to compute min/max with the proper NaN semantics for xmm registers.
@@ -340,7 +323,6 @@ pub enum Inst {
dst: Writable<Reg>,
imm: u8,
is64: bool,
srcloc: Option<SourceLoc>,
},
// =====================================
@@ -350,7 +332,6 @@ pub enum Inst {
dest: ExternalName,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode,
},
@@ -359,7 +340,6 @@ pub enum Inst {
dest: RegMem,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode,
},
@@ -408,23 +388,18 @@ pub enum Inst {
JmpUnknown { target: RegMem },
/// Traps if the condition code is set.
TrapIf {
cc: CC,
trap_code: TrapCode,
srcloc: SourceLoc,
},
TrapIf { cc: CC, trap_code: TrapCode },
/// A debug trap.
Hlt,
/// An instruction that will always trigger the illegal instruction exception.
Ud2 { trap_info: (SourceLoc, TrapCode) },
Ud2 { trap_code: TrapCode },
/// Loads an external symbol in a register, with a relocation: movabsq $name, dst
LoadExtName {
dst: Writable<Reg>,
name: Box<ExternalName>,
srcloc: SourceLoc,
offset: i64,
},
@@ -443,7 +418,6 @@ pub enum Inst {
ty: Type, // I8, I16, I32 or I64
src: Reg,
dst: SyntheticAmode,
srcloc: Option<SourceLoc>,
},
/// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction.
@@ -472,7 +446,6 @@ pub enum Inst {
AtomicRmwSeq {
ty: Type, // I8, I16, I32 or I64
op: inst_common::AtomicRmwOp,
srcloc: Option<SourceLoc>,
},
/// A memory fence (mfence, lfence or sfence).
@@ -617,14 +590,13 @@ impl Inst {
Inst::Neg { size, src }
}
pub(crate) fn div(size: u8, signed: bool, divisor: RegMem, loc: SourceLoc) -> Inst {
pub(crate) fn div(size: u8, signed: bool, divisor: RegMem) -> Inst {
divisor.assert_regclass_is(RegClass::I64);
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
Inst::Div {
size,
signed,
divisor,
loc,
}
}
@@ -639,7 +611,6 @@ impl Inst {
size: u8,
divisor: Writable<Reg>,
tmp: Option<Writable<Reg>>,
loc: SourceLoc,
) -> Inst {
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
debug_assert!(divisor.to_reg().get_class() == RegClass::I64);
@@ -651,7 +622,6 @@ impl Inst {
size,
divisor,
tmp,
loc,
}
}
@@ -679,20 +649,10 @@ impl Inst {
}
// TODO Can be replaced by `Inst::move` (high-level) and `Inst::unary_rm_r` (low-level)
pub(crate) fn xmm_mov(
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
pub(crate) fn xmm_mov(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmUnaryRmR {
op,
src,
dst,
srcloc,
}
Inst::XmmUnaryRmR { op, src, dst }
}
pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable<Reg>, ty: Type) -> Inst {
@@ -705,28 +665,13 @@ impl Inst {
pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmUnaryRmR {
op,
src,
dst,
srcloc: None,
}
Inst::XmmUnaryRmR { op, src, dst }
}
pub(crate) fn xmm_rm_r(
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Self {
pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmRmR {
op,
src,
dst,
srcloc,
}
Inst::XmmRmR { op, src, dst }
}
pub(crate) fn xmm_uninit_value(dst: Writable<Reg>) -> Self {
@@ -734,18 +679,12 @@ impl Inst {
Inst::XmmUninitializedValue { dst }
}
pub(crate) fn xmm_mov_r_m(
op: SseOpcode,
src: Reg,
dst: impl Into<SyntheticAmode>,
srcloc: Option<SourceLoc>,
) -> Inst {
pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
debug_assert!(src.get_class() == RegClass::V128);
Inst::XmmMovRM {
op,
src,
dst: dst.into(),
srcloc,
}
}
@@ -815,7 +754,6 @@ impl Inst {
dst: Writable<Reg>,
tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
) -> Inst {
debug_assert!(src.to_reg().get_class() == RegClass::V128);
debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128);
@@ -829,7 +767,6 @@ impl Inst {
dst,
tmp_gpr,
tmp_xmm,
srcloc,
}
}
@@ -841,7 +778,6 @@ impl Inst {
dst: Writable<Reg>,
tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
) -> Inst {
debug_assert!(src.to_reg().get_class() == RegClass::V128);
debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128);
@@ -855,7 +791,6 @@ impl Inst {
dst,
tmp_gpr,
tmp_xmm,
srcloc,
}
}
@@ -881,7 +816,6 @@ impl Inst {
dst: Writable<Reg>,
imm: u8,
is64: bool,
srcloc: Option<SourceLoc>,
) -> Inst {
Inst::XmmRmRImm {
op,
@@ -889,24 +823,13 @@ impl Inst {
dst,
imm,
is64,
srcloc,
}
}
pub(crate) fn movzx_rm_r(
ext_mode: ExtMode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::MovzxRmR {
ext_mode,
src,
dst,
srcloc,
}
Inst::MovzxRmR { ext_mode, src, dst }
}
pub(crate) fn xmm_rmi_reg(opcode: SseOpcode, src: RegMemImm, dst: Writable<Reg>) -> Inst {
@@ -915,41 +838,26 @@ impl Inst {
Inst::XmmRmiReg { opcode, src, dst }
}
pub(crate) fn movsx_rm_r(
ext_mode: ExtMode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::MovsxRmR {
ext_mode,
src,
dst,
srcloc,
}
Inst::MovsxRmR { ext_mode, src, dst }
}
pub(crate) fn mov64_m_r(
src: impl Into<SyntheticAmode>,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
pub(crate) fn mov64_m_r(src: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::Mov64MR {
src: src.into(),
dst,
srcloc,
}
}
/// A convenience function to be able to use a RegMem as the source of a move.
pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable<Reg>, srcloc: Option<SourceLoc>) -> Inst {
pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64);
match src {
RegMem::Reg { reg } => Self::mov_r_r(true, reg, dst),
RegMem::Mem { addr } => Self::mov64_m_r(addr, dst, srcloc),
RegMem::Mem { addr } => Self::mov64_m_r(addr, dst),
}
}
@@ -957,7 +865,6 @@ impl Inst {
size: u8, // 1, 2, 4 or 8
src: Reg,
dst: impl Into<SyntheticAmode>,
srcloc: Option<SourceLoc>,
) -> Inst {
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
debug_assert!(src.get_class() == RegClass::I64);
@@ -965,7 +872,6 @@ impl Inst {
size,
src,
dst: dst.into(),
srcloc,
}
}
@@ -1011,9 +917,9 @@ impl Inst {
Inst::CmpRmiR { size, src, dst }
}
pub(crate) fn trap(srcloc: SourceLoc, trap_code: TrapCode) -> Inst {
pub(crate) fn trap(trap_code: TrapCode) -> Inst {
Inst::Ud2 {
trap_info: (srcloc, trap_code),
trap_code: trap_code,
}
}
@@ -1053,14 +959,12 @@ impl Inst {
dest: ExternalName,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode,
) -> Inst {
Inst::CallKnown {
dest,
uses,
defs,
loc,
opcode,
}
}
@@ -1069,7 +973,6 @@ impl Inst {
dest: RegMem,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode,
) -> Inst {
dest.assert_regclass_is(RegClass::I64);
@@ -1077,7 +980,6 @@ impl Inst {
dest,
uses,
defs,
loc,
opcode,
}
}
@@ -1111,12 +1013,8 @@ impl Inst {
Inst::JmpUnknown { target }
}
pub(crate) fn trap_if(cc: CC, trap_code: TrapCode, srcloc: SourceLoc) -> Inst {
Inst::TrapIf {
cc,
trap_code,
srcloc,
}
pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
Inst::TrapIf { cc, trap_code }
}
/// Choose which instruction to use for loading a register value from memory. For loads smaller
@@ -1127,7 +1025,6 @@ impl Inst {
from_addr: impl Into<SyntheticAmode>,
to_reg: Writable<Reg>,
ext_kind: ExtKind,
srcloc: Option<SourceLoc>,
) -> Inst {
let rc = to_reg.to_reg().get_class();
match rc {
@@ -1143,10 +1040,10 @@ impl Inst {
// Values smaller than 64 bits must be extended in some way.
match ext_kind {
ExtKind::SignExtend => {
Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg, srcloc)
Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
}
ExtKind::ZeroExtend => {
Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg, srcloc)
Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
}
ExtKind::None => panic!(
"expected an extension kind for extension mode: {:?}",
@@ -1155,7 +1052,7 @@ impl Inst {
}
} else {
// 64-bit values can be moved directly.
Inst::mov64_m_r(from_addr, to_reg, srcloc)
Inst::mov64_m_r(from_addr, to_reg)
}
}
RegClass::V128 => {
@@ -1174,18 +1071,13 @@ impl Inst {
}
/// Choose which instruction to use for storing a register value to memory.
pub(crate) fn store(
ty: Type,
from_reg: Reg,
to_addr: impl Into<SyntheticAmode>,
srcloc: Option<SourceLoc>,
) -> Inst {
pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
let rc = from_reg.get_class();
match rc {
RegClass::I64 => {
// Always store the full register, to ensure that the high bits are properly set
// when doing a full reload.
Inst::mov_r_m(8 /* bytes */, from_reg, to_addr, srcloc)
Inst::mov_r_m(8 /* bytes */, from_reg, to_addr)
}
RegClass::V128 => {
let opcode = match ty {
@@ -1196,7 +1088,7 @@ impl Inst {
_ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqu,
_ => unimplemented!("unable to store type: {}", ty),
};
Inst::xmm_mov_r_m(opcode, from_reg, to_addr, srcloc)
Inst::xmm_mov_r_m(opcode, from_reg, to_addr)
}
_ => panic!("unable to generate store for register class: {:?}", rc),
}
@@ -1247,26 +1139,16 @@ impl Inst {
/// Choose which instruction to use for comparing two values for equality.
pub(crate) fn equals(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty {
types::I8X16 | types::B8X16 => Inst::xmm_rm_r(SseOpcode::Pcmpeqb, from, to, None),
types::I16X8 | types::B16X8 => Inst::xmm_rm_r(SseOpcode::Pcmpeqw, from, to, None),
types::I32X4 | types::B32X4 => Inst::xmm_rm_r(SseOpcode::Pcmpeqd, from, to, None),
types::I64X2 | types::B64X2 => Inst::xmm_rm_r(SseOpcode::Pcmpeqq, from, to, None),
types::F32X4 => Inst::xmm_rm_r_imm(
SseOpcode::Cmpps,
from,
to,
FcmpImm::Equal.encode(),
false,
None,
),
types::F64X2 => Inst::xmm_rm_r_imm(
SseOpcode::Cmppd,
from,
to,
FcmpImm::Equal.encode(),
false,
None,
),
types::I8X16 | types::B8X16 => Inst::xmm_rm_r(SseOpcode::Pcmpeqb, from, to),
types::I16X8 | types::B16X8 => Inst::xmm_rm_r(SseOpcode::Pcmpeqw, from, to),
types::I32X4 | types::B32X4 => Inst::xmm_rm_r(SseOpcode::Pcmpeqd, from, to),
types::I64X2 | types::B64X2 => Inst::xmm_rm_r(SseOpcode::Pcmpeqq, from, to),
types::F32X4 => {
Inst::xmm_rm_r_imm(SseOpcode::Cmpps, from, to, FcmpImm::Equal.encode(), false)
}
types::F64X2 => {
Inst::xmm_rm_r_imm(SseOpcode::Cmppd, from, to, FcmpImm::Equal.encode(), false)
}
_ => unimplemented!("unimplemented type for Inst::equals: {}", ty),
}
}
@@ -1274,11 +1156,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise AND on two values.
pub(crate) fn and(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andps, from, to, None),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andpd, from, to, None),
_ if ty.is_vector() && ty.bits() == 128 => {
Inst::xmm_rm_r(SseOpcode::Pand, from, to, None)
}
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pand, from, to),
_ => unimplemented!("unimplemented type for Inst::and: {}", ty),
}
}
@@ -1286,11 +1166,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise AND NOT on two values.
pub(crate) fn and_not(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andnps, from, to, None),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andnpd, from, to, None),
_ if ty.is_vector() && ty.bits() == 128 => {
Inst::xmm_rm_r(SseOpcode::Pandn, from, to, None)
}
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andnps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andnpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pandn, from, to),
_ => unimplemented!("unimplemented type for Inst::and_not: {}", ty),
}
}
@@ -1298,11 +1176,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise OR on two values.
pub(crate) fn or(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Orps, from, to, None),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Orpd, from, to, None),
_ if ty.is_vector() && ty.bits() == 128 => {
Inst::xmm_rm_r(SseOpcode::Por, from, to, None)
}
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Orps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Orpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Por, from, to),
_ => unimplemented!("unimplemented type for Inst::or: {}", ty),
}
}
@@ -1310,11 +1186,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise XOR on two values.
pub(crate) fn xor(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Xorps, from, to, None),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Xorpd, from, to, None),
_ if ty.is_vector() && ty.bits() == 128 => {
Inst::xmm_rm_r(SseOpcode::Pxor, from, to, None)
}
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Xorps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Xorpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pxor, from, to),
_ => unimplemented!("unimplemented type for Inst::xor: {}", ty),
}
}
@@ -1839,7 +1713,7 @@ impl PrettyPrint for Inst {
Inst::Hlt => "hlt".into(),
Inst::Ud2 { trap_info } => format!("ud2 {}", trap_info.1),
Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code),
}
}
}
@@ -2627,7 +2501,6 @@ impl MachInst for Inst {
SseOpcode::Xorps,
RegMem::reg(to_reg.to_reg()),
to_reg,
None,
));
} else {
let tmp = alloc_tmp(RegClass::I64, types::I32);
@@ -2646,7 +2519,6 @@ impl MachInst for Inst {
SseOpcode::Xorpd,
RegMem::reg(to_reg.to_reg()),
to_reg,
None,
));
} else {
let tmp = alloc_tmp(RegClass::I64, types::I64);
@@ -2717,6 +2589,8 @@ pub struct EmitState {
pub(crate) nominal_sp_to_fp: i64,
/// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
stack_map: Option<StackMap>,
/// Current source location.
cur_srcloc: SourceLoc,
}
/// Constant state used during emissions of a sequence of instructions.
@@ -2757,12 +2631,17 @@ impl MachInstEmitState<Inst> for EmitState {
virtual_sp_offset: 0,
nominal_sp_to_fp: abi.frame_size() as i64,
stack_map: None,
cur_srcloc: SourceLoc::default(),
}
}
fn pre_safepoint(&mut self, stack_map: StackMap) {
self.stack_map = Some(stack_map);
}
fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
self.cur_srcloc = srcloc;
}
}
impl EmitState {
@@ -2773,6 +2652,10 @@ impl EmitState {
fn clear_post_insn(&mut self) {
self.stack_map = None;
}
fn cur_srcloc(&self) -> SourceLoc {
self.cur_srcloc
}
}
/// A label-use (internal relocation) in generated code.

451
cranelift/codegen/src/isa/x64/lower.rs
View File

File diff suppressed because it is too large Load Diff