T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MachInst backends: handle SourceLocs out-of-band, not in Insts.

Browse Source 
In existing MachInst backends, many instructions -- any that can trap or
result in a relocation -- carry `SourceLoc` values in order to propagate
the location-in-original-source to use to describe resulting traps or
relocation errors.

This is quite tedious, and also error-prone: it is likely that the
necessary plumbing will be missed in some cases, and in any case, it's
unnecessarily verbose.

This PR factors out the `SourceLoc` handling so that it is tracked
during emission as part of the `EmitState`, and plumbed through
automatically by the machine-independent framework. Instruction emission
code that directly emits trap or relocation records can query the
current location as necessary. Then we only need to ensure that memory
references and trap instructions, at their (one) emission point rather
than their (many) lowering/generation points, are wired up correctly.

This does have the side-effect that some loads and stores that do not
correspond directly to user code's heap accesses will have unnecessary
but harmless trap metadata. For example, the load that fetches a code
offset from a jump table will have a 'heap out of bounds' trap record
attached to it; but because it is bounds-checked, and will never
actually trap if the lowering is correct, this should be harmless.  The
simplicity improvement here seemed more worthwhile to me than plumbing
through a "corresponds to user-level load/store" bit, because the latter
is a bit complex when we allow for op merging.

Closes #2290: though it does not implement a full "metadata" scheme as
described in that issue, this seems simpler overall.
This commit is contained in:
Chris Fallin
2020-11-10 14:37:11 -08:00
parent 0568f4fb02
commit 4dce51096d
19 changed files with 849 additions and 1714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -3,7 +3,6 @@
use crate::ir; use crate::ir;
use crate::ir::types; use crate::ir::types;
use crate::ir::types::*; use crate::ir::types::*;
use crate::ir::SourceLoc;
use crate::isa; use crate::isa;
use crate::isa::aarch64::{inst::EmitState, inst::*}; use crate::isa::aarch64::{inst::EmitState, inst::*};
use crate::machinst::*; use crate::machinst::*;
@@ -380,7 +379,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
extendop: ExtendOp::UXTX, extendop: ExtendOp::UXTX,
}); });
insts.push(Inst::TrapIf { insts.push(Inst::TrapIf {
trap_info: (ir::SourceLoc::default(), ir::TrapCode::StackOverflow), trap_code: ir::TrapCode::StackOverflow,
// Here `Lo` == "less than" when interpreting the two // Here `Lo` == "less than" when interpreting the two
// operands as unsigned integers. // operands as unsigned integers.
kind: CondBrKind::Cond(Cond::Lo), kind: CondBrKind::Cond(Cond::Lo),
@@ -554,7 +553,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(), stack_reg(),
SImm9::maybe_from_i64((vec_offset + (i * 16)) as i64).unwrap(), SImm9::maybe_from_i64((vec_offset + (i * 16)) as i64).unwrap(),
), ),
srcloc: None,
}); });
} }
@@ -603,7 +601,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(), stack_reg(),
SImm9::maybe_from_i64(((i * 16) + int_save_bytes) as i64).unwrap(), SImm9::maybe_from_i64(((i * 16) + int_save_bytes) as i64).unwrap(),
), ),
srcloc: None,
}); });
} }
@@ -634,7 +631,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
dest: &CallDest, dest: &CallDest,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: ir::Opcode, opcode: ir::Opcode,
tmp: Writable<Reg>, tmp: Writable<Reg>,
callee_conv: isa::CallConv, callee_conv: isa::CallConv,
@@ -649,7 +645,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
dest: name.clone(), dest: name.clone(),
uses, uses,
defs, defs,
loc,
opcode, opcode,
caller_callconv: caller_conv, caller_callconv: caller_conv,
callee_callconv: callee_conv, callee_callconv: callee_conv,
@@ -663,7 +658,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
rd: tmp, rd: tmp,
name: Box::new(name.clone()), name: Box::new(name.clone()),
offset: 0, offset: 0,
srcloc: loc,
}, },
)); ));
insts.push(( insts.push((
@@ -673,7 +667,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
rn: tmp.to_reg(), rn: tmp.to_reg(),
uses, uses,
defs, defs,
loc,
opcode, opcode,
caller_callconv: caller_conv, caller_callconv: caller_conv,
callee_callconv: callee_conv, callee_callconv: callee_conv,
@@ -688,7 +681,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
rn: *reg, rn: *reg,
uses, uses,
defs, defs,
loc,
opcode, opcode,
caller_callconv: caller_conv, caller_callconv: caller_conv,
callee_callconv: callee_conv, callee_callconv: callee_conv,

191
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -462,6 +462,8 @@ pub struct EmitState {
pub(crate) nominal_sp_to_fp: i64, pub(crate) nominal_sp_to_fp: i64,
/// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
stack_map: Option<StackMap>, stack_map: Option<StackMap>,
/// Current source-code location corresponding to instruction to be emitted.
cur_srcloc: SourceLoc,
} }
impl MachInstEmitState<Inst> for EmitState { impl MachInstEmitState<Inst> for EmitState {
@@ -470,12 +472,17 @@ impl MachInstEmitState<Inst> for EmitState {
virtual_sp_offset: 0, virtual_sp_offset: 0,
nominal_sp_to_fp: abi.frame_size() as i64, nominal_sp_to_fp: abi.frame_size() as i64,
stack_map: None, stack_map: None,
cur_srcloc: SourceLoc::default(),
} }
} }
fn pre_safepoint(&mut self, stack_map: StackMap) { fn pre_safepoint(&mut self, stack_map: StackMap) {
self.stack_map = Some(stack_map); self.stack_map = Some(stack_map);
} }
fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
self.cur_srcloc = srcloc;
}
} }
impl EmitState { impl EmitState {
@@ -486,6 +493,10 @@ impl EmitState {
fn clear_post_insn(&mut self) { fn clear_post_insn(&mut self) {
self.stack_map = None; self.stack_map = None;
} }
fn cur_srcloc(&self) -> SourceLoc {
self.cur_srcloc
}
} }
/// Constant state used during function compilation. /// Constant state used during function compilation.
@@ -730,57 +741,16 @@ impl MachInstEmit for Inst {
sink.put4(enc_bit_rr(size, op1, op2, rn, rd)) sink.put4(enc_bit_rr(size, op1, op2, rn, rd))
} }
&Inst::ULoad8 { &Inst::ULoad8 { rd, ref mem }
rd, | &Inst::SLoad8 { rd, ref mem }
ref mem, | &Inst::ULoad16 { rd, ref mem }
srcloc, | &Inst::SLoad16 { rd, ref mem }
} | &Inst::ULoad32 { rd, ref mem }
| &Inst::SLoad8 { | &Inst::SLoad32 { rd, ref mem }
rd, | &Inst::ULoad64 { rd, ref mem, .. }
ref mem, | &Inst::FpuLoad32 { rd, ref mem }
srcloc, | &Inst::FpuLoad64 { rd, ref mem }
} | &Inst::FpuLoad128 { rd, ref mem } => {
| &Inst::ULoad16 {
rd,
ref mem,
srcloc,
}
| &Inst::SLoad16 {
rd,
ref mem,
srcloc,
}
| &Inst::ULoad32 {
rd,
ref mem,
srcloc,
}
| &Inst::SLoad32 {
rd,
ref mem,
srcloc,
}
| &Inst::ULoad64 {
rd,
ref mem,
srcloc,
..
}
| &Inst::FpuLoad32 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuLoad64 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuLoad128 {
rd,
ref mem,
srcloc,
} => {
let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state); let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
for inst in mem_insts.into_iter() { for inst in mem_insts.into_iter() {
@@ -807,7 +777,8 @@ impl MachInstEmit for Inst {
_ => unreachable!(), _ => unreachable!(),
}; };
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the actual load instruction starts. // Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
@@ -890,42 +861,13 @@ impl MachInstEmit for Inst {
} }
} }
&Inst::Store8 { &Inst::Store8 { rd, ref mem }
rd, | &Inst::Store16 { rd, ref mem }
ref mem, | &Inst::Store32 { rd, ref mem }
srcloc, | &Inst::Store64 { rd, ref mem, .. }
} | &Inst::FpuStore32 { rd, ref mem }
| &Inst::Store16 { | &Inst::FpuStore64 { rd, ref mem }
rd, | &Inst::FpuStore128 { rd, ref mem } => {
ref mem,
srcloc,
}
| &Inst::Store32 {
rd,
ref mem,
srcloc,
}
| &Inst::Store64 {
rd,
ref mem,
srcloc,
..
}
| &Inst::FpuStore32 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuStore64 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuStore128 {
rd,
ref mem,
srcloc,
} => {
let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state); let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
for inst in mem_insts.into_iter() { for inst in mem_insts.into_iter() {
@@ -943,7 +885,8 @@ impl MachInstEmit for Inst {
_ => unreachable!(), _ => unreachable!(),
}; };
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the actual load instruction starts. // Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
@@ -1086,7 +1029,7 @@ impl MachInstEmit for Inst {
} => { } => {
sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond)); sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond));
} }
&Inst::AtomicRMW { ty, op, srcloc } => { &Inst::AtomicRMW { ty, op } => {
/* Emit this: /* Emit this:
dmb ish dmb ish
again: again:
@@ -1124,7 +1067,8 @@ impl MachInstEmit for Inst {
// again: // again:
sink.bind_label(again_label); sink.bind_label(again_label);
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
sink.put4(enc_ldxr(ty, x27wr, x25)); // ldxr x27, [x25] sink.put4(enc_ldxr(ty, x27wr, x25)); // ldxr x27, [x25]
@@ -1145,7 +1089,8 @@ impl MachInstEmit for Inst {
sink.put4(enc_arith_rrr(bits_31_21, 0b000000, x28wr, x27, x26)); sink.put4(enc_arith_rrr(bits_31_21, 0b000000, x28wr, x27, x26));
} }
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
sink.put4(enc_stxr(ty, x24wr, x28, x25)); // stxr w24, x28, [x25] sink.put4(enc_stxr(ty, x24wr, x28, x25)); // stxr w24, x28, [x25]
@@ -1162,7 +1107,7 @@ impl MachInstEmit for Inst {
sink.put4(enc_dmb_ish()); // dmb ish sink.put4(enc_dmb_ish()); // dmb ish
} }
&Inst::AtomicCAS { ty, srcloc } => { &Inst::AtomicCAS { ty } => {
/* Emit this: /* Emit this:
dmb ish dmb ish
again: again:
@@ -1195,7 +1140,8 @@ impl MachInstEmit for Inst {
// again: // again:
sink.bind_label(again_label); sink.bind_label(again_label);
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
sink.put4(enc_ldxr(ty, x27wr, x25)); // ldxr x27, [x25] sink.put4(enc_ldxr(ty, x27wr, x25)); // ldxr x27, [x25]
@@ -1230,7 +1176,8 @@ impl MachInstEmit for Inst {
)); ));
sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19); sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19);
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
sink.put4(enc_stxr(ty, x24wr, x28, x25)); // stxr w24, x28, [x25] sink.put4(enc_stxr(ty, x24wr, x28, x25)); // stxr w24, x28, [x25]
@@ -1249,12 +1196,7 @@ impl MachInstEmit for Inst {
sink.bind_label(out_label); sink.bind_label(out_label);
sink.put4(enc_dmb_ish()); // dmb ish sink.put4(enc_dmb_ish()); // dmb ish
} }
&Inst::AtomicLoad { &Inst::AtomicLoad { ty, r_data, r_addr } => {
ty,
r_data,
r_addr,
srcloc,
} => {
let op = match ty { let op = match ty {
I8 => 0b0011100001, I8 => 0b0011100001,
I16 => 0b0111100001, I16 => 0b0111100001,
@@ -1264,7 +1206,8 @@ impl MachInstEmit for Inst {
}; };
sink.put4(enc_dmb_ish()); // dmb ish sink.put4(enc_dmb_ish()); // dmb ish
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
let uimm12scaled_zero = UImm12Scaled::zero(I8 /*irrelevant*/); let uimm12scaled_zero = UImm12Scaled::zero(I8 /*irrelevant*/);
@@ -1275,12 +1218,7 @@ impl MachInstEmit for Inst {
r_data.to_reg(), r_data.to_reg(),
)); ));
} }
&Inst::AtomicStore { &Inst::AtomicStore { ty, r_data, r_addr } => {
ty,
r_data,
r_addr,
srcloc,
} => {
let op = match ty { let op = match ty {
I8 => 0b0011100000, I8 => 0b0011100000,
I16 => 0b0111100000, I16 => 0b0111100000,
@@ -1289,7 +1227,8 @@ impl MachInstEmit for Inst {
_ => unreachable!(), _ => unreachable!(),
}; };
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
let uimm12scaled_zero = UImm12Scaled::zero(I8 /*irrelevant*/); let uimm12scaled_zero = UImm12Scaled::zero(I8 /*irrelevant*/);
@@ -1607,7 +1546,6 @@ impl MachInstEmit for Inst {
let inst = Inst::FpuLoad64 { let inst = Inst::FpuLoad64 {
rd, rd,
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
}; };
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
let inst = Inst::Jump { let inst = Inst::Jump {
@@ -1620,7 +1558,6 @@ impl MachInstEmit for Inst {
let inst = Inst::FpuLoad128 { let inst = Inst::FpuLoad128 {
rd, rd,
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
}; };
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
let inst = Inst::Jump { let inst = Inst::Jump {
@@ -1961,15 +1898,11 @@ impl MachInstEmit for Inst {
}; };
sink.put4(enc_vec_rrr(top11, rm, bit15_10, rn, rd)); sink.put4(enc_vec_rrr(top11, rm, bit15_10, rn, rd));
} }
&Inst::VecLoadReplicate { &Inst::VecLoadReplicate { rd, rn, size } => {
rd,
rn,
size,
srcloc,
} => {
let (q, size) = size.enc_size(); let (q, size) = size.enc_size();
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the actual load instruction starts. // Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
@@ -2073,10 +2006,11 @@ impl MachInstEmit for Inst {
if let Some(s) = state.take_stack_map() { if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s); sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
} }
sink.add_reloc(info.loc, Reloc::Arm64Call, &info.dest, 0); let loc = state.cur_srcloc();
sink.add_reloc(loc, Reloc::Arm64Call, &info.dest, 0);
sink.put4(enc_jump26(0b100101, 0)); sink.put4(enc_jump26(0b100101, 0));
if info.opcode.is_call() { if info.opcode.is_call() {
sink.add_call_site(info.loc, info.opcode); sink.add_call_site(loc, info.opcode);
} }
} }
&Inst::CallInd { ref info } => { &Inst::CallInd { ref info } => {
@@ -2084,8 +2018,9 @@ impl MachInstEmit for Inst {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s); sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
} }
sink.put4(0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(info.rn) << 5)); sink.put4(0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(info.rn) << 5));
let loc = state.cur_srcloc();
if info.opcode.is_call() { if info.opcode.is_call() {
sink.add_call_site(info.loc, info.opcode); sink.add_call_site(loc, info.opcode);
} }
} }
&Inst::CondBr { &Inst::CondBr {
@@ -2110,7 +2045,7 @@ impl MachInstEmit for Inst {
} }
sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero())); sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
} }
&Inst::TrapIf { kind, trap_info } => { &Inst::TrapIf { kind, trap_code } => {
// condbr KIND, LABEL // condbr KIND, LABEL
let off = sink.cur_offset(); let off = sink.cur_offset();
let label = sink.get_label(); let label = sink.get_label();
@@ -2120,7 +2055,7 @@ impl MachInstEmit for Inst {
)); ));
sink.use_label_at_offset(off, label, LabelUse::Branch19); sink.use_label_at_offset(off, label, LabelUse::Branch19);
// udf // udf
let trap = Inst::Udf { trap_info }; let trap = Inst::Udf { trap_code };
trap.emit(sink, emit_info, state); trap.emit(sink, emit_info, state);
// LABEL: // LABEL:
sink.bind_label(label); sink.bind_label(label);
@@ -2135,9 +2070,9 @@ impl MachInstEmit for Inst {
&Inst::Brk => { &Inst::Brk => {
sink.put4(0xd4200000); sink.put4(0xd4200000);
} }
&Inst::Udf { trap_info } => { &Inst::Udf { trap_code } => {
let (srcloc, code) = trap_info; let srcloc = state.cur_srcloc();
sink.add_trap(srcloc, code); sink.add_trap(srcloc, trap_code);
if let Some(s) = state.take_stack_map() { if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s); sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
} }
@@ -2192,7 +2127,6 @@ impl MachInstEmit for Inst {
I32, I32,
ExtendOp::UXTW, ExtendOp::UXTW,
), ),
srcloc: None, // can't cause a user trap.
}; };
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
// Add base of jump table to jump-table-sourced block offset // Add base of jump table to jump-table-sourced block offset
@@ -2235,18 +2169,17 @@ impl MachInstEmit for Inst {
rd, rd,
ref name, ref name,
offset, offset,
srcloc,
} => { } => {
let inst = Inst::ULoad64 { let inst = Inst::ULoad64 {
rd, rd,
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None, // can't cause a user trap.
}; };
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
let inst = Inst::Jump { let inst = Inst::Jump {
dest: BranchTarget::ResolvedOffset(12), dest: BranchTarget::ResolvedOffset(12),
}; };
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
let srcloc = state.cur_srcloc();
sink.add_reloc(srcloc, Reloc::Abs8, name, offset); sink.add_reloc(srcloc, Reloc::Abs8, name, offset);
if emit_info.flags().emit_all_ones_funcaddrs() { if emit_info.flags().emit_all_ones_funcaddrs() {
sink.put8(u64::max_value()); sink.put8(u64::max_value());

109
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -1079,7 +1079,6 @@ fn test_aarch64_binemit() {
Inst::ULoad8 { Inst::ULoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"41004038", "41004038",
"ldurb w1, [x2]", "ldurb w1, [x2]",
@@ -1088,7 +1087,6 @@ fn test_aarch64_binemit() {
Inst::ULoad8 { Inst::ULoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)),
srcloc: None,
}, },
"41004039", "41004039",
"ldrb w1, [x2]", "ldrb w1, [x2]",
@@ -1097,7 +1095,6 @@ fn test_aarch64_binemit() {
Inst::ULoad8 { Inst::ULoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)), mem: AMode::RegReg(xreg(2), xreg(5)),
srcloc: None,
}, },
"41686538", "41686538",
"ldrb w1, [x2, x5]", "ldrb w1, [x2, x5]",
@@ -1106,7 +1103,6 @@ fn test_aarch64_binemit() {
Inst::SLoad8 { Inst::SLoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"41008038", "41008038",
"ldursb x1, [x2]", "ldursb x1, [x2]",
@@ -1115,7 +1111,6 @@ fn test_aarch64_binemit() {
Inst::SLoad8 { Inst::SLoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()),
srcloc: None,
}, },
"41FC8039", "41FC8039",
"ldrsb x1, [x2, #63]", "ldrsb x1, [x2, #63]",
@@ -1124,7 +1119,6 @@ fn test_aarch64_binemit() {
Inst::SLoad8 { Inst::SLoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)), mem: AMode::RegReg(xreg(2), xreg(5)),
srcloc: None,
}, },
"4168A538", "4168A538",
"ldrsb x1, [x2, x5]", "ldrsb x1, [x2, x5]",
@@ -1133,7 +1127,6 @@ fn test_aarch64_binemit() {
Inst::ULoad16 { Inst::ULoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()), mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()),
srcloc: None,
}, },
"41504078", "41504078",
"ldurh w1, [x2, #5]", "ldurh w1, [x2, #5]",
@@ -1142,7 +1135,6 @@ fn test_aarch64_binemit() {
Inst::ULoad16 { Inst::ULoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()),
srcloc: None,
}, },
"41104079", "41104079",
"ldrh w1, [x2, #8]", "ldrh w1, [x2, #8]",
@@ -1151,7 +1143,6 @@ fn test_aarch64_binemit() {
Inst::ULoad16 { Inst::ULoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I16), mem: AMode::RegScaled(xreg(2), xreg(3), I16),
srcloc: None,
}, },
"41786378", "41786378",
"ldrh w1, [x2, x3, LSL #1]", "ldrh w1, [x2, x3, LSL #1]",
@@ -1160,7 +1151,6 @@ fn test_aarch64_binemit() {
Inst::SLoad16 { Inst::SLoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"41008078", "41008078",
"ldursh x1, [x2]", "ldursh x1, [x2]",
@@ -1169,7 +1159,6 @@ fn test_aarch64_binemit() {
Inst::SLoad16 { Inst::SLoad16 {
rd: writable_xreg(28), rd: writable_xreg(28),
mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()), mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()),
srcloc: None,
}, },
"9C328079", "9C328079",
"ldrsh x28, [x20, #24]", "ldrsh x28, [x20, #24]",
@@ -1178,7 +1167,6 @@ fn test_aarch64_binemit() {
Inst::SLoad16 { Inst::SLoad16 {
rd: writable_xreg(28), rd: writable_xreg(28),
mem: AMode::RegScaled(xreg(20), xreg(20), I16), mem: AMode::RegScaled(xreg(20), xreg(20), I16),
srcloc: None,
}, },
"9C7AB478", "9C7AB478",
"ldrsh x28, [x20, x20, LSL #1]", "ldrsh x28, [x20, x20, LSL #1]",
@@ -1187,7 +1175,6 @@ fn test_aarch64_binemit() {
Inst::ULoad32 { Inst::ULoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"410040B8", "410040B8",
"ldur w1, [x2]", "ldur w1, [x2]",
@@ -1196,7 +1183,6 @@ fn test_aarch64_binemit() {
Inst::ULoad32 { Inst::ULoad32 {
rd: writable_xreg(12), rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()), mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()),
srcloc: None,
}, },
"0CCC40B9", "0CCC40B9",
"ldr w12, [x0, #204]", "ldr w12, [x0, #204]",
@@ -1205,7 +1191,6 @@ fn test_aarch64_binemit() {
Inst::ULoad32 { Inst::ULoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(12), I32), mem: AMode::RegScaled(xreg(2), xreg(12), I32),
srcloc: None,
}, },
"41786CB8", "41786CB8",
"ldr w1, [x2, x12, LSL #2]", "ldr w1, [x2, x12, LSL #2]",
@@ -1214,7 +1199,6 @@ fn test_aarch64_binemit() {
Inst::SLoad32 { Inst::SLoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"410080B8", "410080B8",
"ldursw x1, [x2]", "ldursw x1, [x2]",
@@ -1223,7 +1207,6 @@ fn test_aarch64_binemit() {
Inst::SLoad32 { Inst::SLoad32 {
rd: writable_xreg(12), rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
srcloc: None,
}, },
"2CFCBFB9", "2CFCBFB9",
"ldrsw x12, [x1, #16380]", "ldrsw x12, [x1, #16380]",
@@ -1232,7 +1215,6 @@ fn test_aarch64_binemit() {
Inst::SLoad32 { Inst::SLoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(5), xreg(1), I32), mem: AMode::RegScaled(xreg(5), xreg(1), I32),
srcloc: None,
}, },
"A178A1B8", "A178A1B8",
"ldrsw x1, [x5, x1, LSL #2]", "ldrsw x1, [x5, x1, LSL #2]",
@@ -1241,7 +1223,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"410040F8", "410040F8",
"ldur x1, [x2]", "ldur x1, [x2]",
@@ -1250,7 +1231,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()), mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()),
srcloc: None,
}, },
"410050F8", "410050F8",
"ldur x1, [x2, #-256]", "ldur x1, [x2, #-256]",
@@ -1259,7 +1239,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()), mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()),
srcloc: None,
}, },
"41F04FF8", "41F04FF8",
"ldur x1, [x2, #255]", "ldur x1, [x2, #255]",
@@ -1268,7 +1247,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
srcloc: None,
}, },
"41FC7FF9", "41FC7FF9",
"ldr x1, [x2, #32760]", "ldr x1, [x2, #32760]",
@@ -1277,7 +1255,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)), mem: AMode::RegReg(xreg(2), xreg(3)),
srcloc: None,
}, },
"416863F8", "416863F8",
"ldr x1, [x2, x3]", "ldr x1, [x2, x3]",
@@ -1286,7 +1263,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64), mem: AMode::RegScaled(xreg(2), xreg(3), I64),
srcloc: None,
}, },
"417863F8", "417863F8",
"ldr x1, [x2, x3, LSL #3]", "ldr x1, [x2, x3, LSL #3]",
@@ -1295,7 +1271,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW), mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW),
srcloc: None,
}, },
"41D863F8", "41D863F8",
"ldr x1, [x2, w3, SXTW #3]", "ldr x1, [x2, w3, SXTW #3]",
@@ -1304,7 +1279,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW), mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW),
srcloc: None,
}, },
"41C863F8", "41C863F8",
"ldr x1, [x2, w3, SXTW]", "ldr x1, [x2, w3, SXTW]",
@@ -1313,7 +1287,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Label(MemLabel::PCRel(64)), mem: AMode::Label(MemLabel::PCRel(64)),
srcloc: None,
}, },
"01020058", "01020058",
"ldr x1, pc+64", "ldr x1, pc+64",
@@ -1322,7 +1295,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
}, },
"410C41F8", "410C41F8",
"ldr x1, [x2, #16]!", "ldr x1, [x2, #16]!",
@@ -1331,7 +1303,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
}, },
"410441F8", "410441F8",
"ldr x1, [x2], #16", "ldr x1, [x2], #16",
@@ -1340,7 +1311,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(32768, I8), mem: AMode::FPOffset(32768, I8),
srcloc: None,
}, },
"100090D2B063308B010240F9", "100090D2B063308B010240F9",
"movz x16, #32768 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", "movz x16, #32768 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1349,7 +1319,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(-32768, I8), mem: AMode::FPOffset(-32768, I8),
srcloc: None,
}, },
"F0FF8F92B063308B010240F9", "F0FF8F92B063308B010240F9",
"movn x16, #32767 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", "movn x16, #32767 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1358,7 +1327,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(1048576, I8), // 2^20 mem: AMode::FPOffset(1048576, I8), // 2^20
srcloc: None,
}, },
"1002A0D2B063308B010240F9", "1002A0D2B063308B010240F9",
"movz x16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", "movz x16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1367,7 +1335,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1 mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1
srcloc: None,
}, },
"300080521002A072B063308B010240F9", "300080521002A072B063308B010240F9",
"movz w16, #1 ; movk w16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]", "movz w16, #1 ; movk w16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1377,7 +1344,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 8, I64), mem: AMode::RegOffset(xreg(7), 8, I64),
srcloc: None,
}, },
"E18040F8", "E18040F8",
"ldur x1, [x7, #8]", "ldur x1, [x7, #8]",
@@ -1387,7 +1353,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1024, I64), mem: AMode::RegOffset(xreg(7), 1024, I64),
srcloc: None,
}, },
"E10042F9", "E10042F9",
"ldr x1, [x7, #1024]", "ldr x1, [x7, #1024]",
@@ -1397,7 +1362,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1048576, I64), mem: AMode::RegOffset(xreg(7), 1048576, I64),
srcloc: None,
}, },
"1002A0D2F060308B010240F9", "1002A0D2F060308B010240F9",
"movz x16, #16, LSL #16 ; add x16, x7, x16, UXTX ; ldr x1, [x16]", "movz x16, #16, LSL #16 ; add x16, x7, x16, UXTX ; ldr x1, [x16]",
@@ -1407,7 +1371,6 @@ fn test_aarch64_binemit() {
Inst::Store8 { Inst::Store8 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"41000038", "41000038",
"sturb w1, [x2]", "sturb w1, [x2]",
@@ -1416,7 +1379,6 @@ fn test_aarch64_binemit() {
Inst::Store8 { Inst::Store8 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()),
srcloc: None,
}, },
"41FC3F39", "41FC3F39",
"strb w1, [x2, #4095]", "strb w1, [x2, #4095]",
@@ -1425,7 +1387,6 @@ fn test_aarch64_binemit() {
Inst::Store16 { Inst::Store16 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"41000078", "41000078",
"sturh w1, [x2]", "sturh w1, [x2]",
@@ -1434,7 +1395,6 @@ fn test_aarch64_binemit() {
Inst::Store16 { Inst::Store16 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()),
srcloc: None,
}, },
"41FC3F79", "41FC3F79",
"strh w1, [x2, #8190]", "strh w1, [x2, #8190]",
@@ -1443,7 +1403,6 @@ fn test_aarch64_binemit() {
Inst::Store32 { Inst::Store32 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"410000B8", "410000B8",
"stur w1, [x2]", "stur w1, [x2]",
@@ -1452,7 +1411,6 @@ fn test_aarch64_binemit() {
Inst::Store32 { Inst::Store32 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
srcloc: None,
}, },
"41FC3FB9", "41FC3FB9",
"str w1, [x2, #16380]", "str w1, [x2, #16380]",
@@ -1461,7 +1419,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()), mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
}, },
"410000F8", "410000F8",
"stur x1, [x2]", "stur x1, [x2]",
@@ -1470,7 +1427,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
srcloc: None,
}, },
"41FC3FF9", "41FC3FF9",
"str x1, [x2, #32760]", "str x1, [x2, #32760]",
@@ -1479,7 +1435,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)), mem: AMode::RegReg(xreg(2), xreg(3)),
srcloc: None,
}, },
"416823F8", "416823F8",
"str x1, [x2, x3]", "str x1, [x2, x3]",
@@ -1488,7 +1443,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64), mem: AMode::RegScaled(xreg(2), xreg(3), I64),
srcloc: None,
}, },
"417823F8", "417823F8",
"str x1, [x2, x3, LSL #3]", "str x1, [x2, x3, LSL #3]",
@@ -1497,7 +1451,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW), mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW),
srcloc: None,
}, },
"415823F8", "415823F8",
"str x1, [x2, w3, UXTW #3]", "str x1, [x2, w3, UXTW #3]",
@@ -1506,7 +1459,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW), mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW),
srcloc: None,
}, },
"414823F8", "414823F8",
"str x1, [x2, w3, UXTW]", "str x1, [x2, w3, UXTW]",
@@ -1515,7 +1467,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
}, },
"410C01F8", "410C01F8",
"str x1, [x2, #16]!", "str x1, [x2, #16]!",
@@ -1524,7 +1475,6 @@ fn test_aarch64_binemit() {
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()), mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
}, },
"410401F8", "410401F8",
"str x1, [x2], #16", "str x1, [x2], #16",
@@ -3911,7 +3861,7 @@ fn test_aarch64_binemit() {
Inst::VecLoadReplicate { Inst::VecLoadReplicate {
rd: writable_vreg(31), rd: writable_vreg(31),
rn: xreg(0), rn: xreg(0),
srcloc: None,
size: VectorSize::Size64x2, size: VectorSize::Size64x2,
}, },
"1FCC404D", "1FCC404D",
@@ -3922,7 +3872,7 @@ fn test_aarch64_binemit() {
Inst::VecLoadReplicate { Inst::VecLoadReplicate {
rd: writable_vreg(0), rd: writable_vreg(0),
rn: xreg(25), rn: xreg(25),
srcloc: None,
size: VectorSize::Size8x8, size: VectorSize::Size8x8,
}, },
"20C3400D", "20C3400D",
@@ -4050,7 +4000,7 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::NotZero(xreg(8)), kind: CondBrKind::NotZero(xreg(8)),
}, },
"480000B40000A0D4", "480000B40000A0D4",
@@ -4058,7 +4008,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Zero(xreg(8)), kind: CondBrKind::Zero(xreg(8)),
}, },
"480000B50000A0D4", "480000B50000A0D4",
@@ -4066,7 +4016,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ne), kind: CondBrKind::Cond(Cond::Ne),
}, },
"400000540000A0D4", "400000540000A0D4",
@@ -4074,7 +4024,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Eq), kind: CondBrKind::Cond(Cond::Eq),
}, },
"410000540000A0D4", "410000540000A0D4",
@@ -4082,7 +4032,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Lo), kind: CondBrKind::Cond(Cond::Lo),
}, },
"420000540000A0D4", "420000540000A0D4",
@@ -4090,7 +4040,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Hs), kind: CondBrKind::Cond(Cond::Hs),
}, },
"430000540000A0D4", "430000540000A0D4",
@@ -4098,7 +4048,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Pl), kind: CondBrKind::Cond(Cond::Pl),
}, },
"440000540000A0D4", "440000540000A0D4",
@@ -4106,7 +4056,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Mi), kind: CondBrKind::Cond(Cond::Mi),
}, },
"450000540000A0D4", "450000540000A0D4",
@@ -4114,7 +4064,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Vc), kind: CondBrKind::Cond(Cond::Vc),
}, },
"460000540000A0D4", "460000540000A0D4",
@@ -4122,7 +4072,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Vs), kind: CondBrKind::Cond(Cond::Vs),
}, },
"470000540000A0D4", "470000540000A0D4",
@@ -4130,7 +4080,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ls), kind: CondBrKind::Cond(Cond::Ls),
}, },
"480000540000A0D4", "480000540000A0D4",
@@ -4138,7 +4088,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Hi), kind: CondBrKind::Cond(Cond::Hi),
}, },
"490000540000A0D4", "490000540000A0D4",
@@ -4146,7 +4096,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Lt), kind: CondBrKind::Cond(Cond::Lt),
}, },
"4A0000540000A0D4", "4A0000540000A0D4",
@@ -4154,7 +4104,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ge), kind: CondBrKind::Cond(Cond::Ge),
}, },
"4B0000540000A0D4", "4B0000540000A0D4",
@@ -4162,7 +4112,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Le), kind: CondBrKind::Cond(Cond::Le),
}, },
"4C0000540000A0D4", "4C0000540000A0D4",
@@ -4170,7 +4120,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Gt), kind: CondBrKind::Cond(Cond::Gt),
}, },
"4D0000540000A0D4", "4D0000540000A0D4",
@@ -4178,7 +4128,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Nv), kind: CondBrKind::Cond(Cond::Nv),
}, },
"4E0000540000A0D4", "4E0000540000A0D4",
@@ -4186,7 +4136,7 @@ fn test_aarch64_binemit() {
)); ));
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Al), kind: CondBrKind::Cond(Cond::Al),
}, },
"4F0000540000A0D4", "4F0000540000A0D4",
@@ -4209,7 +4159,6 @@ fn test_aarch64_binemit() {
dest: ExternalName::testcase("test0"), dest: ExternalName::testcase("test0"),
uses: Vec::new(), uses: Vec::new(),
defs: Vec::new(), defs: Vec::new(),
loc: SourceLoc::default(),
opcode: Opcode::Call, opcode: Opcode::Call,
caller_callconv: CallConv::SystemV, caller_callconv: CallConv::SystemV,
callee_callconv: CallConv::SystemV, callee_callconv: CallConv::SystemV,
@@ -4225,7 +4174,6 @@ fn test_aarch64_binemit() {
rn: xreg(10), rn: xreg(10),
uses: Vec::new(), uses: Vec::new(),
defs: Vec::new(), defs: Vec::new(),
loc: SourceLoc::default(),
opcode: Opcode::CallIndirect, opcode: Opcode::CallIndirect,
caller_callconv: CallConv::SystemV, caller_callconv: CallConv::SystemV,
callee_callconv: CallConv::SystemV, callee_callconv: CallConv::SystemV,
@@ -4797,7 +4745,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad32 { Inst::FpuLoad32 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32), mem: AMode::RegScaled(xreg(8), xreg(9), F32),
srcloc: None,
}, },
"107969BC", "107969BC",
"ldr s16, [x8, x9, LSL #2]", "ldr s16, [x8, x9, LSL #2]",
@@ -4807,7 +4754,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad64 { Inst::FpuLoad64 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64), mem: AMode::RegScaled(xreg(8), xreg(9), F64),
srcloc: None,
}, },
"107969FC", "107969FC",
"ldr d16, [x8, x9, LSL #3]", "ldr d16, [x8, x9, LSL #3]",
@@ -4817,7 +4763,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad128 { Inst::FpuLoad128 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128), mem: AMode::RegScaled(xreg(8), xreg(9), I128),
srcloc: None,
}, },
"1079E93C", "1079E93C",
"ldr q16, [x8, x9, LSL #4]", "ldr q16, [x8, x9, LSL #4]",
@@ -4827,7 +4772,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad32 { Inst::FpuLoad32 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
}, },
"5000001C", "5000001C",
"ldr s16, pc+8", "ldr s16, pc+8",
@@ -4837,7 +4781,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad64 { Inst::FpuLoad64 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
}, },
"5000005C", "5000005C",
"ldr d16, pc+8", "ldr d16, pc+8",
@@ -4847,7 +4790,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad128 { Inst::FpuLoad128 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
}, },
"5000009C", "5000009C",
"ldr q16, pc+8", "ldr q16, pc+8",
@@ -4857,7 +4799,6 @@ fn test_aarch64_binemit() {
Inst::FpuStore32 { Inst::FpuStore32 {
rd: vreg(16), rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32), mem: AMode::RegScaled(xreg(8), xreg(9), F32),
srcloc: None,
}, },
"107929BC", "107929BC",
"str s16, [x8, x9, LSL #2]", "str s16, [x8, x9, LSL #2]",
@@ -4867,7 +4808,6 @@ fn test_aarch64_binemit() {
Inst::FpuStore64 { Inst::FpuStore64 {
rd: vreg(16), rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64), mem: AMode::RegScaled(xreg(8), xreg(9), F64),
srcloc: None,
}, },
"107929FC", "107929FC",
"str d16, [x8, x9, LSL #3]", "str d16, [x8, x9, LSL #3]",
@@ -4877,7 +4817,6 @@ fn test_aarch64_binemit() {
Inst::FpuStore128 { Inst::FpuStore128 {
rd: vreg(16), rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128), mem: AMode::RegScaled(xreg(8), xreg(9), I128),
srcloc: None,
}, },
"1079A93C", "1079A93C",
"str q16, [x8, x9, LSL #4]", "str q16, [x8, x9, LSL #4]",
@@ -5000,7 +4939,6 @@ fn test_aarch64_binemit() {
Inst::AtomicRMW { Inst::AtomicRMW {
ty: I16, ty: I16,
op: inst_common::AtomicRmwOp::Xor, op: inst_common::AtomicRmwOp::Xor,
srcloc: None,
}, },
"BF3B03D53B7F5F487C031ACA3C7F1848B8FFFFB5BF3B03D5", "BF3B03D53B7F5F487C031ACA3C7F1848B8FFFFB5BF3B03D5",
"atomically { 16_bits_at_[x25]) Xor= x26 ; x27 = old_value_at_[x25]; x24,x28 = trash }", "atomically { 16_bits_at_[x25]) Xor= x26 ; x27 = old_value_at_[x25]; x24,x28 = trash }",
@@ -5010,7 +4948,6 @@ fn test_aarch64_binemit() {
Inst::AtomicRMW { Inst::AtomicRMW {
ty: I32, ty: I32,
op: inst_common::AtomicRmwOp::Xchg, op: inst_common::AtomicRmwOp::Xchg,
srcloc: None,
}, },
"BF3B03D53B7F5F88FC031AAA3C7F1888B8FFFFB5BF3B03D5", "BF3B03D53B7F5F88FC031AAA3C7F1888B8FFFFB5BF3B03D5",
"atomically { 32_bits_at_[x25]) Xchg= x26 ; x27 = old_value_at_[x25]; x24,x28 = trash }", "atomically { 32_bits_at_[x25]) Xchg= x26 ; x27 = old_value_at_[x25]; x24,x28 = trash }",
@@ -5019,7 +4956,6 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::AtomicCAS { Inst::AtomicCAS {
ty: I8, ty: I8,
srcloc: None,
}, },
"BF3B03D53B7F5F08581F40927F0318EB610000543C7F180878FFFFB5BF3B03D5", "BF3B03D53B7F5F08581F40927F0318EB610000543C7F180878FFFFB5BF3B03D5",
"atomically { compare-and-swap(8_bits_at_[x25], x26 -> x28), x27 = old_value_at_[x25]; x24 = trash }" "atomically { compare-and-swap(8_bits_at_[x25], x26 -> x28), x27 = old_value_at_[x25]; x24 = trash }"
@@ -5028,7 +4964,6 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::AtomicCAS { Inst::AtomicCAS {
ty: I64, ty: I64,
srcloc: None,
}, },
"BF3B03D53B7F5FC8F8031AAA7F0318EB610000543C7F18C878FFFFB5BF3B03D5", "BF3B03D53B7F5FC8F8031AAA7F0318EB610000543C7F18C878FFFFB5BF3B03D5",
"atomically { compare-and-swap(64_bits_at_[x25], x26 -> x28), x27 = old_value_at_[x25]; x24 = trash }" "atomically { compare-and-swap(64_bits_at_[x25], x26 -> x28), x27 = old_value_at_[x25]; x24 = trash }"
@@ -5039,7 +4974,6 @@ fn test_aarch64_binemit() {
ty: I8, ty: I8,
r_data: writable_xreg(7), r_data: writable_xreg(7),
r_addr: xreg(28), r_addr: xreg(28),
srcloc: None,
}, },
"BF3B03D587034039", "BF3B03D587034039",
"atomically { x7 = zero_extend_8_bits_at[x28] }", "atomically { x7 = zero_extend_8_bits_at[x28] }",
@@ -5050,7 +4984,6 @@ fn test_aarch64_binemit() {
ty: I64, ty: I64,
r_data: writable_xreg(28), r_data: writable_xreg(28),
r_addr: xreg(7), r_addr: xreg(7),
srcloc: None,
}, },
"BF3B03D5FC0040F9", "BF3B03D5FC0040F9",
"atomically { x28 = zero_extend_64_bits_at[x7] }", "atomically { x28 = zero_extend_64_bits_at[x7] }",
@@ -5061,7 +4994,6 @@ fn test_aarch64_binemit() {
ty: I16, ty: I16,
r_data: xreg(17), r_data: xreg(17),
r_addr: xreg(8), r_addr: xreg(8),
srcloc: None,
}, },
"11010079BF3B03D5", "11010079BF3B03D5",
"atomically { 16_bits_at[x8] = x17 }", "atomically { 16_bits_at[x8] = x17 }",
@@ -5072,7 +5004,6 @@ fn test_aarch64_binemit() {
ty: I32, ty: I32,
r_data: xreg(18), r_data: xreg(18),
r_addr: xreg(7), r_addr: xreg(7),
srcloc: None,
}, },
"F20000B9BF3B03D5", "F20000B9BF3B03D5",
"atomically { 32_bits_at[x7] = x18 }", "atomically { 32_bits_at[x7] = x18 }",

123
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -420,7 +420,6 @@ pub struct CallInfo {
pub dest: ExternalName, pub dest: ExternalName,
pub uses: Vec<Reg>, pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>, pub defs: Vec<Writable<Reg>>,
pub loc: SourceLoc,
pub opcode: Opcode, pub opcode: Opcode,
pub caller_callconv: CallConv, pub caller_callconv: CallConv,
pub callee_callconv: CallConv, pub callee_callconv: CallConv,
@@ -433,7 +432,6 @@ pub struct CallIndInfo {
pub rn: Reg, pub rn: Reg,
pub uses: Vec<Reg>, pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>, pub defs: Vec<Writable<Reg>>,
pub loc: SourceLoc,
pub opcode: Opcode, pub opcode: Opcode,
pub caller_callconv: CallConv, pub caller_callconv: CallConv,
pub callee_callconv: CallConv, pub callee_callconv: CallConv,
@@ -524,68 +522,57 @@ pub enum Inst {
ULoad8 { ULoad8 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A signed (sign-extending) 8-bit load. /// A signed (sign-extending) 8-bit load.
SLoad8 { SLoad8 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// An unsigned (zero-extending) 16-bit load. /// An unsigned (zero-extending) 16-bit load.
ULoad16 { ULoad16 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A signed (sign-extending) 16-bit load. /// A signed (sign-extending) 16-bit load.
SLoad16 { SLoad16 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// An unsigned (zero-extending) 32-bit load. /// An unsigned (zero-extending) 32-bit load.
ULoad32 { ULoad32 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A signed (sign-extending) 32-bit load. /// A signed (sign-extending) 32-bit load.
SLoad32 { SLoad32 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A 64-bit load. /// A 64-bit load.
ULoad64 { ULoad64 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// An 8-bit store. /// An 8-bit store.
Store8 { Store8 {
rd: Reg, rd: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A 16-bit store. /// A 16-bit store.
Store16 { Store16 {
rd: Reg, rd: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A 32-bit store. /// A 32-bit store.
Store32 { Store32 {
rd: Reg, rd: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A 64-bit store. /// A 64-bit store.
Store64 { Store64 {
rd: Reg, rd: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// A store of a pair of registers. /// A store of a pair of registers.
@@ -686,7 +673,6 @@ pub enum Inst {
AtomicRMW { AtomicRMW {
ty: Type, // I8, I16, I32 or I64 ty: Type, // I8, I16, I32 or I64
op: inst_common::AtomicRmwOp, op: inst_common::AtomicRmwOp,
srcloc: Option<SourceLoc>,
}, },
/// Similar to AtomicRMW, a compare-and-swap operation implemented using a load-linked /// Similar to AtomicRMW, a compare-and-swap operation implemented using a load-linked
@@ -703,7 +689,6 @@ pub enum Inst {
/// x24 (wr) scratch reg; value afterwards has no meaning /// x24 (wr) scratch reg; value afterwards has no meaning
AtomicCAS { AtomicCAS {
ty: Type, // I8, I16, I32 or I64 ty: Type, // I8, I16, I32 or I64
srcloc: Option<SourceLoc>,
}, },
/// Read `ty` bits from address `r_addr`, zero extend the loaded value to 64 bits and put it /// Read `ty` bits from address `r_addr`, zero extend the loaded value to 64 bits and put it
@@ -713,7 +698,6 @@ pub enum Inst {
ty: Type, // I8, I16, I32 or I64 ty: Type, // I8, I16, I32 or I64
r_data: Writable<Reg>, r_data: Writable<Reg>,
r_addr: Reg, r_addr: Reg,
srcloc: Option<SourceLoc>,
}, },
/// Write the lowest `ty` bits of `r_data` to address `r_addr`, with a memory fence /// Write the lowest `ty` bits of `r_data` to address `r_addr`, with a memory fence
@@ -723,7 +707,6 @@ pub enum Inst {
ty: Type, // I8, I16, I32 or I64 ty: Type, // I8, I16, I32 or I64
r_data: Reg, r_data: Reg,
r_addr: Reg, r_addr: Reg,
srcloc: Option<SourceLoc>,
}, },
/// A memory fence. This must provide ordering to ensure that, at a minimum, neither loads /// A memory fence. This must provide ordering to ensure that, at a minimum, neither loads
@@ -798,37 +781,31 @@ pub enum Inst {
FpuLoad32 { FpuLoad32 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// Floating-point store, single-precision (32 bit). /// Floating-point store, single-precision (32 bit).
FpuStore32 { FpuStore32 {
rd: Reg, rd: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// Floating-point load, double-precision (64 bit). /// Floating-point load, double-precision (64 bit).
FpuLoad64 { FpuLoad64 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// Floating-point store, double-precision (64 bit). /// Floating-point store, double-precision (64 bit).
FpuStore64 { FpuStore64 {
rd: Reg, rd: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// Floating-point/vector load, 128 bit. /// Floating-point/vector load, 128 bit.
FpuLoad128 { FpuLoad128 {
rd: Writable<Reg>, rd: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
/// Floating-point/vector store, 128 bit. /// Floating-point/vector store, 128 bit.
FpuStore128 { FpuStore128 {
rd: Reg, rd: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
}, },
LoadFpuConst64 { LoadFpuConst64 {
@@ -1037,7 +1014,6 @@ pub enum Inst {
rd: Writable<Reg>, rd: Writable<Reg>,
rn: Reg, rn: Reg,
size: VectorSize, size: VectorSize,
srcloc: Option<SourceLoc>,
}, },
/// Move to the NZCV flags (actually a `MSR NZCV, Xn` insn). /// Move to the NZCV flags (actually a `MSR NZCV, Xn` insn).
@@ -1095,7 +1071,7 @@ pub enum Inst {
/// of this condition in a branch that skips the trap instruction.) /// of this condition in a branch that skips the trap instruction.)
TrapIf { TrapIf {
kind: CondBrKind, kind: CondBrKind,
trap_info: (SourceLoc, TrapCode), trap_code: TrapCode,
}, },
/// An indirect branch through a register, augmented with set of all /// An indirect branch through a register, augmented with set of all
@@ -1111,7 +1087,7 @@ pub enum Inst {
/// An instruction guaranteed to always be undefined and to trigger an illegal instruction at /// An instruction guaranteed to always be undefined and to trigger an illegal instruction at
/// runtime. /// runtime.
Udf { Udf {
trap_info: (SourceLoc, TrapCode), trap_code: TrapCode,
}, },
/// Compute the address (using a PC-relative offset) of a memory location, using the `ADR` /// Compute the address (using a PC-relative offset) of a memory location, using the `ADR`
@@ -1146,7 +1122,6 @@ pub enum Inst {
LoadExtName { LoadExtName {
rd: Writable<Reg>, rd: Writable<Reg>,
name: Box<ExternalName>, name: Box<ExternalName>,
srcloc: SourceLoc,
offset: i64, offset: i64,
}, },
@@ -1457,47 +1432,22 @@ impl Inst {
/// Generic constructor for a load (zero-extending where appropriate). /// Generic constructor for a load (zero-extending where appropriate).
pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type) -> Inst { pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type) -> Inst {
match ty { match ty {
B1 | B8 | I8 => Inst::ULoad8 { B1 | B8 | I8 => Inst::ULoad8 { rd: into_reg, mem },
rd: into_reg, B16 | I16 => Inst::ULoad16 { rd: into_reg, mem },
mem, B32 | I32 | R32 => Inst::ULoad32 { rd: into_reg, mem },
srcloc: None, B64 | I64 | R64 => Inst::ULoad64 { rd: into_reg, mem },
}, F32 => Inst::FpuLoad32 { rd: into_reg, mem },
B16 | I16 => Inst::ULoad16 { F64 => Inst::FpuLoad64 { rd: into_reg, mem },
rd: into_reg,
mem,
srcloc: None,
},
B32 | I32 | R32 => Inst::ULoad32 {
rd: into_reg,
mem,
srcloc: None,
},
B64 | I64 | R64 => Inst::ULoad64 {
rd: into_reg,
mem,
srcloc: None,
},
F32 => Inst::FpuLoad32 {
rd: into_reg,
mem,
srcloc: None,
},
F64 => Inst::FpuLoad64 {
rd: into_reg,
mem,
srcloc: None,
},
_ => { _ => {
if ty.is_vector() { if ty.is_vector() {
let bits = ty_bits(ty); let bits = ty_bits(ty);
let rd = into_reg; let rd = into_reg;
let srcloc = None;
if bits == 128 { if bits == 128 {
Inst::FpuLoad128 { rd, mem, srcloc } Inst::FpuLoad128 { rd, mem }
} else { } else {
assert_eq!(bits, 64); assert_eq!(bits, 64);
Inst::FpuLoad64 { rd, mem, srcloc } Inst::FpuLoad64 { rd, mem }
} }
} else { } else {
unimplemented!("gen_load({})", ty); unimplemented!("gen_load({})", ty);
@@ -1509,47 +1459,22 @@ impl Inst {
/// Generic constructor for a store. /// Generic constructor for a store.
pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type) -> Inst { pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type) -> Inst {
match ty { match ty {
B1 | B8 | I8 => Inst::Store8 { B1 | B8 | I8 => Inst::Store8 { rd: from_reg, mem },
rd: from_reg, B16 | I16 => Inst::Store16 { rd: from_reg, mem },
mem, B32 | I32 | R32 => Inst::Store32 { rd: from_reg, mem },
srcloc: None, B64 | I64 | R64 => Inst::Store64 { rd: from_reg, mem },
}, F32 => Inst::FpuStore32 { rd: from_reg, mem },
B16 | I16 => Inst::Store16 { F64 => Inst::FpuStore64 { rd: from_reg, mem },
rd: from_reg,
mem,
srcloc: None,
},
B32 | I32 | R32 => Inst::Store32 {
rd: from_reg,
mem,
srcloc: None,
},
B64 | I64 | R64 => Inst::Store64 {
rd: from_reg,
mem,
srcloc: None,
},
F32 => Inst::FpuStore32 {
rd: from_reg,
mem,
srcloc: None,
},
F64 => Inst::FpuStore64 {
rd: from_reg,
mem,
srcloc: None,
},
_ => { _ => {
if ty.is_vector() { if ty.is_vector() {
let bits = ty_bits(ty); let bits = ty_bits(ty);
let rd = from_reg; let rd = from_reg;
let srcloc = None;
if bits == 128 { if bits == 128 {
Inst::FpuStore128 { rd, mem, srcloc } Inst::FpuStore128 { rd, mem }
} else { } else {
assert_eq!(bits, 64); assert_eq!(bits, 64);
Inst::FpuStore64 { rd, mem, srcloc } Inst::FpuStore64 { rd, mem }
} }
} else { } else {
unimplemented!("gen_store({})", ty); unimplemented!("gen_store({})", ty);
@@ -3024,37 +2949,30 @@ impl Inst {
&Inst::ULoad8 { &Inst::ULoad8 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::SLoad8 { | &Inst::SLoad8 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::ULoad16 { | &Inst::ULoad16 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::SLoad16 { | &Inst::SLoad16 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::ULoad32 { | &Inst::ULoad32 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::SLoad32 { | &Inst::SLoad32 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::ULoad64 { | &Inst::ULoad64 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
.. ..
} => { } => {
let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru, state); let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru, state);
@@ -3087,22 +3005,18 @@ impl Inst {
&Inst::Store8 { &Inst::Store8 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::Store16 { | &Inst::Store16 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::Store32 { | &Inst::Store32 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
} }
| &Inst::Store64 { | &Inst::Store64 {
rd, rd,
ref mem, ref mem,
srcloc: _srcloc,
.. ..
} => { } => {
let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru, state); let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru, state);
@@ -3841,7 +3755,6 @@ impl Inst {
rd, rd,
ref name, ref name,
offset, offset,
srcloc: _srcloc,
} => { } => {
let rd = rd.show_rru(mb_rru); let rd = rd.show_rru(mb_rru);
format!("ldr {}, 8 ; b 12 ; data {:?} + {}", rd, name, offset) format!("ldr {}, 8 ; b 12 ; data {:?} + {}", rd, name, offset)

147
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -484,9 +484,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// msub rd, rd, rm, rn ; rd = rn - rd * rm // msub rd, rd, rm, rn ; rd = rn - rd * rm
// Check for divide by 0. // Check for divide by 0.
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero); let trap_code = TrapCode::IntegerDivisionByZero;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Zero(rm), kind: CondBrKind::Zero(rm),
}); });
@@ -507,9 +507,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// udf ; signed overflow // udf ; signed overflow
// Check for divide by 0. // Check for divide by 0.
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero); let trap_code = TrapCode::IntegerDivisionByZero;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Zero(rm), kind: CondBrKind::Zero(rm),
}); });
@@ -535,9 +535,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
nzcv: NZCV::new(false, false, false, false), nzcv: NZCV::new(false, false, false, false),
cond: Cond::Eq, cond: Cond::Eq,
}); });
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Cond(Cond::Vs), kind: CondBrKind::Cond(Cond::Vs),
}); });
} else { } else {
@@ -545,9 +545,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// udf ; divide by zero // udf ; divide by zero
// Check for divide by 0. // Check for divide by 0.
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero); let trap_code = TrapCode::IntegerDivisionByZero;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Zero(rm), kind: CondBrKind::Zero(rm),
}); });
} }
@@ -1161,27 +1161,20 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mem = lower_address(ctx, elem_ty, &inputs[..], off); let mem = lower_address(ctx, elem_ty, &inputs[..], off);
let rd = get_output_reg(ctx, outputs[0]); let rd = get_output_reg(ctx, outputs[0]);
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(match (ty_bits(elem_ty), sign_extend, is_float) { ctx.emit(match (ty_bits(elem_ty), sign_extend, is_float) {
(1, _, _) => Inst::ULoad8 { rd, mem, srcloc }, (1, _, _) => Inst::ULoad8 { rd, mem },
(8, false, _) => Inst::ULoad8 { rd, mem, srcloc }, (8, false, _) => Inst::ULoad8 { rd, mem },
(8, true, _) => Inst::SLoad8 { rd, mem, srcloc }, (8, true, _) => Inst::SLoad8 { rd, mem },
(16, false, _) => Inst::ULoad16 { rd, mem, srcloc }, (16, false, _) => Inst::ULoad16 { rd, mem },
(16, true, _) => Inst::SLoad16 { rd, mem, srcloc }, (16, true, _) => Inst::SLoad16 { rd, mem },
(32, false, false) => Inst::ULoad32 { rd, mem, srcloc }, (32, false, false) => Inst::ULoad32 { rd, mem },
(32, true, false) => Inst::SLoad32 { rd, mem, srcloc }, (32, true, false) => Inst::SLoad32 { rd, mem },
(32, _, true) => Inst::FpuLoad32 { rd, mem, srcloc }, (32, _, true) => Inst::FpuLoad32 { rd, mem },
(64, _, false) => Inst::ULoad64 { rd, mem, srcloc }, (64, _, false) => Inst::ULoad64 { rd, mem },
// Note that we treat some of the vector loads as scalar floating-point loads, // Note that we treat some of the vector loads as scalar floating-point loads,
// which is correct in a little endian environment. // which is correct in a little endian environment.
(64, _, true) => Inst::FpuLoad64 { rd, mem, srcloc }, (64, _, true) => Inst::FpuLoad64 { rd, mem },
(128, _, _) => Inst::FpuLoad128 { rd, mem, srcloc }, (128, _, _) => Inst::FpuLoad128 { rd, mem },
_ => panic!("Unsupported size in load"), _ => panic!("Unsupported size in load"),
}); });
@@ -1209,14 +1202,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let off = ctx.data(insn).load_store_offset().unwrap(); let off = ctx.data(insn).load_store_offset().unwrap();
let ty = ty.unwrap(); let ty = ty.unwrap();
let mem = lower_address(ctx, ty.lane_type(), &inputs[..], off); let mem = lower_address(ctx, ty.lane_type(), &inputs[..], off);
let memflags = ctx.memflags(insn).expect("memory flags");
let rd = get_output_reg(ctx, outputs[0]); let rd = get_output_reg(ctx, outputs[0]);
let size = VectorSize::from_ty(ty); let size = VectorSize::from_ty(ty);
let srcloc = if memflags.notrap() {
None
} else {
Some(ctx.srcloc(insn))
};
let tmp = ctx.alloc_tmp(RegClass::I64, I64); let tmp = ctx.alloc_tmp(RegClass::I64, I64);
ctx.emit(Inst::LoadAddr { rd: tmp, mem }); ctx.emit(Inst::LoadAddr { rd: tmp, mem });
@@ -1224,7 +1211,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rd, rd,
rn: tmp.to_reg(), rn: tmp.to_reg(),
size, size,
srcloc,
}); });
} }
@@ -1249,21 +1235,14 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mem = lower_address(ctx, elem_ty, &inputs[1..], off); let mem = lower_address(ctx, elem_ty, &inputs[1..], off);
let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(match (ty_bits(elem_ty), is_float) { ctx.emit(match (ty_bits(elem_ty), is_float) {
(1, _) | (8, _) => Inst::Store8 { rd, mem, srcloc }, (1, _) | (8, _) => Inst::Store8 { rd, mem },
(16, _) => Inst::Store16 { rd, mem, srcloc }, (16, _) => Inst::Store16 { rd, mem },
(32, false) => Inst::Store32 { rd, mem, srcloc }, (32, false) => Inst::Store32 { rd, mem },
(32, true) => Inst::FpuStore32 { rd, mem, srcloc }, (32, true) => Inst::FpuStore32 { rd, mem },
(64, false) => Inst::Store64 { rd, mem, srcloc }, (64, false) => Inst::Store64 { rd, mem },
(64, true) => Inst::FpuStore64 { rd, mem, srcloc }, (64, true) => Inst::FpuStore64 { rd, mem },
(128, _) => Inst::FpuStore128 { rd, mem, srcloc }, (128, _) => Inst::FpuStore128 { rd, mem },
_ => panic!("Unsupported size in store"), _ => panic!("Unsupported size in store"),
}); });
} }
@@ -1291,12 +1270,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mut r_arg2 = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); let mut r_arg2 = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
let ty_access = ty.unwrap(); let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
// Make sure that both args are in virtual regs, since in effect // Make sure that both args are in virtual regs, since in effect
// we have to do a parallel copy to get them safely to the AtomicRMW input // we have to do a parallel copy to get them safely to the AtomicRMW input
// regs, and that's not guaranteed safe if either is in a real reg. // regs, and that's not guaranteed safe if either is in a real reg.
@@ -1307,11 +1280,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::gen_move(Writable::from_reg(xreg(26)), r_arg2, I64)); ctx.emit(Inst::gen_move(Writable::from_reg(xreg(26)), r_arg2, I64));
// Now the AtomicRMW insn itself // Now the AtomicRMW insn itself
let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap()); let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
ctx.emit(Inst::AtomicRMW { ctx.emit(Inst::AtomicRMW { ty: ty_access, op });
ty: ty_access,
op,
srcloc,
});
// And finally, copy the preordained AtomicRMW output reg to its destination. // And finally, copy the preordained AtomicRMW output reg to its destination.
ctx.emit(Inst::gen_move(r_dst, xreg(27), I64)); ctx.emit(Inst::gen_move(r_dst, xreg(27), I64));
// Also, x24 and x28 are trashed. `fn aarch64_get_regs` must mention that. // Also, x24 and x28 are trashed. `fn aarch64_get_regs` must mention that.
@@ -1327,12 +1296,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mut r_replacement = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None); let mut r_replacement = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
let ty_access = ty.unwrap(); let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
// Make sure that all three args are in virtual regs. See corresponding comment // Make sure that all three args are in virtual regs. See corresponding comment
// for `Opcode::AtomicRmw` above. // for `Opcode::AtomicRmw` above.
r_addr = ctx.ensure_in_vreg(r_addr, I64); r_addr = ctx.ensure_in_vreg(r_addr, I64);
@@ -1351,10 +1314,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
I64, I64,
)); ));
// Now the AtomicCAS itself, implemented in the normal way, with an LL-SC loop // Now the AtomicCAS itself, implemented in the normal way, with an LL-SC loop
ctx.emit(Inst::AtomicCAS { ctx.emit(Inst::AtomicCAS { ty: ty_access });
ty: ty_access,
srcloc,
});
// And finally, copy the preordained AtomicCAS output reg to its destination. // And finally, copy the preordained AtomicCAS output reg to its destination.
ctx.emit(Inst::gen_move(r_dst, xreg(27), I64)); ctx.emit(Inst::gen_move(r_dst, xreg(27), I64));
// Also, x24 and x28 are trashed. `fn aarch64_get_regs` must mention that. // Also, x24 and x28 are trashed. `fn aarch64_get_regs` must mention that.
@@ -1365,17 +1325,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let r_addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); let r_addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let ty_access = ty.unwrap(); let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(Inst::AtomicLoad { ctx.emit(Inst::AtomicLoad {
ty: ty_access, ty: ty_access,
r_data, r_data,
r_addr, r_addr,
srcloc,
}); });
} }
@@ -1384,17 +1337,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let r_addr = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); let r_addr = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
let ty_access = ctx.input_ty(insn, 0); let ty_access = ctx.input_ty(insn, 0);
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(Inst::AtomicStore { ctx.emit(Inst::AtomicStore {
ty: ty_access, ty: ty_access,
r_data, r_data,
r_addr, r_addr,
srcloc,
}); });
} }
@@ -1811,12 +1757,12 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Trap | Opcode::ResumableTrap => { Opcode::Trap | Opcode::ResumableTrap => {
let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap()); let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::Udf { trap_info }); ctx.emit_safepoint(Inst::Udf { trap_code });
} }
Opcode::Trapif | Opcode::Trapff => { Opcode::Trapif | Opcode::Trapff => {
let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap()); let trap_code = ctx.data(insn).trap_code().unwrap();
let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() { let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() {
let condcode = ctx.data(insn).cond_code().unwrap(); let condcode = ctx.data(insn).cond_code().unwrap();
@@ -1847,7 +1793,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}; };
ctx.emit_safepoint(Inst::TrapIf { ctx.emit_safepoint(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Cond(cond), kind: CondBrKind::Cond(cond),
}); });
} }
@@ -1864,11 +1810,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let rd = get_output_reg(ctx, outputs[0]); let rd = get_output_reg(ctx, outputs[0]);
let (extname, _) = ctx.call_target(insn).unwrap(); let (extname, _) = ctx.call_target(insn).unwrap();
let extname = extname.clone(); let extname = extname.clone();
let loc = ctx.srcloc(insn);
ctx.emit(Inst::LoadExtName { ctx.emit(Inst::LoadExtName {
rd, rd,
name: Box::new(extname), name: Box::new(extname),
srcloc: loc,
offset: 0, offset: 0,
}); });
} }
@@ -1881,17 +1825,14 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let rd = get_output_reg(ctx, outputs[0]); let rd = get_output_reg(ctx, outputs[0]);
let (extname, _, offset) = ctx.symbol_value(insn).unwrap(); let (extname, _, offset) = ctx.symbol_value(insn).unwrap();
let extname = extname.clone(); let extname = extname.clone();
let loc = ctx.srcloc(insn);
ctx.emit(Inst::LoadExtName { ctx.emit(Inst::LoadExtName {
rd, rd,
name: Box::new(extname), name: Box::new(extname),
srcloc: loc,
offset, offset,
}); });
} }
Opcode::Call | Opcode::CallIndirect => { Opcode::Call | Opcode::CallIndirect => {
let loc = ctx.srcloc(insn);
let caller_conv = ctx.abi().call_conv(); let caller_conv = ctx.abi().call_conv();
let (mut abi, inputs) = match op { let (mut abi, inputs) = match op {
Opcode::Call => { Opcode::Call => {
@@ -1901,7 +1842,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert!(inputs.len() == sig.params.len()); assert!(inputs.len() == sig.params.len());
assert!(outputs.len() == sig.returns.len()); assert!(outputs.len() == sig.returns.len());
( (
AArch64ABICaller::from_func(sig, &extname, dist, loc, caller_conv)?, AArch64ABICaller::from_func(sig, &extname, dist, caller_conv)?,
&inputs[..], &inputs[..],
) )
} }
@@ -1911,7 +1852,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert!(inputs.len() - 1 == sig.params.len()); assert!(inputs.len() - 1 == sig.params.len());
assert!(outputs.len() == sig.returns.len()); assert!(outputs.len() == sig.returns.len());
( (
AArch64ABICaller::from_ptr(sig, ptr, loc, op, caller_conv)?, AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
&inputs[1..], &inputs[1..],
) )
} }
@@ -2687,9 +2628,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else { } else {
ctx.emit(Inst::FpuCmp64 { rn, rm: rn }); ctx.emit(Inst::FpuCmp64 { rn, rm: rn });
} }
let trap_info = (ctx.srcloc(insn), TrapCode::BadConversionToInteger); let trap_code = TrapCode::BadConversionToInteger;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Unordered)), kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Unordered)),
}); });
@@ -2739,9 +2680,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn, rn,
rm: tmp.to_reg(), rm: tmp.to_reg(),
}); });
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()), kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()),
}); });
@@ -2751,9 +2692,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn, rn,
rm: tmp.to_reg(), rm: tmp.to_reg(),
}); });
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()), kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()),
}); });
} else { } else {
@@ -2792,9 +2733,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn, rn,
rm: tmp.to_reg(), rm: tmp.to_reg(),
}); });
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()), kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()),
}); });
@@ -2804,9 +2745,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn, rn,
rm: tmp.to_reg(), rm: tmp.to_reg(),
}); });
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf { ctx.emit(Inst::TrapIf {
trap_info, trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()), kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()),
}); });
}; };

8
cranelift/codegen/src/isa/arm32/abi.rs
View File

@@ -2,7 +2,6 @@
use crate::ir; use crate::ir;
use crate::ir::types::*; use crate::ir::types::*;
use crate::ir::SourceLoc;
use crate::isa; use crate::isa;
use crate::isa::arm32::inst::*; use crate::isa::arm32::inst::*;
use crate::machinst::*; use crate::machinst::*;
@@ -217,7 +216,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
rm: limit_reg, rm: limit_reg,
}); });
insts.push(Inst::TrapIf { insts.push(Inst::TrapIf {
trap_info: (ir::SourceLoc::default(), ir::TrapCode::StackOverflow), trap_info: ir::TrapCode::StackOverflow,
// Here `Lo` == "less than" when interpreting the two // Here `Lo` == "less than" when interpreting the two
// operands as unsigned integers. // operands as unsigned integers.
cond: Cond::Lo, cond: Cond::Lo,
@@ -366,7 +365,6 @@ impl ABIMachineSpec for Arm32MachineDeps {
dest: &CallDest, dest: &CallDest,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: ir::Opcode, opcode: ir::Opcode,
tmp: Writable<Reg>, tmp: Writable<Reg>,
_callee_conv: isa::CallConv, _callee_conv: isa::CallConv,
@@ -381,7 +379,6 @@ impl ABIMachineSpec for Arm32MachineDeps {
dest: name.clone(), dest: name.clone(),
uses, uses,
defs, defs,
loc,
opcode, opcode,
}), }),
}, },
@@ -393,7 +390,6 @@ impl ABIMachineSpec for Arm32MachineDeps {
rt: tmp, rt: tmp,
name: Box::new(name.clone()), name: Box::new(name.clone()),
offset: 0, offset: 0,
srcloc: loc,
}, },
)); ));
insts.push(( insts.push((
@@ -403,7 +399,6 @@ impl ABIMachineSpec for Arm32MachineDeps {
rm: tmp.to_reg(), rm: tmp.to_reg(),
uses, uses,
defs, defs,
loc,
opcode, opcode,
}), }),
}, },
@@ -416,7 +411,6 @@ impl ABIMachineSpec for Arm32MachineDeps {
rm: *reg, rm: *reg,
uses, uses,
defs, defs,
loc,
opcode, opcode,
}), }),
}, },

40
cranelift/codegen/src/isa/arm32/inst/emit.rs
View File

@@ -1,6 +1,7 @@
//! 32-bit ARM ISA: binary code emission. //! 32-bit ARM ISA: binary code emission.
use crate::binemit::{Reloc, StackMap}; use crate::binemit::{Reloc, StackMap};
use crate::ir::SourceLoc;
use crate::isa::arm32::inst::*; use crate::isa::arm32::inst::*;
use core::convert::TryFrom; use core::convert::TryFrom;
@@ -229,6 +230,8 @@ pub struct EmitState {
pub(crate) nominal_sp_to_fp: i64, pub(crate) nominal_sp_to_fp: i64,
/// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
stack_map: Option<StackMap>, stack_map: Option<StackMap>,
/// Source location of next machine code instruction to be emitted.
cur_srcloc: SourceLoc,
} }
impl MachInstEmitState<Inst> for EmitState { impl MachInstEmitState<Inst> for EmitState {
@@ -237,12 +240,17 @@ impl MachInstEmitState<Inst> for EmitState {
virtual_sp_offset: 0, virtual_sp_offset: 0,
nominal_sp_to_fp: abi.frame_size() as i64, nominal_sp_to_fp: abi.frame_size() as i64,
stack_map: None, stack_map: None,
cur_srcloc: SourceLoc::default(),
} }
} }
fn pre_safepoint(&mut self, stack_map: StackMap) { fn pre_safepoint(&mut self, stack_map: StackMap) {
self.stack_map = Some(stack_map); self.stack_map = Some(stack_map);
} }
fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
self.cur_srcloc = srcloc;
}
} }
impl EmitState { impl EmitState {
@@ -253,6 +261,10 @@ impl EmitState {
fn clear_post_insn(&mut self) { fn clear_post_insn(&mut self) {
self.stack_map = None; self.stack_map = None;
} }
fn cur_srcloc(&self) -> SourceLoc {
self.cur_srcloc
}
} }
pub struct EmitInfo { pub struct EmitInfo {
@@ -456,17 +468,13 @@ impl MachInstEmit for Inst {
let inst = enc_32_regs(inst, None, None, None, Some(rn)); let inst = enc_32_regs(inst, None, None, None, Some(rn));
emit_32(inst, sink); emit_32(inst, sink);
} }
&Inst::Store { &Inst::Store { rt, ref mem, bits } => {
rt,
ref mem,
srcloc,
bits,
} => {
let (mem_insts, mem) = mem_finalize(mem, state); let (mem_insts, mem) = mem_finalize(mem, state);
for inst in mem_insts.into_iter() { for inst in mem_insts.into_iter() {
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
} }
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the store instruction starts. // Register the offset at which the store instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
@@ -496,7 +504,6 @@ impl MachInstEmit for Inst {
&Inst::Load { &Inst::Load {
rt, rt,
ref mem, ref mem,
srcloc,
bits, bits,
sign_extend, sign_extend,
} => { } => {
@@ -504,7 +511,8 @@ impl MachInstEmit for Inst {
for inst in mem_insts.into_iter() { for inst in mem_insts.into_iter() {
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
} }
if let Some(srcloc) = srcloc { let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the load instruction starts. // Register the offset at which the load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
} }
@@ -696,23 +704,24 @@ impl MachInstEmit for Inst {
} }
}, },
&Inst::Call { ref info } => { &Inst::Call { ref info } => {
sink.add_reloc(info.loc, Reloc::Arm32Call, &info.dest, 0); let srcloc = state.cur_srcloc();
sink.add_reloc(srcloc, Reloc::Arm32Call, &info.dest, 0);
emit_32(0b11110_0_0000000000_11_0_1_0_00000000000, sink); emit_32(0b11110_0_0000000000_11_0_1_0_00000000000, sink);
if info.opcode.is_call() { if info.opcode.is_call() {
sink.add_call_site(info.loc, info.opcode); sink.add_call_site(srcloc, info.opcode);
} }
} }
&Inst::CallInd { ref info } => { &Inst::CallInd { ref info } => {
let srcloc = state.cur_srcloc();
sink.put2(0b01000111_1_0000_000 | (machreg_to_gpr(info.rm) << 3)); sink.put2(0b01000111_1_0000_000 | (machreg_to_gpr(info.rm) << 3));
if info.opcode.is_call() { if info.opcode.is_call() {
sink.add_call_site(info.loc, info.opcode); sink.add_call_site(srcloc, info.opcode);
} }
} }
&Inst::LoadExtName { &Inst::LoadExtName {
rt, rt,
ref name, ref name,
offset, offset,
srcloc,
} => { } => {
// maybe nop2 (0|2) bytes (pc is now 4-aligned) // maybe nop2 (0|2) bytes (pc is now 4-aligned)
// ldr rt, [pc, #4] 4 bytes // ldr rt, [pc, #4] 4 bytes
@@ -729,7 +738,6 @@ impl MachInstEmit for Inst {
let inst = Inst::Load { let inst = Inst::Load {
rt, rt,
mem, mem,
srcloc: Some(srcloc),
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}; };
@@ -740,6 +748,7 @@ impl MachInstEmit for Inst {
}; };
inst.emit(sink, emit_info, state); inst.emit(sink, emit_info, state);
let srcloc = state.cur_srcloc();
sink.add_reloc(srcloc, Reloc::Abs4, name, offset.into()); sink.add_reloc(srcloc, Reloc::Abs4, name, offset.into());
sink.put4(0); sink.put4(0);
} }
@@ -784,7 +793,8 @@ impl MachInstEmit for Inst {
sink.put2(inst); sink.put2(inst);
} }
&Inst::Udf { trap_info } => { &Inst::Udf { trap_info } => {
let (srcloc, code) = trap_info; let srcloc = state.cur_srcloc();
let code = trap_info;
sink.add_trap(srcloc, code); sink.add_trap(srcloc, code);
sink.put2(0b11011110_00000000); sink.put2(0b11011110_00000000);
} }

48
cranelift/codegen/src/isa/arm32/inst/emit_tests.rs
View File

@@ -1244,7 +1244,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(0), rt: rreg(0),
mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0), mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0),
srcloc: None,
bits: 32, bits: 32,
}, },
"41F80200", "41F80200",
@@ -1254,7 +1253,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(8), rt: rreg(8),
mem: AMode::reg_plus_reg(rreg(9), rreg(10), 3), mem: AMode::reg_plus_reg(rreg(9), rreg(10), 3),
srcloc: None,
bits: 32, bits: 32,
}, },
"49F83A80", "49F83A80",
@@ -1264,7 +1262,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(0), rt: rreg(0),
mem: AMode::RegOffset(rreg(1), 4095), mem: AMode::RegOffset(rreg(1), 4095),
srcloc: None,
bits: 32, bits: 32,
}, },
"C1F8FF0F", "C1F8FF0F",
@@ -1274,7 +1271,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(8), rt: rreg(8),
mem: AMode::RegOffset(rreg(9), 0), mem: AMode::RegOffset(rreg(9), 0),
srcloc: None,
bits: 32, bits: 32,
}, },
"C9F80080", "C9F80080",
@@ -1284,7 +1280,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(7), rt: rreg(7),
mem: AMode::RegOffset(rreg(11), 65535), mem: AMode::RegOffset(rreg(11), 65535),
srcloc: None,
bits: 32, bits: 32,
}, },
"4FF6FF7C4BF80C70", "4FF6FF7C4BF80C70",
@@ -1294,7 +1289,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(10), rt: rreg(10),
mem: AMode::RegOffset(rreg(4), 16777215), mem: AMode::RegOffset(rreg(4), 16777215),
srcloc: None,
bits: 32, bits: 32,
}, },
"4FF6FF7CC0F2FF0C44F80CA0", "4FF6FF7CC0F2FF0C44F80CA0",
@@ -1304,7 +1298,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(0), rt: rreg(0),
mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0), mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0),
srcloc: None,
bits: 16, bits: 16,
}, },
"21F80200", "21F80200",
@@ -1314,7 +1307,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(8), rt: rreg(8),
mem: AMode::reg_plus_reg(rreg(9), rreg(10), 2), mem: AMode::reg_plus_reg(rreg(9), rreg(10), 2),
srcloc: None,
bits: 16, bits: 16,
}, },
"29F82A80", "29F82A80",
@@ -1324,7 +1316,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(0), rt: rreg(0),
mem: AMode::RegOffset(rreg(1), 3210), mem: AMode::RegOffset(rreg(1), 3210),
srcloc: None,
bits: 16, bits: 16,
}, },
"A1F88A0C", "A1F88A0C",
@@ -1334,7 +1325,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(8), rt: rreg(8),
mem: AMode::RegOffset(rreg(9), 1), mem: AMode::RegOffset(rreg(9), 1),
srcloc: None,
bits: 16, bits: 16,
}, },
"A9F80180", "A9F80180",
@@ -1344,7 +1334,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(7), rt: rreg(7),
mem: AMode::RegOffset(rreg(11), 65535), mem: AMode::RegOffset(rreg(11), 65535),
srcloc: None,
bits: 16, bits: 16,
}, },
"4FF6FF7C2BF80C70", "4FF6FF7C2BF80C70",
@@ -1354,7 +1343,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(10), rt: rreg(10),
mem: AMode::RegOffset(rreg(4), 16777215), mem: AMode::RegOffset(rreg(4), 16777215),
srcloc: None,
bits: 16, bits: 16,
}, },
"4FF6FF7CC0F2FF0C24F80CA0", "4FF6FF7CC0F2FF0C24F80CA0",
@@ -1364,7 +1352,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(0), rt: rreg(0),
mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0), mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0),
srcloc: None,
bits: 8, bits: 8,
}, },
"01F80200", "01F80200",
@@ -1374,7 +1361,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(8), rt: rreg(8),
mem: AMode::reg_plus_reg(rreg(9), rreg(10), 1), mem: AMode::reg_plus_reg(rreg(9), rreg(10), 1),
srcloc: None,
bits: 8, bits: 8,
}, },
"09F81A80", "09F81A80",
@@ -1384,7 +1370,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(0), rt: rreg(0),
mem: AMode::RegOffset(rreg(1), 4), mem: AMode::RegOffset(rreg(1), 4),
srcloc: None,
bits: 8, bits: 8,
}, },
"81F80400", "81F80400",
@@ -1394,7 +1379,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(8), rt: rreg(8),
mem: AMode::RegOffset(rreg(9), 777), mem: AMode::RegOffset(rreg(9), 777),
srcloc: None,
bits: 8, bits: 8,
}, },
"89F80983", "89F80983",
@@ -1404,7 +1388,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(7), rt: rreg(7),
mem: AMode::RegOffset(rreg(11), 65535), mem: AMode::RegOffset(rreg(11), 65535),
srcloc: None,
bits: 8, bits: 8,
}, },
"4FF6FF7C0BF80C70", "4FF6FF7C0BF80C70",
@@ -1414,7 +1397,6 @@ fn test_arm32_emit() {
Inst::Store { Inst::Store {
rt: rreg(10), rt: rreg(10),
mem: AMode::RegOffset(rreg(4), 16777215), mem: AMode::RegOffset(rreg(4), 16777215),
srcloc: None,
bits: 8, bits: 8,
}, },
"4FF6FF7CC0F2FF0C04F80CA0", "4FF6FF7CC0F2FF0C04F80CA0",
@@ -1424,7 +1406,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0), mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1435,7 +1416,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::reg_plus_reg(rreg(9), rreg(10), 1), mem: AMode::reg_plus_reg(rreg(9), rreg(10), 1),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1446,7 +1426,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::RegOffset(rreg(1), 55), mem: AMode::RegOffset(rreg(1), 55),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1457,7 +1436,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::RegOffset(rreg(9), 1234), mem: AMode::RegOffset(rreg(9), 1234),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1468,7 +1446,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(7), rt: writable_rreg(7),
mem: AMode::RegOffset(rreg(11), 9876), mem: AMode::RegOffset(rreg(11), 9876),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1479,7 +1456,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(10), rt: writable_rreg(10),
mem: AMode::RegOffset(rreg(4), 252645135), mem: AMode::RegOffset(rreg(4), 252645135),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1490,7 +1466,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::PCRel(-56), mem: AMode::PCRel(-56),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1501,7 +1476,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::PCRel(1024), mem: AMode::PCRel(1024),
srcloc: None,
bits: 32, bits: 32,
sign_extend: false, sign_extend: false,
}, },
@@ -1512,7 +1486,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0), mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0),
srcloc: None,
bits: 16, bits: 16,
sign_extend: true, sign_extend: true,
}, },
@@ -1523,7 +1496,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::reg_plus_reg(rreg(9), rreg(10), 2), mem: AMode::reg_plus_reg(rreg(9), rreg(10), 2),
srcloc: None,
bits: 16, bits: 16,
sign_extend: false, sign_extend: false,
}, },
@@ -1534,7 +1506,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::RegOffset(rreg(1), 55), mem: AMode::RegOffset(rreg(1), 55),
srcloc: None,
bits: 16, bits: 16,
sign_extend: false, sign_extend: false,
}, },
@@ -1545,7 +1516,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::RegOffset(rreg(9), 1234), mem: AMode::RegOffset(rreg(9), 1234),
srcloc: None,
bits: 16, bits: 16,
sign_extend: true, sign_extend: true,
}, },
@@ -1556,7 +1526,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(7), rt: writable_rreg(7),
mem: AMode::RegOffset(rreg(11), 9876), mem: AMode::RegOffset(rreg(11), 9876),
srcloc: None,
bits: 16, bits: 16,
sign_extend: true, sign_extend: true,
}, },
@@ -1567,7 +1536,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(10), rt: writable_rreg(10),
mem: AMode::RegOffset(rreg(4), 252645135), mem: AMode::RegOffset(rreg(4), 252645135),
srcloc: None,
bits: 16, bits: 16,
sign_extend: false, sign_extend: false,
}, },
@@ -1578,7 +1546,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::PCRel(56), mem: AMode::PCRel(56),
srcloc: None,
bits: 16, bits: 16,
sign_extend: false, sign_extend: false,
}, },
@@ -1589,7 +1556,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::PCRel(-1000), mem: AMode::PCRel(-1000),
srcloc: None,
bits: 16, bits: 16,
sign_extend: true, sign_extend: true,
}, },
@@ -1600,7 +1566,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0), mem: AMode::reg_plus_reg(rreg(1), rreg(2), 0),
srcloc: None,
bits: 8, bits: 8,
sign_extend: true, sign_extend: true,
}, },
@@ -1611,7 +1576,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::reg_plus_reg(rreg(9), rreg(10), 3), mem: AMode::reg_plus_reg(rreg(9), rreg(10), 3),
srcloc: None,
bits: 8, bits: 8,
sign_extend: false, sign_extend: false,
}, },
@@ -1622,7 +1586,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::RegOffset(rreg(1), 55), mem: AMode::RegOffset(rreg(1), 55),
srcloc: None,
bits: 8, bits: 8,
sign_extend: false, sign_extend: false,
}, },
@@ -1633,7 +1596,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::RegOffset(rreg(9), 1234), mem: AMode::RegOffset(rreg(9), 1234),
srcloc: None,
bits: 8, bits: 8,
sign_extend: true, sign_extend: true,
}, },
@@ -1644,7 +1606,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(7), rt: writable_rreg(7),
mem: AMode::RegOffset(rreg(11), 9876), mem: AMode::RegOffset(rreg(11), 9876),
srcloc: None,
bits: 8, bits: 8,
sign_extend: true, sign_extend: true,
}, },
@@ -1655,7 +1616,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(10), rt: writable_rreg(10),
mem: AMode::RegOffset(rreg(4), 252645135), mem: AMode::RegOffset(rreg(4), 252645135),
srcloc: None,
bits: 8, bits: 8,
sign_extend: false, sign_extend: false,
}, },
@@ -1666,7 +1626,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(0), rt: writable_rreg(0),
mem: AMode::PCRel(72), mem: AMode::PCRel(72),
srcloc: None,
bits: 8, bits: 8,
sign_extend: false, sign_extend: false,
}, },
@@ -1677,7 +1636,6 @@ fn test_arm32_emit() {
Inst::Load { Inst::Load {
rt: writable_rreg(8), rt: writable_rreg(8),
mem: AMode::PCRel(-1234), mem: AMode::PCRel(-1234),
srcloc: None,
bits: 8, bits: 8,
sign_extend: true, sign_extend: true,
}, },
@@ -1961,7 +1919,7 @@ fn test_arm32_emit() {
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
cond: Cond::Eq, cond: Cond::Eq,
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_info: TrapCode::Interrupt,
}, },
"40F0018000DE", "40F0018000DE",
"bne 2 ; udf #0", "bne 2 ; udf #0",
@@ -1969,14 +1927,14 @@ fn test_arm32_emit() {
insns.push(( insns.push((
Inst::TrapIf { Inst::TrapIf {
cond: Cond::Hs, cond: Cond::Hs,
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_info: TrapCode::Interrupt,
}, },
"C0F0018000DE", "C0F0018000DE",
"blo 2 ; udf #0", "blo 2 ; udf #0",
)); ));
insns.push(( insns.push((
Inst::Udf { Inst::Udf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_info: TrapCode::Interrupt,
}, },
"00DE", "00DE",
"udf #0", "udf #0",

14
cranelift/codegen/src/isa/arm32/inst/mod.rs
View File

@@ -4,7 +4,7 @@
use crate::binemit::CodeOffset; use crate::binemit::CodeOffset;
use crate::ir::types::{B1, B16, B32, B8, I16, I32, I8, IFLAGS}; use crate::ir::types::{B1, B16, B32, B8, I16, I32, I8, IFLAGS};
use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type}; use crate::ir::{ExternalName, Opcode, TrapCode, Type};
use crate::machinst::*; use crate::machinst::*;
use crate::{settings, CodegenError, CodegenResult}; use crate::{settings, CodegenError, CodegenResult};
@@ -83,7 +83,6 @@ pub struct CallInfo {
pub dest: ExternalName, pub dest: ExternalName,
pub uses: Vec<Reg>, pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>, pub defs: Vec<Writable<Reg>>,
pub loc: SourceLoc,
pub opcode: Opcode, pub opcode: Opcode,
} }
@@ -94,7 +93,6 @@ pub struct CallIndInfo {
pub rm: Reg, pub rm: Reg,
pub uses: Vec<Reg>, pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>, pub defs: Vec<Writable<Reg>>,
pub loc: SourceLoc,
pub opcode: Opcode, pub opcode: Opcode,
} }
@@ -218,7 +216,6 @@ pub enum Inst {
Store { Store {
rt: Reg, rt: Reg,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
bits: u8, bits: u8,
}, },
@@ -227,7 +224,6 @@ pub enum Inst {
Load { Load {
rt: Writable<Reg>, rt: Writable<Reg>,
mem: AMode, mem: AMode,
srcloc: Option<SourceLoc>,
bits: u8, bits: u8,
sign_extend: bool, sign_extend: bool,
}, },
@@ -276,7 +272,6 @@ pub enum Inst {
LoadExtName { LoadExtName {
rt: Writable<Reg>, rt: Writable<Reg>,
name: Box<ExternalName>, name: Box<ExternalName>,
srcloc: SourceLoc,
offset: i32, offset: i32,
}, },
@@ -308,13 +303,13 @@ pub enum Inst {
/// unit to the register allocator. /// unit to the register allocator.
TrapIf { TrapIf {
cond: Cond, cond: Cond,
trap_info: (SourceLoc, TrapCode), trap_info: TrapCode,
}, },
/// An instruction guaranteed to always be undefined and to trigger an illegal instruction at /// An instruction guaranteed to always be undefined and to trigger an illegal instruction at
/// runtime. /// runtime.
Udf { Udf {
trap_info: (SourceLoc, TrapCode), trap_info: TrapCode,
}, },
/// A "breakpoint" instruction, used for e.g. traps and debug breakpoints. /// A "breakpoint" instruction, used for e.g. traps and debug breakpoints.
@@ -390,7 +385,6 @@ impl Inst {
Inst::Load { Inst::Load {
rt: into_reg, rt: into_reg,
mem, mem,
srcloc: None,
bits, bits,
sign_extend: false, sign_extend: false,
} }
@@ -405,7 +399,6 @@ impl Inst {
Inst::Store { Inst::Store {
rt: from_reg, rt: from_reg,
mem, mem,
srcloc: None,
bits, bits,
} }
} }
@@ -1189,7 +1182,6 @@ impl Inst {
rt, rt,
ref name, ref name,
offset, offset,
srcloc: _srcloc,
} => { } => {
let rt = rt.show_rru(mb_rru); let rt = rt.show_rru(mb_rru);
format!("ldr {}, [pc, #4] ; b 4 ; data {:?} + {}", rt, name, offset) format!("ldr {}, [pc, #4] ; b 4 ; data {:?} + {}", rt, name, offset)

29
cranelift/codegen/src/isa/arm32/lower_inst.rs
View File

@@ -386,19 +386,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let base = input_to_reg(ctx, inputs[1], NarrowValueMode::None); let base = input_to_reg(ctx, inputs[1], NarrowValueMode::None);
let mem = AMode::RegOffset(base, i64::from(off)); let mem = AMode::RegOffset(base, i64::from(off));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() { ctx.emit(Inst::Store { rt, mem, bits });
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(Inst::Store {
rt,
mem,
srcloc,
bits,
});
} }
Opcode::Load Opcode::Load
| Opcode::Uload8 | Opcode::Uload8
@@ -429,17 +418,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert_eq!(inputs.len(), 2, "only one input for store memory operands"); assert_eq!(inputs.len(), 2, "only one input for store memory operands");
let base = input_to_reg(ctx, inputs[1], NarrowValueMode::None); let base = input_to_reg(ctx, inputs[1], NarrowValueMode::None);
let mem = AMode::RegOffset(base, i64::from(off)); let mem = AMode::RegOffset(base, i64::from(off));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(Inst::Load { ctx.emit(Inst::Load {
rt: out_reg, rt: out_reg,
mem, mem,
srcloc,
bits, bits,
sign_extend, sign_extend,
}); });
@@ -484,7 +466,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::Bkpt); ctx.emit(Inst::Bkpt);
} }
Opcode::Trap => { Opcode::Trap => {
let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); let trap_info = inst_trapcode(ctx.data(insn)).unwrap();
ctx.emit(Inst::Udf { trap_info }) ctx.emit(Inst::Udf { trap_info })
} }
Opcode::Trapif => { Opcode::Trapif => {
@@ -496,7 +478,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp); debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
emit_cmp(ctx, cmp_insn); emit_cmp(ctx, cmp_insn);
let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); let trap_info = inst_trapcode(ctx.data(insn)).unwrap();
let condcode = inst_condcode(ctx.data(insn)).unwrap(); let condcode = inst_condcode(ctx.data(insn)).unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
@@ -512,7 +494,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
} }
Opcode::Call | Opcode::CallIndirect => { Opcode::Call | Opcode::CallIndirect => {
let loc = ctx.srcloc(insn);
let caller_conv = ctx.abi().call_conv(); let caller_conv = ctx.abi().call_conv();
let (mut abi, inputs) = match op { let (mut abi, inputs) = match op {
Opcode::Call => { Opcode::Call => {
@@ -522,7 +503,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert_eq!(inputs.len(), sig.params.len()); assert_eq!(inputs.len(), sig.params.len());
assert_eq!(outputs.len(), sig.returns.len()); assert_eq!(outputs.len(), sig.returns.len());
( (
Arm32ABICaller::from_func(sig, &extname, dist, loc, caller_conv)?, Arm32ABICaller::from_func(sig, &extname, dist, caller_conv)?,
&inputs[..], &inputs[..],
) )
} }
@@ -532,7 +513,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert_eq!(inputs.len() - 1, sig.params.len()); assert_eq!(inputs.len() - 1, sig.params.len());
assert_eq!(outputs.len(), sig.returns.len()); assert_eq!(outputs.len(), sig.returns.len());
( (
Arm32ABICaller::from_ptr(sig, ptr, loc, op, caller_conv)?, Arm32ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
&inputs[1..], &inputs[1..],
) )
} }

26
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -1,7 +1,7 @@
//! Implementation of the standard x64 ABI. //! Implementation of the standard x64 ABI.
use crate::ir::types::*; use crate::ir::types::*;
use crate::ir::{self, types, SourceLoc, TrapCode, Type}; use crate::ir::{self, types, TrapCode, Type};
use crate::isa; use crate::isa;
use crate::isa::{x64::inst::*, CallConv}; use crate::isa::{x64::inst::*, CallConv};
use crate::machinst::abi_impl::*; use crate::machinst::abi_impl::*;
@@ -252,11 +252,11 @@ impl ABIMachineSpec for X64ABIMachineSpec {
_ if ty.bytes() == 16 => ExtKind::None, _ if ty.bytes() == 16 => ExtKind::None,
_ => panic!("load_stack({})", ty), _ => panic!("load_stack({})", ty),
}; };
Inst::load(ty, mem, into_reg, ext_kind, /* infallible */ None) Inst::load(ty, mem, into_reg, ext_kind)
} }
fn gen_store_stack(mem: StackAMode, from_reg: Reg, ty: Type) -> Self::I { fn gen_store_stack(mem: StackAMode, from_reg: Reg, ty: Type) -> Self::I {
Inst::store(ty, from_reg, mem, /* infallible */ None) Inst::store(ty, from_reg, mem)
} }
fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self::I { fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self::I {
@@ -274,9 +274,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
let ext_mode = ExtMode::new(from_bits as u16, to_bits as u16) let ext_mode = ExtMode::new(from_bits as u16, to_bits as u16)
.expect(&format!("invalid extension: {} -> {}", from_bits, to_bits)); .expect(&format!("invalid extension: {} -> {}", from_bits, to_bits));
if is_signed { if is_signed {
Inst::movsx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg, None) Inst::movsx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg)
} else { } else {
Inst::movzx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg, None) Inst::movzx_rm_r(ext_mode, RegMem::reg(from_reg), to_reg)
} }
} }
@@ -308,7 +308,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
Inst::TrapIf { Inst::TrapIf {
// NBE == "> unsigned"; args above are reversed; this tests limit_reg > rsp. // NBE == "> unsigned"; args above are reversed; this tests limit_reg > rsp.
cc: CC::NBE, cc: CC::NBE,
srcloc: SourceLoc::default(),
trap_code: TrapCode::StackOverflow, trap_code: TrapCode::StackOverflow,
}, },
] ]
@@ -335,13 +334,13 @@ impl ABIMachineSpec for X64ABIMachineSpec {
assert_eq!(ty, I64); assert_eq!(ty, I64);
let simm32 = offset as u32; let simm32 = offset as u32;
let mem = Amode::imm_reg(simm32, base); let mem = Amode::imm_reg(simm32, base);
Inst::load(ty, mem, into_reg, ExtKind::None, None) Inst::load(ty, mem, into_reg, ExtKind::None)
} }
fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Self::I { fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Self::I {
let simm32 = offset as u32; let simm32 = offset as u32;
let mem = Amode::imm_reg(simm32, base); let mem = Amode::imm_reg(simm32, base);
Inst::store(ty, from_reg, mem, None) Inst::store(ty, from_reg, mem)
} }
fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Self::I; 2]> { fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Self::I; 2]> {
@@ -426,7 +425,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
/* bytes = */ 8, /* bytes = */ 8,
r_reg.to_reg(), r_reg.to_reg(),
Amode::imm_reg(cur_offset, regs::rsp()), Amode::imm_reg(cur_offset, regs::rsp()),
None,
)); ));
cur_offset += 8; cur_offset += 8;
} }
@@ -461,7 +459,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
insts.push(Inst::mov64_m_r( insts.push(Inst::mov64_m_r(
Amode::imm_reg(cur_offset, regs::rsp()), Amode::imm_reg(cur_offset, regs::rsp()),
Writable::from_reg(rreg.to_reg()), Writable::from_reg(rreg.to_reg()),
None,
)); ));
cur_offset += 8; cur_offset += 8;
} }
@@ -486,7 +483,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
insts.push(Inst::mov64_m_r( insts.push(Inst::mov64_m_r(
Amode::imm_reg(off as u32, regs::rbp()), Amode::imm_reg(off as u32, regs::rbp()),
Writable::from_reg(regs::r14()), Writable::from_reg(regs::r14()),
None,
)); ));
} }
@@ -498,7 +494,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
dest: &CallDest, dest: &CallDest,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: ir::Opcode, opcode: ir::Opcode,
tmp: Writable<Reg>, tmp: Writable<Reg>,
_callee_conv: isa::CallConv, _callee_conv: isa::CallConv,
@@ -509,7 +504,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
&CallDest::ExtName(ref name, RelocDistance::Near) => { &CallDest::ExtName(ref name, RelocDistance::Near) => {
insts.push(( insts.push((
InstIsSafepoint::Yes, InstIsSafepoint::Yes,
Inst::call_known(name.clone(), uses, defs, loc, opcode), Inst::call_known(name.clone(), uses, defs, opcode),
)); ));
} }
&CallDest::ExtName(ref name, RelocDistance::Far) => { &CallDest::ExtName(ref name, RelocDistance::Far) => {
@@ -519,18 +514,17 @@ impl ABIMachineSpec for X64ABIMachineSpec {
dst: tmp, dst: tmp,
name: Box::new(name.clone()), name: Box::new(name.clone()),
offset: 0, offset: 0,
srcloc: loc,
}, },
)); ));
insts.push(( insts.push((
InstIsSafepoint::Yes, InstIsSafepoint::Yes,
Inst::call_unknown(RegMem::reg(tmp.to_reg()), uses, defs, loc, opcode), Inst::call_unknown(RegMem::reg(tmp.to_reg()), uses, defs, opcode),
)); ));
} }
&CallDest::Reg(reg) => { &CallDest::Reg(reg) => {
insts.push(( insts.push((
InstIsSafepoint::Yes, InstIsSafepoint::Yes,
Inst::call_unknown(RegMem::reg(reg), uses, defs, loc, opcode), Inst::call_unknown(RegMem::reg(reg), uses, defs, opcode),
)); ));
} }
} }

366
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -1,4 +1,4 @@
use crate::binemit::Reloc; use crate::binemit::{Addend, Reloc};
use crate::ir::immediates::{Ieee32, Ieee64}; use crate::ir::immediates::{Ieee32, Ieee64};
use crate::ir::TrapCode; use crate::ir::TrapCode;
use crate::isa::x64::inst::args::*; use crate::isa::x64::inst::args::*;
@@ -181,6 +181,7 @@ impl LegacyPrefixes {
/// indicate a 64-bit operation. /// indicate a 64-bit operation.
fn emit_std_enc_mem( fn emit_std_enc_mem(
sink: &mut MachBuffer<Inst>, sink: &mut MachBuffer<Inst>,
state: &EmitState,
prefixes: LegacyPrefixes, prefixes: LegacyPrefixes,
opcodes: u32, opcodes: u32,
mut num_opcodes: usize, mut num_opcodes: usize,
@@ -192,6 +193,11 @@ fn emit_std_enc_mem(
// 64-bit integer registers, because they are part of an address // 64-bit integer registers, because they are part of an address
// expression. But `enc_g` can be derived from a register of any class. // expression. But `enc_g` can be derived from a register of any class.
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
prefixes.emit(sink); prefixes.emit(sink);
match mem_e { match mem_e {
@@ -342,6 +348,7 @@ fn emit_std_enc_enc(
fn emit_std_reg_mem( fn emit_std_reg_mem(
sink: &mut MachBuffer<Inst>, sink: &mut MachBuffer<Inst>,
state: &EmitState,
prefixes: LegacyPrefixes, prefixes: LegacyPrefixes,
opcodes: u32, opcodes: u32,
num_opcodes: usize, num_opcodes: usize,
@@ -350,7 +357,16 @@ fn emit_std_reg_mem(
rex: RexFlags, rex: RexFlags,
) { ) {
let enc_g = reg_enc(reg_g); let enc_g = reg_enc(reg_g);
emit_std_enc_mem(sink, prefixes, opcodes, num_opcodes, enc_g, mem_e, rex); emit_std_enc_mem(
sink,
state,
prefixes,
opcodes,
num_opcodes,
enc_g,
mem_e,
rex,
);
} }
fn emit_std_reg_reg( fn emit_std_reg_reg(
@@ -407,6 +423,18 @@ fn one_way_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
sink.put4(0x0); sink.put4(0x0);
} }
/// Emits a relocation, attaching the current source location as well.
fn emit_reloc(
sink: &mut MachBuffer<Inst>,
state: &EmitState,
kind: Reloc,
name: &ExternalName,
addend: Addend,
) {
let srcloc = state.cur_srcloc();
sink.add_reloc(srcloc, kind, name, addend);
}
/// The top-level emit function. /// The top-level emit function.
/// ///
/// Important! Do not add improved (shortened) encoding cases to existing /// Important! Do not add improved (shortened) encoding cases to existing
@@ -505,13 +533,15 @@ pub(crate) fn emit(
} }
RegMemImm::Mem { addr } => { RegMemImm::Mem { addr } => {
let amode = addr.finalize(state);
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
0x0FAF, 0x0FAF,
2, 2,
reg_g.to_reg(), reg_g.to_reg(),
&addr.finalize(state), &amode,
rex, rex,
); );
} }
@@ -562,13 +592,15 @@ pub(crate) fn emit(
RegMemImm::Mem { addr } => { RegMemImm::Mem { addr } => {
// Here we revert to the "normal" G-E ordering. // Here we revert to the "normal" G-E ordering.
let amode = addr.finalize(state);
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
opcode_m, opcode_m,
1, 1,
reg_g.to_reg(), reg_g.to_reg(),
&addr.finalize(state), &amode,
rex, rex,
); );
} }
@@ -616,15 +648,19 @@ pub(crate) fn emit(
*src, *src,
rex_flags, rex_flags,
), ),
RegMem::Mem { addr: src } => emit_std_reg_mem( RegMem::Mem { addr: src } => {
sink, let amode = src.finalize(state);
prefix, emit_std_reg_mem(
opcode, sink,
num_opcodes, state,
dst.to_reg(), prefix,
&src.finalize(state), opcode,
rex_flags, num_opcodes,
), dst.to_reg(),
&amode,
rex_flags,
);
}
} }
} }
@@ -660,7 +696,6 @@ pub(crate) fn emit(
size, size,
signed, signed,
divisor, divisor,
loc,
} => { } => {
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix, rex_flags) = match size {
1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()), 1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
@@ -670,7 +705,8 @@ pub(crate) fn emit(
_ => unreachable!("{}", size), _ => unreachable!("{}", size),
}; };
sink.add_trap(*loc, TrapCode::IntegerDivisionByZero); let loc = state.cur_srcloc();
sink.add_trap(loc, TrapCode::IntegerDivisionByZero);
let subopcode = if *signed { 7 } else { 6 }; let subopcode = if *signed { 7 } else { 6 };
match divisor { match divisor {
@@ -678,15 +714,10 @@ pub(crate) fn emit(
let src = int_reg_enc(*reg); let src = int_reg_enc(*reg);
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
} }
RegMem::Mem { addr: src } => emit_std_enc_mem( RegMem::Mem { addr: src } => {
sink, let amode = src.finalize(state);
prefix, emit_std_enc_mem(sink, state, prefix, opcode, 1, subopcode, &amode, rex_flags);
opcode, }
1,
subopcode,
&src.finalize(state),
rex_flags,
),
} }
} }
@@ -704,15 +735,10 @@ pub(crate) fn emit(
let src = int_reg_enc(*reg); let src = int_reg_enc(*reg);
emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags) emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags)
} }
RegMem::Mem { addr: src } => emit_std_enc_mem( RegMem::Mem { addr: src } => {
sink, let amode = src.finalize(state);
prefix, emit_std_enc_mem(sink, state, prefix, 0xF7, 1, subopcode, &amode, rex_flags);
0xF7, }
1,
subopcode,
&src.finalize(state),
rex_flags,
),
} }
} }
@@ -737,7 +763,6 @@ pub(crate) fn emit(
kind, kind,
size, size,
divisor, divisor,
loc,
tmp, tmp,
} => { } => {
// Generates the following code sequence: // Generates the following code sequence:
@@ -773,7 +798,7 @@ pub(crate) fn emit(
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg()); let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg());
inst.emit(sink, info, state); inst.emit(sink, info, state);
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero, *loc); let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero);
inst.emit(sink, info, state); inst.emit(sink, info, state);
let (do_op, done_label) = if kind.is_signed() { let (do_op, done_label) = if kind.is_signed() {
@@ -818,7 +843,7 @@ pub(crate) fn emit(
} }
// If not equal, jump over the trap. // If not equal, jump over the trap.
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerOverflow, *loc); let inst = Inst::trap_if(CC::Z, TrapCode::IntegerOverflow);
inst.emit(sink, info, state); inst.emit(sink, info, state);
(Some(do_op), None) (Some(do_op), None)
@@ -847,7 +872,7 @@ pub(crate) fn emit(
inst.emit(sink, info, state); inst.emit(sink, info, state);
} }
let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg()), *loc); let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg()));
inst.emit(sink, info, state); inst.emit(sink, info, state);
// Lowering takes care of moving the result back into the right register, see comment // Lowering takes care of moving the result back into the right register, see comment
@@ -900,12 +925,7 @@ pub(crate) fn emit(
emit_std_reg_reg(sink, LegacyPrefixes::None, 0x89, 1, *src, dst.to_reg(), rex); emit_std_reg_reg(sink, LegacyPrefixes::None, 0x89, 1, *src, dst.to_reg(), rex);
} }
Inst::MovzxRmR { Inst::MovzxRmR { ext_mode, src, dst } => {
ext_mode,
src,
dst,
srcloc,
} => {
let (opcodes, num_opcodes, mut rex_flags) = match ext_mode { let (opcodes, num_opcodes, mut rex_flags) = match ext_mode {
ExtMode::BL => { ExtMode::BL => {
// MOVZBL is (REX.W==0) 0F B6 /r // MOVZBL is (REX.W==0) 0F B6 /r
@@ -963,13 +983,9 @@ pub(crate) fn emit(
RegMem::Mem { addr: src } => { RegMem::Mem { addr: src } => {
let src = &src.finalize(state); let src = &src.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
opcodes, opcodes,
num_opcodes, num_opcodes,
@@ -981,16 +997,12 @@ pub(crate) fn emit(
} }
} }
Inst::Mov64MR { src, dst, srcloc } => { Inst::Mov64MR { src, dst } => {
let src = &src.finalize(state); let src = &src.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
0x8B, 0x8B,
1, 1,
@@ -1000,22 +1012,22 @@ pub(crate) fn emit(
) )
} }
Inst::LoadEffectiveAddress { addr, dst } => emit_std_reg_mem( Inst::LoadEffectiveAddress { addr, dst } => {
sink, let amode = addr.finalize(state);
LegacyPrefixes::None,
0x8D,
1,
dst.to_reg(),
&addr.finalize(state),
RexFlags::set_w(),
),
Inst::MovsxRmR { emit_std_reg_mem(
ext_mode, sink,
src, state,
dst, LegacyPrefixes::None,
srcloc, 0x8D,
} => { 1,
dst.to_reg(),
&amode,
RexFlags::set_w(),
);
}
Inst::MovsxRmR { ext_mode, src, dst } => {
let (opcodes, num_opcodes, mut rex_flags) = match ext_mode { let (opcodes, num_opcodes, mut rex_flags) = match ext_mode {
ExtMode::BL => { ExtMode::BL => {
// MOVSBL is (REX.W==0) 0F BE /r // MOVSBL is (REX.W==0) 0F BE /r
@@ -1065,13 +1077,9 @@ pub(crate) fn emit(
RegMem::Mem { addr: src } => { RegMem::Mem { addr: src } => {
let src = &src.finalize(state); let src = &src.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
opcodes, opcodes,
num_opcodes, num_opcodes,
@@ -1083,19 +1091,9 @@ pub(crate) fn emit(
} }
} }
Inst::MovRM { Inst::MovRM { size, src, dst } => {
size,
src,
dst,
srcloc,
} => {
let dst = &dst.finalize(state); let dst = &dst.finalize(state);
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
match size { match size {
1 => { 1 => {
// This is one of the few places where the presence of a // This is one of the few places where the presence of a
@@ -1109,13 +1107,14 @@ pub(crate) fn emit(
}; };
// MOV r8, r/m8 is (REX.W==0) 88 /r // MOV r8, r/m8 is (REX.W==0) 88 /r
emit_std_reg_mem(sink, LegacyPrefixes::None, 0x88, 1, *src, dst, rex) emit_std_reg_mem(sink, state, LegacyPrefixes::None, 0x88, 1, *src, dst, rex)
} }
2 => { 2 => {
// MOV r16, r/m16 is 66 (REX.W==0) 89 /r // MOV r16, r/m16 is 66 (REX.W==0) 89 /r
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::_66, LegacyPrefixes::_66,
0x89, 0x89,
1, 1,
@@ -1129,6 +1128,7 @@ pub(crate) fn emit(
// MOV r32, r/m32 is (REX.W==0) 89 /r // MOV r32, r/m32 is (REX.W==0) 89 /r
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
0x89, 0x89,
1, 1,
@@ -1142,6 +1142,7 @@ pub(crate) fn emit(
// MOV r64, r/m64 is (REX.W==1) 89 /r // MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem( emit_std_reg_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
0x89, 0x89,
1, 1,
@@ -1248,7 +1249,16 @@ pub(crate) fn emit(
} }
RegMemImm::Mem { addr } => { RegMemImm::Mem { addr } => {
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode_bytes, 2, dst.to_reg(), addr, rex); emit_std_reg_mem(
sink,
state,
prefix,
opcode_bytes,
2,
dst.to_reg(),
addr,
rex,
);
} }
RegMemImm::Imm { .. } => unreachable!(), RegMemImm::Imm { .. } => unreachable!(),
} }
@@ -1300,7 +1310,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
// Whereas here we revert to the "normal" G-E ordering. // Whereas here we revert to the "normal" G-E ordering.
let opcode = if *size == 1 { 0x3A } else { 0x3B }; let opcode = if *size == 1 { 0x3A } else { 0x3B };
emit_std_reg_mem(sink, prefix, opcode, 1, *reg_g, addr, rex); emit_std_reg_mem(sink, state, prefix, opcode, 1, *reg_g, addr, rex);
} }
RegMemImm::Imm { simm32 } => { RegMemImm::Imm { simm32 } => {
@@ -1358,7 +1368,16 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex_flags); emit_std_reg_mem(
sink,
state,
prefix,
opcode,
2,
reg_g.to_reg(),
addr,
rex_flags,
);
} }
} }
} }
@@ -1402,6 +1421,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_enc_mem( emit_std_enc_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
0xFF, 0xFF,
1, 1,
@@ -1432,25 +1452,22 @@ pub(crate) fn emit(
sink.put1(0x58 + (enc_dst & 7)); sink.put1(0x58 + (enc_dst & 7));
} }
Inst::CallKnown { Inst::CallKnown { dest, opcode, .. } => {
dest, loc, opcode, ..
} => {
if let Some(s) = state.take_stack_map() { if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s); sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s);
} }
sink.put1(0xE8); sink.put1(0xE8);
// The addend adjusts for the difference between the end of the instruction and the // The addend adjusts for the difference between the end of the instruction and the
// beginning of the immediate field. // beginning of the immediate field.
sink.add_reloc(*loc, Reloc::X86CallPCRel4, &dest, -4); emit_reloc(sink, state, Reloc::X86CallPCRel4, &dest, -4);
sink.put4(0); sink.put4(0);
if opcode.is_call() { if opcode.is_call() {
sink.add_call_site(*loc, *opcode); let loc = state.cur_srcloc();
sink.add_call_site(loc, *opcode);
} }
} }
Inst::CallUnknown { Inst::CallUnknown { dest, opcode, .. } => {
dest, opcode, loc, ..
} => {
let start_offset = sink.cur_offset(); let start_offset = sink.cur_offset();
match dest { match dest {
RegMem::Reg { reg } => { RegMem::Reg { reg } => {
@@ -1470,6 +1487,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_enc_mem( emit_std_enc_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
0xFF, 0xFF,
1, 1,
@@ -1483,7 +1501,8 @@ pub(crate) fn emit(
sink.add_stack_map(StackMapExtent::StartedAtOffset(start_offset), s); sink.add_stack_map(StackMapExtent::StartedAtOffset(start_offset), s);
} }
if opcode.is_call() { if opcode.is_call() {
sink.add_call_site(*loc, *opcode); let loc = state.cur_srcloc();
sink.add_call_site(loc, *opcode);
} }
} }
@@ -1566,6 +1585,7 @@ pub(crate) fn emit(
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_enc_mem( emit_std_enc_mem(
sink, sink,
state,
LegacyPrefixes::None, LegacyPrefixes::None,
0xFF, 0xFF,
1, 1,
@@ -1609,7 +1629,7 @@ pub(crate) fn emit(
one_way_jmp(sink, CC::NB, *default_target); // idx unsigned >= jmp table size one_way_jmp(sink, CC::NB, *default_target); // idx unsigned >= jmp table size
// Copy the index (and make sure to clear the high 32-bits lane of tmp2). // Copy the index (and make sure to clear the high 32-bits lane of tmp2).
let inst = Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(*idx), *tmp2, None); let inst = Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(*idx), *tmp2);
inst.emit(sink, info, state); inst.emit(sink, info, state);
// Load base address of jump table. // Load base address of jump table.
@@ -1623,7 +1643,6 @@ pub(crate) fn emit(
ExtMode::LQ, ExtMode::LQ,
RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)), RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)),
*tmp2, *tmp2,
None,
); );
inst.emit(sink, info, state); inst.emit(sink, info, state);
@@ -1655,18 +1674,14 @@ pub(crate) fn emit(
} }
} }
Inst::TrapIf { Inst::TrapIf { cc, trap_code } => {
cc,
trap_code,
srcloc,
} => {
let else_label = sink.get_label(); let else_label = sink.get_label();
// Jump over if the invert of CC is set (i.e. CC is not set). // Jump over if the invert of CC is set (i.e. CC is not set).
one_way_jmp(sink, cc.invert(), else_label); one_way_jmp(sink, cc.invert(), else_label);
// Trap! // Trap!
let inst = Inst::trap(*srcloc, *trap_code); let inst = Inst::trap(*trap_code);
inst.emit(sink, info, state); inst.emit(sink, info, state);
sink.bind_label(else_label); sink.bind_label(else_label);
@@ -1676,7 +1691,6 @@ pub(crate) fn emit(
op, op,
src: src_e, src: src_e,
dst: reg_g, dst: reg_g,
srcloc,
} => { } => {
let rex = RexFlags::clear_w(); let rex = RexFlags::clear_w();
@@ -1715,11 +1729,16 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
if let Some(srcloc) = *srcloc { emit_std_reg_mem(
// Register the offset at which the actual load instruction starts. sink,
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); state,
} prefix,
emit_std_reg_mem(sink, prefix, opcode, num_opcodes, reg_g.to_reg(), addr, rex); opcode,
num_opcodes,
reg_g.to_reg(),
addr,
rex,
);
} }
}; };
} }
@@ -1728,7 +1747,6 @@ pub(crate) fn emit(
op, op,
src: src_e, src: src_e,
dst: reg_g, dst: reg_g,
srcloc,
} => { } => {
let rex = RexFlags::clear_w(); let rex = RexFlags::clear_w();
let (prefix, opcode, length) = match op { let (prefix, opcode, length) = match op {
@@ -1823,12 +1841,17 @@ pub(crate) fn emit(
emit_std_reg_reg(sink, prefix, opcode, length, reg_g.to_reg(), *reg_e, rex); emit_std_reg_reg(sink, prefix, opcode, length, reg_g.to_reg(), *reg_e, rex);
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, length, reg_g.to_reg(), addr, rex); emit_std_reg_mem(
sink,
state,
prefix,
opcode,
length,
reg_g.to_reg(),
addr,
rex,
);
} }
} }
} }
@@ -1897,7 +1920,7 @@ pub(crate) fn emit(
// and negative zero. These instructions merge the sign bits in that // and negative zero. These instructions merge the sign bits in that
// case, and are no-ops otherwise. // case, and are no-ops otherwise.
let op = if *is_min { or_op } else { and_op }; let op = if *is_min { or_op } else { and_op };
let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst, None); let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst);
inst.emit(sink, info, state); inst.emit(sink, info, state);
let inst = Inst::jmp_known(done); let inst = Inst::jmp_known(done);
@@ -1907,13 +1930,13 @@ pub(crate) fn emit(
// read-only operand: perform an addition between the two operands, which has the // read-only operand: perform an addition between the two operands, which has the
// desired NaN propagation effects. // desired NaN propagation effects.
sink.bind_label(propagate_nan); sink.bind_label(propagate_nan);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst, None); let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst);
inst.emit(sink, info, state); inst.emit(sink, info, state);
one_way_jmp(sink, CC::P, done); one_way_jmp(sink, CC::P, done);
sink.bind_label(do_min_max); sink.bind_label(do_min_max);
let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst, None); let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst);
inst.emit(sink, info, state); inst.emit(sink, info, state);
sink.bind_label(done); sink.bind_label(done);
@@ -1925,7 +1948,6 @@ pub(crate) fn emit(
dst, dst,
imm, imm,
is64, is64,
srcloc,
} => { } => {
let (prefix, opcode, len) = match op { let (prefix, opcode, len) = match op {
SseOpcode::Cmpps => (LegacyPrefixes::None, 0x0FC2, 2), SseOpcode::Cmpps => (LegacyPrefixes::None, 0x0FC2, 2),
@@ -1964,16 +1986,12 @@ pub(crate) fn emit(
} }
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
if let Some(srcloc) = *srcloc {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
assert!( assert!(
!regs_swapped, !regs_swapped,
"No existing way to encode a mem argument in the ModRM r/m field." "No existing way to encode a mem argument in the ModRM r/m field."
); );
emit_std_reg_mem(sink, prefix, opcode, len, dst.to_reg(), addr, rex); emit_std_reg_mem(sink, state, prefix, opcode, len, dst.to_reg(), addr, rex);
} }
} }
sink.put1(*imm); sink.put1(*imm);
@@ -1981,7 +1999,7 @@ pub(crate) fn emit(
Inst::XmmLoadConst { src, dst, ty } => { Inst::XmmLoadConst { src, dst, ty } => {
let load_offset = Amode::rip_relative(sink.get_label_for_constant(*src)); let load_offset = Amode::rip_relative(sink.get_label_for_constant(*src));
let load = Inst::load(*ty, load_offset, *dst, ExtKind::None, None); let load = Inst::load(*ty, load_offset, *dst, ExtKind::None);
load.emit(sink, info, state); load.emit(sink, info, state);
} }
@@ -1990,12 +2008,7 @@ pub(crate) fn emit(
// emitted. // emitted.
} }
Inst::XmmMovRM { Inst::XmmMovRM { op, src, dst } => {
op,
src,
dst,
srcloc,
} => {
let (prefix, opcode) = match op { let (prefix, opcode) = match op {
SseOpcode::Movaps => (LegacyPrefixes::None, 0x0F29), SseOpcode::Movaps => (LegacyPrefixes::None, 0x0F29),
SseOpcode::Movapd => (LegacyPrefixes::_66, 0x0F29), SseOpcode::Movapd => (LegacyPrefixes::_66, 0x0F29),
@@ -2008,11 +2021,16 @@ pub(crate) fn emit(
_ => unimplemented!("Opcode {:?} not implemented", op), _ => unimplemented!("Opcode {:?} not implemented", op),
}; };
let dst = &dst.finalize(state); let dst = &dst.finalize(state);
if let Some(srcloc) = *srcloc { emit_std_reg_mem(
// Register the offset at which the actual load instruction starts. sink,
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); state,
} prefix,
emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, RexFlags::clear_w()); opcode,
2,
*src,
dst,
RexFlags::clear_w(),
);
} }
Inst::XmmToGpr { Inst::XmmToGpr {
@@ -2070,7 +2088,7 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex); emit_std_reg_mem(sink, state, prefix, opcode, 2, reg_g.to_reg(), addr, rex);
} }
} }
} }
@@ -2090,7 +2108,7 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state); let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, len, *dst, addr, rex); emit_std_reg_mem(sink, state, prefix, opcode, len, *dst, addr, rex);
} }
} }
} }
@@ -2185,7 +2203,7 @@ pub(crate) fn emit(
} else { } else {
SseOpcode::Addss SseOpcode::Addss
}; };
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst, None); let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst.to_reg()), *dst);
inst.emit(sink, info, state); inst.emit(sink, info, state);
sink.bind_label(done); sink.bind_label(done);
@@ -2199,7 +2217,6 @@ pub(crate) fn emit(
dst, dst,
tmp_gpr, tmp_gpr,
tmp_xmm, tmp_xmm,
srcloc,
} => { } => {
// Emits the following common sequence: // Emits the following common sequence:
// //
@@ -2292,12 +2309,8 @@ pub(crate) fn emit(
// If the input was positive, saturate to INT_MAX. // If the input was positive, saturate to INT_MAX.
// Zero out tmp_xmm. // Zero out tmp_xmm.
let inst = Inst::xmm_rm_r( let inst =
SseOpcode::Xorpd, Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm);
RegMem::reg(tmp_xmm.to_reg()),
*tmp_xmm,
None,
);
inst.emit(sink, info, state); inst.emit(sink, info, state);
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg());
@@ -2317,7 +2330,7 @@ pub(crate) fn emit(
} else { } else {
let check_positive = sink.get_label(); let check_positive = sink.get_label();
let inst = Inst::trap(*srcloc, TrapCode::BadConversionToInteger); let inst = Inst::trap(TrapCode::BadConversionToInteger);
inst.emit(sink, info, state); inst.emit(sink, info, state);
// Check if INT_MIN was the correct result: determine the smallest floating point // Check if INT_MIN was the correct result: determine the smallest floating point
@@ -2360,7 +2373,7 @@ pub(crate) fn emit(
// jump over trap if src >= or > threshold // jump over trap if src >= or > threshold
one_way_jmp(sink, no_overflow_cc, check_positive); one_way_jmp(sink, no_overflow_cc, check_positive);
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow); let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state); inst.emit(sink, info, state);
// If positive, it was a real overflow. // If positive, it was a real overflow.
@@ -2368,12 +2381,8 @@ pub(crate) fn emit(
sink.bind_label(check_positive); sink.bind_label(check_positive);
// Zero out the tmp_xmm register. // Zero out the tmp_xmm register.
let inst = Inst::xmm_rm_r( let inst =
SseOpcode::Xorpd, Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::reg(tmp_xmm.to_reg()), *tmp_xmm);
RegMem::reg(tmp_xmm.to_reg()),
*tmp_xmm,
None,
);
inst.emit(sink, info, state); inst.emit(sink, info, state);
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg()); let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(src), tmp_xmm.to_reg());
@@ -2381,7 +2390,7 @@ pub(crate) fn emit(
one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src one_way_jmp(sink, CC::NB, done); // jump over trap if 0 >= src
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow); let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state); inst.emit(sink, info, state);
} }
@@ -2396,7 +2405,6 @@ pub(crate) fn emit(
dst, dst,
tmp_gpr, tmp_gpr,
tmp_xmm, tmp_xmm,
srcloc,
} => { } => {
// The only difference in behavior between saturating and non-saturating is how we // The only difference in behavior between saturating and non-saturating is how we
// handle errors. Emits the following sequence: // handle errors. Emits the following sequence:
@@ -2487,7 +2495,7 @@ pub(crate) fn emit(
inst.emit(sink, info, state); inst.emit(sink, info, state);
} else { } else {
// Trap. // Trap.
let inst = Inst::trap(*srcloc, TrapCode::BadConversionToInteger); let inst = Inst::trap(TrapCode::BadConversionToInteger);
inst.emit(sink, info, state); inst.emit(sink, info, state);
} }
@@ -2519,7 +2527,7 @@ pub(crate) fn emit(
inst.emit(sink, info, state); inst.emit(sink, info, state);
} else { } else {
// Trap. // Trap.
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow); let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state); inst.emit(sink, info, state);
} }
@@ -2527,7 +2535,7 @@ pub(crate) fn emit(
sink.bind_label(handle_large); sink.bind_label(handle_large);
let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src, None); let inst = Inst::xmm_rm_r(sub_op, RegMem::reg(tmp_xmm.to_reg()), *src);
inst.emit(sink, info, state); inst.emit(sink, info, state);
let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size); let inst = Inst::xmm_to_gpr(trunc_op, src.to_reg(), *dst, *dst_size);
@@ -2556,7 +2564,7 @@ pub(crate) fn emit(
let inst = Inst::jmp_known(done); let inst = Inst::jmp_known(done);
inst.emit(sink, info, state); inst.emit(sink, info, state);
} else { } else {
let inst = Inst::trap(*srcloc, TrapCode::IntegerOverflow); let inst = Inst::trap(TrapCode::IntegerOverflow);
inst.emit(sink, info, state); inst.emit(sink, info, state);
} }
@@ -2582,18 +2590,13 @@ pub(crate) fn emit(
sink.bind_label(done); sink.bind_label(done);
} }
Inst::LoadExtName { Inst::LoadExtName { dst, name, offset } => {
dst,
name,
offset,
srcloc,
} => {
// The full address can be encoded in the register, with a relocation. // The full address can be encoded in the register, with a relocation.
// Generates: movabsq $name, %dst // Generates: movabsq $name, %dst
let enc_dst = int_reg_enc(dst.to_reg()); let enc_dst = int_reg_enc(dst.to_reg());
sink.put1(0x48 | ((enc_dst >> 3) & 1)); sink.put1(0x48 | ((enc_dst >> 3) & 1));
sink.put1(0xB8 | (enc_dst & 7)); sink.put1(0xB8 | (enc_dst & 7));
sink.add_reloc(*srcloc, Reloc::Abs8, name, *offset); emit_reloc(sink, state, Reloc::Abs8, name, *offset);
if info.flags().emit_all_ones_funcaddrs() { if info.flags().emit_all_ones_funcaddrs() {
sink.put8(u64::max_value()); sink.put8(u64::max_value());
} else { } else {
@@ -2601,15 +2604,7 @@ pub(crate) fn emit(
} }
} }
Inst::LockCmpxchg { Inst::LockCmpxchg { ty, src, dst } => {
ty,
src,
dst,
srcloc,
} => {
if let Some(srcloc) = srcloc {
sink.add_trap(*srcloc, TrapCode::HeapOutOfBounds);
}
// lock cmpxchg{b,w,l,q} %src, (dst) // lock cmpxchg{b,w,l,q} %src, (dst)
// Note that 0xF0 is the Lock prefix. // Note that 0xF0 is the Lock prefix.
let (prefix, rex, opcodes) = match *ty { let (prefix, rex, opcodes) = match *ty {
@@ -2626,10 +2621,11 @@ pub(crate) fn emit(
types::I64 => (LegacyPrefixes::_F0, RexFlags::set_w(), 0x0FB1), types::I64 => (LegacyPrefixes::_F0, RexFlags::set_w(), 0x0FB1),
_ => unreachable!(), _ => unreachable!(),
}; };
emit_std_reg_mem(sink, prefix, opcodes, 2, *src, &dst.finalize(state), rex); let amode = dst.finalize(state);
emit_std_reg_mem(sink, state, prefix, opcodes, 2, *src, &amode, rex);
} }
Inst::AtomicRmwSeq { ty, op, srcloc } => { Inst::AtomicRmwSeq { ty, op } => {
// Emit this: // Emit this:
// //
// mov{zbq,zwq,zlq,q} (%r9), %rax // rax = old value // mov{zbq,zwq,zlq,q} (%r9), %rax // rax = old value
@@ -2657,7 +2653,7 @@ pub(crate) fn emit(
// mov{zbq,zwq,zlq,q} (%r9), %rax // mov{zbq,zwq,zlq,q} (%r9), %rax
// No need to call `add_trap` here, since the `i1` emit will do that. // No need to call `add_trap` here, since the `i1` emit will do that.
let i1 = Inst::load(*ty, amode.clone(), rax_w, ExtKind::ZeroExtend, *srcloc); let i1 = Inst::load(*ty, amode.clone(), rax_w, ExtKind::ZeroExtend);
i1.emit(sink, info, state); i1.emit(sink, info, state);
// again: // again:
@@ -2690,7 +2686,6 @@ pub(crate) fn emit(
ty: *ty, ty: *ty,
src: r11, src: r11,
dst: amode.into(), dst: amode.into(),
srcloc: *srcloc,
}; };
i4.emit(sink, info, state); i4.emit(sink, info, state);
@@ -2712,8 +2707,9 @@ pub(crate) fn emit(
sink.put1(0xcc); sink.put1(0xcc);
} }
Inst::Ud2 { trap_info } => { Inst::Ud2 { trap_code } => {
sink.add_trap(trap_info.0, trap_info.1); let cur_srcloc = state.cur_srcloc();
sink.add_trap(cur_srcloc, *trap_code);
if let Some(s) = state.take_stack_map() { if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s); sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
} }

744
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

File diff suppressed because it is too large Load Diff

237
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -71,7 +71,6 @@ pub enum Inst {
size: u8, // 1, 2, 4 or 8 size: u8, // 1, 2, 4 or 8
signed: bool, signed: bool,
divisor: RegMem, divisor: RegMem,
loc: SourceLoc,
}, },
/// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs.
@@ -95,7 +94,6 @@ pub enum Inst {
/// different from the temporary. /// different from the temporary.
divisor: Writable<Reg>, divisor: Writable<Reg>,
tmp: Option<Writable<Reg>>, tmp: Option<Writable<Reg>>,
loc: SourceLoc,
}, },
/// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo) /// Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo)
@@ -126,16 +124,12 @@ pub enum Inst {
ext_mode: ExtMode, ext_mode: ExtMode,
src: RegMem, src: RegMem,
dst: Writable<Reg>, dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
}, },
/// A plain 64-bit integer load, since MovZX_RM_R can't represent that. /// A plain 64-bit integer load, since MovZX_RM_R can't represent that.
Mov64MR { Mov64MR {
src: SyntheticAmode, src: SyntheticAmode,
dst: Writable<Reg>, dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
}, },
/// Loads the memory address of addr into dst. /// Loads the memory address of addr into dst.
@@ -149,8 +143,6 @@ pub enum Inst {
ext_mode: ExtMode, ext_mode: ExtMode,
src: RegMem, src: RegMem,
dst: Writable<Reg>, dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
}, },
/// Integer stores: mov (b w l q) reg addr. /// Integer stores: mov (b w l q) reg addr.
@@ -158,8 +150,6 @@ pub enum Inst {
size: u8, // 1, 2, 4 or 8. size: u8, // 1, 2, 4 or 8.
src: Reg, src: Reg,
dst: SyntheticAmode, dst: SyntheticAmode,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
}, },
/// Arithmetic shifts: (shl shr sar) (b w l q) imm reg. /// Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
@@ -213,7 +203,6 @@ pub enum Inst {
op: SseOpcode, op: SseOpcode,
src: RegMem, src: RegMem,
dst: Writable<Reg>, dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
}, },
/// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt, /// XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg, sqrt,
@@ -226,8 +215,6 @@ pub enum Inst {
op: SseOpcode, op: SseOpcode,
src: RegMem, src: RegMem,
dst: Writable<Reg>, dst: Writable<Reg>,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
}, },
/// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq
@@ -235,8 +222,6 @@ pub enum Inst {
op: SseOpcode, op: SseOpcode,
src: Reg, src: Reg,
dst: SyntheticAmode, dst: SyntheticAmode,
/// Source location, if the memory access can be out-of-bounds.
srcloc: Option<SourceLoc>,
}, },
/// XMM (vector) unary op (to move a constant value into an xmm register): movups /// XMM (vector) unary op (to move a constant value into an xmm register): movups
@@ -289,7 +274,6 @@ pub enum Inst {
dst: Writable<Reg>, dst: Writable<Reg>,
tmp_gpr: Writable<Reg>, tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>, tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
}, },
/// Converts a scalar xmm to an unsigned int32/int64. /// Converts a scalar xmm to an unsigned int32/int64.
@@ -305,7 +289,6 @@ pub enum Inst {
dst: Writable<Reg>, dst: Writable<Reg>,
tmp_gpr: Writable<Reg>, tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>, tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
}, },
/// A sequence to compute min/max with the proper NaN semantics for xmm registers. /// A sequence to compute min/max with the proper NaN semantics for xmm registers.
@@ -340,7 +323,6 @@ pub enum Inst {
dst: Writable<Reg>, dst: Writable<Reg>,
imm: u8, imm: u8,
is64: bool, is64: bool,
srcloc: Option<SourceLoc>,
}, },
// ===================================== // =====================================
@@ -350,7 +332,6 @@ pub enum Inst {
dest: ExternalName, dest: ExternalName,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode, opcode: Opcode,
}, },
@@ -359,7 +340,6 @@ pub enum Inst {
dest: RegMem, dest: RegMem,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode, opcode: Opcode,
}, },
@@ -408,23 +388,18 @@ pub enum Inst {
JmpUnknown { target: RegMem }, JmpUnknown { target: RegMem },
/// Traps if the condition code is set. /// Traps if the condition code is set.
TrapIf { TrapIf { cc: CC, trap_code: TrapCode },
cc: CC,
trap_code: TrapCode,
srcloc: SourceLoc,
},
/// A debug trap. /// A debug trap.
Hlt, Hlt,
/// An instruction that will always trigger the illegal instruction exception. /// An instruction that will always trigger the illegal instruction exception.
Ud2 { trap_info: (SourceLoc, TrapCode) }, Ud2 { trap_code: TrapCode },
/// Loads an external symbol in a register, with a relocation: movabsq $name, dst /// Loads an external symbol in a register, with a relocation: movabsq $name, dst
LoadExtName { LoadExtName {
dst: Writable<Reg>, dst: Writable<Reg>,
name: Box<ExternalName>, name: Box<ExternalName>,
srcloc: SourceLoc,
offset: i64, offset: i64,
}, },
@@ -443,7 +418,6 @@ pub enum Inst {
ty: Type, // I8, I16, I32 or I64 ty: Type, // I8, I16, I32 or I64
src: Reg, src: Reg,
dst: SyntheticAmode, dst: SyntheticAmode,
srcloc: Option<SourceLoc>,
}, },
/// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction. /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction.
@@ -472,7 +446,6 @@ pub enum Inst {
AtomicRmwSeq { AtomicRmwSeq {
ty: Type, // I8, I16, I32 or I64 ty: Type, // I8, I16, I32 or I64
op: inst_common::AtomicRmwOp, op: inst_common::AtomicRmwOp,
srcloc: Option<SourceLoc>,
}, },
/// A memory fence (mfence, lfence or sfence). /// A memory fence (mfence, lfence or sfence).
@@ -617,14 +590,13 @@ impl Inst {
Inst::Neg { size, src } Inst::Neg { size, src }
} }
pub(crate) fn div(size: u8, signed: bool, divisor: RegMem, loc: SourceLoc) -> Inst { pub(crate) fn div(size: u8, signed: bool, divisor: RegMem) -> Inst {
divisor.assert_regclass_is(RegClass::I64); divisor.assert_regclass_is(RegClass::I64);
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
Inst::Div { Inst::Div {
size, size,
signed, signed,
divisor, divisor,
loc,
} }
} }
@@ -639,7 +611,6 @@ impl Inst {
size: u8, size: u8,
divisor: Writable<Reg>, divisor: Writable<Reg>,
tmp: Option<Writable<Reg>>, tmp: Option<Writable<Reg>>,
loc: SourceLoc,
) -> Inst { ) -> Inst {
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
debug_assert!(divisor.to_reg().get_class() == RegClass::I64); debug_assert!(divisor.to_reg().get_class() == RegClass::I64);
@@ -651,7 +622,6 @@ impl Inst {
size, size,
divisor, divisor,
tmp, tmp,
loc,
} }
} }
@@ -679,20 +649,10 @@ impl Inst {
} }
// TODO Can be replaced by `Inst::move` (high-level) and `Inst::unary_rm_r` (low-level) // TODO Can be replaced by `Inst::move` (high-level) and `Inst::unary_rm_r` (low-level)
pub(crate) fn xmm_mov( pub(crate) fn xmm_mov(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
src.assert_regclass_is(RegClass::V128); src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmUnaryRmR { Inst::XmmUnaryRmR { op, src, dst }
op,
src,
dst,
srcloc,
}
} }
pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable<Reg>, ty: Type) -> Inst { pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable<Reg>, ty: Type) -> Inst {
@@ -705,28 +665,13 @@ impl Inst {
pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst { pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::V128); src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmUnaryRmR { Inst::XmmUnaryRmR { op, src, dst }
op,
src,
dst,
srcloc: None,
}
} }
pub(crate) fn xmm_rm_r( pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Self {
src.assert_regclass_is(RegClass::V128); src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmRmR { Inst::XmmRmR { op, src, dst }
op,
src,
dst,
srcloc,
}
} }
pub(crate) fn xmm_uninit_value(dst: Writable<Reg>) -> Self { pub(crate) fn xmm_uninit_value(dst: Writable<Reg>) -> Self {
@@ -734,18 +679,12 @@ impl Inst {
Inst::XmmUninitializedValue { dst } Inst::XmmUninitializedValue { dst }
} }
pub(crate) fn xmm_mov_r_m( pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
op: SseOpcode,
src: Reg,
dst: impl Into<SyntheticAmode>,
srcloc: Option<SourceLoc>,
) -> Inst {
debug_assert!(src.get_class() == RegClass::V128); debug_assert!(src.get_class() == RegClass::V128);
Inst::XmmMovRM { Inst::XmmMovRM {
op, op,
src, src,
dst: dst.into(), dst: dst.into(),
srcloc,
} }
} }
@@ -815,7 +754,6 @@ impl Inst {
dst: Writable<Reg>, dst: Writable<Reg>,
tmp_gpr: Writable<Reg>, tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>, tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
) -> Inst { ) -> Inst {
debug_assert!(src.to_reg().get_class() == RegClass::V128); debug_assert!(src.to_reg().get_class() == RegClass::V128);
debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128); debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128);
@@ -829,7 +767,6 @@ impl Inst {
dst, dst,
tmp_gpr, tmp_gpr,
tmp_xmm, tmp_xmm,
srcloc,
} }
} }
@@ -841,7 +778,6 @@ impl Inst {
dst: Writable<Reg>, dst: Writable<Reg>,
tmp_gpr: Writable<Reg>, tmp_gpr: Writable<Reg>,
tmp_xmm: Writable<Reg>, tmp_xmm: Writable<Reg>,
srcloc: SourceLoc,
) -> Inst { ) -> Inst {
debug_assert!(src.to_reg().get_class() == RegClass::V128); debug_assert!(src.to_reg().get_class() == RegClass::V128);
debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128); debug_assert!(tmp_xmm.to_reg().get_class() == RegClass::V128);
@@ -855,7 +791,6 @@ impl Inst {
dst, dst,
tmp_gpr, tmp_gpr,
tmp_xmm, tmp_xmm,
srcloc,
} }
} }
@@ -881,7 +816,6 @@ impl Inst {
dst: Writable<Reg>, dst: Writable<Reg>,
imm: u8, imm: u8,
is64: bool, is64: bool,
srcloc: Option<SourceLoc>,
) -> Inst { ) -> Inst {
Inst::XmmRmRImm { Inst::XmmRmRImm {
op, op,
@@ -889,24 +823,13 @@ impl Inst {
dst, dst,
imm, imm,
is64, is64,
srcloc,
} }
} }
pub(crate) fn movzx_rm_r( pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
ext_mode: ExtMode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
src.assert_regclass_is(RegClass::I64); src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::MovzxRmR { Inst::MovzxRmR { ext_mode, src, dst }
ext_mode,
src,
dst,
srcloc,
}
} }
pub(crate) fn xmm_rmi_reg(opcode: SseOpcode, src: RegMemImm, dst: Writable<Reg>) -> Inst { pub(crate) fn xmm_rmi_reg(opcode: SseOpcode, src: RegMemImm, dst: Writable<Reg>) -> Inst {
@@ -915,41 +838,26 @@ impl Inst {
Inst::XmmRmiReg { opcode, src, dst } Inst::XmmRmiReg { opcode, src, dst }
} }
pub(crate) fn movsx_rm_r( pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
ext_mode: ExtMode,
src: RegMem,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
src.assert_regclass_is(RegClass::I64); src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::MovsxRmR { Inst::MovsxRmR { ext_mode, src, dst }
ext_mode,
src,
dst,
srcloc,
}
} }
pub(crate) fn mov64_m_r( pub(crate) fn mov64_m_r(src: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
src: impl Into<SyntheticAmode>,
dst: Writable<Reg>,
srcloc: Option<SourceLoc>,
) -> Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::Mov64MR { Inst::Mov64MR {
src: src.into(), src: src.into(),
dst, dst,
srcloc,
} }
} }
/// A convenience function to be able to use a RegMem as the source of a move. /// A convenience function to be able to use a RegMem as the source of a move.
pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable<Reg>, srcloc: Option<SourceLoc>) -> Inst { pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64); src.assert_regclass_is(RegClass::I64);
match src { match src {
RegMem::Reg { reg } => Self::mov_r_r(true, reg, dst), RegMem::Reg { reg } => Self::mov_r_r(true, reg, dst),
RegMem::Mem { addr } => Self::mov64_m_r(addr, dst, srcloc), RegMem::Mem { addr } => Self::mov64_m_r(addr, dst),
} }
} }
@@ -957,7 +865,6 @@ impl Inst {
size: u8, // 1, 2, 4 or 8 size: u8, // 1, 2, 4 or 8
src: Reg, src: Reg,
dst: impl Into<SyntheticAmode>, dst: impl Into<SyntheticAmode>,
srcloc: Option<SourceLoc>,
) -> Inst { ) -> Inst {
debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); debug_assert!(size == 8 || size == 4 || size == 2 || size == 1);
debug_assert!(src.get_class() == RegClass::I64); debug_assert!(src.get_class() == RegClass::I64);
@@ -965,7 +872,6 @@ impl Inst {
size, size,
src, src,
dst: dst.into(), dst: dst.into(),
srcloc,
} }
} }
@@ -1011,9 +917,9 @@ impl Inst {
Inst::CmpRmiR { size, src, dst } Inst::CmpRmiR { size, src, dst }
} }
pub(crate) fn trap(srcloc: SourceLoc, trap_code: TrapCode) -> Inst { pub(crate) fn trap(trap_code: TrapCode) -> Inst {
Inst::Ud2 { Inst::Ud2 {
trap_info: (srcloc, trap_code), trap_code: trap_code,
} }
} }
@@ -1053,14 +959,12 @@ impl Inst {
dest: ExternalName, dest: ExternalName,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode, opcode: Opcode,
) -> Inst { ) -> Inst {
Inst::CallKnown { Inst::CallKnown {
dest, dest,
uses, uses,
defs, defs,
loc,
opcode, opcode,
} }
} }
@@ -1069,7 +973,6 @@ impl Inst {
dest: RegMem, dest: RegMem,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: Opcode, opcode: Opcode,
) -> Inst { ) -> Inst {
dest.assert_regclass_is(RegClass::I64); dest.assert_regclass_is(RegClass::I64);
@@ -1077,7 +980,6 @@ impl Inst {
dest, dest,
uses, uses,
defs, defs,
loc,
opcode, opcode,
} }
} }
@@ -1111,12 +1013,8 @@ impl Inst {
Inst::JmpUnknown { target } Inst::JmpUnknown { target }
} }
pub(crate) fn trap_if(cc: CC, trap_code: TrapCode, srcloc: SourceLoc) -> Inst { pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
Inst::TrapIf { Inst::TrapIf { cc, trap_code }
cc,
trap_code,
srcloc,
}
} }
/// Choose which instruction to use for loading a register value from memory. For loads smaller /// Choose which instruction to use for loading a register value from memory. For loads smaller
@@ -1127,7 +1025,6 @@ impl Inst {
from_addr: impl Into<SyntheticAmode>, from_addr: impl Into<SyntheticAmode>,
to_reg: Writable<Reg>, to_reg: Writable<Reg>,
ext_kind: ExtKind, ext_kind: ExtKind,
srcloc: Option<SourceLoc>,
) -> Inst { ) -> Inst {
let rc = to_reg.to_reg().get_class(); let rc = to_reg.to_reg().get_class();
match rc { match rc {
@@ -1143,10 +1040,10 @@ impl Inst {
// Values smaller than 64 bits must be extended in some way. // Values smaller than 64 bits must be extended in some way.
match ext_kind { match ext_kind {
ExtKind::SignExtend => { ExtKind::SignExtend => {
Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg, srcloc) Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
} }
ExtKind::ZeroExtend => { ExtKind::ZeroExtend => {
Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg, srcloc) Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
} }
ExtKind::None => panic!( ExtKind::None => panic!(
"expected an extension kind for extension mode: {:?}", "expected an extension kind for extension mode: {:?}",
@@ -1155,7 +1052,7 @@ impl Inst {
} }
} else { } else {
// 64-bit values can be moved directly. // 64-bit values can be moved directly.
Inst::mov64_m_r(from_addr, to_reg, srcloc) Inst::mov64_m_r(from_addr, to_reg)
} }
} }
RegClass::V128 => { RegClass::V128 => {
@@ -1174,18 +1071,13 @@ impl Inst {
} }
/// Choose which instruction to use for storing a register value to memory. /// Choose which instruction to use for storing a register value to memory.
pub(crate) fn store( pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
ty: Type,
from_reg: Reg,
to_addr: impl Into<SyntheticAmode>,
srcloc: Option<SourceLoc>,
) -> Inst {
let rc = from_reg.get_class(); let rc = from_reg.get_class();
match rc { match rc {
RegClass::I64 => { RegClass::I64 => {
// Always store the full register, to ensure that the high bits are properly set // Always store the full register, to ensure that the high bits are properly set
// when doing a full reload. // when doing a full reload.
Inst::mov_r_m(8 /* bytes */, from_reg, to_addr, srcloc) Inst::mov_r_m(8 /* bytes */, from_reg, to_addr)
} }
RegClass::V128 => { RegClass::V128 => {
let opcode = match ty { let opcode = match ty {
@@ -1196,7 +1088,7 @@ impl Inst {
_ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqu, _ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqu,
_ => unimplemented!("unable to store type: {}", ty), _ => unimplemented!("unable to store type: {}", ty),
}; };
Inst::xmm_mov_r_m(opcode, from_reg, to_addr, srcloc) Inst::xmm_mov_r_m(opcode, from_reg, to_addr)
} }
_ => panic!("unable to generate store for register class: {:?}", rc), _ => panic!("unable to generate store for register class: {:?}", rc),
} }
@@ -1247,26 +1139,16 @@ impl Inst {
/// Choose which instruction to use for comparing two values for equality. /// Choose which instruction to use for comparing two values for equality.
pub(crate) fn equals(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst { pub(crate) fn equals(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty { match ty {
types::I8X16 | types::B8X16 => Inst::xmm_rm_r(SseOpcode::Pcmpeqb, from, to, None), types::I8X16 | types::B8X16 => Inst::xmm_rm_r(SseOpcode::Pcmpeqb, from, to),
types::I16X8 | types::B16X8 => Inst::xmm_rm_r(SseOpcode::Pcmpeqw, from, to, None), types::I16X8 | types::B16X8 => Inst::xmm_rm_r(SseOpcode::Pcmpeqw, from, to),
types::I32X4 | types::B32X4 => Inst::xmm_rm_r(SseOpcode::Pcmpeqd, from, to, None), types::I32X4 | types::B32X4 => Inst::xmm_rm_r(SseOpcode::Pcmpeqd, from, to),
types::I64X2 | types::B64X2 => Inst::xmm_rm_r(SseOpcode::Pcmpeqq, from, to, None), types::I64X2 | types::B64X2 => Inst::xmm_rm_r(SseOpcode::Pcmpeqq, from, to),
types::F32X4 => Inst::xmm_rm_r_imm( types::F32X4 => {
SseOpcode::Cmpps, Inst::xmm_rm_r_imm(SseOpcode::Cmpps, from, to, FcmpImm::Equal.encode(), false)
from, }
to, types::F64X2 => {
FcmpImm::Equal.encode(), Inst::xmm_rm_r_imm(SseOpcode::Cmppd, from, to, FcmpImm::Equal.encode(), false)
false, }
None,
),
types::F64X2 => Inst::xmm_rm_r_imm(
SseOpcode::Cmppd,
from,
to,
FcmpImm::Equal.encode(),
false,
None,
),
_ => unimplemented!("unimplemented type for Inst::equals: {}", ty), _ => unimplemented!("unimplemented type for Inst::equals: {}", ty),
} }
} }
@@ -1274,11 +1156,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise AND on two values. /// Choose which instruction to use for computing a bitwise AND on two values.
pub(crate) fn and(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst { pub(crate) fn and(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty { match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andps, from, to, None), types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andpd, from, to, None), types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => { _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pand, from, to),
Inst::xmm_rm_r(SseOpcode::Pand, from, to, None)
}
_ => unimplemented!("unimplemented type for Inst::and: {}", ty), _ => unimplemented!("unimplemented type for Inst::and: {}", ty),
} }
} }
@@ -1286,11 +1166,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise AND NOT on two values. /// Choose which instruction to use for computing a bitwise AND NOT on two values.
pub(crate) fn and_not(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst { pub(crate) fn and_not(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty { match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andnps, from, to, None), types::F32X4 => Inst::xmm_rm_r(SseOpcode::Andnps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andnpd, from, to, None), types::F64X2 => Inst::xmm_rm_r(SseOpcode::Andnpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => { _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pandn, from, to),
Inst::xmm_rm_r(SseOpcode::Pandn, from, to, None)
}
_ => unimplemented!("unimplemented type for Inst::and_not: {}", ty), _ => unimplemented!("unimplemented type for Inst::and_not: {}", ty),
} }
} }
@@ -1298,11 +1176,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise OR on two values. /// Choose which instruction to use for computing a bitwise OR on two values.
pub(crate) fn or(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst { pub(crate) fn or(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty { match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Orps, from, to, None), types::F32X4 => Inst::xmm_rm_r(SseOpcode::Orps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Orpd, from, to, None), types::F64X2 => Inst::xmm_rm_r(SseOpcode::Orpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => { _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Por, from, to),
Inst::xmm_rm_r(SseOpcode::Por, from, to, None)
}
_ => unimplemented!("unimplemented type for Inst::or: {}", ty), _ => unimplemented!("unimplemented type for Inst::or: {}", ty),
} }
} }
@@ -1310,11 +1186,9 @@ impl Inst {
/// Choose which instruction to use for computing a bitwise XOR on two values. /// Choose which instruction to use for computing a bitwise XOR on two values.
pub(crate) fn xor(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst { pub(crate) fn xor(ty: Type, from: RegMem, to: Writable<Reg>) -> Inst {
match ty { match ty {
types::F32X4 => Inst::xmm_rm_r(SseOpcode::Xorps, from, to, None), types::F32X4 => Inst::xmm_rm_r(SseOpcode::Xorps, from, to),
types::F64X2 => Inst::xmm_rm_r(SseOpcode::Xorpd, from, to, None), types::F64X2 => Inst::xmm_rm_r(SseOpcode::Xorpd, from, to),
_ if ty.is_vector() && ty.bits() == 128 => { _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pxor, from, to),
Inst::xmm_rm_r(SseOpcode::Pxor, from, to, None)
}
_ => unimplemented!("unimplemented type for Inst::xor: {}", ty), _ => unimplemented!("unimplemented type for Inst::xor: {}", ty),
} }
} }
@@ -1839,7 +1713,7 @@ impl PrettyPrint for Inst {
Inst::Hlt => "hlt".into(), Inst::Hlt => "hlt".into(),
Inst::Ud2 { trap_info } => format!("ud2 {}", trap_info.1), Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code),
} }
} }
} }
@@ -2627,7 +2501,6 @@ impl MachInst for Inst {
SseOpcode::Xorps, SseOpcode::Xorps,
RegMem::reg(to_reg.to_reg()), RegMem::reg(to_reg.to_reg()),
to_reg, to_reg,
None,
)); ));
} else { } else {
let tmp = alloc_tmp(RegClass::I64, types::I32); let tmp = alloc_tmp(RegClass::I64, types::I32);
@@ -2646,7 +2519,6 @@ impl MachInst for Inst {
SseOpcode::Xorpd, SseOpcode::Xorpd,
RegMem::reg(to_reg.to_reg()), RegMem::reg(to_reg.to_reg()),
to_reg, to_reg,
None,
)); ));
} else { } else {
let tmp = alloc_tmp(RegClass::I64, types::I64); let tmp = alloc_tmp(RegClass::I64, types::I64);
@@ -2717,6 +2589,8 @@ pub struct EmitState {
pub(crate) nominal_sp_to_fp: i64, pub(crate) nominal_sp_to_fp: i64,
/// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
stack_map: Option<StackMap>, stack_map: Option<StackMap>,
/// Current source location.
cur_srcloc: SourceLoc,
} }
/// Constant state used during emissions of a sequence of instructions. /// Constant state used during emissions of a sequence of instructions.
@@ -2757,12 +2631,17 @@ impl MachInstEmitState<Inst> for EmitState {
virtual_sp_offset: 0, virtual_sp_offset: 0,
nominal_sp_to_fp: abi.frame_size() as i64, nominal_sp_to_fp: abi.frame_size() as i64,
stack_map: None, stack_map: None,
cur_srcloc: SourceLoc::default(),
} }
} }
fn pre_safepoint(&mut self, stack_map: StackMap) { fn pre_safepoint(&mut self, stack_map: StackMap) {
self.stack_map = Some(stack_map); self.stack_map = Some(stack_map);
} }
fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
self.cur_srcloc = srcloc;
}
} }
impl EmitState { impl EmitState {
@@ -2773,6 +2652,10 @@ impl EmitState {
fn clear_post_insn(&mut self) { fn clear_post_insn(&mut self) {
self.stack_map = None; self.stack_map = None;
} }
fn cur_srcloc(&self) -> SourceLoc {
self.cur_srcloc
}
} }
/// A label-use (internal relocation) in generated code. /// A label-use (internal relocation) in generated code.

451
cranelift/codegen/src/isa/x64/lower.rs
View File

File diff suppressed because it is too large Load Diff

10
cranelift/codegen/src/machinst/abi_impl.rs
View File

@@ -111,7 +111,7 @@
use super::abi::*; use super::abi::*;
use crate::binemit::StackMap; use crate::binemit::StackMap;
use crate::ir::types::*; use crate::ir::types::*;
use crate::ir::{ArgumentExtension, SourceLoc, StackSlot}; use crate::ir::{ArgumentExtension, StackSlot};
use crate::machinst::*; use crate::machinst::*;
use crate::settings; use crate::settings;
use crate::CodegenResult; use crate::CodegenResult;
@@ -350,7 +350,6 @@ pub trait ABIMachineSpec {
dest: &CallDest, dest: &CallDest,
uses: Vec<Reg>, uses: Vec<Reg>,
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: ir::Opcode, opcode: ir::Opcode,
tmp: Writable<Reg>, tmp: Writable<Reg>,
callee_conv: isa::CallConv, callee_conv: isa::CallConv,
@@ -1102,8 +1101,6 @@ pub struct ABICallerImpl<M: ABIMachineSpec> {
defs: Vec<Writable<Reg>>, defs: Vec<Writable<Reg>>,
/// Call destination. /// Call destination.
dest: CallDest, dest: CallDest,
/// Location of callsite.
loc: ir::SourceLoc,
/// Actual call opcode; used to distinguish various types of calls. /// Actual call opcode; used to distinguish various types of calls.
opcode: ir::Opcode, opcode: ir::Opcode,
/// Caller's calling convention. /// Caller's calling convention.
@@ -1127,7 +1124,6 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
sig: &ir::Signature, sig: &ir::Signature,
extname: &ir::ExternalName, extname: &ir::ExternalName,
dist: RelocDistance, dist: RelocDistance,
loc: ir::SourceLoc,
caller_conv: isa::CallConv, caller_conv: isa::CallConv,
) -> CodegenResult<ABICallerImpl<M>> { ) -> CodegenResult<ABICallerImpl<M>> {
let sig = ABISig::from_func_sig::<M>(sig)?; let sig = ABISig::from_func_sig::<M>(sig)?;
@@ -1137,7 +1133,6 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
uses, uses,
defs, defs,
dest: CallDest::ExtName(extname.clone(), dist), dest: CallDest::ExtName(extname.clone(), dist),
loc,
opcode: ir::Opcode::Call, opcode: ir::Opcode::Call,
caller_conv, caller_conv,
_mach: PhantomData, _mach: PhantomData,
@@ -1149,7 +1144,6 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
pub fn from_ptr( pub fn from_ptr(
sig: &ir::Signature, sig: &ir::Signature,
ptr: Reg, ptr: Reg,
loc: ir::SourceLoc,
opcode: ir::Opcode, opcode: ir::Opcode,
caller_conv: isa::CallConv, caller_conv: isa::CallConv,
) -> CodegenResult<ABICallerImpl<M>> { ) -> CodegenResult<ABICallerImpl<M>> {
@@ -1160,7 +1154,6 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
uses, uses,
defs, defs,
dest: CallDest::Reg(ptr), dest: CallDest::Reg(ptr),
loc,
opcode, opcode,
caller_conv, caller_conv,
_mach: PhantomData, _mach: PhantomData,
@@ -1311,7 +1304,6 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
&self.dest, &self.dest,
uses, uses,
defs, defs,
self.loc,
self.opcode, self.opcode,
tmp, tmp,
self.sig.call_conv, self.sig.call_conv,

4
cranelift/codegen/src/machinst/buffer.rs
View File

@@ -1535,7 +1535,7 @@ mod test {
buf.bind_label(label(1)); buf.bind_label(label(1));
let inst = Inst::Udf { let inst = Inst::Udf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
}; };
inst.emit(&mut buf, &info, &mut state); inst.emit(&mut buf, &info, &mut state);
@@ -1551,7 +1551,7 @@ mod test {
let mut state = Default::default(); let mut state = Default::default();
let inst = Inst::TrapIf { let inst = Inst::TrapIf {
kind: CondBrKind::NotZero(xreg(0)), kind: CondBrKind::NotZero(xreg(0)),
trap_info: (SourceLoc::default(), TrapCode::Interrupt), trap_code: TrapCode::Interrupt,
}; };
inst.emit(&mut buf2, &info, &mut state); inst.emit(&mut buf2, &info, &mut state);
let inst = Inst::Nop4; let inst = Inst::Nop4;

5
cranelift/codegen/src/machinst/mod.rs
View File

@@ -98,7 +98,7 @@
use crate::binemit::{CodeInfo, CodeOffset, StackMap}; use crate::binemit::{CodeInfo, CodeOffset, StackMap};
use crate::ir::condcodes::IntCC; use crate::ir::condcodes::IntCC;
use crate::ir::{Function, Type}; use crate::ir::{Function, SourceLoc, Type};
use crate::isa::unwind::input as unwind_input; use crate::isa::unwind::input as unwind_input;
use crate::result::CodegenResult; use crate::result::CodegenResult;
use crate::settings::Flags; use crate::settings::Flags;
@@ -302,6 +302,9 @@ pub trait MachInstEmitState<I: MachInst>: Default + Clone + Debug {
/// Update the emission state before emitting an instruction that is a /// Update the emission state before emitting an instruction that is a
/// safepoint. /// safepoint.
fn pre_safepoint(&mut self, _stack_map: StackMap) {} fn pre_safepoint(&mut self, _stack_map: StackMap) {}
/// Update the emission state to indicate instructions are associated with a
/// particular SourceLoc.
fn pre_sourceloc(&mut self, _srcloc: SourceLoc) {}
} }
/// The result of a `MachBackend::compile_function()` call. Contains machine /// The result of a `MachBackend::compile_function()` call. Contains machine

1
cranelift/codegen/src/machinst/vcode.rs
View File

@@ -509,6 +509,7 @@ impl<I: VCodeInst> VCode<I> {
buffer.start_srcloc(srcloc); buffer.start_srcloc(srcloc);
cur_srcloc = Some(srcloc); cur_srcloc = Some(srcloc);
} }
state.pre_sourceloc(cur_srcloc.unwrap_or(SourceLoc::default()));
if safepoint_idx < self.safepoint_insns.len() if safepoint_idx < self.safepoint_insns.len()
&& self.safepoint_insns[safepoint_idx] == iix && self.safepoint_insns[safepoint_idx] == iix