T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MachInst backends: handle SourceLocs out-of-band, not in Insts.

Browse Source 
In existing MachInst backends, many instructions -- any that can trap or
result in a relocation -- carry `SourceLoc` values in order to propagate
the location-in-original-source to use to describe resulting traps or
relocation errors.

This is quite tedious, and also error-prone: it is likely that the
necessary plumbing will be missed in some cases, and in any case, it's
unnecessarily verbose.

This PR factors out the `SourceLoc` handling so that it is tracked
during emission as part of the `EmitState`, and plumbed through
automatically by the machine-independent framework. Instruction emission
code that directly emits trap or relocation records can query the
current location as necessary. Then we only need to ensure that memory
references and trap instructions, at their (one) emission point rather
than their (many) lowering/generation points, are wired up correctly.

This does have the side-effect that some loads and stores that do not
correspond directly to user code's heap accesses will have unnecessary
but harmless trap metadata. For example, the load that fetches a code
offset from a jump table will have a 'heap out of bounds' trap record
attached to it; but because it is bounds-checked, and will never
actually trap if the lowering is correct, this should be harmless.  The
simplicity improvement here seemed more worthwhile to me than plumbing
through a "corresponds to user-level load/store" bit, because the latter
is a bit complex when we allow for op merging.

Closes #2290: though it does not implement a full "metadata" scheme as
described in that issue, this seems simpler overall.
This commit is contained in:
Chris Fallin
2020-11-10 14:37:11 -08:00
parent 0568f4fb02
commit 4dce51096d
19 changed files with 849 additions and 1714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -3,7 +3,6 @@
use crate::ir;
use crate::ir::types;
use crate::ir::types::*;
use crate::ir::SourceLoc;
use crate::isa;
use crate::isa::aarch64::{inst::EmitState, inst::*};
use crate::machinst::*;
@@ -380,7 +379,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
extendop: ExtendOp::UXTX,
});
insts.push(Inst::TrapIf {
trap_info: (ir::SourceLoc::default(), ir::TrapCode::StackOverflow),
trap_code: ir::TrapCode::StackOverflow,
// Here `Lo` == "less than" when interpreting the two
// operands as unsigned integers.
kind: CondBrKind::Cond(Cond::Lo),
@@ -554,7 +553,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(),
SImm9::maybe_from_i64((vec_offset + (i * 16)) as i64).unwrap(),
),
srcloc: None,
});
}
@@ -603,7 +601,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(),
SImm9::maybe_from_i64(((i * 16) + int_save_bytes) as i64).unwrap(),
),
srcloc: None,
});
}
@@ -634,7 +631,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
dest: &CallDest,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
loc: SourceLoc,
opcode: ir::Opcode,
tmp: Writable<Reg>,
callee_conv: isa::CallConv,
@@ -649,7 +645,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
dest: name.clone(),
uses,
defs,
loc,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
@@ -663,7 +658,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
rd: tmp,
name: Box::new(name.clone()),
offset: 0,
srcloc: loc,
},
));
insts.push((
@@ -673,7 +667,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
rn: tmp.to_reg(),
uses,
defs,
loc,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
@@ -688,7 +681,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
rn: *reg,
uses,
defs,
loc,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,

191
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -462,6 +462,8 @@ pub struct EmitState {
pub(crate) nominal_sp_to_fp: i64,
/// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
stack_map: Option<StackMap>,
/// Current source-code location corresponding to instruction to be emitted.
cur_srcloc: SourceLoc,
}
impl MachInstEmitState<Inst> for EmitState {
@@ -470,12 +472,17 @@ impl MachInstEmitState<Inst> for EmitState {
virtual_sp_offset: 0,
nominal_sp_to_fp: abi.frame_size() as i64,
stack_map: None,
cur_srcloc: SourceLoc::default(),
}
}
fn pre_safepoint(&mut self, stack_map: StackMap) {
self.stack_map = Some(stack_map);
}
fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
self.cur_srcloc = srcloc;
}
}
impl EmitState {
@@ -486,6 +493,10 @@ impl EmitState {
fn clear_post_insn(&mut self) {
self.stack_map = None;
}
fn cur_srcloc(&self) -> SourceLoc {
self.cur_srcloc
}
}
/// Constant state used during function compilation.
@@ -730,57 +741,16 @@ impl MachInstEmit for Inst {
sink.put4(enc_bit_rr(size, op1, op2, rn, rd))
}
&Inst::ULoad8 {
rd,
ref mem,
srcloc,
}
| &Inst::SLoad8 {
rd,
ref mem,
srcloc,
}
| &Inst::ULoad16 {
rd,
ref mem,
srcloc,
}
| &Inst::SLoad16 {
rd,
ref mem,
srcloc,
}
| &Inst::ULoad32 {
rd,
ref mem,
srcloc,
}
| &Inst::SLoad32 {
rd,
ref mem,
srcloc,
}
| &Inst::ULoad64 {
rd,
ref mem,
srcloc,
..
}
| &Inst::FpuLoad32 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuLoad64 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuLoad128 {
rd,
ref mem,
srcloc,
} => {
&Inst::ULoad8 { rd, ref mem }
| &Inst::SLoad8 { rd, ref mem }
| &Inst::ULoad16 { rd, ref mem }
| &Inst::SLoad16 { rd, ref mem }
| &Inst::ULoad32 { rd, ref mem }
| &Inst::SLoad32 { rd, ref mem }
| &Inst::ULoad64 { rd, ref mem, .. }
| &Inst::FpuLoad32 { rd, ref mem }
| &Inst::FpuLoad64 { rd, ref mem }
| &Inst::FpuLoad128 { rd, ref mem } => {
let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
for inst in mem_insts.into_iter() {
@@ -807,7 +777,8 @@ impl MachInstEmit for Inst {
_ => unreachable!(),
};
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
@@ -890,42 +861,13 @@ impl MachInstEmit for Inst {
}
}
&Inst::Store8 {
rd,
ref mem,
srcloc,
}
| &Inst::Store16 {
rd,
ref mem,
srcloc,
}
| &Inst::Store32 {
rd,
ref mem,
srcloc,
}
| &Inst::Store64 {
rd,
ref mem,
srcloc,
..
}
| &Inst::FpuStore32 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuStore64 {
rd,
ref mem,
srcloc,
}
| &Inst::FpuStore128 {
rd,
ref mem,
srcloc,
} => {
&Inst::Store8 { rd, ref mem }
| &Inst::Store16 { rd, ref mem }
| &Inst::Store32 { rd, ref mem }
| &Inst::Store64 { rd, ref mem, .. }
| &Inst::FpuStore32 { rd, ref mem }
| &Inst::FpuStore64 { rd, ref mem }
| &Inst::FpuStore128 { rd, ref mem } => {
let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
for inst in mem_insts.into_iter() {
@@ -943,7 +885,8 @@ impl MachInstEmit for Inst {
_ => unreachable!(),
};
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
@@ -1086,7 +1029,7 @@ impl MachInstEmit for Inst {
} => {
sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond));
}
&Inst::AtomicRMW { ty, op, srcloc } => {
&Inst::AtomicRMW { ty, op } => {
/* Emit this:
dmb ish
again:
@@ -1124,7 +1067,8 @@ impl MachInstEmit for Inst {
// again:
sink.bind_label(again_label);
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
sink.put4(enc_ldxr(ty, x27wr, x25)); // ldxr x27, [x25]
@@ -1145,7 +1089,8 @@ impl MachInstEmit for Inst {
sink.put4(enc_arith_rrr(bits_31_21, 0b000000, x28wr, x27, x26));
}
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
sink.put4(enc_stxr(ty, x24wr, x28, x25)); // stxr w24, x28, [x25]
@@ -1162,7 +1107,7 @@ impl MachInstEmit for Inst {
sink.put4(enc_dmb_ish()); // dmb ish
}
&Inst::AtomicCAS { ty, srcloc } => {
&Inst::AtomicCAS { ty } => {
/* Emit this:
dmb ish
again:
@@ -1195,7 +1140,8 @@ impl MachInstEmit for Inst {
// again:
sink.bind_label(again_label);
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
sink.put4(enc_ldxr(ty, x27wr, x25)); // ldxr x27, [x25]
@@ -1230,7 +1176,8 @@ impl MachInstEmit for Inst {
));
sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19);
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
sink.put4(enc_stxr(ty, x24wr, x28, x25)); // stxr w24, x28, [x25]
@@ -1249,12 +1196,7 @@ impl MachInstEmit for Inst {
sink.bind_label(out_label);
sink.put4(enc_dmb_ish()); // dmb ish
}
&Inst::AtomicLoad {
ty,
r_data,
r_addr,
srcloc,
} => {
&Inst::AtomicLoad { ty, r_data, r_addr } => {
let op = match ty {
I8 => 0b0011100001,
I16 => 0b0111100001,
@@ -1264,7 +1206,8 @@ impl MachInstEmit for Inst {
};
sink.put4(enc_dmb_ish()); // dmb ish
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
let uimm12scaled_zero = UImm12Scaled::zero(I8 /*irrelevant*/);
@@ -1275,12 +1218,7 @@ impl MachInstEmit for Inst {
r_data.to_reg(),
));
}
&Inst::AtomicStore {
ty,
r_data,
r_addr,
srcloc,
} => {
&Inst::AtomicStore { ty, r_data, r_addr } => {
let op = match ty {
I8 => 0b0011100000,
I16 => 0b0111100000,
@@ -1289,7 +1227,8 @@ impl MachInstEmit for Inst {
_ => unreachable!(),
};
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
let uimm12scaled_zero = UImm12Scaled::zero(I8 /*irrelevant*/);
@@ -1607,7 +1546,6 @@ impl MachInstEmit for Inst {
let inst = Inst::FpuLoad64 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
};
inst.emit(sink, emit_info, state);
let inst = Inst::Jump {
@@ -1620,7 +1558,6 @@ impl MachInstEmit for Inst {
let inst = Inst::FpuLoad128 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
};
inst.emit(sink, emit_info, state);
let inst = Inst::Jump {
@@ -1961,15 +1898,11 @@ impl MachInstEmit for Inst {
};
sink.put4(enc_vec_rrr(top11, rm, bit15_10, rn, rd));
}
&Inst::VecLoadReplicate {
rd,
rn,
size,
srcloc,
} => {
&Inst::VecLoadReplicate { rd, rn, size } => {
let (q, size) = size.enc_size();
if let Some(srcloc) = srcloc {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
@@ -2073,10 +2006,11 @@ impl MachInstEmit for Inst {
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
}
sink.add_reloc(info.loc, Reloc::Arm64Call, &info.dest, 0);
let loc = state.cur_srcloc();
sink.add_reloc(loc, Reloc::Arm64Call, &info.dest, 0);
sink.put4(enc_jump26(0b100101, 0));
if info.opcode.is_call() {
sink.add_call_site(info.loc, info.opcode);
sink.add_call_site(loc, info.opcode);
}
}
&Inst::CallInd { ref info } => {
@@ -2084,8 +2018,9 @@ impl MachInstEmit for Inst {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
}
sink.put4(0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(info.rn) << 5));
let loc = state.cur_srcloc();
if info.opcode.is_call() {
sink.add_call_site(info.loc, info.opcode);
sink.add_call_site(loc, info.opcode);
}
}
&Inst::CondBr {
@@ -2110,7 +2045,7 @@ impl MachInstEmit for Inst {
}
sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
}
&Inst::TrapIf { kind, trap_info } => {
&Inst::TrapIf { kind, trap_code } => {
// condbr KIND, LABEL
let off = sink.cur_offset();
let label = sink.get_label();
@@ -2120,7 +2055,7 @@ impl MachInstEmit for Inst {
));
sink.use_label_at_offset(off, label, LabelUse::Branch19);
// udf
let trap = Inst::Udf { trap_info };
let trap = Inst::Udf { trap_code };
trap.emit(sink, emit_info, state);
// LABEL:
sink.bind_label(label);
@@ -2135,9 +2070,9 @@ impl MachInstEmit for Inst {
&Inst::Brk => {
sink.put4(0xd4200000);
}
&Inst::Udf { trap_info } => {
let (srcloc, code) = trap_info;
sink.add_trap(srcloc, code);
&Inst::Udf { trap_code } => {
let srcloc = state.cur_srcloc();
sink.add_trap(srcloc, trap_code);
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
}
@@ -2192,7 +2127,6 @@ impl MachInstEmit for Inst {
I32,
ExtendOp::UXTW,
),
srcloc: None, // can't cause a user trap.
};
inst.emit(sink, emit_info, state);
// Add base of jump table to jump-table-sourced block offset
@@ -2235,18 +2169,17 @@ impl MachInstEmit for Inst {
rd,
ref name,
offset,
srcloc,
} => {
let inst = Inst::ULoad64 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None, // can't cause a user trap.
};
inst.emit(sink, emit_info, state);
let inst = Inst::Jump {
dest: BranchTarget::ResolvedOffset(12),
};
inst.emit(sink, emit_info, state);
let srcloc = state.cur_srcloc();
sink.add_reloc(srcloc, Reloc::Abs8, name, offset);
if emit_info.flags().emit_all_ones_funcaddrs() {
sink.put8(u64::max_value());

109
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -1079,7 +1079,6 @@ fn test_aarch64_binemit() {
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"41004038",
"ldurb w1, [x2]",
@@ -1088,7 +1087,6 @@ fn test_aarch64_binemit() {
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)),
srcloc: None,
},
"41004039",
"ldrb w1, [x2]",
@@ -1097,7 +1095,6 @@ fn test_aarch64_binemit() {
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)),
srcloc: None,
},
"41686538",
"ldrb w1, [x2, x5]",
@@ -1106,7 +1103,6 @@ fn test_aarch64_binemit() {
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"41008038",
"ldursb x1, [x2]",
@@ -1115,7 +1111,6 @@ fn test_aarch64_binemit() {
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()),
srcloc: None,
},
"41FC8039",
"ldrsb x1, [x2, #63]",
@@ -1124,7 +1119,6 @@ fn test_aarch64_binemit() {
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)),
srcloc: None,
},
"4168A538",
"ldrsb x1, [x2, x5]",
@@ -1133,7 +1127,6 @@ fn test_aarch64_binemit() {
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()),
srcloc: None,
},
"41504078",
"ldurh w1, [x2, #5]",
@@ -1142,7 +1135,6 @@ fn test_aarch64_binemit() {
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()),
srcloc: None,
},
"41104079",
"ldrh w1, [x2, #8]",
@@ -1151,7 +1143,6 @@ fn test_aarch64_binemit() {
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I16),
srcloc: None,
},
"41786378",
"ldrh w1, [x2, x3, LSL #1]",
@@ -1160,7 +1151,6 @@ fn test_aarch64_binemit() {
Inst::SLoad16 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"41008078",
"ldursh x1, [x2]",
@@ -1169,7 +1159,6 @@ fn test_aarch64_binemit() {
Inst::SLoad16 {
rd: writable_xreg(28),
mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()),
srcloc: None,
},
"9C328079",
"ldrsh x28, [x20, #24]",
@@ -1178,7 +1167,6 @@ fn test_aarch64_binemit() {
Inst::SLoad16 {
rd: writable_xreg(28),
mem: AMode::RegScaled(xreg(20), xreg(20), I16),
srcloc: None,
},
"9C7AB478",
"ldrsh x28, [x20, x20, LSL #1]",
@@ -1187,7 +1175,6 @@ fn test_aarch64_binemit() {
Inst::ULoad32 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"410040B8",
"ldur w1, [x2]",
@@ -1196,7 +1183,6 @@ fn test_aarch64_binemit() {
Inst::ULoad32 {
rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()),
srcloc: None,
},
"0CCC40B9",
"ldr w12, [x0, #204]",
@@ -1205,7 +1191,6 @@ fn test_aarch64_binemit() {
Inst::ULoad32 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(12), I32),
srcloc: None,
},
"41786CB8",
"ldr w1, [x2, x12, LSL #2]",
@@ -1214,7 +1199,6 @@ fn test_aarch64_binemit() {
Inst::SLoad32 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"410080B8",
"ldursw x1, [x2]",
@@ -1223,7 +1207,6 @@ fn test_aarch64_binemit() {
Inst::SLoad32 {
rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
srcloc: None,
},
"2CFCBFB9",
"ldrsw x12, [x1, #16380]",
@@ -1232,7 +1215,6 @@ fn test_aarch64_binemit() {
Inst::SLoad32 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(5), xreg(1), I32),
srcloc: None,
},
"A178A1B8",
"ldrsw x1, [x5, x1, LSL #2]",
@@ -1241,7 +1223,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"410040F8",
"ldur x1, [x2]",
@@ -1250,7 +1231,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()),
srcloc: None,
},
"410050F8",
"ldur x1, [x2, #-256]",
@@ -1259,7 +1239,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()),
srcloc: None,
},
"41F04FF8",
"ldur x1, [x2, #255]",
@@ -1268,7 +1247,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
srcloc: None,
},
"41FC7FF9",
"ldr x1, [x2, #32760]",
@@ -1277,7 +1255,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)),
srcloc: None,
},
"416863F8",
"ldr x1, [x2, x3]",
@@ -1286,7 +1263,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64),
srcloc: None,
},
"417863F8",
"ldr x1, [x2, x3, LSL #3]",
@@ -1295,7 +1271,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW),
srcloc: None,
},
"41D863F8",
"ldr x1, [x2, w3, SXTW #3]",
@@ -1304,7 +1279,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW),
srcloc: None,
},
"41C863F8",
"ldr x1, [x2, w3, SXTW]",
@@ -1313,7 +1287,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Label(MemLabel::PCRel(64)),
srcloc: None,
},
"01020058",
"ldr x1, pc+64",
@@ -1322,7 +1295,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
},
"410C41F8",
"ldr x1, [x2, #16]!",
@@ -1331,7 +1303,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
},
"410441F8",
"ldr x1, [x2], #16",
@@ -1340,7 +1311,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(32768, I8),
srcloc: None,
},
"100090D2B063308B010240F9",
"movz x16, #32768 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1349,7 +1319,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(-32768, I8),
srcloc: None,
},
"F0FF8F92B063308B010240F9",
"movn x16, #32767 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1358,7 +1327,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(1048576, I8), // 2^20
srcloc: None,
},
"1002A0D2B063308B010240F9",
"movz x16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1367,7 +1335,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1
srcloc: None,
},
"300080521002A072B063308B010240F9",
"movz w16, #1 ; movk w16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1377,7 +1344,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 8, I64),
srcloc: None,
},
"E18040F8",
"ldur x1, [x7, #8]",
@@ -1387,7 +1353,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1024, I64),
srcloc: None,
},
"E10042F9",
"ldr x1, [x7, #1024]",
@@ -1397,7 +1362,6 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1048576, I64),
srcloc: None,
},
"1002A0D2F060308B010240F9",
"movz x16, #16, LSL #16 ; add x16, x7, x16, UXTX ; ldr x1, [x16]",
@@ -1407,7 +1371,6 @@ fn test_aarch64_binemit() {
Inst::Store8 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"41000038",
"sturb w1, [x2]",
@@ -1416,7 +1379,6 @@ fn test_aarch64_binemit() {
Inst::Store8 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()),
srcloc: None,
},
"41FC3F39",
"strb w1, [x2, #4095]",
@@ -1425,7 +1387,6 @@ fn test_aarch64_binemit() {
Inst::Store16 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"41000078",
"sturh w1, [x2]",
@@ -1434,7 +1395,6 @@ fn test_aarch64_binemit() {
Inst::Store16 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()),
srcloc: None,
},
"41FC3F79",
"strh w1, [x2, #8190]",
@@ -1443,7 +1403,6 @@ fn test_aarch64_binemit() {
Inst::Store32 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"410000B8",
"stur w1, [x2]",
@@ -1452,7 +1411,6 @@ fn test_aarch64_binemit() {
Inst::Store32 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
srcloc: None,
},
"41FC3FB9",
"str w1, [x2, #16380]",
@@ -1461,7 +1419,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
srcloc: None,
},
"410000F8",
"stur x1, [x2]",
@@ -1470,7 +1427,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
srcloc: None,
},
"41FC3FF9",
"str x1, [x2, #32760]",
@@ -1479,7 +1435,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)),
srcloc: None,
},
"416823F8",
"str x1, [x2, x3]",
@@ -1488,7 +1443,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64),
srcloc: None,
},
"417823F8",
"str x1, [x2, x3, LSL #3]",
@@ -1497,7 +1451,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW),
srcloc: None,
},
"415823F8",
"str x1, [x2, w3, UXTW #3]",
@@ -1506,7 +1459,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW),
srcloc: None,
},
"414823F8",
"str x1, [x2, w3, UXTW]",
@@ -1515,7 +1467,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
},
"410C01F8",
"str x1, [x2, #16]!",
@@ -1524,7 +1475,6 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
srcloc: None,
},
"410401F8",
"str x1, [x2], #16",
@@ -3911,7 +3861,7 @@ fn test_aarch64_binemit() {
Inst::VecLoadReplicate {
rd: writable_vreg(31),
rn: xreg(0),
srcloc: None,
size: VectorSize::Size64x2,
},
"1FCC404D",
@@ -3922,7 +3872,7 @@ fn test_aarch64_binemit() {
Inst::VecLoadReplicate {
rd: writable_vreg(0),
rn: xreg(25),
srcloc: None,
size: VectorSize::Size8x8,
},
"20C3400D",
@@ -4050,7 +4000,7 @@ fn test_aarch64_binemit() {
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::NotZero(xreg(8)),
},
"480000B40000A0D4",
@@ -4058,7 +4008,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Zero(xreg(8)),
},
"480000B50000A0D4",
@@ -4066,7 +4016,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ne),
},
"400000540000A0D4",
@@ -4074,7 +4024,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Eq),
},
"410000540000A0D4",
@@ -4082,7 +4032,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Lo),
},
"420000540000A0D4",
@@ -4090,7 +4040,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Hs),
},
"430000540000A0D4",
@@ -4098,7 +4048,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Pl),
},
"440000540000A0D4",
@@ -4106,7 +4056,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Mi),
},
"450000540000A0D4",
@@ -4114,7 +4064,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Vc),
},
"460000540000A0D4",
@@ -4122,7 +4072,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Vs),
},
"470000540000A0D4",
@@ -4130,7 +4080,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ls),
},
"480000540000A0D4",
@@ -4138,7 +4088,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Hi),
},
"490000540000A0D4",
@@ -4146,7 +4096,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Lt),
},
"4A0000540000A0D4",
@@ -4154,7 +4104,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Ge),
},
"4B0000540000A0D4",
@@ -4162,7 +4112,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Le),
},
"4C0000540000A0D4",
@@ -4170,7 +4120,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Gt),
},
"4D0000540000A0D4",
@@ -4178,7 +4128,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Nv),
},
"4E0000540000A0D4",
@@ -4186,7 +4136,7 @@ fn test_aarch64_binemit() {
));
insns.push((
Inst::TrapIf {
trap_info: (SourceLoc::default(), TrapCode::Interrupt),
trap_code: TrapCode::Interrupt,
kind: CondBrKind::Cond(Cond::Al),
},
"4F0000540000A0D4",
@@ -4209,7 +4159,6 @@ fn test_aarch64_binemit() {
dest: ExternalName::testcase("test0"),
uses: Vec::new(),
defs: Vec::new(),
loc: SourceLoc::default(),
opcode: Opcode::Call,
caller_callconv: CallConv::SystemV,
callee_callconv: CallConv::SystemV,
@@ -4225,7 +4174,6 @@ fn test_aarch64_binemit() {
rn: xreg(10),
uses: Vec::new(),
defs: Vec::new(),
loc: SourceLoc::default(),
opcode: Opcode::CallIndirect,
caller_callconv: CallConv::SystemV,
callee_callconv: CallConv::SystemV,
@@ -4797,7 +4745,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad32 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32),
srcloc: None,
},
"107969BC",
"ldr s16, [x8, x9, LSL #2]",
@@ -4807,7 +4754,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad64 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64),
srcloc: None,
},
"107969FC",
"ldr d16, [x8, x9, LSL #3]",
@@ -4817,7 +4763,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad128 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128),
srcloc: None,
},
"1079E93C",
"ldr q16, [x8, x9, LSL #4]",
@@ -4827,7 +4772,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad32 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
},
"5000001C",
"ldr s16, pc+8",
@@ -4837,7 +4781,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad64 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
},
"5000005C",
"ldr d16, pc+8",
@@ -4847,7 +4790,6 @@ fn test_aarch64_binemit() {
Inst::FpuLoad128 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
srcloc: None,
},
"5000009C",
"ldr q16, pc+8",
@@ -4857,7 +4799,6 @@ fn test_aarch64_binemit() {
Inst::FpuStore32 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32),
srcloc: None,
},
"107929BC",
"str s16, [x8, x9, LSL #2]",
@@ -4867,7 +4808,6 @@ fn test_aarch64_binemit() {
Inst::FpuStore64 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64),
srcloc: None,
},
"107929FC",
"str d16, [x8, x9, LSL #3]",
@@ -4877,7 +4817,6 @@ fn test_aarch64_binemit() {
Inst::FpuStore128 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128),
srcloc: None,
},
"1079A93C",
"str q16, [x8, x9, LSL #4]",
@@ -5000,7 +4939,6 @@ fn test_aarch64_binemit() {
Inst::AtomicRMW {
ty: I16,
op: inst_common::AtomicRmwOp::Xor,
srcloc: None,
},
"BF3B03D53B7F5F487C031ACA3C7F1848B8FFFFB5BF3B03D5",
"atomically { 16_bits_at_[x25]) Xor= x26 ; x27 = old_value_at_[x25]; x24,x28 = trash }",
@@ -5010,7 +4948,6 @@ fn test_aarch64_binemit() {
Inst::AtomicRMW {
ty: I32,
op: inst_common::AtomicRmwOp::Xchg,
srcloc: None,
},
"BF3B03D53B7F5F88FC031AAA3C7F1888B8FFFFB5BF3B03D5",
"atomically { 32_bits_at_[x25]) Xchg= x26 ; x27 = old_value_at_[x25]; x24,x28 = trash }",
@@ -5019,7 +4956,6 @@ fn test_aarch64_binemit() {
insns.push((
Inst::AtomicCAS {
ty: I8,
srcloc: None,
},
"BF3B03D53B7F5F08581F40927F0318EB610000543C7F180878FFFFB5BF3B03D5",
"atomically { compare-and-swap(8_bits_at_[x25], x26 -> x28), x27 = old_value_at_[x25]; x24 = trash }"
@@ -5028,7 +4964,6 @@ fn test_aarch64_binemit() {
insns.push((
Inst::AtomicCAS {
ty: I64,
srcloc: None,
},
"BF3B03D53B7F5FC8F8031AAA7F0318EB610000543C7F18C878FFFFB5BF3B03D5",
"atomically { compare-and-swap(64_bits_at_[x25], x26 -> x28), x27 = old_value_at_[x25]; x24 = trash }"
@@ -5039,7 +4974,6 @@ fn test_aarch64_binemit() {
ty: I8,
r_data: writable_xreg(7),
r_addr: xreg(28),
srcloc: None,
},
"BF3B03D587034039",
"atomically { x7 = zero_extend_8_bits_at[x28] }",
@@ -5050,7 +4984,6 @@ fn test_aarch64_binemit() {
ty: I64,
r_data: writable_xreg(28),
r_addr: xreg(7),
srcloc: None,
},
"BF3B03D5FC0040F9",
"atomically { x28 = zero_extend_64_bits_at[x7] }",
@@ -5061,7 +4994,6 @@ fn test_aarch64_binemit() {
ty: I16,
r_data: xreg(17),
r_addr: xreg(8),
srcloc: None,
},
"11010079BF3B03D5",
"atomically { 16_bits_at[x8] = x17 }",
@@ -5072,7 +5004,6 @@ fn test_aarch64_binemit() {
ty: I32,
r_data: xreg(18),
r_addr: xreg(7),
srcloc: None,
},
"F20000B9BF3B03D5",
"atomically { 32_bits_at[x7] = x18 }",

123
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -420,7 +420,6 @@ pub struct CallInfo {
pub dest: ExternalName,
pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>,
pub loc: SourceLoc,
pub opcode: Opcode,
pub caller_callconv: CallConv,
pub callee_callconv: CallConv,
@@ -433,7 +432,6 @@ pub struct CallIndInfo {
pub rn: Reg,
pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>,
pub loc: SourceLoc,
pub opcode: Opcode,
pub caller_callconv: CallConv,
pub callee_callconv: CallConv,
@@ -524,68 +522,57 @@ pub enum Inst {
ULoad8 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A signed (sign-extending) 8-bit load.
SLoad8 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// An unsigned (zero-extending) 16-bit load.
ULoad16 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A signed (sign-extending) 16-bit load.
SLoad16 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// An unsigned (zero-extending) 32-bit load.
ULoad32 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A signed (sign-extending) 32-bit load.
SLoad32 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A 64-bit load.
ULoad64 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// An 8-bit store.
Store8 {
rd: Reg,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A 16-bit store.
Store16 {
rd: Reg,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A 32-bit store.
Store32 {
rd: Reg,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A 64-bit store.
Store64 {
rd: Reg,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// A store of a pair of registers.
@@ -686,7 +673,6 @@ pub enum Inst {
AtomicRMW {
ty: Type, // I8, I16, I32 or I64
op: inst_common::AtomicRmwOp,
srcloc: Option<SourceLoc>,
},
/// Similar to AtomicRMW, a compare-and-swap operation implemented using a load-linked
@@ -703,7 +689,6 @@ pub enum Inst {
/// x24 (wr) scratch reg; value afterwards has no meaning
AtomicCAS {
ty: Type, // I8, I16, I32 or I64
srcloc: Option<SourceLoc>,
},
/// Read `ty` bits from address `r_addr`, zero extend the loaded value to 64 bits and put it
@@ -713,7 +698,6 @@ pub enum Inst {
ty: Type, // I8, I16, I32 or I64
r_data: Writable<Reg>,
r_addr: Reg,
srcloc: Option<SourceLoc>,
},
/// Write the lowest `ty` bits of `r_data` to address `r_addr`, with a memory fence
@@ -723,7 +707,6 @@ pub enum Inst {
ty: Type, // I8, I16, I32 or I64
r_data: Reg,
r_addr: Reg,
srcloc: Option<SourceLoc>,
},
/// A memory fence. This must provide ordering to ensure that, at a minimum, neither loads
@@ -798,37 +781,31 @@ pub enum Inst {
FpuLoad32 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// Floating-point store, single-precision (32 bit).
FpuStore32 {
rd: Reg,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// Floating-point load, double-precision (64 bit).
FpuLoad64 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// Floating-point store, double-precision (64 bit).
FpuStore64 {
rd: Reg,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// Floating-point/vector load, 128 bit.
FpuLoad128 {
rd: Writable<Reg>,
mem: AMode,
srcloc: Option<SourceLoc>,
},
/// Floating-point/vector store, 128 bit.
FpuStore128 {
rd: Reg,
mem: AMode,
srcloc: Option<SourceLoc>,
},
LoadFpuConst64 {
@@ -1037,7 +1014,6 @@ pub enum Inst {
rd: Writable<Reg>,
rn: Reg,
size: VectorSize,
srcloc: Option<SourceLoc>,
},
/// Move to the NZCV flags (actually a `MSR NZCV, Xn` insn).
@@ -1095,7 +1071,7 @@ pub enum Inst {
/// of this condition in a branch that skips the trap instruction.)
TrapIf {
kind: CondBrKind,
trap_info: (SourceLoc, TrapCode),
trap_code: TrapCode,
},
/// An indirect branch through a register, augmented with set of all
@@ -1111,7 +1087,7 @@ pub enum Inst {
/// An instruction guaranteed to always be undefined and to trigger an illegal instruction at
/// runtime.
Udf {
trap_info: (SourceLoc, TrapCode),
trap_code: TrapCode,
},
/// Compute the address (using a PC-relative offset) of a memory location, using the `ADR`
@@ -1146,7 +1122,6 @@ pub enum Inst {
LoadExtName {
rd: Writable<Reg>,
name: Box<ExternalName>,
srcloc: SourceLoc,
offset: i64,
},
@@ -1457,47 +1432,22 @@ impl Inst {
/// Generic constructor for a load (zero-extending where appropriate).
pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type) -> Inst {
match ty {
B1 | B8 | I8 => Inst::ULoad8 {
rd: into_reg,
mem,
srcloc: None,
},
B16 | I16 => Inst::ULoad16 {
rd: into_reg,
mem,
srcloc: None,
},
B32 | I32 | R32 => Inst::ULoad32 {
rd: into_reg,
mem,
srcloc: None,
},
B64 | I64 | R64 => Inst::ULoad64 {
rd: into_reg,
mem,
srcloc: None,
},
F32 => Inst::FpuLoad32 {
rd: into_reg,
mem,
srcloc: None,
},
F64 => Inst::FpuLoad64 {
rd: into_reg,
mem,
srcloc: None,
},
B1 | B8 | I8 => Inst::ULoad8 { rd: into_reg, mem },
B16 | I16 => Inst::ULoad16 { rd: into_reg, mem },
B32 | I32 | R32 => Inst::ULoad32 { rd: into_reg, mem },
B64 | I64 | R64 => Inst::ULoad64 { rd: into_reg, mem },
F32 => Inst::FpuLoad32 { rd: into_reg, mem },
F64 => Inst::FpuLoad64 { rd: into_reg, mem },
_ => {
if ty.is_vector() {
let bits = ty_bits(ty);
let rd = into_reg;
let srcloc = None;
if bits == 128 {
Inst::FpuLoad128 { rd, mem, srcloc }
Inst::FpuLoad128 { rd, mem }
} else {
assert_eq!(bits, 64);
Inst::FpuLoad64 { rd, mem, srcloc }
Inst::FpuLoad64 { rd, mem }
}
} else {
unimplemented!("gen_load({})", ty);
@@ -1509,47 +1459,22 @@ impl Inst {
/// Generic constructor for a store.
pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type) -> Inst {
match ty {
B1 | B8 | I8 => Inst::Store8 {
rd: from_reg,
mem,
srcloc: None,
},
B16 | I16 => Inst::Store16 {
rd: from_reg,
mem,
srcloc: None,
},
B32 | I32 | R32 => Inst::Store32 {
rd: from_reg,
mem,
srcloc: None,
},
B64 | I64 | R64 => Inst::Store64 {
rd: from_reg,
mem,
srcloc: None,
},
F32 => Inst::FpuStore32 {
rd: from_reg,
mem,
srcloc: None,
},
F64 => Inst::FpuStore64 {
rd: from_reg,
mem,
srcloc: None,
},
B1 | B8 | I8 => Inst::Store8 { rd: from_reg, mem },
B16 | I16 => Inst::Store16 { rd: from_reg, mem },
B32 | I32 | R32 => Inst::Store32 { rd: from_reg, mem },
B64 | I64 | R64 => Inst::Store64 { rd: from_reg, mem },
F32 => Inst::FpuStore32 { rd: from_reg, mem },
F64 => Inst::FpuStore64 { rd: from_reg, mem },
_ => {
if ty.is_vector() {
let bits = ty_bits(ty);
let rd = from_reg;
let srcloc = None;
if bits == 128 {
Inst::FpuStore128 { rd, mem, srcloc }
Inst::FpuStore128 { rd, mem }
} else {
assert_eq!(bits, 64);
Inst::FpuStore64 { rd, mem, srcloc }
Inst::FpuStore64 { rd, mem }
}
} else {
unimplemented!("gen_store({})", ty);
@@ -3024,37 +2949,30 @@ impl Inst {
&Inst::ULoad8 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::SLoad8 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::ULoad16 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::SLoad16 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::ULoad32 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::SLoad32 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::ULoad64 {
rd,
ref mem,
srcloc: _srcloc,
..
} => {
let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru, state);
@@ -3087,22 +3005,18 @@ impl Inst {
&Inst::Store8 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::Store16 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::Store32 {
rd,
ref mem,
srcloc: _srcloc,
}
| &Inst::Store64 {
rd,
ref mem,
srcloc: _srcloc,
..
} => {
let (mem_str, mem) = mem_finalize_for_show(mem, mb_rru, state);
@@ -3841,7 +3755,6 @@ impl Inst {
rd,
ref name,
offset,
srcloc: _srcloc,
} => {
let rd = rd.show_rru(mb_rru);
format!("ldr {}, 8 ; b 12 ; data {:?} + {}", rd, name, offset)

147
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -484,9 +484,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// msub rd, rd, rm, rn ; rd = rn - rd * rm
// Check for divide by 0.
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero);
let trap_code = TrapCode::IntegerDivisionByZero;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Zero(rm),
});
@@ -507,9 +507,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// udf ; signed overflow
// Check for divide by 0.
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero);
let trap_code = TrapCode::IntegerDivisionByZero;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Zero(rm),
});
@@ -535,9 +535,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
nzcv: NZCV::new(false, false, false, false),
cond: Cond::Eq,
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Cond(Cond::Vs),
});
} else {
@@ -545,9 +545,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// udf ; divide by zero
// Check for divide by 0.
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero);
let trap_code = TrapCode::IntegerDivisionByZero;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Zero(rm),
});
}
@@ -1161,27 +1161,20 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mem = lower_address(ctx, elem_ty, &inputs[..], off);
let rd = get_output_reg(ctx, outputs[0]);
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(match (ty_bits(elem_ty), sign_extend, is_float) {
(1, _, _) => Inst::ULoad8 { rd, mem, srcloc },
(8, false, _) => Inst::ULoad8 { rd, mem, srcloc },
(8, true, _) => Inst::SLoad8 { rd, mem, srcloc },
(16, false, _) => Inst::ULoad16 { rd, mem, srcloc },
(16, true, _) => Inst::SLoad16 { rd, mem, srcloc },
(32, false, false) => Inst::ULoad32 { rd, mem, srcloc },
(32, true, false) => Inst::SLoad32 { rd, mem, srcloc },
(32, _, true) => Inst::FpuLoad32 { rd, mem, srcloc },
(64, _, false) => Inst::ULoad64 { rd, mem, srcloc },
(1, _, _) => Inst::ULoad8 { rd, mem },
(8, false, _) => Inst::ULoad8 { rd, mem },
(8, true, _) => Inst::SLoad8 { rd, mem },
(16, false, _) => Inst::ULoad16 { rd, mem },
(16, true, _) => Inst::SLoad16 { rd, mem },
(32, false, false) => Inst::ULoad32 { rd, mem },
(32, true, false) => Inst::SLoad32 { rd, mem },
(32, _, true) => Inst::FpuLoad32 { rd, mem },
(64, _, false) => Inst::ULoad64 { rd, mem },
// Note that we treat some of the vector loads as scalar floating-point loads,
// which is correct in a little endian environment.
(64, _, true) => Inst::FpuLoad64 { rd, mem, srcloc },
(128, _, _) => Inst::FpuLoad128 { rd, mem, srcloc },
(64, _, true) => Inst::FpuLoad64 { rd, mem },
(128, _, _) => Inst::FpuLoad128 { rd, mem },
_ => panic!("Unsupported size in load"),
});
@@ -1209,14 +1202,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let off = ctx.data(insn).load_store_offset().unwrap();
let ty = ty.unwrap();
let mem = lower_address(ctx, ty.lane_type(), &inputs[..], off);
let memflags = ctx.memflags(insn).expect("memory flags");
let rd = get_output_reg(ctx, outputs[0]);
let size = VectorSize::from_ty(ty);
let srcloc = if memflags.notrap() {
None
} else {
Some(ctx.srcloc(insn))
};
let tmp = ctx.alloc_tmp(RegClass::I64, I64);
ctx.emit(Inst::LoadAddr { rd: tmp, mem });
@@ -1224,7 +1211,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rd,
rn: tmp.to_reg(),
size,
srcloc,
});
}
@@ -1249,21 +1235,14 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mem = lower_address(ctx, elem_ty, &inputs[1..], off);
let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(match (ty_bits(elem_ty), is_float) {
(1, _) | (8, _) => Inst::Store8 { rd, mem, srcloc },
(16, _) => Inst::Store16 { rd, mem, srcloc },
(32, false) => Inst::Store32 { rd, mem, srcloc },
(32, true) => Inst::FpuStore32 { rd, mem, srcloc },
(64, false) => Inst::Store64 { rd, mem, srcloc },
(64, true) => Inst::FpuStore64 { rd, mem, srcloc },
(128, _) => Inst::FpuStore128 { rd, mem, srcloc },
(1, _) | (8, _) => Inst::Store8 { rd, mem },
(16, _) => Inst::Store16 { rd, mem },
(32, false) => Inst::Store32 { rd, mem },
(32, true) => Inst::FpuStore32 { rd, mem },
(64, false) => Inst::Store64 { rd, mem },
(64, true) => Inst::FpuStore64 { rd, mem },
(128, _) => Inst::FpuStore128 { rd, mem },
_ => panic!("Unsupported size in store"),
});
}
@@ -1291,12 +1270,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mut r_arg2 = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
// Make sure that both args are in virtual regs, since in effect
// we have to do a parallel copy to get them safely to the AtomicRMW input
// regs, and that's not guaranteed safe if either is in a real reg.
@@ -1307,11 +1280,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::gen_move(Writable::from_reg(xreg(26)), r_arg2, I64));
// Now the AtomicRMW insn itself
let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
ctx.emit(Inst::AtomicRMW {
ty: ty_access,
op,
srcloc,
});
ctx.emit(Inst::AtomicRMW { ty: ty_access, op });
// And finally, copy the preordained AtomicRMW output reg to its destination.
ctx.emit(Inst::gen_move(r_dst, xreg(27), I64));
// Also, x24 and x28 are trashed. `fn aarch64_get_regs` must mention that.
@@ -1327,12 +1296,6 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let mut r_replacement = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
// Make sure that all three args are in virtual regs. See corresponding comment
// for `Opcode::AtomicRmw` above.
r_addr = ctx.ensure_in_vreg(r_addr, I64);
@@ -1351,10 +1314,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
I64,
));
// Now the AtomicCAS itself, implemented in the normal way, with an LL-SC loop
ctx.emit(Inst::AtomicCAS {
ty: ty_access,
srcloc,
});
ctx.emit(Inst::AtomicCAS { ty: ty_access });
// And finally, copy the preordained AtomicCAS output reg to its destination.
ctx.emit(Inst::gen_move(r_dst, xreg(27), I64));
// Also, x24 and x28 are trashed. `fn aarch64_get_regs` must mention that.
@@ -1365,17 +1325,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let r_addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(Inst::AtomicLoad {
ty: ty_access,
r_data,
r_addr,
srcloc,
});
}
@@ -1384,17 +1337,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let r_addr = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
let ty_access = ctx.input_ty(insn, 0);
assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags");
let srcloc = if !memflags.notrap() {
Some(ctx.srcloc(insn))
} else {
None
};
ctx.emit(Inst::AtomicStore {
ty: ty_access,
r_data,
r_addr,
srcloc,
});
}
@@ -1811,12 +1757,12 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}
Opcode::Trap | Opcode::ResumableTrap => {
let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap());
ctx.emit_safepoint(Inst::Udf { trap_info });
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::Udf { trap_code });
}
Opcode::Trapif | Opcode::Trapff => {
let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap());
let trap_code = ctx.data(insn).trap_code().unwrap();
let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() {
let condcode = ctx.data(insn).cond_code().unwrap();
@@ -1847,7 +1793,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
};
ctx.emit_safepoint(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Cond(cond),
});
}
@@ -1864,11 +1810,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let rd = get_output_reg(ctx, outputs[0]);
let (extname, _) = ctx.call_target(insn).unwrap();
let extname = extname.clone();
let loc = ctx.srcloc(insn);
ctx.emit(Inst::LoadExtName {
rd,
name: Box::new(extname),
srcloc: loc,
offset: 0,
});
}
@@ -1881,17 +1825,14 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let rd = get_output_reg(ctx, outputs[0]);
let (extname, _, offset) = ctx.symbol_value(insn).unwrap();
let extname = extname.clone();
let loc = ctx.srcloc(insn);
ctx.emit(Inst::LoadExtName {
rd,
name: Box::new(extname),
srcloc: loc,
offset,
});
}
Opcode::Call | Opcode::CallIndirect => {
let loc = ctx.srcloc(insn);
let caller_conv = ctx.abi().call_conv();
let (mut abi, inputs) = match op {
Opcode::Call => {
@@ -1901,7 +1842,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert!(inputs.len() == sig.params.len());
assert!(outputs.len() == sig.returns.len());
(
AArch64ABICaller::from_func(sig, &extname, dist, loc, caller_conv)?,
AArch64ABICaller::from_func(sig, &extname, dist, caller_conv)?,
&inputs[..],
)
}
@@ -1911,7 +1852,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert!(inputs.len() - 1 == sig.params.len());
assert!(outputs.len() == sig.returns.len());
(
AArch64ABICaller::from_ptr(sig, ptr, loc, op, caller_conv)?,
AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
&inputs[1..],
)
}
@@ -2687,9 +2628,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else {
ctx.emit(Inst::FpuCmp64 { rn, rm: rn });
}
let trap_info = (ctx.srcloc(insn), TrapCode::BadConversionToInteger);
let trap_code = TrapCode::BadConversionToInteger;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Unordered)),
});
@@ -2739,9 +2680,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()),
});
@@ -2751,9 +2692,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()),
});
} else {
@@ -2792,9 +2733,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()),
});
@@ -2804,9 +2745,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
rn,
rm: tmp.to_reg(),
});
let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow);
let trap_code = TrapCode::IntegerOverflow;
ctx.emit(Inst::TrapIf {
trap_info,
trap_code,
kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()),
});
};