T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x64 and aarch64: carry MemFlags on loads/stores; don't emit trap info unless an op can trap.

Browse Source 
This end result was previously enacted by carrying a `SourceLoc` on
every load/store, which was somewhat cumbersome, and only indirectly
encoded metadata about a memory reference (can it trap) by its presence
or absence. We have a type for this -- `MemFlags` -- that tells us
everything we might want to know about a load or store, and we should
plumb it through to code emission instead.

This PR attaches a `MemFlags` to an `Amode` on x64, and puts it on load
and store `Inst` variants on aarch64. These two choices seem to factor
things out in the nicest way: there are relatively few load/store insts
on aarch64 but many addressing modes, while the opposite is true on x64.
This commit is contained in:
Chris Fallin
2020-11-17 09:17:12 -08:00
parent e7df081696
commit 073c727a74
11 changed files with 340 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -3,6 +3,7 @@
use crate::ir;
use crate::ir::types;
use crate::ir::types::*;
use crate::ir::MemFlags;
use crate::isa;
use crate::isa::aarch64::{inst::EmitState, inst::*};
use crate::machinst::*;
@@ -312,11 +313,11 @@ impl ABIMachineSpec for AArch64MachineDeps {
}
fn gen_load_stack(mem: StackAMode, into_reg: Writable<Reg>, ty: Type) -> Inst {
Inst::gen_load(into_reg, mem.into(), ty)
Inst::gen_load(into_reg, mem.into(), ty, MemFlags::trusted())
}
fn gen_store_stack(mem: StackAMode, from_reg: Reg, ty: Type) -> Inst {
Inst::gen_store(mem.into(), from_reg, ty)
Inst::gen_store(mem.into(), from_reg, ty, MemFlags::trusted())
}
fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
@@ -402,12 +403,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn gen_load_base_offset(into_reg: Writable<Reg>, base: Reg, offset: i32, ty: Type) -> Inst {
let mem = AMode::RegOffset(base, offset as i64, ty);
Inst::gen_load(into_reg, mem, ty)
Inst::gen_load(into_reg, mem, ty, MemFlags::trusted())
}
fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst {
let mem = AMode::RegOffset(base, offset as i64, ty);
Inst::gen_store(mem, from_reg, ty)
Inst::gen_store(mem, from_reg, ty, MemFlags::trusted())
}
fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Inst; 2]> {
@@ -464,6 +465,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
writable_stack_reg(),
SImm7Scaled::maybe_from_i64(-16, types::I64).unwrap(),
),
flags: MemFlags::trusted(),
});
// mov fp (x29), sp. This uses the ADDI rd, rs, 0 form of `MOV` because
// the usual encoding (`ORR`) does not work with SP.
@@ -500,6 +502,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
writable_stack_reg(),
SImm7Scaled::maybe_from_i64(16, types::I64).unwrap(),
),
flags: MemFlags::trusted(),
});
insts
@@ -542,6 +545,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(),
SImm7Scaled::maybe_from_i64((i * 16) as i64, types::I64).unwrap(),
),
flags: MemFlags::trusted(),
});
}
@@ -553,6 +557,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(),
SImm9::maybe_from_i64((vec_offset + (i * 16)) as i64).unwrap(),
),
flags: MemFlags::trusted(),
});
}
@@ -591,6 +596,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(),
SImm7Scaled::maybe_from_i64((i * 16) as i64, types::I64).unwrap(),
),
flags: MemFlags::trusted(),
});
}
@@ -601,6 +607,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
stack_reg(),
SImm9::maybe_from_i64(((i * 16) + int_save_bytes) as i64).unwrap(),
),
flags: MemFlags::trusted(),
});
}
@@ -621,6 +628,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
writable_xreg(BALDRDASH_TLS_REG),
AMode::UnsignedOffset(fp_reg(), UImm12Scaled::maybe_from_i64(off, I64).unwrap()),
I64,
MemFlags::trusted(),
));
}

75
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -3,7 +3,7 @@
use crate::binemit::{CodeOffset, Reloc, StackMap};
use crate::ir::constant::ConstantData;
use crate::ir::types::*;
use crate::ir::TrapCode;
use crate::ir::{MemFlags, TrapCode};
use crate::isa::aarch64::inst::*;
use crate::machinst::ty_bits;
@@ -741,16 +741,18 @@ impl MachInstEmit for Inst {
sink.put4(enc_bit_rr(size, op1, op2, rn, rd))
}
&Inst::ULoad8 { rd, ref mem }
| &Inst::SLoad8 { rd, ref mem }
| &Inst::ULoad16 { rd, ref mem }
| &Inst::SLoad16 { rd, ref mem }
| &Inst::ULoad32 { rd, ref mem }
| &Inst::SLoad32 { rd, ref mem }
| &Inst::ULoad64 { rd, ref mem, .. }
| &Inst::FpuLoad32 { rd, ref mem }
| &Inst::FpuLoad64 { rd, ref mem }
| &Inst::FpuLoad128 { rd, ref mem } => {
&Inst::ULoad8 { rd, ref mem, flags }
| &Inst::SLoad8 { rd, ref mem, flags }
| &Inst::ULoad16 { rd, ref mem, flags }
| &Inst::SLoad16 { rd, ref mem, flags }
| &Inst::ULoad32 { rd, ref mem, flags }
| &Inst::SLoad32 { rd, ref mem, flags }
| &Inst::ULoad64 {
rd, ref mem, flags, ..
}
| &Inst::FpuLoad32 { rd, ref mem, flags }
| &Inst::FpuLoad64 { rd, ref mem, flags }
| &Inst::FpuLoad128 { rd, ref mem, flags } => {
let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
for inst in mem_insts.into_iter() {
@@ -778,7 +780,7 @@ impl MachInstEmit for Inst {
};
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
if srcloc != SourceLoc::default() && !flags.notrap() {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
@@ -861,13 +863,13 @@ impl MachInstEmit for Inst {
}
}
&Inst::Store8 { rd, ref mem }
| &Inst::Store16 { rd, ref mem }
| &Inst::Store32 { rd, ref mem }
| &Inst::Store64 { rd, ref mem, .. }
| &Inst::FpuStore32 { rd, ref mem }
| &Inst::FpuStore64 { rd, ref mem }
| &Inst::FpuStore128 { rd, ref mem } => {
&Inst::Store8 { rd, ref mem, flags }
| &Inst::Store16 { rd, ref mem, flags }
| &Inst::Store32 { rd, ref mem, flags }
| &Inst::Store64 { rd, ref mem, flags }
| &Inst::FpuStore32 { rd, ref mem, flags }
| &Inst::FpuStore64 { rd, ref mem, flags }
| &Inst::FpuStore128 { rd, ref mem, flags } => {
let (mem_insts, mem) = mem_finalize(sink.cur_offset(), mem, state);
for inst in mem_insts.into_iter() {
@@ -886,7 +888,7 @@ impl MachInstEmit for Inst {
};
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
if srcloc != SourceLoc::default() && !flags.notrap() {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
@@ -943,7 +945,18 @@ impl MachInstEmit for Inst {
}
}
&Inst::StoreP64 { rt, rt2, ref mem } => match mem {
&Inst::StoreP64 {
rt,
rt2,
ref mem,
flags,
} => {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() && !flags.notrap() {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
match mem {
&PairAMode::SignedOffset(reg, simm7) => {
assert_eq!(simm7.scale_ty, I64);
sink.put4(enc_ldst_pair(0b1010100100, simm7, reg, rt, rt2));
@@ -956,8 +969,20 @@ impl MachInstEmit for Inst {
assert_eq!(simm7.scale_ty, I64);
sink.put4(enc_ldst_pair(0b1010100010, simm7, reg.to_reg(), rt, rt2));
}
},
&Inst::LoadP64 { rt, rt2, ref mem } => {
}
}
&Inst::LoadP64 {
rt,
rt2,
ref mem,
flags,
} => {
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() && !flags.notrap() {
// Register the offset at which the actual load instruction starts.
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
let rt = rt.to_reg();
let rt2 = rt2.to_reg();
match mem {
@@ -1546,6 +1571,7 @@ impl MachInstEmit for Inst {
let inst = Inst::FpuLoad64 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
flags: MemFlags::trusted(),
};
inst.emit(sink, emit_info, state);
let inst = Inst::Jump {
@@ -1558,6 +1584,7 @@ impl MachInstEmit for Inst {
let inst = Inst::FpuLoad128 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
flags: MemFlags::trusted(),
};
inst.emit(sink, emit_info, state);
let inst = Inst::Jump {
@@ -2169,6 +2196,7 @@ impl MachInstEmit for Inst {
I32,
ExtendOp::UXTW,
),
flags: MemFlags::trusted(),
};
inst.emit(sink, emit_info, state);
// Add base of jump table to jump-table-sourced block offset
@@ -2215,6 +2243,7 @@ impl MachInstEmit for Inst {
let inst = Inst::ULoad64 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
flags: MemFlags::trusted(),
};
inst.emit(sink, emit_info, state);
let inst = Inst::Jump {

71
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -1079,6 +1079,7 @@ fn test_aarch64_binemit() {
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"41004038",
"ldurb w1, [x2]",
@@ -1087,6 +1088,7 @@ fn test_aarch64_binemit() {
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)),
flags: MemFlags::trusted(),
},
"41004039",
"ldrb w1, [x2]",
@@ -1095,6 +1097,7 @@ fn test_aarch64_binemit() {
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)),
flags: MemFlags::trusted(),
},
"41686538",
"ldrb w1, [x2, x5]",
@@ -1103,6 +1106,7 @@ fn test_aarch64_binemit() {
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"41008038",
"ldursb x1, [x2]",
@@ -1111,6 +1115,7 @@ fn test_aarch64_binemit() {
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()),
flags: MemFlags::trusted(),
},
"41FC8039",
"ldrsb x1, [x2, #63]",
@@ -1119,6 +1124,7 @@ fn test_aarch64_binemit() {
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)),
flags: MemFlags::trusted(),
},
"4168A538",
"ldrsb x1, [x2, x5]",
@@ -1127,6 +1133,7 @@ fn test_aarch64_binemit() {
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()),
flags: MemFlags::trusted(),
},
"41504078",
"ldurh w1, [x2, #5]",
@@ -1135,6 +1142,7 @@ fn test_aarch64_binemit() {
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()),
flags: MemFlags::trusted(),
},
"41104079",
"ldrh w1, [x2, #8]",
@@ -1143,6 +1151,7 @@ fn test_aarch64_binemit() {
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I16),
flags: MemFlags::trusted(),
},
"41786378",
"ldrh w1, [x2, x3, LSL #1]",
@@ -1151,6 +1160,7 @@ fn test_aarch64_binemit() {
Inst::SLoad16 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"41008078",
"ldursh x1, [x2]",
@@ -1159,6 +1169,7 @@ fn test_aarch64_binemit() {
Inst::SLoad16 {
rd: writable_xreg(28),
mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()),
flags: MemFlags::trusted(),
},
"9C328079",
"ldrsh x28, [x20, #24]",
@@ -1167,6 +1178,7 @@ fn test_aarch64_binemit() {
Inst::SLoad16 {
rd: writable_xreg(28),
mem: AMode::RegScaled(xreg(20), xreg(20), I16),
flags: MemFlags::trusted(),
},
"9C7AB478",
"ldrsh x28, [x20, x20, LSL #1]",
@@ -1175,6 +1187,7 @@ fn test_aarch64_binemit() {
Inst::ULoad32 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"410040B8",
"ldur w1, [x2]",
@@ -1183,6 +1196,7 @@ fn test_aarch64_binemit() {
Inst::ULoad32 {
rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()),
flags: MemFlags::trusted(),
},
"0CCC40B9",
"ldr w12, [x0, #204]",
@@ -1191,6 +1205,7 @@ fn test_aarch64_binemit() {
Inst::ULoad32 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(12), I32),
flags: MemFlags::trusted(),
},
"41786CB8",
"ldr w1, [x2, x12, LSL #2]",
@@ -1199,6 +1214,7 @@ fn test_aarch64_binemit() {
Inst::SLoad32 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"410080B8",
"ldursw x1, [x2]",
@@ -1207,6 +1223,7 @@ fn test_aarch64_binemit() {
Inst::SLoad32 {
rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
flags: MemFlags::trusted(),
},
"2CFCBFB9",
"ldrsw x12, [x1, #16380]",
@@ -1215,6 +1232,7 @@ fn test_aarch64_binemit() {
Inst::SLoad32 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(5), xreg(1), I32),
flags: MemFlags::trusted(),
},
"A178A1B8",
"ldrsw x1, [x5, x1, LSL #2]",
@@ -1223,6 +1241,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"410040F8",
"ldur x1, [x2]",
@@ -1231,6 +1250,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()),
flags: MemFlags::trusted(),
},
"410050F8",
"ldur x1, [x2, #-256]",
@@ -1239,6 +1259,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()),
flags: MemFlags::trusted(),
},
"41F04FF8",
"ldur x1, [x2, #255]",
@@ -1247,6 +1268,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
flags: MemFlags::trusted(),
},
"41FC7FF9",
"ldr x1, [x2, #32760]",
@@ -1255,6 +1277,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)),
flags: MemFlags::trusted(),
},
"416863F8",
"ldr x1, [x2, x3]",
@@ -1263,6 +1286,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64),
flags: MemFlags::trusted(),
},
"417863F8",
"ldr x1, [x2, x3, LSL #3]",
@@ -1271,6 +1295,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW),
flags: MemFlags::trusted(),
},
"41D863F8",
"ldr x1, [x2, w3, SXTW #3]",
@@ -1279,6 +1304,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW),
flags: MemFlags::trusted(),
},
"41C863F8",
"ldr x1, [x2, w3, SXTW]",
@@ -1287,6 +1313,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Label(MemLabel::PCRel(64)),
flags: MemFlags::trusted(),
},
"01020058",
"ldr x1, pc+64",
@@ -1295,6 +1322,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
flags: MemFlags::trusted(),
},
"410C41F8",
"ldr x1, [x2, #16]!",
@@ -1303,6 +1331,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
flags: MemFlags::trusted(),
},
"410441F8",
"ldr x1, [x2], #16",
@@ -1311,6 +1340,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(32768, I8),
flags: MemFlags::trusted(),
},
"100090D2B063308B010240F9",
"movz x16, #32768 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1319,6 +1349,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(-32768, I8),
flags: MemFlags::trusted(),
},
"F0FF8F92B063308B010240F9",
"movn x16, #32767 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1327,6 +1358,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(1048576, I8), // 2^20
flags: MemFlags::trusted(),
},
"1002A0D2B063308B010240F9",
"movz x16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1335,6 +1367,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1
flags: MemFlags::trusted(),
},
"300080521002A072B063308B010240F9",
"movz w16, #1 ; movk w16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
@@ -1344,6 +1377,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 8, I64),
flags: MemFlags::trusted(),
},
"E18040F8",
"ldur x1, [x7, #8]",
@@ -1353,6 +1387,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1024, I64),
flags: MemFlags::trusted(),
},
"E10042F9",
"ldr x1, [x7, #1024]",
@@ -1362,6 +1397,7 @@ fn test_aarch64_binemit() {
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1048576, I64),
flags: MemFlags::trusted(),
},
"1002A0D2F060308B010240F9",
"movz x16, #16, LSL #16 ; add x16, x7, x16, UXTX ; ldr x1, [x16]",
@@ -1371,6 +1407,7 @@ fn test_aarch64_binemit() {
Inst::Store8 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"41000038",
"sturb w1, [x2]",
@@ -1379,6 +1416,7 @@ fn test_aarch64_binemit() {
Inst::Store8 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()),
flags: MemFlags::trusted(),
},
"41FC3F39",
"strb w1, [x2, #4095]",
@@ -1387,6 +1425,7 @@ fn test_aarch64_binemit() {
Inst::Store16 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"41000078",
"sturh w1, [x2]",
@@ -1395,6 +1434,7 @@ fn test_aarch64_binemit() {
Inst::Store16 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()),
flags: MemFlags::trusted(),
},
"41FC3F79",
"strh w1, [x2, #8190]",
@@ -1403,6 +1443,7 @@ fn test_aarch64_binemit() {
Inst::Store32 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"410000B8",
"stur w1, [x2]",
@@ -1411,6 +1452,7 @@ fn test_aarch64_binemit() {
Inst::Store32 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
flags: MemFlags::trusted(),
},
"41FC3FB9",
"str w1, [x2, #16380]",
@@ -1419,6 +1461,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::zero()),
flags: MemFlags::trusted(),
},
"410000F8",
"stur x1, [x2]",
@@ -1427,6 +1470,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
flags: MemFlags::trusted(),
},
"41FC3FF9",
"str x1, [x2, #32760]",
@@ -1435,6 +1479,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)),
flags: MemFlags::trusted(),
},
"416823F8",
"str x1, [x2, x3]",
@@ -1443,6 +1488,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64),
flags: MemFlags::trusted(),
},
"417823F8",
"str x1, [x2, x3, LSL #3]",
@@ -1451,6 +1497,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW),
flags: MemFlags::trusted(),
},
"415823F8",
"str x1, [x2, w3, UXTW #3]",
@@ -1459,6 +1506,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW),
flags: MemFlags::trusted(),
},
"414823F8",
"str x1, [x2, w3, UXTW]",
@@ -1467,6 +1515,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::PreIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
flags: MemFlags::trusted(),
},
"410C01F8",
"str x1, [x2, #16]!",
@@ -1475,6 +1524,7 @@ fn test_aarch64_binemit() {
Inst::Store64 {
rd: xreg(1),
mem: AMode::PostIndexed(writable_xreg(2), SImm9::maybe_from_i64(16).unwrap()),
flags: MemFlags::trusted(),
},
"410401F8",
"str x1, [x2], #16",
@@ -1485,6 +1535,7 @@ fn test_aarch64_binemit() {
rt: xreg(8),
rt2: xreg(9),
mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::zero(I64)),
flags: MemFlags::trusted(),
},
"482500A9",
"stp x8, x9, [x10]",
@@ -1494,6 +1545,7 @@ fn test_aarch64_binemit() {
rt: xreg(8),
rt2: xreg(9),
mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(504, I64).unwrap()),
flags: MemFlags::trusted(),
},
"48A51FA9",
"stp x8, x9, [x10, #504]",
@@ -1503,6 +1555,7 @@ fn test_aarch64_binemit() {
rt: xreg(8),
rt2: xreg(9),
mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap()),
flags: MemFlags::trusted(),
},
"48253CA9",
"stp x8, x9, [x10, #-64]",
@@ -1512,6 +1565,7 @@ fn test_aarch64_binemit() {
rt: xreg(21),
rt2: xreg(28),
mem: PairAMode::SignedOffset(xreg(1), SImm7Scaled::maybe_from_i64(-512, I64).unwrap()),
flags: MemFlags::trusted(),
},
"357020A9",
"stp x21, x28, [x1, #-512]",
@@ -1524,6 +1578,7 @@ fn test_aarch64_binemit() {
writable_xreg(10),
SImm7Scaled::maybe_from_i64(-64, I64).unwrap(),
),
flags: MemFlags::trusted(),
},
"4825BCA9",
"stp x8, x9, [x10, #-64]!",
@@ -1536,6 +1591,7 @@ fn test_aarch64_binemit() {
writable_xreg(20),
SImm7Scaled::maybe_from_i64(504, I64).unwrap(),
),
flags: MemFlags::trusted(),
},
"8FC29FA8",
"stp x15, x16, [x20], #504",
@@ -1546,6 +1602,7 @@ fn test_aarch64_binemit() {
rt: writable_xreg(8),
rt2: writable_xreg(9),
mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::zero(I64)),
flags: MemFlags::trusted(),
},
"482540A9",
"ldp x8, x9, [x10]",
@@ -1555,6 +1612,7 @@ fn test_aarch64_binemit() {
rt: writable_xreg(8),
rt2: writable_xreg(9),
mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(504, I64).unwrap()),
flags: MemFlags::trusted(),
},
"48A55FA9",
"ldp x8, x9, [x10, #504]",
@@ -1564,6 +1622,7 @@ fn test_aarch64_binemit() {
rt: writable_xreg(8),
rt2: writable_xreg(9),
mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-64, I64).unwrap()),
flags: MemFlags::trusted(),
},
"48257CA9",
"ldp x8, x9, [x10, #-64]",
@@ -1573,6 +1632,7 @@ fn test_aarch64_binemit() {
rt: writable_xreg(8),
rt2: writable_xreg(9),
mem: PairAMode::SignedOffset(xreg(10), SImm7Scaled::maybe_from_i64(-512, I64).unwrap()),
flags: MemFlags::trusted(),
},
"482560A9",
"ldp x8, x9, [x10, #-512]",
@@ -1585,6 +1645,7 @@ fn test_aarch64_binemit() {
writable_xreg(10),
SImm7Scaled::maybe_from_i64(-64, I64).unwrap(),
),
flags: MemFlags::trusted(),
},
"4825FCA9",
"ldp x8, x9, [x10, #-64]!",
@@ -1597,6 +1658,7 @@ fn test_aarch64_binemit() {
writable_xreg(12),
SImm7Scaled::maybe_from_i64(504, I64).unwrap(),
),
flags: MemFlags::trusted(),
},
"88E5DFA8",
"ldp x8, x25, [x12], #504",
@@ -4756,6 +4818,7 @@ fn test_aarch64_binemit() {
Inst::FpuLoad32 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32),
flags: MemFlags::trusted(),
},
"107969BC",
"ldr s16, [x8, x9, LSL #2]",
@@ -4765,6 +4828,7 @@ fn test_aarch64_binemit() {
Inst::FpuLoad64 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64),
flags: MemFlags::trusted(),
},
"107969FC",
"ldr d16, [x8, x9, LSL #3]",
@@ -4774,6 +4838,7 @@ fn test_aarch64_binemit() {
Inst::FpuLoad128 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128),
flags: MemFlags::trusted(),
},
"1079E93C",
"ldr q16, [x8, x9, LSL #4]",
@@ -4783,6 +4848,7 @@ fn test_aarch64_binemit() {
Inst::FpuLoad32 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
flags: MemFlags::trusted(),
},
"5000001C",
"ldr s16, pc+8",
@@ -4792,6 +4858,7 @@ fn test_aarch64_binemit() {
Inst::FpuLoad64 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
flags: MemFlags::trusted(),
},
"5000005C",
"ldr d16, pc+8",
@@ -4801,6 +4868,7 @@ fn test_aarch64_binemit() {
Inst::FpuLoad128 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
flags: MemFlags::trusted(),
},
"5000009C",
"ldr q16, pc+8",
@@ -4810,6 +4878,7 @@ fn test_aarch64_binemit() {
Inst::FpuStore32 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32),
flags: MemFlags::trusted(),
},
"107929BC",
"str s16, [x8, x9, LSL #2]",
@@ -4819,6 +4888,7 @@ fn test_aarch64_binemit() {
Inst::FpuStore64 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64),
flags: MemFlags::trusted(),
},
"107929FC",
"str d16, [x8, x9, LSL #3]",
@@ -4828,6 +4898,7 @@ fn test_aarch64_binemit() {
Inst::FpuStore128 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128),
flags: MemFlags::trusted(),
},
"1079A93C",
"str q16, [x8, x9, LSL #4]",

120
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -9,7 +9,7 @@ use crate::ir::types::{
F64X2, FFLAGS, I16, I16X4, I16X8, I32, I32X2, I32X4, I64, I64X2, I8, I8X16, I8X8, IFLAGS, R32,
R64,
};
use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type};
use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type};
use crate::isa::CallConv;
use crate::machinst::*;
use crate::{settings, CodegenError, CodegenResult};
@@ -523,57 +523,68 @@ pub enum Inst {
ULoad8 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// A signed (sign-extending) 8-bit load.
SLoad8 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// An unsigned (zero-extending) 16-bit load.
ULoad16 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// A signed (sign-extending) 16-bit load.
SLoad16 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// An unsigned (zero-extending) 32-bit load.
ULoad32 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// A signed (sign-extending) 32-bit load.
SLoad32 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// A 64-bit load.
ULoad64 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// An 8-bit store.
Store8 {
rd: Reg,
mem: AMode,
flags: MemFlags,
},
/// A 16-bit store.
Store16 {
rd: Reg,
mem: AMode,
flags: MemFlags,
},
/// A 32-bit store.
Store32 {
rd: Reg,
mem: AMode,
flags: MemFlags,
},
/// A 64-bit store.
Store64 {
rd: Reg,
mem: AMode,
flags: MemFlags,
},
/// A store of a pair of registers.
@@ -581,12 +592,14 @@ pub enum Inst {
rt: Reg,
rt2: Reg,
mem: PairAMode,
flags: MemFlags,
},
/// A load of a pair of registers.
LoadP64 {
rt: Writable<Reg>,
rt2: Writable<Reg>,
mem: PairAMode,
flags: MemFlags,
},
/// A MOV instruction. These are encoded as ORR's (AluRRR form) but we
@@ -782,31 +795,37 @@ pub enum Inst {
FpuLoad32 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// Floating-point store, single-precision (32 bit).
FpuStore32 {
rd: Reg,
mem: AMode,
flags: MemFlags,
},
/// Floating-point load, double-precision (64 bit).
FpuLoad64 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// Floating-point store, double-precision (64 bit).
FpuStore64 {
rd: Reg,
mem: AMode,
flags: MemFlags,
},
/// Floating-point/vector load, 128 bit.
FpuLoad128 {
rd: Writable<Reg>,
mem: AMode,
flags: MemFlags,
},
/// Floating-point/vector store, 128 bit.
FpuStore128 {
rd: Reg,
mem: AMode,
flags: MemFlags,
},
LoadFpuConst64 {
@@ -1411,24 +1430,48 @@ impl Inst {
}
/// Generic constructor for a load (zero-extending where appropriate).
pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type) -> Inst {
pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst {
match ty {
B1 | B8 | I8 => Inst::ULoad8 { rd: into_reg, mem },
B16 | I16 => Inst::ULoad16 { rd: into_reg, mem },
B32 | I32 | R32 => Inst::ULoad32 { rd: into_reg, mem },
B64 | I64 | R64 => Inst::ULoad64 { rd: into_reg, mem },
F32 => Inst::FpuLoad32 { rd: into_reg, mem },
F64 => Inst::FpuLoad64 { rd: into_reg, mem },
B1 | B8 | I8 => Inst::ULoad8 {
rd: into_reg,
mem,
flags,
},
B16 | I16 => Inst::ULoad16 {
rd: into_reg,
mem,
flags,
},
B32 | I32 | R32 => Inst::ULoad32 {
rd: into_reg,
mem,
flags,
},
B64 | I64 | R64 => Inst::ULoad64 {
rd: into_reg,
mem,
flags,
},
F32 => Inst::FpuLoad32 {
rd: into_reg,
mem,
flags,
},
F64 => Inst::FpuLoad64 {
rd: into_reg,
mem,
flags,
},
_ => {
if ty.is_vector() {
let bits = ty_bits(ty);
let rd = into_reg;
if bits == 128 {
Inst::FpuLoad128 { rd, mem }
Inst::FpuLoad128 { rd, mem, flags }
} else {
assert_eq!(bits, 64);
Inst::FpuLoad64 { rd, mem }
Inst::FpuLoad64 { rd, mem, flags }
}
} else {
unimplemented!("gen_load({})", ty);
@@ -1438,24 +1481,48 @@ impl Inst {
}
/// Generic constructor for a store.
pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type) -> Inst {
pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst {
match ty {
B1 | B8 | I8 => Inst::Store8 { rd: from_reg, mem },
B16 | I16 => Inst::Store16 { rd: from_reg, mem },
B32 | I32 | R32 => Inst::Store32 { rd: from_reg, mem },
B64 | I64 | R64 => Inst::Store64 { rd: from_reg, mem },
F32 => Inst::FpuStore32 { rd: from_reg, mem },
F64 => Inst::FpuStore64 { rd: from_reg, mem },
B1 | B8 | I8 => Inst::Store8 {
rd: from_reg,
mem,
flags,
},
B16 | I16 => Inst::Store16 {
rd: from_reg,
mem,
flags,
},
B32 | I32 | R32 => Inst::Store32 {
rd: from_reg,
mem,
flags,
},
B64 | I64 | R64 => Inst::Store64 {
rd: from_reg,
mem,
flags,
},
F32 => Inst::FpuStore32 {
rd: from_reg,
mem,
flags,
},
F64 => Inst::FpuStore64 {
rd: from_reg,
mem,
flags,
},
_ => {
if ty.is_vector() {
let bits = ty_bits(ty);
let rd = from_reg;
if bits == 128 {
Inst::FpuStore128 { rd, mem }
Inst::FpuStore128 { rd, mem, flags }
} else {
assert_eq!(bits, 64);
Inst::FpuStore64 { rd, mem }
Inst::FpuStore64 { rd, mem, flags }
}
} else {
unimplemented!("gen_store({})", ty);
@@ -2126,6 +2193,7 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
ref mut rt,
ref mut rt2,
ref mut mem,
..
} => {
map_use(mapper, rt);
map_use(mapper, rt2);
@@ -2135,6 +2203,7 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
ref mut rt,
ref mut rt2,
ref mut mem,
..
} => {
map_def(mapper, rt);
map_def(mapper, rt2);
@@ -2979,26 +3048,32 @@ impl Inst {
&Inst::ULoad8 {
rd,
ref mem,
..
}
| &Inst::SLoad8 {
rd,
ref mem,
..
}
| &Inst::ULoad16 {
rd,
ref mem,
..
}
| &Inst::SLoad16 {
rd,
ref mem,
..
}
| &Inst::ULoad32 {
rd,
ref mem,
..
}
| &Inst::SLoad32 {
rd,
ref mem,
..
}
| &Inst::ULoad64 {
rd,
@@ -3035,14 +3110,17 @@ impl Inst {
&Inst::Store8 {
rd,
ref mem,
..
}
| &Inst::Store16 {
rd,
ref mem,
..
}
| &Inst::Store32 {
rd,
ref mem,
..
}
| &Inst::Store64 {
rd,
@@ -3070,13 +3148,13 @@ impl Inst {
let mem = mem.show_rru(mb_rru);
format!("{}{} {}, {}", mem_str, op, rd, mem)
}
&Inst::StoreP64 { rt, rt2, ref mem } => {
&Inst::StoreP64 { rt, rt2, ref mem, .. } => {
let rt = rt.show_rru(mb_rru);
let rt2 = rt2.show_rru(mb_rru);
let mem = mem.show_rru(mb_rru);
format!("stp {}, {}, {}", rt, rt2, mem)
}
&Inst::LoadP64 { rt, rt2, ref mem } => {
&Inst::LoadP64 { rt, rt2, ref mem, .. } => {
let rt = rt.to_reg().show_rru(mb_rru);
let rt2 = rt2.to_reg().show_rru(mb_rru);
let mem = mem.show_rru(mb_rru);

2
cranelift/codegen/src/isa/aarch64/inst/unwind.rs
View File

@@ -29,6 +29,7 @@ impl UnwindInfoGenerator<Inst> for AArch64UnwindInfo {
rt,
rt2,
mem: PairAMode::PreIndexed(rn, imm7),
..
} if *rt == regs::fp_reg()
&& *rt2 == regs::link_reg()
&& *rn == regs::writable_stack_reg()
@@ -60,6 +61,7 @@ impl UnwindInfoGenerator<Inst> for AArch64UnwindInfo {
rt,
rt2,
mem: PairAMode::PreIndexed(rn, imm7),
..
} if rn.to_reg() == regs::stack_reg() && imm7.value % (pair_size as i16) == 0 => {
// stp r1, r2, [sp, #(i * #16)]
let stack_offset = imm7.value as u32;

42
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -1130,6 +1130,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::Sload32Complex => true,
_ => false,
};
let flags = ctx
.memflags(insn)
.expect("Load instruction should have memflags");
lower_load(
ctx,
@@ -1139,19 +1142,19 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
|ctx, rd, elem_ty, mem| {
let is_float = ty_has_float_or_vec_representation(elem_ty);
ctx.emit(match (ty_bits(elem_ty), sign_extend, is_float) {
(1, _, _) => Inst::ULoad8 { rd, mem },
(8, false, _) => Inst::ULoad8 { rd, mem },
(8, true, _) => Inst::SLoad8 { rd, mem },
(16, false, _) => Inst::ULoad16 { rd, mem },
(16, true, _) => Inst::SLoad16 { rd, mem },
(32, false, false) => Inst::ULoad32 { rd, mem },
(32, true, false) => Inst::SLoad32 { rd, mem },
(32, _, true) => Inst::FpuLoad32 { rd, mem },
(64, _, false) => Inst::ULoad64 { rd, mem },
(1, _, _) => Inst::ULoad8 { rd, mem, flags },
(8, false, _) => Inst::ULoad8 { rd, mem, flags },
(8, true, _) => Inst::SLoad8 { rd, mem, flags },
(16, false, _) => Inst::ULoad16 { rd, mem, flags },
(16, true, _) => Inst::SLoad16 { rd, mem, flags },
(32, false, false) => Inst::ULoad32 { rd, mem, flags },
(32, true, false) => Inst::SLoad32 { rd, mem, flags },
(32, _, true) => Inst::FpuLoad32 { rd, mem, flags },
(64, _, false) => Inst::ULoad64 { rd, mem, flags },
// Note that we treat some of the vector loads as scalar floating-point loads,
// which is correct in a little endian environment.
(64, _, true) => Inst::FpuLoad64 { rd, mem },
(128, _, _) => Inst::FpuLoad128 { rd, mem },
(64, _, true) => Inst::FpuLoad64 { rd, mem, flags },
(128, _, _) => Inst::FpuLoad128 { rd, mem, flags },
_ => panic!("Unsupported size in load"),
});
@@ -1200,18 +1203,21 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
_ => unreachable!(),
};
let is_float = ty_has_float_or_vec_representation(elem_ty);
let flags = ctx
.memflags(insn)
.expect("Store instruction should have memflags");
let mem = lower_address(ctx, elem_ty, &inputs[1..], off);
let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
ctx.emit(match (ty_bits(elem_ty), is_float) {
(1, _) | (8, _) => Inst::Store8 { rd, mem },
(16, _) => Inst::Store16 { rd, mem },
(32, false) => Inst::Store32 { rd, mem },
(32, true) => Inst::FpuStore32 { rd, mem },
(64, false) => Inst::Store64 { rd, mem },
(64, true) => Inst::FpuStore64 { rd, mem },
(128, _) => Inst::FpuStore128 { rd, mem },
(1, _) | (8, _) => Inst::Store8 { rd, mem, flags },
(16, _) => Inst::Store16 { rd, mem, flags },
(32, false) => Inst::Store32 { rd, mem, flags },
(32, true) => Inst::FpuStore32 { rd, mem, flags },
(64, false) => Inst::Store64 { rd, mem, flags },
(64, true) => Inst::FpuStore64 { rd, mem, flags },
(128, _) => Inst::FpuStore128 { rd, mem, flags },
_ => panic!("Unsupported size in store"),
});
}

4
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -1,7 +1,7 @@
//! Implementation of the standard x64 ABI.
use crate::ir::types::*;
use crate::ir::{self, types, TrapCode, Type};
use crate::ir::{self, types, MemFlags, TrapCode, Type};
use crate::isa;
use crate::isa::{x64::inst::*, CallConv};
use crate::machinst::abi_impl::*;
@@ -618,6 +618,7 @@ impl From<StackAMode> for SyntheticAmode {
SyntheticAmode::Real(Amode::ImmReg {
simm32,
base: regs::rbp(),
flags: MemFlags::trusted(),
})
}
StackAMode::NominalSPOffset(off, _ty) => {
@@ -634,6 +635,7 @@ impl From<StackAMode> for SyntheticAmode {
SyntheticAmode::Real(Amode::ImmReg {
simm32,
base: regs::rsp(),
flags: MemFlags::trusted(),
})
}
}

56
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -3,6 +3,7 @@
use super::regs::{self, show_ireg_sized};
use super::EmitState;
use crate::ir::condcodes::{FloatCC, IntCC};
use crate::ir::MemFlags;
use crate::isa::x64::inst::Inst;
use crate::machinst::*;
use regalloc::{
@@ -14,10 +15,14 @@ use std::string::String;
/// A possible addressing mode (amode) that can be used in instructions.
/// These denote a 64-bit value only.
#[derive(Clone)]
#[derive(Clone, Debug)]
pub enum Amode {
/// Immediate sign-extended and a Register.
ImmReg { simm32: u32, base: Reg },
ImmReg {
simm32: u32,
base: Reg,
flags: MemFlags,
},
/// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift)
ImmRegRegShift {
@@ -25,6 +30,7 @@ pub enum Amode {
base: Reg,
index: Reg,
shift: u8, /* 0 .. 3 only */
flags: MemFlags,
},
/// sign-extend-32-to-64(Immediate) + RIP (instruction pointer).
@@ -35,7 +41,11 @@ pub enum Amode {
impl Amode {
pub(crate) fn imm_reg(simm32: u32, base: Reg) -> Self {
debug_assert!(base.get_class() == RegClass::I64);
Self::ImmReg { simm32, base }
Self::ImmReg {
simm32,
base,
flags: MemFlags::trusted(),
}
}
pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self {
@@ -47,6 +57,7 @@ impl Amode {
base,
index,
shift,
flags: MemFlags::trusted(),
}
}
@@ -54,6 +65,30 @@ impl Amode {
Self::RipRelative { target }
}
pub(crate) fn with_flags(&self, flags: MemFlags) -> Self {
match self {
&Self::ImmReg { simm32, base, .. } => Self::ImmReg {
simm32,
base,
flags,
},
&Self::ImmRegRegShift {
simm32,
base,
index,
shift,
..
} => Self::ImmRegRegShift {
simm32,
base,
index,
shift,
flags,
},
_ => panic!("Amode {:?} cannot take memflags", self),
}
}
/// Add the regs mentioned by `self` to `collector`.
pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
match self {
@@ -69,12 +104,24 @@ impl Amode {
}
}
}
pub(crate) fn get_flags(&self) -> MemFlags {
match self {
Amode::ImmReg { flags, .. } => *flags,
Amode::ImmRegRegShift { flags, .. } => *flags,
Amode::RipRelative { .. } => MemFlags::trusted(),
}
}
pub(crate) fn can_trap(&self) -> bool {
!self.get_flags().notrap()
}
}
impl PrettyPrint for Amode {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
match self {
Amode::ImmReg { simm32, base } => {
Amode::ImmReg { simm32, base, .. } => {
format!("{}({})", *simm32 as i32, base.show_rru(mb_rru))
}
Amode::ImmRegRegShift {
@@ -82,6 +129,7 @@ impl PrettyPrint for Amode {
base,
index,
shift,
..
} => format!(
"{}({},{},{})",
*simm32 as i32,

5
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -194,14 +194,14 @@ fn emit_std_enc_mem(
// expression. But `enc_g` can be derived from a register of any class.
let srcloc = state.cur_srcloc();
if srcloc != SourceLoc::default() {
if srcloc != SourceLoc::default() && mem_e.can_trap() {
sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
}
prefixes.emit(sink);
match mem_e {
Amode::ImmReg { simm32, base } => {
Amode::ImmReg { simm32, base, .. } => {
// First, the REX byte.
let enc_e = int_reg_enc(*base);
rex.emit_two_op(sink, enc_g, enc_e);
@@ -260,6 +260,7 @@ fn emit_std_enc_mem(
base: reg_base,
index: reg_index,
shift,
..
} => {
let enc_base = int_reg_enc(*reg_base);
let enc_index = int_reg_enc(*reg_index);

2
cranelift/codegen/src/isa/x64/inst/unwind.rs
View File

@@ -61,7 +61,7 @@ impl UnwindInfoGenerator<Inst> for X64UnwindInfo {
}
Inst::MovRM {
src,
dst: SyntheticAmode::Real(Amode::ImmReg { simm32, base }),
dst: SyntheticAmode::Real(Amode::ImmReg { simm32, base, .. }),
..
} if *base == regs::rsp() => {
// `mov reg, imm(rsp)`

18
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -579,6 +579,10 @@ fn matches_small_constant_shift<C: LowerCtx<I = Inst>>(
///
/// Note: the 32-bit offset in Cranelift has to be sign-extended, which maps x86's behavior.
fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i32) -> Amode {
let flags = ctx
.memflags(spec.insn)
.expect("Instruction with amode should have memflags");
// We now either have an add that we must materialize, or some other input; as well as the
// final offset.
if let Some(add) = matches_input(ctx, spec, Opcode::Iadd) {
@@ -632,7 +636,7 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i
let final_offset = (offset as i64).wrapping_add(uext_cst as i64);
if low32_will_sign_extend_to_64(final_offset as u64) {
let base = put_input_in_reg(ctx, add_inputs[1 - i]);
return Amode::imm_reg(final_offset as u32, base);
return Amode::imm_reg(final_offset as u32, base).with_flags(flags);
}
}
}
@@ -642,7 +646,7 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i
let final_offset = (offset as i64).wrapping_add(cst as i64);
if low32_will_sign_extend_to_64(final_offset as u64) {
let base = put_input_in_reg(ctx, add_inputs[1 - i]);
return Amode::imm_reg(final_offset as u32, base);
return Amode::imm_reg(final_offset as u32, base).with_flags(flags);
}
}
}
@@ -654,11 +658,11 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i
)
};
return Amode::imm_reg_reg_shift(offset as u32, base, index, shift);
return Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags);
}
let input = put_input_in_reg(ctx, spec);
Amode::imm_reg(offset as u32, input)
Amode::imm_reg(offset as u32, input).with_flags(flags)
}
//=============================================================================
@@ -3060,7 +3064,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let base = put_input_in_reg(ctx, inputs[0]);
let index = put_input_in_reg(ctx, inputs[1]);
let shift = 0;
Amode::imm_reg_reg_shift(offset as u32, base, index, shift)
let flags = ctx.memflags(insn).expect("load should have memflags");
Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags)
}
_ => unreachable!(),
@@ -3132,7 +3137,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let base = put_input_in_reg(ctx, inputs[1]);
let index = put_input_in_reg(ctx, inputs[2]);
let shift = 0;
Amode::imm_reg_reg_shift(offset as u32, base, index, shift)
let flags = ctx.memflags(insn).expect("store should have memflags");
Amode::imm_reg_reg_shift(offset as u32, base, index, shift).with_flags(flags)
}
_ => unreachable!(),