T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch Cranelift over to regalloc2. (#3989)

Browse Source 
This PR switches Cranelift over to the new register allocator, regalloc2.

See [this document](https://gist.github.com/cfallin/08553421a91f150254fe878f67301801)
for a summary of the design changes. This switchover has implications for
core VCode/MachInst types and the lowering pass.

Overall, this change brings improvements to both compile time and speed of
generated code (runtime), as reported in #3942:

```
Benchmark       Compilation (wallclock)     Execution (wallclock)
blake3-scalar   25% faster                  28% faster
blake3-simd     no diff                     no diff
meshoptimizer   19% faster                  17% faster
pulldown-cmark  17% faster                  no diff
bz2             15% faster                  no diff
SpiderMonkey,   21% faster                  2% faster
  fib(30)
clang.wasm      42% faster                  N/A
```
This commit is contained in:
Chris Fallin
2022-04-14 10:28:21 -07:00
committed by GitHub
parent bfae6384aa
commit a0318f36f0
181 changed files with 16887 additions and 21587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

255
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -14,7 +14,7 @@ use crate::settings;
use crate::{CodegenError, CodegenResult};
use alloc::boxed::Box;
use alloc::vec::Vec;
use regalloc::{RealReg, Reg, RegClass, Set, Writable};
use regalloc2::VReg;
use smallvec::{smallvec, SmallVec};
// We use a generic implementation that factors out AArch64 and x64 ABI commonalities, because
@@ -80,7 +80,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
&ir::ArgumentPurpose::VMContext => {
// This is SpiderMonkey's `WasmTlsReg`.
Some(ABIArg::reg(
xreg(BALDRDASH_TLS_REG).to_real_reg(),
xreg(BALDRDASH_TLS_REG).to_real_reg().unwrap(),
ir::types::I64,
param.extension,
param.purpose,
@@ -89,7 +89,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
&ir::ArgumentPurpose::SignatureId => {
// This is SpiderMonkey's `WasmTableCallSigReg`.
Some(ABIArg::reg(
xreg(BALDRDASH_SIG_REG).to_real_reg(),
xreg(BALDRDASH_SIG_REG).to_real_reg().unwrap(),
ir::types::I64,
param.extension,
param.purpose,
@@ -268,7 +268,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
let (rcs, reg_types) = Inst::rc_for_type(param.value_type)?;
if let Some(param) = try_fill_baldrdash_reg(call_conv, param) {
assert!(rcs[0] == RegClass::I64);
assert!(rcs[0] == RegClass::Int);
ret.push(param);
continue;
}
@@ -313,7 +313,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
"Unable to handle multi reg params with more than 2 regs"
);
assert!(
rcs == &[RegClass::I64, RegClass::I64],
rcs == &[RegClass::Int, RegClass::Int],
"Unable to handle non i64 regs"
);
@@ -335,12 +335,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
ret.push(ABIArg::Slots {
slots: vec![
ABIArgSlot::Reg {
reg: lower_reg.to_real_reg(),
reg: lower_reg.to_real_reg().unwrap(),
ty: param.value_type,
extension: param.extension,
},
ABIArgSlot::Reg {
reg: upper_reg.to_real_reg(),
reg: upper_reg.to_real_reg().unwrap(),
ty: param.value_type,
extension: param.extension,
},
@@ -356,19 +356,17 @@ impl ABIMachineSpec for AArch64MachineDeps {
// Single Register parameters
let rc = rcs[0];
let next_reg = match rc {
RegClass::I64 => &mut next_xreg,
RegClass::V128 => &mut next_vreg,
_ => panic!("Invalid register class: {:?}", rc),
RegClass::Int => &mut next_xreg,
RegClass::Float => &mut next_vreg,
};
if *next_reg < max_per_class_reg_vals && remaining_reg_vals > 0 {
let reg = match rc {
RegClass::I64 => xreg(*next_reg),
RegClass::V128 => vreg(*next_reg),
_ => unreachable!(),
RegClass::Int => xreg(*next_reg),
RegClass::Float => vreg(*next_reg),
};
ret.push(ABIArg::reg(
reg.to_real_reg(),
reg.to_real_reg().unwrap(),
param.value_type,
param.extension,
param.purpose,
@@ -435,7 +433,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
debug_assert!(args_or_rets == ArgsOrRets::Args);
if next_xreg < max_per_class_reg_vals && remaining_reg_vals > 0 {
ret.push(ABIArg::reg(
xreg(next_xreg).to_real_reg(),
xreg(next_xreg).to_real_reg().unwrap(),
I64,
ir::ArgumentExtension::None,
ir::ArgumentPurpose::Normal,
@@ -505,8 +503,8 @@ impl ABIMachineSpec for AArch64MachineDeps {
}
}
fn gen_ret() -> Inst {
Inst::Ret
fn gen_ret(rets: Vec<Reg>) -> Inst {
Inst::Ret { rets }
}
fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallInstVec<Inst> {
@@ -708,7 +706,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
call_conv: isa::CallConv,
setup_frame: bool,
flags: &settings::Flags,
clobbered_callee_saves: &Vec<Writable<RealReg>>,
clobbered_callee_saves: &[Writable<RealReg>],
fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> (u64, SmallVec<[Inst; 16]>) {
@@ -716,10 +714,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
let mut clobbered_vec = vec![];
for &reg in clobbered_callee_saves.iter() {
match reg.to_reg().get_class() {
RegClass::I64 => clobbered_int.push(reg),
RegClass::V128 => clobbered_vec.push(reg),
class => panic!("Unexpected RegClass: {:?}", class),
match reg.to_reg().class() {
RegClass::Int => clobbered_int.push(reg),
RegClass::Float => clobbered_vec.push(reg),
}
}
@@ -758,9 +755,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
let iter = clobbered_int.chunks_exact(2);
if let [rd] = iter.remainder() {
let rd = rd.to_reg().to_reg();
let rd: Reg = rd.to_reg().into();
debug_assert_eq!(rd.get_class(), RegClass::I64);
debug_assert_eq!(rd.class(), RegClass::Int);
// str rd, [sp, #-16]!
insts.push(Inst::Store64 {
rd,
@@ -776,7 +773,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset,
reg: rd.to_real_reg(),
reg: rd.to_real_reg().unwrap(),
},
});
}
@@ -785,12 +782,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
let mut iter = iter.rev();
while let Some([rt, rt2]) = iter.next() {
// .to_reg().to_reg(): Writable<RealReg> --> RealReg --> Reg
let rt = rt.to_reg().to_reg();
let rt2 = rt2.to_reg().to_reg();
// .to_reg().into(): Writable<RealReg> --> RealReg --> Reg
let rt: Reg = rt.to_reg().into();
let rt2: Reg = rt2.to_reg().into();
debug_assert!(rt.get_class() == RegClass::I64);
debug_assert!(rt2.get_class() == RegClass::I64);
debug_assert!(rt.class() == RegClass::Int);
debug_assert!(rt2.class() == RegClass::Int);
// stp rt, rt2, [sp, #-16]!
insts.push(Inst::StoreP64 {
@@ -808,13 +805,13 @@ impl ABIMachineSpec for AArch64MachineDeps {
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset,
reg: rt.to_real_reg(),
reg: rt.to_real_reg().unwrap(),
},
});
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset: clobber_offset + (clobber_offset_change / 2) as u32,
reg: rt2.to_real_reg(),
reg: rt2.to_real_reg().unwrap(),
},
});
}
@@ -844,9 +841,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
let iter = clobbered_vec.chunks_exact(2);
if let [rd] = iter.remainder() {
let rd = rd.to_reg().to_reg();
let rd: Reg = rd.to_reg().into();
debug_assert_eq!(rd.get_class(), RegClass::V128);
debug_assert_eq!(rd.class(), RegClass::Float);
insts.push(store_vec_reg(rd));
if flags.unwind_info() {
@@ -854,7 +851,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset,
reg: rd.to_real_reg(),
reg: rd.to_real_reg().unwrap(),
},
});
}
@@ -896,11 +893,11 @@ impl ABIMachineSpec for AArch64MachineDeps {
let mut iter = iter.rev();
while let Some([rt, rt2]) = iter.next() {
let rt = rt.to_reg().to_reg();
let rt2 = rt2.to_reg().to_reg();
let rt: Reg = rt.to_reg().into();
let rt2: Reg = rt2.to_reg().into();
debug_assert_eq!(rt.get_class(), RegClass::V128);
debug_assert_eq!(rt2.get_class(), RegClass::V128);
debug_assert_eq!(rt.class(), RegClass::Float);
debug_assert_eq!(rt2.class(), RegClass::Float);
let (inst, clobber_offset_change) = store_vec_reg_pair(rt, rt2);
@@ -911,13 +908,13 @@ impl ABIMachineSpec for AArch64MachineDeps {
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset,
reg: rt.to_real_reg(),
reg: rt.to_real_reg().unwrap(),
},
});
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset: clobber_offset + clobber_offset_change / 2,
reg: rt2.to_real_reg(),
reg: rt2.to_real_reg().unwrap(),
},
});
}
@@ -934,7 +931,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn gen_clobber_restore(
call_conv: isa::CallConv,
flags: &settings::Flags,
clobbers: &Set<Writable<RealReg>>,
clobbers: &[Writable<RealReg>],
fixed_frame_storage_size: u32,
_outgoing_args_size: u32,
) -> SmallVec<[Inst; 16]> {
@@ -994,31 +991,31 @@ impl ABIMachineSpec for AArch64MachineDeps {
let mut iter = clobbered_vec.chunks_exact(2);
while let Some([rt, rt2]) = iter.next() {
let rt = rt.map(|r| r.to_reg());
let rt2 = rt2.map(|r| r.to_reg());
let rt: Writable<Reg> = rt.map(|r| r.into());
let rt2: Writable<Reg> = rt2.map(|r| r.into());
debug_assert_eq!(rt.to_reg().get_class(), RegClass::V128);
debug_assert_eq!(rt2.to_reg().get_class(), RegClass::V128);
debug_assert_eq!(rt.to_reg().class(), RegClass::Float);
debug_assert_eq!(rt2.to_reg().class(), RegClass::Float);
insts.push(load_vec_reg_pair(rt, rt2));
}
debug_assert!(iter.remainder().len() <= 1);
if let [rd] = iter.remainder() {
let rd = rd.map(|r| r.to_reg());
let rd: Writable<Reg> = rd.map(|r| r.into());
debug_assert_eq!(rd.to_reg().get_class(), RegClass::V128);
debug_assert_eq!(rd.to_reg().class(), RegClass::Float);
insts.push(load_vec_reg(rd));
}
let mut iter = clobbered_int.chunks_exact(2);
while let Some([rt, rt2]) = iter.next() {
let rt = rt.map(|r| r.to_reg());
let rt2 = rt2.map(|r| r.to_reg());
let rt: Writable<Reg> = rt.map(|r| r.into());
let rt2: Writable<Reg> = rt2.map(|r| r.into());
debug_assert_eq!(rt.to_reg().get_class(), RegClass::I64);
debug_assert_eq!(rt2.to_reg().get_class(), RegClass::I64);
debug_assert_eq!(rt.to_reg().class(), RegClass::Int);
debug_assert_eq!(rt2.to_reg().class(), RegClass::Int);
// ldp rt, rt2, [sp], #16
insts.push(Inst::LoadP64 {
rt,
@@ -1034,9 +1031,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
debug_assert!(iter.remainder().len() <= 1);
if let [rd] = iter.remainder() {
let rd = rd.map(|r| r.to_reg());
let rd: Writable<Reg> = rd.map(|r| r.into());
debug_assert_eq!(rd.to_reg().get_class(), RegClass::I64);
debug_assert_eq!(rd.to_reg().class(), RegClass::Int);
// ldr rd, [sp], #16
insts.push(Inst::ULoad64 {
rd,
@@ -1069,58 +1066,46 @@ impl ABIMachineSpec for AArch64MachineDeps {
tmp: Writable<Reg>,
callee_conv: isa::CallConv,
caller_conv: isa::CallConv,
) -> SmallVec<[(InstIsSafepoint, Inst); 2]> {
) -> SmallVec<[Inst; 2]> {
let mut insts = SmallVec::new();
match &dest {
&CallDest::ExtName(ref name, RelocDistance::Near) => insts.push((
InstIsSafepoint::Yes,
Inst::Call {
info: Box::new(CallInfo {
dest: name.clone(),
uses,
defs,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
}),
},
)),
&CallDest::ExtName(ref name, RelocDistance::Near) => insts.push(Inst::Call {
info: Box::new(CallInfo {
dest: name.clone(),
uses,
defs,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
}),
}),
&CallDest::ExtName(ref name, RelocDistance::Far) => {
insts.push((
InstIsSafepoint::No,
Inst::LoadExtName {
rd: tmp,
name: Box::new(name.clone()),
offset: 0,
},
));
insts.push((
InstIsSafepoint::Yes,
Inst::CallInd {
info: Box::new(CallIndInfo {
rn: tmp.to_reg(),
uses,
defs,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
}),
},
));
}
&CallDest::Reg(reg) => insts.push((
InstIsSafepoint::Yes,
Inst::CallInd {
insts.push(Inst::LoadExtName {
rd: tmp,
name: Box::new(name.clone()),
offset: 0,
});
insts.push(Inst::CallInd {
info: Box::new(CallIndInfo {
rn: *reg,
rn: tmp.to_reg(),
uses,
defs,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
}),
},
)),
});
}
&CallDest::Reg(reg) => insts.push(Inst::CallInd {
info: Box::new(CallIndInfo {
rn: *reg,
uses,
defs,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
}),
}),
}
insts
@@ -1157,9 +1142,8 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn get_number_of_spillslots_for_value(rc: RegClass) -> u32 {
// We allocate in terms of 8-byte slots.
match rc {
RegClass::I64 => 1,
RegClass::V128 => 2,
_ => panic!("Unexpected register class!"),
RegClass::Int => 1,
RegClass::Float => 2,
}
}
@@ -1177,13 +1161,13 @@ impl ABIMachineSpec for AArch64MachineDeps {
let mut caller_saved = Vec::new();
for i in 0..29 {
let x = writable_xreg(i);
if is_reg_clobbered_by_call(call_conv_of_callee, x.to_reg().to_real_reg()) {
if is_reg_clobbered_by_call(call_conv_of_callee, x.to_reg().to_real_reg().unwrap()) {
caller_saved.push(x);
}
}
for i in 0..32 {
let v = writable_vreg(i);
if is_reg_clobbered_by_call(call_conv_of_callee, v.to_reg().to_real_reg()) {
if is_reg_clobbered_by_call(call_conv_of_callee, v.to_reg().to_real_reg().unwrap()) {
caller_saved.push(v);
}
}
@@ -1205,7 +1189,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn get_clobbered_callee_saves(
call_conv: isa::CallConv,
regs: &Set<Writable<RealReg>>,
regs: &[Writable<RealReg>],
) -> Vec<Writable<RealReg>> {
let mut regs: Vec<Writable<RealReg>> = regs
.iter()
@@ -1215,7 +1199,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
// Sort registers for deterministic code output. We can do an unstable
// sort because the registers will be unique (there are no dups).
regs.sort_unstable_by_key(|r| r.to_reg().get_index());
regs.sort_unstable_by_key(|r| VReg::from(r.to_reg()).vreg());
regs
}
@@ -1247,29 +1231,27 @@ fn legal_type_for_machine(ty: Type) -> bool {
/// callee-save?
fn is_reg_saved_in_prologue(call_conv: isa::CallConv, r: RealReg) -> bool {
if call_conv.extends_baldrdash() {
match r.get_class() {
RegClass::I64 => {
let enc = r.get_hw_encoding();
return BALDRDASH_JIT_CALLEE_SAVED_GPR[enc];
match r.class() {
RegClass::Int => {
let enc = r.hw_enc() & 31;
return BALDRDASH_JIT_CALLEE_SAVED_GPR[enc as usize];
}
RegClass::V128 => {
let enc = r.get_hw_encoding();
return BALDRDASH_JIT_CALLEE_SAVED_FPU[enc];
RegClass::Float => {
let enc = r.hw_enc() & 31;
return BALDRDASH_JIT_CALLEE_SAVED_FPU[enc as usize];
}
_ => unimplemented!("baldrdash callee saved on non-i64 reg classes"),
};
}
match r.get_class() {
RegClass::I64 => {
match r.class() {
RegClass::Int => {
// x19 - x28 inclusive are callee-saves.
r.get_hw_encoding() >= 19 && r.get_hw_encoding() <= 28
r.hw_enc() >= 19 && r.hw_enc() <= 28
}
RegClass::V128 => {
RegClass::Float => {
// v8 - v15 inclusive are callee-saves.
r.get_hw_encoding() >= 8 && r.get_hw_encoding() <= 15
r.hw_enc() >= 8 && r.hw_enc() <= 15
}
_ => panic!("Unexpected RegClass"),
}
}
@@ -1278,53 +1260,51 @@ fn is_reg_saved_in_prologue(call_conv: isa::CallConv, r: RealReg) -> bool {
/// written by the function's body.
fn get_regs_restored_in_epilogue(
call_conv: isa::CallConv,
regs: &Set<Writable<RealReg>>,
regs: &[Writable<RealReg>],
) -> (Vec<Writable<RealReg>>, Vec<Writable<RealReg>>) {
let mut int_saves = vec![];
let mut vec_saves = vec![];
for &reg in regs.iter() {
for &reg in regs {
if is_reg_saved_in_prologue(call_conv, reg.to_reg()) {
match reg.to_reg().get_class() {
RegClass::I64 => int_saves.push(reg),
RegClass::V128 => vec_saves.push(reg),
_ => panic!("Unexpected RegClass"),
match reg.to_reg().class() {
RegClass::Int => int_saves.push(reg),
RegClass::Float => vec_saves.push(reg),
}
}
}
// Sort registers for deterministic code output. We can do an unstable sort because the
// registers will be unique (there are no dups).
int_saves.sort_unstable_by_key(|r| r.to_reg().get_index());
vec_saves.sort_unstable_by_key(|r| r.to_reg().get_index());
int_saves.sort_unstable_by_key(|r| VReg::from(r.to_reg()).vreg());
vec_saves.sort_unstable_by_key(|r| VReg::from(r.to_reg()).vreg());
(int_saves, vec_saves)
}
fn is_reg_clobbered_by_call(call_conv_of_callee: isa::CallConv, r: RealReg) -> bool {
if call_conv_of_callee.extends_baldrdash() {
match r.get_class() {
RegClass::I64 => {
let enc = r.get_hw_encoding();
if !BALDRDASH_JIT_CALLEE_SAVED_GPR[enc] {
match r.class() {
RegClass::Int => {
let enc = r.hw_enc() & 31;
if !BALDRDASH_JIT_CALLEE_SAVED_GPR[enc as usize] {
return true;
}
// Otherwise, fall through to preserve native's ABI caller-saved.
}
RegClass::V128 => {
let enc = r.get_hw_encoding();
if !BALDRDASH_JIT_CALLEE_SAVED_FPU[enc] {
RegClass::Float => {
let enc = r.hw_enc() & 31;
if !BALDRDASH_JIT_CALLEE_SAVED_FPU[enc as usize] {
return true;
}
// Otherwise, fall through to preserve native's ABI caller-saved.
}
_ => unimplemented!("baldrdash callee saved on non-i64 reg classes"),
};
}
match r.get_class() {
RegClass::I64 => {
match r.class() {
RegClass::Int => {
// x0 - x17 inclusive are caller-saves.
r.get_hw_encoding() <= 17
r.hw_enc() <= 17
}
RegClass::V128 => {
RegClass::Float => {
// v0 - v7 inclusive and v16 - v31 inclusive are caller-saves. The
// upper 64 bits of v8 - v15 inclusive are also caller-saves.
// However, because we cannot currently represent partial registers
@@ -1341,6 +1321,5 @@ fn is_reg_clobbered_by_call(call_conv_of_callee: isa::CallConv, r: RealReg) -> b
// include them as defs here.
true
}
_ => panic!("Unexpected RegClass"),
}
}

1511
cranelift/codegen/src/isa/aarch64/inst.isle
View File

File diff suppressed because it is too large Load Diff

155
cranelift/codegen/src/isa/aarch64/inst/args.rs
View File

@@ -6,10 +6,7 @@
use crate::ir::types::*;
use crate::ir::Type;
use crate::isa::aarch64::inst::*;
use crate::machinst::{ty_bits, MachLabel};
use regalloc::{PrettyPrint, RealRegUniverse, Reg, Writable};
use crate::machinst::{ty_bits, MachLabel, PrettyPrint, Reg, Writable};
use core::convert::Into;
use std::string::String;
@@ -222,6 +219,29 @@ impl AMode {
_ => None,
}
}
pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
// This should match `memarg_operands()`.
match self {
&AMode::Unscaled(reg, imm9) => AMode::Unscaled(allocs.next(reg), imm9),
&AMode::UnsignedOffset(r, uimm12) => AMode::UnsignedOffset(allocs.next(r), uimm12),
&AMode::RegReg(r1, r2) => AMode::RegReg(allocs.next(r1), allocs.next(r2)),
&AMode::RegScaled(r1, r2, ty) => AMode::RegScaled(allocs.next(r1), allocs.next(r2), ty),
&AMode::RegScaledExtended(r1, r2, ty, ext) => {
AMode::RegScaledExtended(allocs.next(r1), allocs.next(r2), ty, ext)
}
&AMode::RegExtended(r1, r2, ext) => {
AMode::RegExtended(allocs.next(r1), allocs.next(r2), ext)
}
&AMode::PreIndexed(reg, simm9) => AMode::PreIndexed(allocs.next_writable(reg), simm9),
&AMode::PostIndexed(reg, simm9) => AMode::PostIndexed(allocs.next_writable(reg), simm9),
&AMode::RegOffset(r, off, ty) => AMode::RegOffset(allocs.next(r), off, ty),
&AMode::FPOffset(..)
| &AMode::SPOffset(..)
| &AMode::NominalSPOffset(..)
| AMode::Label(..) => self.clone(),
}
}
}
/// A memory argument to a load/store-pair.
@@ -232,6 +252,23 @@ pub enum PairAMode {
PostIndexed(Writable<Reg>, SImm7Scaled),
}
impl PairAMode {
pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
// Should match `pairmemarg_operands()`.
match self {
&PairAMode::SignedOffset(reg, simm7scaled) => {
PairAMode::SignedOffset(allocs.next(reg), simm7scaled)
}
&PairAMode::PreIndexed(reg, simm7scaled) => {
PairAMode::PreIndexed(allocs.next_writable(reg), simm7scaled)
}
&PairAMode::PostIndexed(reg, simm7scaled) => {
PairAMode::PostIndexed(allocs.next_writable(reg), simm7scaled)
}
}
}
}
//=============================================================================
// Instruction sub-components (conditions, branches and branch targets):
// definitions
@@ -362,19 +399,19 @@ impl BranchTarget {
}
impl PrettyPrint for ShiftOpAndAmt {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("{:?} {}", self.op(), self.amt().value())
}
}
impl PrettyPrint for ExtendOp {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("{:?}", self)
}
}
impl PrettyPrint for MemLabel {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
match self {
&MemLabel::PCRel(off) => format!("pc+{}", off),
}
@@ -393,33 +430,36 @@ fn shift_for_type(ty: Type) -> usize {
}
impl PrettyPrint for AMode {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self {
&AMode::Unscaled(reg, simm9) => {
let reg = pretty_print_reg(reg, allocs);
if simm9.value != 0 {
format!("[{}, {}]", reg.show_rru(mb_rru), simm9.show_rru(mb_rru))
let simm9 = simm9.pretty_print(8, allocs);
format!("[{}, {}]", reg, simm9)
} else {
format!("[{}]", reg.show_rru(mb_rru))
format!("[{}]", reg)
}
}
&AMode::UnsignedOffset(reg, uimm12) => {
let reg = pretty_print_reg(reg, allocs);
if uimm12.value != 0 {
format!("[{}, {}]", reg.show_rru(mb_rru), uimm12.show_rru(mb_rru))
let uimm12 = uimm12.pretty_print(8, allocs);
format!("[{}, {}]", reg, uimm12)
} else {
format!("[{}]", reg.show_rru(mb_rru))
format!("[{}]", reg)
}
}
&AMode::RegReg(r1, r2) => {
format!("[{}, {}]", r1.show_rru(mb_rru), r2.show_rru(mb_rru),)
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_reg(r2, allocs);
format!("[{}, {}]", r1, r2)
}
&AMode::RegScaled(r1, r2, ty) => {
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_reg(r2, allocs);
let shift = shift_for_type(ty);
format!(
"[{}, {}, LSL #{}]",
r1.show_rru(mb_rru),
r2.show_rru(mb_rru),
shift,
)
format!("[{}, {}, LSL #{}]", r1, r2, shift)
}
&AMode::RegScaledExtended(r1, r2, ty, op) => {
let shift = shift_for_type(ty);
@@ -427,39 +467,32 @@ impl PrettyPrint for AMode {
ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32,
_ => OperandSize::Size64,
};
let op = op.show_rru(mb_rru);
format!(
"[{}, {}, {} #{}]",
r1.show_rru(mb_rru),
show_ireg_sized(r2, mb_rru, size),
op,
shift
)
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_ireg(r2, size, allocs);
let op = op.pretty_print(0, allocs);
format!("[{}, {}, {} #{}]", r1, r2, op, shift)
}
&AMode::RegExtended(r1, r2, op) => {
let size = match op {
ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32,
_ => OperandSize::Size64,
};
let op = op.show_rru(mb_rru);
format!(
"[{}, {}, {}]",
r1.show_rru(mb_rru),
show_ireg_sized(r2, mb_rru, size),
op,
)
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_ireg(r2, size, allocs);
let op = op.pretty_print(0, allocs);
format!("[{}, {}, {}]", r1, r2, op)
}
&AMode::Label(ref label) => label.pretty_print(0, allocs),
&AMode::PreIndexed(r, simm9) => {
let r = pretty_print_reg(r.to_reg(), allocs);
let simm9 = simm9.pretty_print(8, allocs);
format!("[{}, {}]!", r, simm9)
}
&AMode::PostIndexed(r, simm9) => {
let r = pretty_print_reg(r.to_reg(), allocs);
let simm9 = simm9.pretty_print(8, allocs);
format!("[{}], {}", r, simm9)
}
&AMode::Label(ref label) => label.show_rru(mb_rru),
&AMode::PreIndexed(r, simm9) => format!(
"[{}, {}]!",
r.to_reg().show_rru(mb_rru),
simm9.show_rru(mb_rru)
),
&AMode::PostIndexed(r, simm9) => format!(
"[{}], {}",
r.to_reg().show_rru(mb_rru),
simm9.show_rru(mb_rru)
),
// Eliminated by `mem_finalize()`.
&AMode::SPOffset(..)
| &AMode::FPOffset(..)
@@ -472,31 +505,33 @@ impl PrettyPrint for AMode {
}
impl PrettyPrint for PairAMode {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self {
&PairAMode::SignedOffset(reg, simm7) => {
let reg = pretty_print_reg(reg, allocs);
if simm7.value != 0 {
format!("[{}, {}]", reg.show_rru(mb_rru), simm7.show_rru(mb_rru))
let simm7 = simm7.pretty_print(8, allocs);
format!("[{}, {}]", reg, simm7)
} else {
format!("[{}]", reg.show_rru(mb_rru))
format!("[{}]", reg)
}
}
&PairAMode::PreIndexed(reg, simm7) => format!(
"[{}, {}]!",
reg.to_reg().show_rru(mb_rru),
simm7.show_rru(mb_rru)
),
&PairAMode::PostIndexed(reg, simm7) => format!(
"[{}], {}",
reg.to_reg().show_rru(mb_rru),
simm7.show_rru(mb_rru)
),
&PairAMode::PreIndexed(reg, simm7) => {
let reg = pretty_print_reg(reg.to_reg(), allocs);
let simm7 = simm7.pretty_print(8, allocs);
format!("[{}, {}]!", reg, simm7)
}
&PairAMode::PostIndexed(reg, simm7) => {
let reg = pretty_print_reg(reg.to_reg(), allocs);
let simm7 = simm7.pretty_print(8, allocs);
format!("[{}], {}", reg, simm7)
}
}
}
}
impl PrettyPrint for Cond {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
let mut s = format!("{:?}", self);
s.make_ascii_lowercase();
s
@@ -504,7 +539,7 @@ impl PrettyPrint for Cond {
}
impl PrettyPrint for BranchTarget {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
match self {
&BranchTarget::Label(label) => format!("label{:?}", label.get()),
&BranchTarget::ResolvedOffset(off) => format!("{}", off),

498
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

File diff suppressed because it is too large Load Diff

10
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -27,7 +27,7 @@ fn test_aarch64_binemit() {
// Then:
//
// $ echo "mov x1, x2" | aarch64inst.sh
insns.push((Inst::Ret, "C0035FD6", "ret"));
insns.push((Inst::Ret { rets: vec![] }, "C0035FD6", "ret"));
insns.push((Inst::Nop0, "", "nop-zero-len"));
insns.push((Inst::Nop4, "1F2003D5", "nop"));
insns.push((
@@ -1631,7 +1631,7 @@ fn test_aarch64_binemit() {
flags: MemFlags::trusted(),
},
"E18040F8",
"ldur x1, [x7, #8]",
"ldr x1, [x7, #8]",
));
insns.push((
@@ -6794,7 +6794,6 @@ fn test_aarch64_binemit() {
insns.push((Inst::Fence {}, "BF3B03D5", "dmb ish"));
let flags = settings::Flags::new(settings::builder());
let rru = create_reg_universe(&flags);
let emit_info = EmitInfo::new(flags);
for (insn, expected_encoding, expected_printing) in insns {
println!(
@@ -6803,11 +6802,12 @@ fn test_aarch64_binemit() {
);
// Check the printed text is as expected.
let actual_printing = insn.show_rru(Some(&rru));
let actual_printing =
insn.print_with_state(&mut EmitState::default(), &mut AllocationConsumer::new(&[]));
assert_eq!(expected_printing, actual_printing);
let mut buffer = MachBuffer::new();
insn.emit(&mut buffer, &emit_info, &mut Default::default());
insn.emit(&[], &mut buffer, &emit_info, &mut Default::default());
let buffer = buffer.finish();
let actual_encoding = &buffer.stringify_code_bytes();
assert_eq!(expected_encoding, actual_encoding);

29
cranelift/codegen/src/isa/aarch64/inst/imms.rs
View File

@@ -5,8 +5,7 @@
use crate::ir::types::*;
use crate::ir::Type;
use crate::isa::aarch64::inst::{OperandSize, ScalarSize};
use regalloc::{PrettyPrint, RealRegUniverse};
use crate::machinst::{AllocationConsumer, PrettyPrint};
use core::convert::TryFrom;
use std::string::String;
@@ -871,7 +870,7 @@ impl ASIMDFPModImm {
}
impl PrettyPrint for NZCV {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
let fmt = |c: char, v| if v { c.to_ascii_uppercase() } else { c };
format!(
"#{}{}{}{}",
@@ -884,13 +883,13 @@ impl PrettyPrint for NZCV {
}
impl PrettyPrint for UImm5 {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.value)
}
}
impl PrettyPrint for Imm12 {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
let shift = if self.shift12 { 12 } else { 0 };
let value = u32::from(self.bits) << shift;
format!("#{}", value)
@@ -898,49 +897,49 @@ impl PrettyPrint for Imm12 {
}
impl PrettyPrint for SImm7Scaled {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.value)
}
}
impl PrettyPrint for FPULeftShiftImm {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.amount)
}
}
impl PrettyPrint for FPURightShiftImm {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.amount)
}
}
impl PrettyPrint for SImm9 {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.value)
}
}
impl PrettyPrint for UImm12Scaled {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.value)
}
}
impl PrettyPrint for ImmLogic {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.value())
}
}
impl PrettyPrint for ImmShift {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
format!("#{}", self.imm)
}
}
impl PrettyPrint for MoveWideConst {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
if self.shift == 0 {
format!("#{}", self.bits)
} else {
@@ -950,7 +949,7 @@ impl PrettyPrint for MoveWideConst {
}
impl PrettyPrint for ASIMDMovModImm {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
if self.is_64bit {
debug_assert_eq!(self.shift, 0);
@@ -974,7 +973,7 @@ impl PrettyPrint for ASIMDMovModImm {
}
impl PrettyPrint for ASIMDFPModImm {
fn show_rru(&self, _mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
if self.is_64bit {
format!("#{}", f64::from_bits(Self::value64(self.imm)))
} else {

1956
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

File diff suppressed because it is too large Load Diff

431
cranelift/codegen/src/isa/aarch64/inst/regs.rs
View File

@@ -3,11 +3,13 @@
use crate::isa::aarch64::inst::OperandSize;
use crate::isa::aarch64::inst::ScalarSize;
use crate::isa::aarch64::inst::VectorSize;
use crate::machinst::AllocationConsumer;
use crate::machinst::RealReg;
use crate::machinst::{Reg, RegClass, Writable};
use crate::settings;
use regalloc::{
PrettyPrint, RealRegUniverse, Reg, RegClass, RegClassInfo, Writable, NUM_REG_CLASSES,
};
use regalloc2::MachineEnv;
use regalloc2::PReg;
use regalloc2::VReg;
use std::string::{String, ToString};
@@ -19,40 +21,12 @@ use std::string::{String, ToString};
/// https://searchfox.org/mozilla-central/source/js/src/jit/arm64/Assembler-arm64.h#103
pub const PINNED_REG: u8 = 21;
#[rustfmt::skip]
const XREG_INDICES: [u8; 31] = [
// X0 - X7
32, 33, 34, 35, 36, 37, 38, 39,
// X8 - X15
40, 41, 42, 43, 44, 45, 46, 47,
// X16, X17
58, 59,
// X18
60,
// X19, X20
48, 49,
// X21, put aside because it's the pinned register.
57,
// X22 - X28
50, 51, 52, 53, 54, 55, 56,
// X29 (FP)
61,
// X30 (LR)
62,
];
const ZERO_REG_INDEX: u8 = 63;
const SP_REG_INDEX: u8 = 64;
/// Get a reference to an X-register (integer register).
/// Get a reference to an X-register (integer register). Do not use
/// this for xsp / xzr; we have two special registers for those.
pub fn xreg(num: u8) -> Reg {
assert!(num < 31);
Reg::new_real(
RegClass::I64,
/* enc = */ num,
/* index = */ XREG_INDICES[num as usize],
)
let preg = PReg::new(num as usize, RegClass::Int);
Reg::from(VReg::new(preg.index(), RegClass::Int))
}
/// Get a writable reference to an X-register.
@@ -63,7 +37,8 @@ pub fn writable_xreg(num: u8) -> Writable<Reg> {
/// Get a reference to a V-register (vector/FP register).
pub fn vreg(num: u8) -> Reg {
assert!(num < 32);
Reg::new_real(RegClass::V128, /* enc = */ num, /* index = */ num)
let preg = PReg::new(num as usize, RegClass::Float);
Reg::from(VReg::new(preg.index(), RegClass::Float))
}
/// Get a writable reference to a V-register.
@@ -73,13 +48,8 @@ pub fn writable_vreg(num: u8) -> Writable<Reg> {
/// Get a reference to the zero-register.
pub fn zero_reg() -> Reg {
// This should be the same as what xreg(31) returns, except that
// we use the special index into the register index space.
Reg::new_real(
RegClass::I64,
/* enc = */ 31,
/* index = */ ZERO_REG_INDEX,
)
let preg = PReg::new(31, RegClass::Int);
Reg::from(VReg::new(preg.index(), RegClass::Int))
}
/// Get a writable reference to the zero-register (this discards a result).
@@ -89,16 +59,19 @@ pub fn writable_zero_reg() -> Writable<Reg> {
/// Get a reference to the stack-pointer register.
pub fn stack_reg() -> Reg {
// XSP (stack) and XZR (zero) are logically different registers which have
// the same hardware encoding, and whose meaning, in real aarch64
// instructions, is context-dependent. For convenience of
// universe-construction and for correct printing, we make them be two
// different real registers.
Reg::new_real(
RegClass::I64,
/* enc = */ 31,
/* index = */ SP_REG_INDEX,
)
// XSP (stack) and XZR (zero) are logically different registers
// which have the same hardware encoding, and whose meaning, in
// real aarch64 instructions, is context-dependent. For extra
// correctness assurances and for correct printing, we make them
// be two different real registers from a regalloc perspective.
//
// We represent XZR as if it were xreg(31); XSP is xreg(31 +
// 32). The PReg bit-packing allows 6 bits (64 registers) so we
// make use of this extra space to distinguish xzr and xsp. We
// mask off the 6th bit (hw_enc & 31) to get the actual hardware
// register encoding.
let preg = PReg::new(31 + 32, RegClass::Int);
Reg::from(VReg::new(preg.index(), RegClass::Int))
}
/// Get a writable reference to the stack-pointer register.
@@ -159,158 +132,193 @@ pub fn writable_tmp2_reg() -> Writable<Reg> {
}
/// Create the register universe for AArch64.
pub fn create_reg_universe(flags: &settings::Flags) -> RealRegUniverse {
let mut regs = vec![];
let mut allocable_by_class = [None; NUM_REG_CLASSES];
// Numbering Scheme: we put V-regs first, then X-regs. The X-regs exclude several registers:
// x18 (globally reserved for platform-specific purposes), x29 (frame pointer), x30 (link
// register), x31 (stack pointer or zero register, depending on context).
let v_reg_base = 0u8; // in contiguous real-register index space
let v_reg_count = 32;
for i in 0u8..v_reg_count {
let reg = Reg::new_real(
RegClass::V128,
/* enc = */ i,
/* index = */ v_reg_base + i,
)
.to_real_reg();
let name = format!("v{}", i);
regs.push((reg, name));
pub fn create_reg_env(flags: &settings::Flags) -> MachineEnv {
fn preg(r: Reg) -> PReg {
r.to_real_reg().unwrap().into()
}
let v_reg_last = v_reg_base + v_reg_count - 1;
// Add the X registers. N.B.: the order here must match the order implied
// by XREG_INDICES, ZERO_REG_INDEX, and SP_REG_INDEX above.
let x_reg_base = 32u8; // in contiguous real-register index space
let mut x_reg_count = 0;
let uses_pinned_reg = flags.enable_pinned_reg();
for i in 0u8..32u8 {
// See above for excluded registers.
if i == 16 || i == 17 || i == 18 || i == 29 || i == 30 || i == 31 || i == PINNED_REG {
continue;
}
let reg = Reg::new_real(
RegClass::I64,
/* enc = */ i,
/* index = */ x_reg_base + x_reg_count,
)
.to_real_reg();
let name = format!("x{}", i);
regs.push((reg, name));
x_reg_count += 1;
}
let x_reg_last = x_reg_base + x_reg_count - 1;
allocable_by_class[RegClass::I64.rc_to_usize()] = Some(RegClassInfo {
first: x_reg_base as usize,
last: x_reg_last as usize,
suggested_scratch: Some(XREG_INDICES[19] as usize),
});
allocable_by_class[RegClass::V128.rc_to_usize()] = Some(RegClassInfo {
first: v_reg_base as usize,
last: v_reg_last as usize,
suggested_scratch: Some(/* V31: */ 31),
});
// Other regs, not available to the allocator.
let allocable = if uses_pinned_reg {
// The pinned register is not allocatable in this case, so record the length before adding
// it.
let len = regs.len();
regs.push((xreg(PINNED_REG).to_real_reg(), "x21/pinned_reg".to_string()));
len
} else {
regs.push((xreg(PINNED_REG).to_real_reg(), "x21".to_string()));
regs.len()
let mut env = MachineEnv {
preferred_regs_by_class: [
vec![
preg(xreg(0)),
preg(xreg(1)),
preg(xreg(2)),
preg(xreg(3)),
preg(xreg(4)),
preg(xreg(5)),
preg(xreg(6)),
preg(xreg(7)),
preg(xreg(8)),
preg(xreg(9)),
preg(xreg(10)),
preg(xreg(11)),
preg(xreg(12)),
preg(xreg(13)),
preg(xreg(14)),
preg(xreg(15)),
// x16 and x17 are spilltmp and tmp2 (see above).
// x19-28 are callee-saved and so not preferred.
// x21 is the pinned register (if enabled) and not allocatable if so.
// x29 is FP, x30 is LR, x31 is SP/ZR.
],
vec![
preg(vreg(0)),
preg(vreg(1)),
preg(vreg(2)),
preg(vreg(3)),
preg(vreg(4)),
preg(vreg(5)),
preg(vreg(6)),
preg(vreg(7)),
preg(vreg(8)),
preg(vreg(9)),
preg(vreg(10)),
preg(vreg(11)),
preg(vreg(12)),
preg(vreg(13)),
preg(vreg(14)),
preg(vreg(15)),
],
],
non_preferred_regs_by_class: [
vec![
preg(xreg(19)),
preg(xreg(20)),
// x21 is pinned reg if enabled; we add to this list below if not.
preg(xreg(22)),
preg(xreg(23)),
preg(xreg(24)),
preg(xreg(25)),
preg(xreg(26)),
preg(xreg(27)),
preg(xreg(28)),
],
vec![
preg(vreg(16)),
preg(vreg(17)),
preg(vreg(18)),
preg(vreg(19)),
preg(vreg(20)),
preg(vreg(21)),
preg(vreg(22)),
preg(vreg(23)),
preg(vreg(24)),
preg(vreg(25)),
preg(vreg(26)),
preg(vreg(27)),
preg(vreg(28)),
preg(vreg(29)),
preg(vreg(30)),
// v31 is the scratch reg, to allow for parallel moves.
],
],
scratch_by_class: [
// We use tmp2 (x17) as the regalloc scratch register,
// used to resolve cyclic parallel moves. This is valid
// because tmp2 is never live between regalloc-visible
// instructions, only within them (i.e. in expansion into
// multiple machine instructions when that
// occurs). spilltmp is used for moves to/from spillslots,
// but tmp2 never is, so it is available for this
// purpose. (Its only other use is in prologue stack
// checks, and the prologue is prepended after regalloc
// runs.)
preg(tmp2_reg()),
// We use v31 for Float/Vec-class parallel moves.
preg(vreg(31)),
],
fixed_stack_slots: vec![],
};
regs.push((xreg(16).to_real_reg(), "x16".to_string()));
regs.push((xreg(17).to_real_reg(), "x17".to_string()));
regs.push((xreg(18).to_real_reg(), "x18".to_string()));
regs.push((fp_reg().to_real_reg(), "fp".to_string()));
regs.push((link_reg().to_real_reg(), "lr".to_string()));
regs.push((zero_reg().to_real_reg(), "xzr".to_string()));
regs.push((stack_reg().to_real_reg(), "sp".to_string()));
// FIXME JRS 2020Feb06: unfortunately this pushes the number of real regs
// to 65, which is potentially inconvenient from a compiler performance
// standpoint. We could possibly drop back to 64 by "losing" a vector
// register in future.
// Assert sanity: the indices in the register structs must match their
// actual indices in the array.
for (i, reg) in regs.iter().enumerate() {
assert_eq!(i, reg.0.get_index());
if !flags.enable_pinned_reg() {
debug_assert_eq!(PINNED_REG, 21); // We assumed this above in hardcoded reg list.
env.non_preferred_regs_by_class[0].push(preg(xreg(PINNED_REG)));
}
RealRegUniverse {
regs,
allocable,
allocable_by_class,
env
}
// PrettyPrint cannot be implemented for Reg; we need to invoke
// backend-specific functions from higher level (inst, arg, ...)
// types.
fn show_ireg(reg: RealReg) -> String {
match reg.hw_enc() {
29 => "fp".to_string(),
30 => "lr".to_string(),
31 => "xzr".to_string(),
63 => "sp".to_string(),
x => {
debug_assert!(x < 29);
format!("x{}", x)
}
}
}
/// If `ireg` denotes an I64-classed reg, make a best-effort attempt to show
fn show_vreg(reg: RealReg) -> String {
format!("v{}", reg.hw_enc() & 31)
}
fn show_reg(reg: Reg) -> String {
if let Some(rreg) = reg.to_real_reg() {
match rreg.class() {
RegClass::Int => show_ireg(rreg),
RegClass::Float => show_vreg(rreg),
}
} else {
format!("%{:?}", reg)
}
}
pub fn pretty_print_reg(reg: Reg, allocs: &mut AllocationConsumer<'_>) -> String {
let reg = allocs.next(reg);
show_reg(reg)
}
/// If `ireg` denotes an Int-classed reg, make a best-effort attempt to show
/// its name at the 32-bit size.
pub fn show_ireg_sized(reg: Reg, mb_rru: Option<&RealRegUniverse>, size: OperandSize) -> String {
let mut s = reg.show_rru(mb_rru);
if reg.get_class() != RegClass::I64 || !size.is32() {
pub fn show_ireg_sized(reg: Reg, size: OperandSize) -> String {
let mut s = show_reg(reg);
if reg.class() != RegClass::Int || !size.is32() {
// We can't do any better.
return s;
}
if reg.is_real() {
// Change (eg) "x42" into "w42" as appropriate
if reg.get_class() == RegClass::I64 && size.is32() && s.starts_with("x") {
s = "w".to_string() + &s[1..];
}
} else {
// Add a "w" suffix to RegClass::I64 vregs used in a 32-bit role
if reg.get_class() == RegClass::I64 && size.is32() {
s.push('w');
}
// Change (eg) "x42" into "w42" as appropriate
if reg.class() == RegClass::Int && size.is32() && s.starts_with("x") {
s = "w".to_string() + &s[1..];
}
s
}
/// Show a vector register used in a scalar context.
pub fn show_vreg_scalar(reg: Reg, mb_rru: Option<&RealRegUniverse>, size: ScalarSize) -> String {
let mut s = reg.show_rru(mb_rru);
if reg.get_class() != RegClass::V128 {
pub fn show_vreg_scalar(reg: Reg, size: ScalarSize) -> String {
let mut s = show_reg(reg);
if reg.class() != RegClass::Float {
// We can't do any better.
return s;
}
if reg.is_real() {
// Change (eg) "v0" into "d0".
if s.starts_with("v") {
let replacement = match size {
ScalarSize::Size8 => "b",
ScalarSize::Size16 => "h",
ScalarSize::Size32 => "s",
ScalarSize::Size64 => "d",
ScalarSize::Size128 => "q",
};
s.replace_range(0..1, replacement);
}
} else {
// Add a "d" suffix to RegClass::V128 vregs.
if reg.get_class() == RegClass::V128 {
s.push('d');
}
// Change (eg) "v0" into "d0".
if s.starts_with("v") {
let replacement = match size {
ScalarSize::Size8 => "b",
ScalarSize::Size16 => "h",
ScalarSize::Size32 => "s",
ScalarSize::Size64 => "d",
ScalarSize::Size128 => "q",
};
s.replace_range(0..1, replacement);
}
s
}
/// Show a vector register.
pub fn show_vreg_vector(reg: Reg, mb_rru: Option<&RealRegUniverse>, size: VectorSize) -> String {
assert_eq!(RegClass::V128, reg.get_class());
let mut s = reg.show_rru(mb_rru);
pub fn show_vreg_vector(reg: Reg, size: VectorSize) -> String {
assert_eq!(RegClass::Float, reg.class());
let mut s = show_reg(reg);
let suffix = match size {
VectorSize::Size8x8 => ".8b",
@@ -327,25 +335,54 @@ pub fn show_vreg_vector(reg: Reg, mb_rru: Option<&RealRegUniverse>, size: Vector
}
/// Show an indexed vector element.
pub fn show_vreg_element(
reg: Reg,
mb_rru: Option<&RealRegUniverse>,
idx: u8,
size: VectorSize,
) -> String {
assert_eq!(RegClass::V128, reg.get_class());
let mut s = reg.show_rru(mb_rru);
pub fn show_vreg_element(reg: Reg, idx: u8, size: VectorSize) -> String {
assert_eq!(RegClass::Float, reg.class());
let s = show_reg(reg);
let suffix = match size {
VectorSize::Size8x8 => "b",
VectorSize::Size8x16 => "b",
VectorSize::Size16x4 => "h",
VectorSize::Size16x8 => "h",
VectorSize::Size32x2 => "s",
VectorSize::Size32x4 => "s",
VectorSize::Size64x2 => "d",
VectorSize::Size8x8 => ".b",
VectorSize::Size8x16 => ".b",
VectorSize::Size16x4 => ".h",
VectorSize::Size16x8 => ".h",
VectorSize::Size32x2 => ".s",
VectorSize::Size32x4 => ".s",
VectorSize::Size64x2 => ".d",
};
s.push_str(&format!(".{}[{}]", suffix, idx));
s
format!("{}{}[{}]", s, suffix, idx)
}
pub fn pretty_print_ireg(
reg: Reg,
size: OperandSize,
allocs: &mut AllocationConsumer<'_>,
) -> String {
let reg = allocs.next(reg);
show_ireg_sized(reg, size)
}
pub fn pretty_print_vreg_scalar(
reg: Reg,
size: ScalarSize,
allocs: &mut AllocationConsumer<'_>,
) -> String {
let reg = allocs.next(reg);
show_vreg_scalar(reg, size)
}
pub fn pretty_print_vreg_vector(
reg: Reg,
size: VectorSize,
allocs: &mut AllocationConsumer<'_>,
) -> String {
let reg = allocs.next(reg);
show_vreg_vector(reg, size)
}
pub fn pretty_print_vreg_element(
reg: Reg,
idx: usize,
size: VectorSize,
allocs: &mut AllocationConsumer<'_>,
) -> String {
let reg = allocs.next(reg);
show_vreg_element(reg, idx as u8, size)
}

23
cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs
View File

@@ -2,8 +2,8 @@
use crate::isa::aarch64::inst::regs;
use crate::isa::unwind::systemv::RegisterMappingError;
use crate::machinst::{Reg, RegClass};
use gimli::{write::CommonInformationEntry, Encoding, Format, Register};
use regalloc::{Reg, RegClass};
/// Creates a new aarch64 common information entry (CIE).
pub fn create_cie() -> CommonInformationEntry {
@@ -17,11 +17,11 @@ pub fn create_cie() -> CommonInformationEntry {
},
4, // Code alignment factor
-8, // Data alignment factor
Register(regs::link_reg().get_hw_encoding().into()),
Register(regs::link_reg().to_real_reg().unwrap().hw_enc().into()),
);
// Every frame will start with the call frame address (CFA) at SP
let sp = Register(regs::stack_reg().get_hw_encoding().into());
let sp = Register((regs::stack_reg().to_real_reg().unwrap().hw_enc() & 31).into());
entry.add_instruction(CallFrameInstruction::Cfa(sp, 0));
entry
@@ -34,16 +34,15 @@ pub fn map_reg(reg: Reg) -> Result<Register, RegisterMappingError> {
// https://developer.arm.com/documentation/ihi0057/e/?lang=en#dwarf-register-names
//
// X0--X31 is 0--31; V0--V31 is 64--95.
match reg.get_class() {
RegClass::I64 => {
let reg = reg.get_hw_encoding() as u16;
match reg.class() {
RegClass::Int => {
let reg = (reg.to_real_reg().unwrap().hw_enc() & 31) as u16;
Ok(Register(reg))
}
RegClass::V128 => {
let reg = reg.get_hw_encoding() as u16;
RegClass::Float => {
let reg = reg.to_real_reg().unwrap().hw_enc() as u16;
Ok(Register(64 + reg))
}
_ => Err(RegisterMappingError::UnsupportedRegisterBank("class?")),
}
}
@@ -54,13 +53,13 @@ impl crate::isa::unwind::systemv::RegisterMapper<Reg> for RegisterMapper {
Ok(map_reg(reg)?.0)
}
fn sp(&self) -> u16 {
regs::stack_reg().get_hw_encoding().into()
(regs::stack_reg().to_real_reg().unwrap().hw_enc() & 31).into()
}
fn fp(&self) -> Option<u16> {
Some(regs::fp_reg().get_hw_encoding().into())
Some(regs::fp_reg().to_real_reg().unwrap().hw_enc().into())
}
fn lr(&self) -> Option<u16> {
Some(regs::link_reg().get_hw_encoding().into())
Some(regs::link_reg().to_real_reg().unwrap().hw_enc().into())
}
fn lr_offset(&self) -> Option<u32> {
Some(8)

15
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -7,21 +7,18 @@
//!
//! - Floating-point immediates (FIMM instruction).
use super::lower_inst;
use crate::data_value::DataValue;
use crate::ir::condcodes::{FloatCC, IntCC};
use crate::ir::types::*;
use crate::ir::Inst as IRInst;
use crate::ir::{Opcode, Type, Value};
use crate::machinst::lower::*;
use crate::machinst::*;
use crate::{CodegenError, CodegenResult};
use crate::isa::aarch64::inst::*;
use crate::isa::aarch64::AArch64Backend;
use super::lower_inst;
use crate::data_value::DataValue;
use regalloc::{Reg, Writable};
use crate::machinst::lower::*;
use crate::machinst::*;
use crate::machinst::{Reg, Writable};
use crate::{CodegenError, CodegenResult};
use smallvec::SmallVec;
use std::cmp;

21
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -17,9 +17,8 @@ use crate::{
binemit::CodeOffset,
ir::{
immediates::*, types::*, ExternalName, Inst, InstructionData, MemFlags, TrapCode, Value,
ValueLabel, ValueList,
ValueList,
},
isa::aarch64::inst::aarch64_map_regs,
isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
isa::unwind::UnwindInst,
machinst::{ty_bits, InsnOutput, LowerCtx},
@@ -45,15 +44,9 @@ pub(crate) fn lower<C>(
where
C: LowerCtx<I = MInst>,
{
lower_common(
lower_ctx,
flags,
isa_flags,
outputs,
inst,
|cx, insn| generated_code::constructor_lower(cx, insn),
aarch64_map_regs,
)
lower_common(lower_ctx, flags, isa_flags, outputs, inst, |cx, insn| {
generated_code::constructor_lower(cx, insn)
})
}
pub struct ExtendedValue {
@@ -200,11 +193,7 @@ where
}
fn emit(&mut self, inst: &MInst) -> Unit {
self.emitted_insts.push((inst.clone(), false));
}
fn emit_safepoint(&mut self, inst: &MInst) -> Unit {
self.emitted_insts.push((inst.clone(), true));
self.lower_ctx.emit(inst.clone());
}
fn cond_br_zero(&mut self, reg: Reg) -> CondBrKind {

4
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle c0751050a11e2686
src/isa/aarch64/inst.isle 19ccefb6a496d392
src/prelude.isle afd037c4d91c875c
src/isa/aarch64/inst.isle 544b7126192140d5
src/isa/aarch64/lower.isle d88b62dd6b40622

362
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

68
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -1,27 +1,22 @@
//! Lower a single Cranelift instruction into vcode.
use super::lower::*;
use crate::binemit::CodeOffset;
use crate::ir::condcodes::FloatCC;
use crate::ir::types::*;
use crate::ir::Inst as IRInst;
use crate::ir::{InstructionData, Opcode, TrapCode};
use crate::isa::aarch64::abi::*;
use crate::isa::aarch64::inst::*;
use crate::isa::aarch64::settings as aarch64_settings;
use crate::machinst::lower::*;
use crate::machinst::*;
use crate::settings::{Flags, TlsModel};
use crate::{CodegenError, CodegenResult};
use crate::isa::aarch64::abi::*;
use crate::isa::aarch64::inst::*;
use regalloc::Writable;
use alloc::boxed::Box;
use alloc::vec::Vec;
use core::convert::TryFrom;
use super::lower::*;
/// Actually codegen an instruction's results into registers.
pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx: &mut C,
@@ -766,7 +761,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::Trap | Opcode::ResumableTrap => {
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::Udf { trap_code });
ctx.emit(Inst::Udf { trap_code });
}
Opcode::Trapif | Opcode::Trapff => {
@@ -797,7 +792,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
cond
};
ctx.emit_safepoint(Inst::TrapIf {
ctx.emit(Inst::TrapIf {
trap_code,
kind: CondBrKind::Cond(cond),
});
@@ -1507,35 +1502,34 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let lane_type = ty.lane_type();
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let mut match_long_pair =
|ext_low_op, ext_high_op| -> Option<(VecRRPairLongOp, regalloc::Reg)> {
if let Some(lhs) = maybe_input_insn(ctx, inputs[0], ext_low_op) {
if let Some(rhs) = maybe_input_insn(ctx, inputs[1], ext_high_op) {
let lhs_inputs = insn_inputs(ctx, lhs);
let rhs_inputs = insn_inputs(ctx, rhs);
let low = put_input_in_reg(ctx, lhs_inputs[0], NarrowValueMode::None);
let high = put_input_in_reg(ctx, rhs_inputs[0], NarrowValueMode::None);
if low == high {
match (lane_type, ext_low_op) {
(I16, Opcode::SwidenLow) => {
return Some((VecRRPairLongOp::Saddlp8, low))
}
(I32, Opcode::SwidenLow) => {
return Some((VecRRPairLongOp::Saddlp16, low))
}
(I16, Opcode::UwidenLow) => {
return Some((VecRRPairLongOp::Uaddlp8, low))
}
(I32, Opcode::UwidenLow) => {
return Some((VecRRPairLongOp::Uaddlp16, low))
}
_ => (),
};
}
let mut match_long_pair = |ext_low_op, ext_high_op| -> Option<(VecRRPairLongOp, Reg)> {
if let Some(lhs) = maybe_input_insn(ctx, inputs[0], ext_low_op) {
if let Some(rhs) = maybe_input_insn(ctx, inputs[1], ext_high_op) {
let lhs_inputs = insn_inputs(ctx, lhs);
let rhs_inputs = insn_inputs(ctx, rhs);
let low = put_input_in_reg(ctx, lhs_inputs[0], NarrowValueMode::None);
let high = put_input_in_reg(ctx, rhs_inputs[0], NarrowValueMode::None);
if low == high {
match (lane_type, ext_low_op) {
(I16, Opcode::SwidenLow) => {
return Some((VecRRPairLongOp::Saddlp8, low))
}
(I32, Opcode::SwidenLow) => {
return Some((VecRRPairLongOp::Saddlp16, low))
}
(I16, Opcode::UwidenLow) => {
return Some((VecRRPairLongOp::Uaddlp8, low))
}
(I32, Opcode::UwidenLow) => {
return Some((VecRRPairLongOp::Uaddlp16, low))
}
_ => (),
};
}
}
None
};
}
None
};
if let Some((op, rn)) = match_long_pair(Opcode::SwidenLow, Opcode::SwidenHigh) {
ctx.emit(Inst::VecRRPairLong { op, rd, rn });

80
cranelift/codegen/src/isa/aarch64/mod.rs
View File

@@ -11,7 +11,7 @@ use crate::result::CodegenResult;
use crate::settings as shared_settings;
use alloc::{boxed::Box, vec::Vec};
use core::fmt;
use regalloc::{PrettyPrint, RealRegUniverse};
use regalloc2::MachineEnv;
use target_lexicon::{Aarch64Architecture, Architecture, Triple};
// New backend:
@@ -21,7 +21,7 @@ mod lower;
mod lower_inst;
mod settings;
use inst::create_reg_universe;
use inst::create_reg_env;
use self::inst::EmitInfo;
@@ -30,7 +30,7 @@ pub struct AArch64Backend {
triple: Triple,
flags: shared_settings::Flags,
isa_flags: aarch64_settings::Flags,
reg_universe: RealRegUniverse,
machine_env: MachineEnv,
}
impl AArch64Backend {
@@ -40,12 +40,12 @@ impl AArch64Backend {
flags: shared_settings::Flags,
isa_flags: aarch64_settings::Flags,
) -> AArch64Backend {
let reg_universe = create_reg_universe(&flags);
let machine_env = create_reg_env(&flags);
AArch64Backend {
triple,
flags,
isa_flags,
reg_universe,
machine_env,
}
}
@@ -55,10 +55,10 @@ impl AArch64Backend {
&self,
func: &Function,
flags: shared_settings::Flags,
) -> CodegenResult<VCode<inst::Inst>> {
) -> CodegenResult<(VCode<inst::Inst>, regalloc2::Output)> {
let emit_info = EmitInfo::new(flags.clone());
let abi = Box::new(abi::AArch64ABICallee::new(func, flags, self.isa_flags())?);
compile::compile::<AArch64Backend>(func, self, abi, &self.reg_universe, emit_info)
compile::compile::<AArch64Backend>(func, self, abi, &self.machine_env, emit_info)
}
}
@@ -69,28 +69,27 @@ impl TargetIsa for AArch64Backend {
want_disasm: bool,
) -> CodegenResult<MachCompileResult> {
let flags = self.flags();
let vcode = self.compile_vcode(func, flags.clone())?;
let (vcode, regalloc_result) = self.compile_vcode(func, flags.clone())?;
let (buffer, bb_starts, bb_edges) = vcode.emit();
let frame_size = vcode.frame_size();
let stackslot_offsets = vcode.stackslot_offsets().clone();
let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug);
let emit_result = vcode.emit(&regalloc_result, want_disasm, flags.machine_code_cfg_info());
let frame_size = emit_result.frame_size;
let value_labels_ranges = emit_result.value_labels_ranges;
let buffer = emit_result.buffer.finish();
let stackslot_offsets = emit_result.stackslot_offsets;
let disasm = if want_disasm {
Some(vcode.show_rru(Some(&create_reg_universe(flags))))
} else {
None
};
let buffer = buffer.finish();
if let Some(disasm) = emit_result.disasm.as_ref() {
log::debug!("disassembly:\n{}", disasm);
}
Ok(MachCompileResult {
buffer,
frame_size,
disasm,
value_labels_ranges: Default::default(),
disasm: emit_result.disasm,
value_labels_ranges,
stackslot_offsets,
bb_starts,
bb_edges,
bb_starts: emit_result.bb_offsets,
bb_edges: emit_result.bb_edges,
})
}
@@ -218,11 +217,11 @@ mod test {
let buffer = backend.compile_function(&mut func, false).unwrap().buffer;
let code = buffer.data();
// mov x1, #0x1234
// add w0, w0, w1
// mov x3, #0x1234
// add w0, w0, w3
// ret
let golden = vec![
0x81, 0x46, 0x82, 0xd2, 0x00, 0x00, 0x01, 0x0b, 0xc0, 0x03, 0x5f, 0xd6,
0x83, 0x46, 0x82, 0xd2, 0x00, 0x00, 0x03, 0x0b, 0xc0, 0x03, 0x5f, 0xd6,
];
assert_eq!(code, &golden[..]);
@@ -273,23 +272,24 @@ mod test {
.unwrap();
let code = result.buffer.data();
// mov x1, #0x1234 // #4660
// add w0, w0, w1
// mov w1, w0
// cbnz x1, 0x28
// mov x1, #0x1234 // #4660
// add w1, w0, w1
// mov w1, w1
// cbnz x1, 0x18
// mov w1, w0
// cbnz x1, 0x18
// mov x1, #0x1234 // #4660
// sub w0, w0, w1
// mov x10, #0x1234 // #4660
// add w12, w0, w10
// mov w11, w12
// cbnz x11, 0x20
// mov x13, #0x1234 // #4660
// add w15, w12, w13
// mov w14, w15
// cbnz x14, 0x10
// mov w1, w12
// cbnz x1, 0x10
// mov x2, #0x1234 // #4660
// sub w0, w12, w2
// ret
let golden = vec![
129, 70, 130, 210, 0, 0, 1, 11, 225, 3, 0, 42, 161, 0, 0, 181, 129, 70, 130, 210, 1, 0,
1, 11, 225, 3, 1, 42, 161, 255, 255, 181, 225, 3, 0, 42, 97, 255, 255, 181, 129, 70,
130, 210, 0, 0, 1, 75, 192, 3, 95, 214,
138, 70, 130, 210, 12, 0, 10, 11, 235, 3, 12, 42, 171, 0, 0, 181, 141, 70, 130, 210,
143, 1, 13, 11, 238, 3, 15, 42, 174, 255, 255, 181, 225, 3, 12, 42, 97, 255, 255, 181,
130, 70, 130, 210, 128, 1, 2, 75, 192, 3, 95, 214,
];
assert_eq!(code, &golden[..]);