T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cranelift: update to latest regalloc2: (#4324)

Browse Source 
- Handle call instructions' clobbers with the clobbers API, using RA2's
  clobbers bitmask (bytecodealliance/regalloc2#58) rather than clobbers
  list;

- Pull in changes from bytecodealliance/regalloc2#59 for much more sane
  edge-case behavior w.r.t. liverange splitting.
This commit is contained in:
Chris Fallin
2022-06-28 09:01:59 -07:00
committed by GitHub
parent 66b829b1bf
commit b2e28b917a
21 changed files with 402 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -14,7 +14,7 @@ use crate::settings;
use crate::{CodegenError, CodegenResult};
use alloc::boxed::Box;
use alloc::vec::Vec;
use regalloc2::VReg;
use regalloc2::{PRegSet, VReg};
use smallvec::{smallvec, SmallVec};
// We use a generic implementation that factors out AArch64 and x64 ABI commonalities, because
@@ -1062,8 +1062,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn gen_call(
dest: &CallDest,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
clobbers: PRegSet,
opcode: ir::Opcode,
tmp: Writable<Reg>,
callee_conv: isa::CallConv,
@@ -1076,6 +1077,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
dest: name.clone(),
uses,
defs,
clobbers,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
@@ -1092,6 +1094,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
rn: tmp.to_reg(),
uses,
defs,
clobbers,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
@@ -1103,6 +1106,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
rn: *reg,
uses,
defs,
clobbers,
opcode,
caller_callconv: caller_conv,
callee_callconv: callee_conv,
@@ -1131,8 +1135,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
insts.push(Inst::Call {
info: Box::new(CallInfo {
dest: ExternalName::LibCall(LibCall::Memcpy),
uses: vec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()],
defs: Self::get_regs_clobbered_by_call(call_conv),
uses: smallvec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()],
defs: smallvec![],
clobbers: Self::get_regs_clobbered_by_call(call_conv),
opcode: Opcode::Call,
caller_callconv: call_conv,
callee_callconv: call_conv,
@@ -1159,21 +1164,19 @@ impl ABIMachineSpec for AArch64MachineDeps {
s.nominal_sp_to_fp
}
fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> Vec<Writable<Reg>> {
let mut caller_saved = Vec::new();
for i in 0..29 {
let x = writable_xreg(i);
if is_reg_clobbered_by_call(call_conv_of_callee, x.to_reg().to_real_reg().unwrap()) {
caller_saved.push(x);
fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet {
let mut clobbers = DEFAULT_AAPCS_CLOBBERS;
if call_conv_of_callee.extends_baldrdash() {
// Every X-register except for x16, x17, x18 is
// caller-saved (clobbered by a call).
for i in 19..=28 {
clobbers.add(xreg_preg(i));
}
clobbers.add(vreg_preg(31));
}
for i in 0..32 {
let v = writable_vreg(i);
if is_reg_clobbered_by_call(call_conv_of_callee, v.to_reg().to_real_reg().unwrap()) {
caller_saved.push(v);
}
}
caller_saved
clobbers
}
fn get_ext_mode(
@@ -1290,47 +1293,74 @@ fn get_regs_restored_in_epilogue(
(int_saves, vec_saves)
}
fn is_reg_clobbered_by_call(call_conv_of_callee: isa::CallConv, r: RealReg) -> bool {
if call_conv_of_callee.extends_baldrdash() {
match r.class() {
RegClass::Int => {
let enc = r.hw_enc() & 31;
if !BALDRDASH_JIT_CALLEE_SAVED_GPR[enc as usize] {
return true;
}
// Otherwise, fall through to preserve native's ABI caller-saved.
}
RegClass::Float => {
let enc = r.hw_enc() & 31;
if !BALDRDASH_JIT_CALLEE_SAVED_FPU[enc as usize] {
return true;
}
// Otherwise, fall through to preserve native's ABI caller-saved.
}
};
}
match r.class() {
RegClass::Int => {
// x0 - x17 inclusive are caller-saves.
r.hw_enc() <= 17
}
RegClass::Float => {
// v0 - v7 inclusive and v16 - v31 inclusive are caller-saves. The
// upper 64 bits of v8 - v15 inclusive are also caller-saves.
// However, because we cannot currently represent partial registers
// to regalloc.rs, we indicate here that every vector register is
// caller-save. Because this function is used at *callsites*,
// approximating in this direction (save more than necessary) is
// conservative and thus safe.
//
// Note that we set the 'not included in clobber set' flag in the
// regalloc.rs API when a call instruction's callee has the same ABI
// as the caller (the current function body); this is safe (anything
// clobbered by callee can be clobbered by caller as well) and
// avoids unnecessary saves of v8-v15 in the prologue even though we
// include them as defs here.
true
}
}
const fn default_aapcs_clobbers() -> PRegSet {
PRegSet::empty()
// x0 - x17 inclusive are caller-saves.
.with(xreg_preg(0))
.with(xreg_preg(1))
.with(xreg_preg(2))
.with(xreg_preg(3))
.with(xreg_preg(4))
.with(xreg_preg(5))
.with(xreg_preg(6))
.with(xreg_preg(7))
.with(xreg_preg(8))
.with(xreg_preg(9))
.with(xreg_preg(10))
.with(xreg_preg(11))
.with(xreg_preg(12))
.with(xreg_preg(13))
.with(xreg_preg(14))
.with(xreg_preg(15))
.with(xreg_preg(16))
.with(xreg_preg(17))
// v0 - v7 inclusive and v16 - v31 inclusive are
// caller-saves. The upper 64 bits of v8 - v15 inclusive are
// also caller-saves. However, because we cannot currently
// represent partial registers to regalloc2, we indicate here
// that every vector register is caller-save. Because this
// function is used at *callsites*, approximating in this
// direction (save more than necessary) is conservative and
// thus safe.
//
// Note that we exclude clobbers from a call instruction when
// a call instruction's callee has the same ABI as the caller
// (the current function body); this is safe (anything
// clobbered by callee can be clobbered by caller as well) and
// avoids unnecessary saves of v8-v15 in the prologue even
// though we include them as defs here.
.with(vreg_preg(0))
.with(vreg_preg(1))
.with(vreg_preg(2))
.with(vreg_preg(3))
.with(vreg_preg(4))
.with(vreg_preg(5))
.with(vreg_preg(6))
.with(vreg_preg(7))
.with(vreg_preg(8))
.with(vreg_preg(9))
.with(vreg_preg(10))
.with(vreg_preg(11))
.with(vreg_preg(12))
.with(vreg_preg(13))
.with(vreg_preg(14))
.with(vreg_preg(15))
.with(vreg_preg(16))
.with(vreg_preg(17))
.with(vreg_preg(18))
.with(vreg_preg(19))
.with(vreg_preg(20))
.with(vreg_preg(21))
.with(vreg_preg(22))
.with(vreg_preg(23))
.with(vreg_preg(24))
.with(vreg_preg(25))
.with(vreg_preg(26))
.with(vreg_preg(27))
.with(vreg_preg(28))
.with(vreg_preg(29))
.with(vreg_preg(30))
.with(vreg_preg(31))
}
const DEFAULT_AAPCS_CLOBBERS: PRegSet = default_aapcs_clobbers();

10
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -5286,8 +5286,9 @@ fn test_aarch64_binemit() {
Inst::Call {
info: Box::new(CallInfo {
dest: ExternalName::testcase("test0"),
uses: Vec::new(),
defs: Vec::new(),
uses: smallvec![],
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::Call,
caller_callconv: CallConv::SystemV,
callee_callconv: CallConv::SystemV,
@@ -5301,8 +5302,9 @@ fn test_aarch64_binemit() {
Inst::CallInd {
info: Box::new(CallIndInfo {
rn: xreg(10),
uses: Vec::new(),
defs: Vec::new(),
uses: smallvec![],
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::CallIndirect,
caller_callconv: CallConv::SystemV,
callee_callconv: CallConv::SystemV,

20
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -13,7 +13,7 @@ use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
use alloc::vec::Vec;
use core::convert::TryFrom;
use regalloc2::VReg;
use regalloc2::{PRegSet, VReg};
use smallvec::{smallvec, SmallVec};
use std::string::{String, ToString};
@@ -70,8 +70,9 @@ impl BitOp {
#[derive(Clone, Debug)]
pub struct CallInfo {
pub dest: ExternalName,
pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>,
pub uses: SmallVec<[Reg; 8]>,
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub clobbers: PRegSet,
pub opcode: Opcode,
pub caller_callconv: CallConv,
pub callee_callconv: CallConv,
@@ -82,8 +83,9 @@ pub struct CallInfo {
#[derive(Clone, Debug)]
pub struct CallIndInfo {
pub rn: Reg,
pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>,
pub uses: SmallVec<[Reg; 8]>,
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub clobbers: PRegSet,
pub opcode: Opcode,
pub caller_callconv: CallConv,
pub callee_callconv: CallConv,
@@ -983,11 +985,13 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
&Inst::Call { ref info, .. } => {
collector.reg_uses(&info.uses[..]);
collector.reg_defs(&info.defs[..]);
collector.reg_clobbers(info.clobbers);
}
&Inst::CallInd { ref info, .. } => {
collector.reg_use(info.rn);
collector.reg_uses(&info.uses[..]);
collector.reg_defs(&info.defs[..]);
collector.reg_clobbers(info.clobbers);
}
&Inst::CondBr { ref kind, .. } => match kind {
CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => {
@@ -1028,9 +1032,9 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
&Inst::VirtualSPOffsetAdj { .. } => {}
&Inst::ElfTlsGetAddr { .. } => {
for reg in AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV) {
collector.reg_def(reg);
}
collector.reg_clobbers(AArch64MachineDeps::get_regs_clobbered_by_call(
CallConv::SystemV,
));
}
&Inst::Unwind { .. } => {}
&Inst::EmitIsland { .. } => {}

16
cranelift/codegen/src/isa/aarch64/inst/regs.rs
View File

@@ -24,9 +24,13 @@ pub const PINNED_REG: u8 = 21;
/// Get a reference to an X-register (integer register). Do not use
/// this for xsp / xzr; we have two special registers for those.
pub fn xreg(num: u8) -> Reg {
Reg::from(xreg_preg(num))
}
/// Get the given X-register as a PReg.
pub(crate) const fn xreg_preg(num: u8) -> PReg {
assert!(num < 31);
let preg = PReg::new(num as usize, RegClass::Int);
Reg::from(VReg::new(preg.index(), RegClass::Int))
PReg::new(num as usize, RegClass::Int)
}
/// Get a writable reference to an X-register.
@@ -36,9 +40,13 @@ pub fn writable_xreg(num: u8) -> Writable<Reg> {
/// Get a reference to a V-register (vector/FP register).
pub fn vreg(num: u8) -> Reg {
Reg::from(vreg_preg(num))
}
/// Get the given V-register as a PReg.
pub(crate) const fn vreg_preg(num: u8) -> PReg {
assert!(num < 32);
let preg = PReg::new(num as usize, RegClass::Float);
Reg::from(VReg::new(preg.index(), RegClass::Float))
PReg::new(num as usize, RegClass::Float)
}
/// Get a writable reference to a V-register.

57
cranelift/codegen/src/isa/s390x/abi.rs
View File

@@ -71,8 +71,7 @@ use crate::settings;
use crate::{CodegenError, CodegenResult};
use alloc::boxed::Box;
use alloc::vec::Vec;
use regalloc2::PReg;
use regalloc2::VReg;
use regalloc2::{PReg, PRegSet, VReg};
use smallvec::{smallvec, SmallVec};
use std::convert::TryFrom;
@@ -618,8 +617,9 @@ impl ABIMachineSpec for S390xMachineDeps {
fn gen_call(
dest: &CallDest,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
clobbers: PRegSet,
opcode: ir::Opcode,
tmp: Writable<Reg>,
_callee_conv: isa::CallConv,
@@ -633,6 +633,7 @@ impl ABIMachineSpec for S390xMachineDeps {
dest: name.clone(),
uses,
defs,
clobbers,
opcode,
}),
}),
@@ -648,6 +649,7 @@ impl ABIMachineSpec for S390xMachineDeps {
rn: tmp.to_reg(),
uses,
defs,
clobbers,
opcode,
}),
});
@@ -658,6 +660,7 @@ impl ABIMachineSpec for S390xMachineDeps {
rn: *reg,
uses,
defs,
clobbers,
opcode,
}),
}),
@@ -693,21 +696,8 @@ impl ABIMachineSpec for S390xMachineDeps {
s.initial_sp_offset
}
fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> Vec<Writable<Reg>> {
let mut caller_saved = Vec::new();
for i in 0..15 {
let x = writable_gpr(i);
if is_reg_clobbered_by_call(call_conv_of_callee, x.to_reg().to_real_reg().unwrap()) {
caller_saved.push(x);
}
}
for i in 0..15 {
let v = writable_fpr(i);
if is_reg_clobbered_by_call(call_conv_of_callee, v.to_reg().to_real_reg().unwrap()) {
caller_saved.push(v);
}
}
caller_saved
fn get_regs_clobbered_by_call(_call_conv_of_callee: isa::CallConv) -> PRegSet {
CLOBBERS
}
fn get_ext_mode(
@@ -783,15 +773,22 @@ fn get_regs_saved_in_prologue(
(int_saves, fpr_saves)
}
fn is_reg_clobbered_by_call(_call_conv: isa::CallConv, r: RealReg) -> bool {
match r.class() {
RegClass::Int => {
// r0 - r5 inclusive are caller-saves.
r.hw_enc() <= 5
}
RegClass::Float => {
// f0 - f7 inclusive are caller-saves.
r.hw_enc() <= 7
}
}
const fn clobbers() -> PRegSet {
PRegSet::empty()
.with(gpr_preg(0))
.with(gpr_preg(1))
.with(gpr_preg(2))
.with(gpr_preg(3))
.with(gpr_preg(4))
.with(gpr_preg(5))
.with(fpr_preg(0))
.with(fpr_preg(1))
.with(fpr_preg(2))
.with(fpr_preg(3))
.with(fpr_preg(4))
.with(fpr_preg(5))
.with(fpr_preg(6))
.with(fpr_preg(7))
}
const CLOBBERS: PRegSet = clobbers();

10
cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
View File

@@ -6804,8 +6804,9 @@ fn test_s390x_binemit() {
link: writable_gpr(14),
info: Box::new(CallInfo {
dest: ExternalName::testcase("test0"),
uses: Vec::new(),
defs: Vec::new(),
uses: smallvec![],
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::Call,
}),
},
@@ -6818,8 +6819,9 @@ fn test_s390x_binemit() {
link: writable_gpr(14),
info: Box::new(CallIndInfo {
rn: gpr(1),
uses: Vec::new(),
defs: Vec::new(),
uses: smallvec![],
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::CallIndirect,
}),
},

14
cranelift/codegen/src/isa/s390x/inst/mod.rs
View File

@@ -7,7 +7,7 @@ use crate::{settings, CodegenError, CodegenResult};
use alloc::boxed::Box;
use alloc::vec::Vec;
use core::convert::TryFrom;
use regalloc2::VReg;
use regalloc2::{PRegSet, VReg};
use smallvec::{smallvec, SmallVec};
use std::string::{String, ToString};
pub mod regs;
@@ -36,8 +36,9 @@ pub use crate::isa::s390x::lower::isle::generated_code::{
#[derive(Clone, Debug)]
pub struct CallInfo {
pub dest: ExternalName,
pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>,
pub uses: SmallVec<[Reg; 8]>,
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub clobbers: PRegSet,
pub opcode: Opcode,
}
@@ -46,8 +47,9 @@ pub struct CallInfo {
#[derive(Clone, Debug)]
pub struct CallIndInfo {
pub rn: Reg,
pub uses: Vec<Reg>,
pub defs: Vec<Writable<Reg>>,
pub uses: SmallVec<[Reg; 8]>,
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub clobbers: PRegSet,
pub opcode: Opcode,
}
@@ -660,12 +662,14 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
collector.reg_def(link);
collector.reg_uses(&*info.uses);
collector.reg_defs(&*info.defs);
collector.reg_clobbers(info.clobbers);
}
&Inst::CallInd { link, ref info } => {
collector.reg_def(link);
collector.reg_use(info.rn);
collector.reg_uses(&*info.uses);
collector.reg_defs(&*info.defs);
collector.reg_clobbers(info.clobbers);
}
&Inst::Ret { link, ref rets } => {
collector.reg_use(link);

19
cranelift/codegen/src/isa/s390x/inst/regs.rs
View File

@@ -13,11 +13,15 @@ use crate::settings;
/// Get a reference to a GPR (integer register).
pub fn gpr(num: u8) -> Reg {
assert!(num < 16);
let preg = PReg::new(num as usize, RegClass::Int);
let preg = gpr_preg(num);
Reg::from(VReg::new(preg.index(), RegClass::Int))
}
pub(crate) const fn gpr_preg(num: u8) -> PReg {
assert!(num < 16);
PReg::new(num as usize, RegClass::Int)
}
/// Get a writable reference to a GPR.
pub fn writable_gpr(num: u8) -> Writable<Reg> {
Writable::from_reg(gpr(num))
@@ -25,12 +29,17 @@ pub fn writable_gpr(num: u8) -> Writable<Reg> {
/// Get a reference to a FPR (floating-point register).
pub fn fpr(num: u8) -> Reg {
assert!(num < 16);
let preg = PReg::new(num as usize, RegClass::Float);
let preg = fpr_preg(num);
Reg::from(VReg::new(preg.index(), RegClass::Float))
}
/// Get a writable reference to a V-register.
pub(crate) const fn fpr_preg(num: u8) -> PReg {
assert!(num < 16);
PReg::new(num as usize, RegClass::Float)
}
/// Get a writable reference to a FPR.
#[allow(dead_code)] // used by tests.
pub fn writable_fpr(num: u8) -> Writable<Reg> {
Writable::from_reg(fpr(num))
}

133
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -11,7 +11,7 @@ use crate::{CodegenError, CodegenResult};
use alloc::boxed::Box;
use alloc::vec::Vec;
use args::*;
use regalloc2::VReg;
use regalloc2::{PRegSet, VReg};
use smallvec::{smallvec, SmallVec};
use std::convert::TryFrom;
@@ -488,9 +488,12 @@ impl ABIMachineSpec for X64ABIMachineSpec {
));
insts.push(Inst::CallKnown {
dest: ExternalName::LibCall(LibCall::Probestack),
uses: vec![regs::rax()],
defs: vec![],
opcode: Opcode::Call,
info: Box::new(CallInfo {
uses: smallvec![regs::rax()],
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::Call,
}),
});
insts
}
@@ -633,8 +636,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
/// Generate a call instruction/sequence.
fn gen_call(
dest: &CallDest,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
clobbers: PRegSet,
opcode: ir::Opcode,
tmp: Writable<Reg>,
_callee_conv: isa::CallConv,
@@ -643,7 +647,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
let mut insts = SmallVec::new();
match dest {
&CallDest::ExtName(ref name, RelocDistance::Near) => {
insts.push(Inst::call_known(name.clone(), uses, defs, opcode));
insts.push(Inst::call_known(name.clone(), uses, defs, clobbers, opcode));
}
&CallDest::ExtName(ref name, RelocDistance::Far) => {
insts.push(Inst::LoadExtName {
@@ -655,11 +659,18 @@ impl ABIMachineSpec for X64ABIMachineSpec {
RegMem::reg(tmp.to_reg()),
uses,
defs,
clobbers,
opcode,
));
}
&CallDest::Reg(reg) => {
insts.push(Inst::call_unknown(RegMem::reg(reg), uses, defs, opcode));
insts.push(Inst::call_unknown(
RegMem::reg(reg),
uses,
defs,
clobbers,
opcode,
));
}
}
insts
@@ -703,8 +714,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
});
insts.push(Inst::call_unknown(
RegMem::reg(memcpy_addr),
/* uses = */ vec![arg0, arg1, arg2],
/* defs = */ Self::get_regs_clobbered_by_call(call_conv),
/* uses = */ smallvec![arg0, arg1, arg2],
/* defs = */ smallvec![],
/* clobbers = */ Self::get_regs_clobbered_by_call(call_conv),
Opcode::Call,
));
insts
@@ -726,51 +738,21 @@ impl ABIMachineSpec for X64ABIMachineSpec {
s.nominal_sp_to_fp
}
fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> Vec<Writable<Reg>> {
let mut caller_saved = vec![
// intersection of Systemv and FastCall calling conventions:
// - GPR: all except RDI, RSI, RBX, RBP, R12 to R15.
// SysV adds RDI, RSI (FastCall makes these callee-saved).
Writable::from_reg(regs::rax()),
Writable::from_reg(regs::rcx()),
Writable::from_reg(regs::rdx()),
Writable::from_reg(regs::r8()),
Writable::from_reg(regs::r9()),
Writable::from_reg(regs::r10()),
Writable::from_reg(regs::r11()),
// - XMM: XMM0-5. SysV adds the rest (XMM6-XMM15).
Writable::from_reg(regs::xmm0()),
Writable::from_reg(regs::xmm1()),
Writable::from_reg(regs::xmm2()),
Writable::from_reg(regs::xmm3()),
Writable::from_reg(regs::xmm4()),
Writable::from_reg(regs::xmm5()),
];
if !call_conv_of_callee.extends_windows_fastcall() {
caller_saved.push(Writable::from_reg(regs::rsi()));
caller_saved.push(Writable::from_reg(regs::rdi()));
caller_saved.push(Writable::from_reg(regs::xmm6()));
caller_saved.push(Writable::from_reg(regs::xmm7()));
caller_saved.push(Writable::from_reg(regs::xmm8()));
caller_saved.push(Writable::from_reg(regs::xmm9()));
caller_saved.push(Writable::from_reg(regs::xmm10()));
caller_saved.push(Writable::from_reg(regs::xmm11()));
caller_saved.push(Writable::from_reg(regs::xmm12()));
caller_saved.push(Writable::from_reg(regs::xmm13()));
caller_saved.push(Writable::from_reg(regs::xmm14()));
caller_saved.push(Writable::from_reg(regs::xmm15()));
}
fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet {
let mut clobbers = if call_conv_of_callee.extends_windows_fastcall() {
WINDOWS_CLOBBERS
} else {
SYSV_CLOBBERS
};
if call_conv_of_callee.extends_baldrdash() {
caller_saved.push(Writable::from_reg(regs::r12()));
caller_saved.push(Writable::from_reg(regs::r13()));
// Not r14; implicitly preserved in the entry.
caller_saved.push(Writable::from_reg(regs::r15()));
caller_saved.push(Writable::from_reg(regs::rbx()));
clobbers.add(regs::gpr_preg(regs::ENC_R12));
clobbers.add(regs::gpr_preg(regs::ENC_R13));
clobbers.add(regs::gpr_preg(regs::ENC_R15));
clobbers.add(regs::gpr_preg(regs::ENC_RBX));
}
caller_saved
clobbers
}
fn get_ext_mode(
@@ -1032,3 +1014,52 @@ fn compute_clobber_size(clobbers: &[Writable<RealReg>]) -> u32 {
}
align_to(clobbered_size, 16)
}
const WINDOWS_CLOBBERS: PRegSet = windows_clobbers();
const SYSV_CLOBBERS: PRegSet = sysv_clobbers();
const fn windows_clobbers() -> PRegSet {
PRegSet::empty()
.with(regs::gpr_preg(regs::ENC_RAX))
.with(regs::gpr_preg(regs::ENC_RCX))
.with(regs::gpr_preg(regs::ENC_RDX))
.with(regs::gpr_preg(regs::ENC_R8))
.with(regs::gpr_preg(regs::ENC_R9))
.with(regs::gpr_preg(regs::ENC_R10))
.with(regs::gpr_preg(regs::ENC_R11))
.with(regs::fpr_preg(0))
.with(regs::fpr_preg(1))
.with(regs::fpr_preg(2))
.with(regs::fpr_preg(3))
.with(regs::fpr_preg(4))
.with(regs::fpr_preg(5))
}
const fn sysv_clobbers() -> PRegSet {
PRegSet::empty()
.with(regs::gpr_preg(regs::ENC_RAX))
.with(regs::gpr_preg(regs::ENC_RCX))
.with(regs::gpr_preg(regs::ENC_RDX))
.with(regs::gpr_preg(regs::ENC_RSI))
.with(regs::gpr_preg(regs::ENC_RDI))
.with(regs::gpr_preg(regs::ENC_R8))
.with(regs::gpr_preg(regs::ENC_R9))
.with(regs::gpr_preg(regs::ENC_R10))
.with(regs::gpr_preg(regs::ENC_R11))
.with(regs::fpr_preg(0))
.with(regs::fpr_preg(1))
.with(regs::fpr_preg(2))
.with(regs::fpr_preg(3))
.with(regs::fpr_preg(4))
.with(regs::fpr_preg(5))
.with(regs::fpr_preg(6))
.with(regs::fpr_preg(7))
.with(regs::fpr_preg(8))
.with(regs::fpr_preg(9))
.with(regs::fpr_preg(10))
.with(regs::fpr_preg(11))
.with(regs::fpr_preg(12))
.with(regs::fpr_preg(13))
.with(regs::fpr_preg(14))
.with(regs::fpr_preg(15))
}

10
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -315,15 +315,11 @@
;; Direct call: call simm32.
(CallKnown (dest ExternalName)
(uses VecReg)
(defs VecWritableReg)
(opcode Opcode))
(info BoxCallInfo))
;; Indirect call: callq (reg mem)
(CallUnknown (dest RegMem)
(uses VecReg)
(defs VecWritableReg)
(opcode Opcode))
(info BoxCallInfo))
;; Return.
(Ret (rets VecReg))
@@ -502,6 +498,8 @@
LFence
SFence))
(type BoxCallInfo extern (enum))
;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits.
(decl operand_size_of_type_32_64 (Type) OperandSize)
(extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64)

20
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -1202,7 +1202,11 @@ pub(crate) fn emit(
sink.put1(0x58 + (enc_dst & 7));
}
Inst::CallKnown { dest, opcode, .. } => {
Inst::CallKnown {
dest,
info: call_info,
..
} => {
if info.flags.enable_probestack() {
sink.add_trap(TrapCode::StackOverflow);
}
@@ -1214,12 +1218,16 @@ pub(crate) fn emit(
// beginning of the immediate field.
emit_reloc(sink, Reloc::X86CallPCRel4, &dest, -4);
sink.put4(0);
if opcode.is_call() {
sink.add_call_site(*opcode);
if call_info.opcode.is_call() {
sink.add_call_site(call_info.opcode);
}
}
Inst::CallUnknown { dest, opcode, .. } => {
Inst::CallUnknown {
dest,
info: call_info,
..
} => {
let dest = dest.with_allocs(allocs);
if info.flags.enable_probestack() {
@@ -1258,8 +1266,8 @@ pub(crate) fn emit(
if let Some(s) = state.take_stack_map() {
sink.add_stack_map(StackMapExtent::StartedAtOffset(start_offset), s);
}
if opcode.is_call() {
sink.add_call_site(*opcode);
if call_info.opcode.is_call() {
sink.add_call_site(call_info.opcode);
}
}

13
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

@@ -3572,8 +3572,9 @@ fn test_x64_emit() {
namespace: 0,
index: 0,
},
Vec::new(),
Vec::new(),
smallvec![],
smallvec![],
PRegSet::default(),
Opcode::Call,
),
"E800000000",
@@ -3583,7 +3584,13 @@ fn test_x64_emit() {
// ========================================================
// CallUnknown
fn call_unknown(rm: RegMem) -> Inst {
Inst::call_unknown(rm, Vec::new(), Vec::new(), Opcode::CallIndirect)
Inst::call_unknown(
rm,
smallvec![],
smallvec![],
PRegSet::default(),
Opcode::CallIndirect,
)
}
insns.push((call_unknown(RegMem::reg(rbp)), "FFD5", "call *%rbp"));

76
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -8,8 +8,9 @@ use crate::isa::x64::settings as x64_settings;
use crate::isa::CallConv;
use crate::machinst::*;
use crate::{settings, CodegenError, CodegenResult};
use alloc::boxed::Box;
use alloc::vec::Vec;
use regalloc2::{Allocation, VReg};
use regalloc2::{Allocation, PRegSet, VReg};
use smallvec::{smallvec, SmallVec};
use std::fmt;
use std::string::{String, ToString};
@@ -29,6 +30,19 @@ use args::*;
// `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here.
pub use super::lower::isle::generated_code::MInst as Inst;
// Out-of-line data for calls, to keep the size of `Inst` downn.
#[derive(Clone, Debug)]
pub struct CallInfo {
/// Register uses of this call.
pub uses: SmallVec<[Reg; 8]>,
/// Register defs of this call.
pub defs: SmallVec<[Writable<Reg>; 8]>,
/// Registers clobbered by this call, as per its calling convention.
pub clobbers: PRegSet,
/// The opcode of this call.
pub opcode: Opcode,
}
pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool {
let xs = x as i64;
xs == ((xs << 32) >> 32)
@@ -646,30 +660,38 @@ impl Inst {
pub(crate) fn call_known(
dest: ExternalName,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
clobbers: PRegSet,
opcode: Opcode,
) -> Inst {
Inst::CallKnown {
dest,
uses,
defs,
opcode,
info: Box::new(CallInfo {
uses,
defs,
clobbers,
opcode,
}),
}
}
pub(crate) fn call_unknown(
dest: RegMem,
uses: Vec<Reg>,
defs: Vec<Writable<Reg>>,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
clobbers: PRegSet,
opcode: Opcode,
) -> Inst {
dest.assert_regclass_is(RegClass::Int);
Inst::CallUnknown {
dest,
uses,
defs,
opcode,
info: Box::new(CallInfo {
uses,
defs,
clobbers,
opcode,
}),
}
}
@@ -1977,34 +1999,25 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_def(dst.to_writable_reg());
}
Inst::CallKnown {
ref uses, ref defs, ..
} => {
for &u in uses {
Inst::CallKnown { ref info, .. } => {
for &u in &info.uses {
collector.reg_use(u);
}
for &d in defs {
for &d in &info.defs {
collector.reg_def(d);
}
// FIXME: keep clobbers separate in the Inst and use
// `reg_clobber()`.
collector.reg_clobbers(info.clobbers);
}
Inst::CallUnknown {
ref uses,
ref defs,
dest,
..
} => {
Inst::CallUnknown { ref info, dest, .. } => {
dest.get_operands(collector);
for &u in uses {
for &u in &info.uses {
collector.reg_use(u);
}
for &d in defs {
for &d in &info.defs {
collector.reg_def(d);
}
// FIXME: keep clobbers separate in the Inst and use
// `reg_clobber()`.
collector.reg_clobbers(info.clobbers);
}
Inst::JmpTableSeq {
@@ -2076,10 +2089,9 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
// pseudoinstruction (and relocation that it emits) is specific to
// ELF systems; other x86-64 targets with other conventions (i.e.,
// Windows) use different TLS strategies.
for reg in X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV) {
// FIXME: use actual clobber functionality.
collector.reg_def(reg);
}
collector.reg_clobbers(X64ABIMachineSpec::get_regs_clobbered_by_call(
CallConv::SystemV,
));
}
Inst::Unwind { .. } => {}

11
cranelift/codegen/src/isa/x64/inst/regs.rs
View File

@@ -33,9 +33,12 @@ pub const ENC_R15: u8 = 15;
// Constructors for Regs.
fn gpr(enc: u8) -> Reg {
let preg = PReg::new(enc as usize, RegClass::Int);
let preg = gpr_preg(enc);
Reg::from(VReg::new(preg.index(), RegClass::Int))
}
pub(crate) const fn gpr_preg(enc: u8) -> PReg {
PReg::new(enc as usize, RegClass::Int)
}
pub(crate) fn rsi() -> Reg {
gpr(ENC_RSI)
@@ -96,10 +99,14 @@ pub(crate) fn pinned_reg() -> Reg {
}
fn fpr(enc: u8) -> Reg {
let preg = PReg::new(enc as usize, RegClass::Float);
let preg = fpr_preg(enc);
Reg::from(VReg::new(preg.index(), RegClass::Float))
}
pub(crate) const fn fpr_preg(enc: u8) -> PReg {
PReg::new(enc as usize, RegClass::Float)
}
pub(crate) fn xmm0() -> Reg {
fpr(0)
}

5
cranelift/codegen/src/isa/x64/lower/isle.rs
View File

@@ -18,7 +18,7 @@ use crate::{
settings::Flags,
unwind::UnwindInst,
x64::{
inst::{args::*, regs},
inst::{args::*, regs, CallInfo},
settings::Flags as IsaFlags,
},
},
@@ -26,8 +26,11 @@ use crate::{
isle::*, AtomicRmwOp, InsnInput, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData,
},
};
use std::boxed::Box;
use std::convert::TryFrom;
type BoxCallInfo = Box<CallInfo>;
pub struct SinkableLoad {
inst: Inst,
addr_input: InsnInput,