T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cranelift: use regalloc2 constraints on caller side of ABI code. (#4892)

Browse Source 
* Cranelift: use regalloc2 constraints on caller side of ABI code.

This PR updates the shared ABI code and backends to use register-operand
constraints rather than explicit pinned-vreg moves for register
arguments and return values.

The s390x backend was not updated, because it has its own implementation
of ABI code. Ideally we could converge back to the code shared by x64
and aarch64 (which didn't exist when s390x ported calls to ISLE, so the
current situation is underestandable, to be clear!). I'll leave this for
future work.

This PR exposed several places where regalloc2 needed to be a bit more
flexible with constraints; it requires regalloc2#74 to be merged and
pulled in.

* Update to regalloc2 0.3.3.

In addition to version bump, this required removing two asserts as
`SpillSlot`s no longer carry their class (so we can't assert that they
have the correct class).

* Review comments.

* Filetest updates.

* Add cargo-vet audit for regalloc2 0.3.2 -> 0.3.3 upgrade.

* Update to regalloc2 0.4.0.
This commit is contained in:
Chris Fallin
2022-09-20 18:17:04 -07:00
committed by GitHub
parent 8b245178a5
commit 05cbd667c7
63 changed files with 1476 additions and 1177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -919,8 +919,8 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn gen_call(
dest: &CallDest,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
uses: CallArgList,
defs: CallRetList,
clobbers: PRegSet,
opcode: ir::Opcode,
tmp: Writable<Reg>,
@@ -978,19 +978,32 @@ impl ABIMachineSpec for AArch64MachineDeps {
call_conv: isa::CallConv,
dst: Reg,
src: Reg,
tmp: Writable<Reg>,
_tmp2: Writable<Reg>,
size: usize,
) -> SmallVec<[Self::I; 8]> {
let mut insts = SmallVec::new();
let arg0 = writable_xreg(0);
let arg1 = writable_xreg(1);
let arg2 = writable_xreg(2);
insts.push(Inst::gen_move(arg0, dst, I64));
insts.push(Inst::gen_move(arg1, src, I64));
insts.extend(Inst::load_constant(arg2, size as u64).into_iter());
insts.extend(Inst::load_constant(tmp, size as u64).into_iter());
insts.push(Inst::Call {
info: Box::new(CallInfo {
dest: ExternalName::LibCall(LibCall::Memcpy),
uses: smallvec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()],
uses: smallvec![
CallArgPair {
vreg: dst,
preg: arg0.to_reg()
},
CallArgPair {
vreg: src,
preg: arg1.to_reg()
},
CallArgPair {
vreg: tmp.to_reg(),
preg: arg2.to_reg()
}
],
defs: smallvec![],
clobbers: Self::get_regs_clobbered_by_call(call_conv),
opcode: Opcode::Call,

24
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -78,8 +78,8 @@ impl BitOp {
#[derive(Clone, Debug)]
pub struct CallInfo {
pub dest: ExternalName,
pub uses: SmallVec<[Reg; 8]>,
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub uses: CallArgList,
pub defs: CallRetList,
pub clobbers: PRegSet,
pub opcode: Opcode,
pub caller_callconv: CallConv,
@@ -91,8 +91,8 @@ pub struct CallInfo {
#[derive(Clone, Debug)]
pub struct CallIndInfo {
pub rn: Reg,
pub uses: SmallVec<[Reg; 8]>,
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub uses: SmallVec<[CallArgPair; 8]>,
pub defs: SmallVec<[CallRetPair; 8]>,
pub clobbers: PRegSet,
pub opcode: Opcode,
pub caller_callconv: CallConv,
@@ -1027,14 +1027,22 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
}
&Inst::Jump { .. } => {}
&Inst::Call { ref info, .. } => {
collector.reg_uses(&info.uses[..]);
collector.reg_defs(&info.defs[..]);
for u in &info.uses {
collector.reg_fixed_use(u.vreg, u.preg);
}
for d in &info.defs {
collector.reg_fixed_def(d.vreg, d.preg);
}
collector.reg_clobbers(info.clobbers);
}
&Inst::CallInd { ref info, .. } => {
collector.reg_use(info.rn);
collector.reg_uses(&info.uses[..]);
collector.reg_defs(&info.defs[..]);
for u in &info.uses {
collector.reg_fixed_use(u.vreg, u.preg);
}
for d in &info.defs {
collector.reg_fixed_def(d.vreg, d.preg);
}
collector.reg_clobbers(info.clobbers);
}
&Inst::CondBr { ref kind, .. } => match kind {

6
cranelift/codegen/src/isa/s390x/abi.rs
View File

@@ -743,8 +743,8 @@ impl ABIMachineSpec for S390xMachineDeps {
fn gen_call(
_dest: &CallDest,
_uses: SmallVec<[Reg; 8]>,
_defs: SmallVec<[Writable<Reg>; 8]>,
_uses: CallArgList,
_defs: CallRetList,
_clobbers: PRegSet,
_opcode: ir::Opcode,
_tmp: Writable<Reg>,
@@ -758,6 +758,8 @@ impl ABIMachineSpec for S390xMachineDeps {
_call_conv: isa::CallConv,
_dst: Reg,
_src: Reg,
_tmp1: Writable<Reg>,
_tmp2: Writable<Reg>,
_size: usize,
) -> SmallVec<[Self::I; 8]> {
unimplemented!("StructArgs not implemented for S390X yet");

44
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -429,7 +429,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
insts.push(Inst::CallKnown {
dest: ExternalName::LibCall(LibCall::Probestack),
info: Box::new(CallInfo {
uses: smallvec![regs::rax()],
// No need to include arg here: we are post-regalloc
// so no constraints will be seen anyway.
uses: smallvec![],
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::Call,
@@ -584,8 +586,8 @@ impl ABIMachineSpec for X64ABIMachineSpec {
/// Generate a call instruction/sequence.
fn gen_call(
dest: &CallDest,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
uses: CallArgList,
defs: CallRetList,
clobbers: PRegSet,
opcode: ir::Opcode,
tmp: Writable<Reg>,
@@ -628,39 +630,47 @@ impl ABIMachineSpec for X64ABIMachineSpec {
call_conv: isa::CallConv,
dst: Reg,
src: Reg,
temp: Writable<Reg>,
temp2: Writable<Reg>,
size: usize,
) -> SmallVec<[Self::I; 8]> {
let mut insts = SmallVec::new();
let arg0 = get_intreg_for_arg(&call_conv, 0, 0).unwrap();
let arg1 = get_intreg_for_arg(&call_conv, 1, 1).unwrap();
let arg2 = get_intreg_for_arg(&call_conv, 2, 2).unwrap();
// We need a register to load the address of `memcpy()` below and we
// don't have a lowering context to allocate a temp here; so just use a
// register we know we are free to mutate as part of this sequence
// (because it is clobbered by the call as per the ABI anyway).
let memcpy_addr = get_intreg_for_arg(&call_conv, 3, 3).unwrap();
insts.push(Inst::gen_move(Writable::from_reg(arg0), dst, I64));
insts.push(Inst::gen_move(Writable::from_reg(arg1), src, I64));
insts.extend(
Inst::gen_constant(
ValueRegs::one(Writable::from_reg(arg2)),
size as u128,
I64,
|_| panic!("tmp should not be needed"),
)
Inst::gen_constant(ValueRegs::one(temp), size as u128, I64, |_| {
panic!("tmp should not be needed")
})
.into_iter(),
);
// We use an indirect call and a full LoadExtName because we do not have
// information about the libcall `RelocDistance` here, so we
// conservatively use the more flexible calling sequence.
insts.push(Inst::LoadExtName {
dst: Writable::from_reg(memcpy_addr),
dst: temp2,
name: Box::new(ExternalName::LibCall(LibCall::Memcpy)),
offset: 0,
});
insts.push(Inst::call_unknown(
RegMem::reg(memcpy_addr),
/* uses = */ smallvec![arg0, arg1, arg2],
RegMem::reg(temp2.to_reg()),
/* uses = */
smallvec![
CallArgPair {
vreg: dst,
preg: arg0
},
CallArgPair {
vreg: src,
preg: arg1
},
CallArgPair {
vreg: temp.to_reg(),
preg: arg2
},
],
/* defs = */ smallvec![],
/* clobbers = */ Self::get_regs_clobbered_by_call(call_conv),
Opcode::Call,

41
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -1,7 +1,7 @@
//! This module defines x86_64-specific machine instruction types.
use crate::binemit::{Addend, CodeOffset, Reloc, StackMap};
use crate::ir::{types, ExternalName, Opcode, RelSourceLoc, TrapCode, Type};
use crate::ir::{types, ExternalName, LibCall, Opcode, RelSourceLoc, TrapCode, Type};
use crate::isa::x64::abi::X64ABIMachineSpec;
use crate::isa::x64::inst::regs::pretty_print_reg;
use crate::isa::x64::settings as x64_settings;
@@ -34,9 +34,9 @@ pub use super::lower::isle::generated_code::MInst as Inst;
#[derive(Clone, Debug)]
pub struct CallInfo {
/// Register uses of this call.
pub uses: SmallVec<[Reg; 8]>,
pub uses: CallArgList,
/// Register defs of this call.
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub defs: CallRetList,
/// Registers clobbered by this call, as per its calling convention.
pub clobbers: PRegSet,
/// The opcode of this call.
@@ -490,8 +490,8 @@ impl Inst {
pub(crate) fn call_known(
dest: ExternalName,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
uses: CallArgList,
defs: CallRetList,
clobbers: PRegSet,
opcode: Opcode,
) -> Inst {
@@ -508,8 +508,8 @@ impl Inst {
pub(crate) fn call_unknown(
dest: RegMem,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
uses: CallArgList,
defs: CallRetList,
clobbers: PRegSet,
opcode: Opcode,
) -> Inst {
@@ -1446,7 +1446,9 @@ impl PrettyPrint for Inst {
format!("{} {}", ljustify("popq".to_string()), dst)
}
Inst::CallKnown { dest, .. } => format!("{} {:?}", ljustify("call".to_string()), dest),
Inst::CallKnown { dest, .. } => {
format!("{} {:?}", ljustify("call".to_string()), dest)
}
Inst::CallUnknown { dest, .. } => {
let dest = dest.pretty_print(8, allocs);
@@ -1981,23 +1983,28 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_early_def(*tmp);
}
Inst::CallKnown { ref info, .. } => {
for &u in &info.uses {
collector.reg_use(u);
Inst::CallKnown { dest, ref info, .. } => {
// Probestack is special and is only inserted after
// regalloc, so we do not need to represent its ABI to the
// register allocator. Assert that we don't alter that
// arrangement.
debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
for u in &info.uses {
collector.reg_fixed_use(u.vreg, u.preg);
}
for &d in &info.defs {
collector.reg_def(d);
for d in &info.defs {
collector.reg_fixed_def(d.vreg, d.preg);
}
collector.reg_clobbers(info.clobbers);
}
Inst::CallUnknown { ref info, dest, .. } => {
dest.get_operands(collector);
for &u in &info.uses {
collector.reg_use(u);
for u in &info.uses {
collector.reg_fixed_use(u.vreg, u.preg);
}
for &d in &info.defs {
collector.reg_def(d);
for d in &info.defs {
collector.reg_fixed_def(d.vreg, d.preg);
}
collector.reg_clobbers(info.clobbers);
}

15
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -8,11 +8,12 @@ use crate::isa::x64::abi::*;
use crate::isa::x64::inst::args::*;
use crate::isa::x64::inst::*;
use crate::isa::{x64::settings as x64_settings, x64::X64Backend, CallConv};
use crate::machinst::abi::SmallInstVec;
use crate::machinst::lower::*;
use crate::machinst::*;
use crate::result::CodegenResult;
use crate::settings::Flags;
use smallvec::SmallVec;
use smallvec::{smallvec, SmallVec};
use target_lexicon::Triple;
//=============================================================================
@@ -168,16 +169,18 @@ fn emit_vm_call(
assert_eq!(inputs.len(), abi.num_args(ctx.sigs()));
for (i, input) in inputs.iter().enumerate() {
for inst in abi.gen_copy_regs_to_arg(ctx, i, ValueRegs::one(*input)) {
for inst in abi.gen_arg(ctx, i, ValueRegs::one(*input)) {
ctx.emit(inst);
}
}
abi.emit_call(ctx);
let mut retval_insts: SmallInstVec<_> = smallvec![];
for (i, output) in outputs.iter().enumerate() {
for inst in abi.gen_copy_retval_to_regs(ctx, i, ValueRegs::one(*output)) {
ctx.emit(inst);
}
retval_insts.extend(abi.gen_retval(ctx, i, ValueRegs::one(*output)).into_iter());
}
abi.emit_call(ctx);
for inst in retval_insts {
ctx.emit(inst);
}
abi.emit_stack_post_adjust(ctx);

242
cranelift/codegen/src/machinst/abi.rs
View File

@@ -506,8 +506,8 @@ pub trait ABIMachineSpec {
/// temporary register to use to synthesize the called address, if needed.
fn gen_call(
dest: &CallDest,
uses: SmallVec<[Reg; 8]>,
defs: SmallVec<[Writable<Reg>; 8]>,
uses: CallArgList,
defs: CallRetList,
clobbers: PRegSet,
opcode: ir::Opcode,
tmp: Writable<Reg>,
@@ -515,13 +515,16 @@ pub trait ABIMachineSpec {
callee_conv: isa::CallConv,
) -> SmallVec<[Self::I; 2]>;
/// Generate a memcpy invocation. Used to set up struct args. May clobber
/// caller-save registers; we only memcpy before we start to set up args for
/// a call.
/// Generate a memcpy invocation. Used to set up struct
/// args. Takes `src`, `dst` as read-only inputs and requires two
/// temporaries to generate the call (for the size immediate and
/// possibly for the address of `memcpy` itself).
fn gen_memcpy(
call_conv: isa::CallConv,
dst: Reg,
src: Reg,
tmp1: Writable<Reg>,
tmp2: Writable<Reg>,
size: usize,
) -> SmallVec<[Self::I; 8]>;
@@ -623,6 +626,9 @@ impl SigData {
/// Return all uses (i.e, function args), defs (i.e., return values
/// and caller-saved registers), and clobbers for the callsite.
///
/// FIXME: used only by s390x; remove once that backend moves to
/// `call_clobbers` and constraint-based calls.
pub fn call_uses_defs_clobbers<M: ABIMachineSpec>(
&self,
) -> (SmallVec<[Reg; 8]>, SmallVec<[Writable<Reg>; 8]>, PRegSet) {
@@ -682,6 +688,30 @@ impl SigData {
(uses, defs, clobbers)
}
/// Return all clobbers for the callsite.
pub fn call_clobbers<M: ABIMachineSpec>(&self) -> PRegSet {
// Get clobbers: all caller-saves. These may include return value
// regs, which we will remove from the clobber set below.
let mut clobbers = M::get_regs_clobbered_by_call(self.call_conv);
// Remove retval regs from clobbers.
for ret in &self.rets {
if let &ABIArg::Slots { ref slots, .. } = ret {
for slot in slots {
match slot {
&ABIArgSlot::Reg { reg, .. } => {
log::trace!("call_clobbers: retval reg {:?}", reg);
clobbers.remove(PReg::from(reg));
}
_ => {}
}
}
}
}
clobbers
}
/// Get the number of arguments expected.
pub fn num_args(&self) -> usize {
if self.stack_ret_arg.is_some() {
@@ -1848,14 +1878,38 @@ impl<M: ABIMachineSpec> Callee<M> {
}
}
/// An input argument to a call instruction: the vreg that is used,
/// and the preg it is constrained to (per the ABI).
#[derive(Clone, Debug)]
pub struct CallArgPair {
/// The virtual register to use for the argument.
pub vreg: Reg,
/// The real register into which the arg goes.
pub preg: Reg,
}
/// An output return value from a call instruction: the vreg that is
/// defined, and the preg it is constrained to (per the ABI).
#[derive(Clone, Debug)]
pub struct CallRetPair {
/// The virtual register to define from this return value.
pub vreg: Writable<Reg>,
/// The real register from which the return value is read.
pub preg: Reg,
}
pub type CallArgList = SmallVec<[CallArgPair; 8]>;
pub type CallRetList = SmallVec<[CallRetPair; 8]>;
/// ABI object for a callsite.
pub struct Caller<M: ABIMachineSpec> {
/// The called function's signature.
sig: Sig,
/// All uses for the callsite, i.e., function args.
uses: SmallVec<[Reg; 8]>,
/// All register uses for the callsite, i.e., function args, with
/// VReg and the physical register it is constrained to.
uses: CallArgList,
/// All defs for the callsite, i.e., return values.
defs: SmallVec<[Writable<Reg>; 8]>,
defs: CallRetList,
/// Caller-save clobbers.
clobbers: PRegSet,
/// Call destination.
@@ -1890,11 +1944,11 @@ impl<M: ABIMachineSpec> Caller<M> {
flags: settings::Flags,
) -> CodegenResult<Caller<M>> {
let sig = sigs.abi_sig_for_sig_ref(sig_ref);
let (uses, defs, clobbers) = sigs[sig].call_uses_defs_clobbers::<M>();
let clobbers = sigs[sig].call_clobbers::<M>();
Ok(Caller {
sig,
uses,
defs,
uses: smallvec![],
defs: smallvec![],
clobbers,
dest: CallDest::ExtName(extname.clone(), dist),
opcode: ir::Opcode::Call,
@@ -1915,11 +1969,11 @@ impl<M: ABIMachineSpec> Caller<M> {
flags: settings::Flags,
) -> CodegenResult<Caller<M>> {
let sig = sigs.abi_sig_for_signature(sig);
let (uses, defs, clobbers) = sigs[sig].call_uses_defs_clobbers::<M>();
let clobbers = sigs[sig].call_clobbers::<M>();
Ok(Caller {
sig,
uses,
defs,
uses: smallvec![],
defs: smallvec![],
clobbers,
dest: CallDest::ExtName(extname.clone(), dist),
opcode: ir::Opcode::Call,
@@ -1940,11 +1994,11 @@ impl<M: ABIMachineSpec> Caller<M> {
flags: settings::Flags,
) -> CodegenResult<Caller<M>> {
let sig = sigs.abi_sig_for_sig_ref(sig_ref);
let (uses, defs, clobbers) = sigs[sig].call_uses_defs_clobbers::<M>();
let clobbers = sigs[sig].call_clobbers::<M>();
Ok(Caller {
sig,
uses,
defs,
uses: smallvec![],
defs: smallvec![],
clobbers,
dest: CallDest::Reg(ptr),
opcode,
@@ -2018,9 +2072,17 @@ impl<M: ABIMachineSpec> Caller<M> {
// arg regs.
let memcpy_call_conv =
isa::CallConv::for_libcall(&self.flags, ctx.sigs()[self.sig].call_conv);
for insn in
M::gen_memcpy(memcpy_call_conv, dst_ptr.to_reg(), src_ptr, size as usize)
.into_iter()
let tmp1 = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
let tmp2 = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
for insn in M::gen_memcpy(
memcpy_call_conv,
dst_ptr.to_reg(),
src_ptr,
tmp1,
tmp2,
size as usize,
)
.into_iter()
{
ctx.emit(insn);
}
@@ -2029,19 +2091,48 @@ impl<M: ABIMachineSpec> Caller<M> {
}
}
/// Generate a copy of an argument value from a source register, prior to
/// the call. For large arguments with associated stack buffer, this may
/// load the address of the buffer into the argument register, if required
/// by the ABI.
pub fn gen_copy_regs_to_arg(
&self,
ctx: &Lower<M::I>,
/// Add a constraint for an argument value from a source register.
/// For large arguments with associated stack buffer, this may
/// load the address of the buffer into the argument register, if
/// required by the ABI.
pub fn gen_arg(
&mut self,
ctx: &mut Lower<M::I>,
idx: usize,
from_regs: ValueRegs<Reg>,
) -> SmallInstVec<M::I> {
let mut insts = smallvec![];
let word_rc = M::word_reg_class();
let word_bits = M::word_bits() as usize;
// How many temps do we need for extends? Allocate them ahead
// of time, since we can't do it while we're iterating over
// the sig and immutably borrowing `ctx`.
let needed_tmps = match &ctx.sigs()[self.sig].args[idx] {
&ABIArg::Slots { ref slots, .. } => slots
.iter()
.map(|slot| match slot {
&ABIArgSlot::Reg { extension, .. }
if extension != ir::ArgumentExtension::None =>
{
1
}
&ABIArgSlot::Reg { ty, .. } if ty.is_ref() => 1,
&ABIArgSlot::Reg { .. } => 0,
&ABIArgSlot::Stack { extension, .. }
if extension != ir::ArgumentExtension::None =>
{
1
}
&ABIArgSlot::Stack { .. } => 0,
})
.sum(),
_ => 0,
};
let mut temps: SmallVec<[Writable<Reg>; 16]> = (0..needed_tmps)
.map(|_| ctx.alloc_tmp(M::word_type()).only_reg().unwrap())
.collect();
match &ctx.sigs()[self.sig].args[idx] {
&ABIArg::Slots { ref slots, .. } => {
assert_eq!(from_regs.len(), slots.len());
@@ -2058,19 +2149,36 @@ impl<M: ABIMachineSpec> Caller<M> {
ir::ArgumentExtension::Sext => true,
_ => unreachable!(),
};
let extend_result =
temps.pop().expect("Must have allocated enough temps");
insts.push(M::gen_extend(
Writable::from_reg(Reg::from(reg)),
extend_result,
*from_reg,
signed,
ty_bits(ty) as u8,
word_bits as u8,
));
self.uses.push(CallArgPair {
vreg: extend_result.to_reg(),
preg: reg.into(),
});
} else if ty.is_ref() {
// Reference-typed args need to be
// passed as a copy; the original vreg
// is constrained to the stack and
// this copy is in a reg.
let ref_copy =
temps.pop().expect("Must have allocated enough temps");
insts.push(M::gen_move(ref_copy, *from_reg, M::word_type()));
self.uses.push(CallArgPair {
vreg: ref_copy.to_reg(),
preg: reg.into(),
});
} else {
insts.push(M::gen_move(
Writable::from_reg(Reg::from(reg)),
*from_reg,
ty,
));
self.uses.push(CallArgPair {
vreg: *from_reg,
preg: reg.into(),
});
}
}
&ABIArgSlot::Stack {
@@ -2079,31 +2187,32 @@ impl<M: ABIMachineSpec> Caller<M> {
extension,
..
} => {
let mut ty = ty;
let ext = M::get_ext_mode(ctx.sigs()[self.sig].call_conv, extension);
if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
assert_eq!(word_rc, from_reg.class());
let signed = match ext {
ir::ArgumentExtension::Uext => false,
ir::ArgumentExtension::Sext => true,
_ => unreachable!(),
let (data, ty) =
if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
assert_eq!(word_rc, from_reg.class());
let signed = match ext {
ir::ArgumentExtension::Uext => false,
ir::ArgumentExtension::Sext => true,
_ => unreachable!(),
};
let extend_result =
temps.pop().expect("Must have allocated enough temps");
insts.push(M::gen_extend(
extend_result,
*from_reg,
signed,
ty_bits(ty) as u8,
word_bits as u8,
));
// Store the extended version.
(extend_result.to_reg(), M::word_type())
} else {
(*from_reg, ty)
};
// Extend in place in the source register. Our convention is to
// treat high bits as undefined for values in registers, so this
// is safe, even for an argument that is nominally read-only.
insts.push(M::gen_extend(
Writable::from_reg(*from_reg),
*from_reg,
signed,
ty_bits(ty) as u8,
word_bits as u8,
));
// Store the extended version.
ty = M::word_type();
}
insts.push(M::gen_store_stack(
StackAMode::SPOffset(offset, ty),
*from_reg,
data,
ty,
));
}
@@ -2118,9 +2227,9 @@ impl<M: ABIMachineSpec> Caller<M> {
insts
}
/// Emit a copy a return value into a destination register, after the call returns.
pub fn gen_copy_retval_to_regs(
&self,
/// Define a return value after the call returns.
pub fn gen_retval(
&mut self,
ctx: &Lower<M::I>,
idx: usize,
into_regs: ValueRegs<Writable<Reg>>,
@@ -2133,8 +2242,11 @@ impl<M: ABIMachineSpec> Caller<M> {
match slot {
// Extension mode doesn't matter because we're copying out, not in,
// and we ignore high bits in our own registers by convention.
&ABIArgSlot::Reg { reg, ty, .. } => {
insts.push(M::gen_move(*into_reg, Reg::from(reg), ty));
&ABIArgSlot::Reg { reg, .. } => {
self.defs.push(CallRetPair {
vreg: *into_reg,
preg: reg.into(),
});
}
&ABIArgSlot::Stack { offset, ty, .. } => {
let ret_area_base = ctx.sigs()[self.sig].sized_stack_arg_space;
@@ -2171,10 +2283,6 @@ impl<M: ABIMachineSpec> Caller<M> {
/// This function should only be called once, as it is allowed to re-use
/// parts of the `Caller` object in emitting instructions.
pub fn emit_call(&mut self, ctx: &mut Lower<M::I>) {
let (uses, defs) = (
mem::replace(&mut self.uses, Default::default()),
mem::replace(&mut self.defs, Default::default()),
);
let word_type = M::word_type();
if let Some(i) = ctx.sigs()[self.sig].stack_ret_arg {
let rd = ctx.alloc_tmp(word_type).only_reg().unwrap();
@@ -2184,10 +2292,16 @@ impl<M: ABIMachineSpec> Caller<M> {
rd,
I8,
));
for inst in self.gen_copy_regs_to_arg(ctx, i, ValueRegs::one(rd.to_reg())) {
for inst in self.gen_arg(ctx, i, ValueRegs::one(rd.to_reg())) {
ctx.emit(inst);
}
}
let (uses, defs) = (
mem::replace(&mut self.uses, Default::default()),
mem::replace(&mut self.defs, Default::default()),
);
let tmp = ctx.alloc_tmp(word_type).only_reg().unwrap();
for inst in M::gen_call(
&self.dest,

23
cranelift/codegen/src/machinst/isle.rs
View File

@@ -1208,21 +1208,32 @@ macro_rules! isle_prelude_method_helpers {
caller.emit_copy_regs_to_buffer(self.lower_ctx, i, *arg_regs);
}
for (i, arg_regs) in arg_regs.iter().enumerate() {
for inst in caller.gen_copy_regs_to_arg(self.lower_ctx, i, *arg_regs) {
for inst in caller.gen_arg(self.lower_ctx, i, *arg_regs) {
self.lower_ctx.emit(inst);
}
}
caller.emit_call(self.lower_ctx);
// Handle retvals prior to emitting call, so the
// constraints are on the call instruction; but buffer the
// instructions till after the call.
let mut outputs = InstOutput::new();
let mut retval_insts: crate::machinst::abi::SmallInstVec<_> = smallvec::smallvec![];
for i in 0..num_rets {
let ret = self.lower_ctx.sigs()[abi].get_ret(i);
let retval_regs = self.abi_arg_slot_regs(&ret).unwrap();
for inst in caller.gen_copy_retval_to_regs(self.lower_ctx, i, retval_regs.clone()) {
self.lower_ctx.emit(inst);
}
retval_insts.extend(
caller
.gen_retval(self.lower_ctx, i, retval_regs.clone())
.into_iter(),
);
outputs.push(valueregs::non_writable_value_regs(retval_regs));
}
caller.emit_call(self.lower_ctx);
for inst in retval_insts {
self.lower_ctx.emit(inst);
}
caller.emit_stack_post_adjust(self.lower_ctx);
outputs

2
cranelift/codegen/src/machinst/vcode.rs
View File

@@ -1027,7 +1027,6 @@ impl<I: VCodeInst> VCode<I> {
// Spill from register to spillslot.
let to = to.as_stack().unwrap();
let from_rreg = RealReg::from(from);
debug_assert_eq!(from.class(), to.class());
let spill = self.abi.gen_spill(to, from_rreg);
do_emit(&spill, &[], &mut disasm, &mut buffer, &mut state);
}
@@ -1035,7 +1034,6 @@ impl<I: VCodeInst> VCode<I> {
// Load from spillslot to register.
let from = from.as_stack().unwrap();
let to_rreg = Writable::from_reg(RealReg::from(to));
debug_assert_eq!(from.class(), to.class());
let reload = self.abi.gen_reload(to_rreg, from);
do_emit(&reload, &[], &mut disasm, &mut buffer, &mut state);
}