T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2081 from cfallin/aarch64-baldrdash-fix

Browse Source 
Aarch64: fix narrow integer-register extension with Baldrdash ABI.
This commit is contained in:
Chris Fallin
2020-07-31 12:13:38 -07:00
committed by GitHub
parent dd09865611 1fbdf169b5
commit 9a9b5015d0
5 changed files with 258 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -113,9 +113,9 @@ use log::{debug, trace};
#[derive(Clone, Copy, Debug)]
enum ABIArg {
/// In a real register.
Reg(RealReg, ir::Type),
Reg(RealReg, ir::Type, ir::ArgumentExtension),
/// Arguments only: on stack, at given offset from SP at entry.
Stack(i64, ir::Type),
Stack(i64, ir::Type, ir::ArgumentExtension),
}
/// AArch64 ABI information shared between body (callee) and caller.
@@ -187,6 +187,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
Some(ABIArg::Reg(
xreg(BALDRDASH_TLS_REG).to_real_reg(),
ir::types::I64,
param.extension,
))
}
&ir::ArgumentPurpose::SignatureId => {
@@ -194,6 +195,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
Some(ABIArg::Reg(
xreg(BALDRDASH_SIG_REG).to_real_reg(),
ir::types::I64,
param.extension,
))
}
_ => None,
@@ -279,7 +281,11 @@ fn compute_arg_locs(
} else {
vreg(*next_reg)
};
ret.push(ABIArg::Reg(reg.to_real_reg(), param.value_type));
ret.push(ABIArg::Reg(
reg.to_real_reg(),
param.value_type,
param.extension,
));
*next_reg += 1;
} else {
// Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte
@@ -289,7 +295,11 @@ fn compute_arg_locs(
// Align.
debug_assert!(size.is_power_of_two());
next_stack = (next_stack + size - 1) & !(size - 1);
ret.push(ABIArg::Stack(next_stack as i64, param.value_type));
ret.push(ABIArg::Stack(
next_stack as i64,
param.value_type,
param.extension,
));
next_stack += size;
}
}
@@ -301,9 +311,17 @@ fn compute_arg_locs(
let extra_arg = if add_ret_area_ptr {
debug_assert!(args_or_rets == ArgsOrRets::Args);
if next_xreg < max_reg_vals {
ret.push(ABIArg::Reg(xreg(next_xreg).to_real_reg(), I64));
ret.push(ABIArg::Reg(
xreg(next_xreg).to_real_reg(),
I64,
ir::ArgumentExtension::None,
));
} else {
ret.push(ABIArg::Stack(next_stack as i64, I64));
ret.push(ABIArg::Stack(
next_stack as i64,
I64,
ir::ArgumentExtension::None,
));
next_stack += 8;
}
Some(ret.len() - 1)
@@ -491,7 +509,7 @@ fn get_special_purpose_param_register(
) -> Option<Reg> {
let idx = f.signature.special_param_index(purpose)?;
match abi.args[idx] {
ABIArg::Reg(reg, _) => Some(reg.to_reg()),
ABIArg::Reg(reg, ..) => Some(reg.to_reg()),
ABIArg::Stack(..) => None,
}
}
@@ -866,7 +884,7 @@ impl ABIBody for AArch64ABIBody {
fn liveins(&self) -> Set<RealReg> {
let mut set: Set<RealReg> = Set::empty();
for &arg in &self.sig.args {
if let ABIArg::Reg(r, _) = arg {
if let ABIArg::Reg(r, ..) = arg {
set.insert(r);
}
}
@@ -876,7 +894,7 @@ impl ABIBody for AArch64ABIBody {
fn liveouts(&self) -> Set<RealReg> {
let mut set: Set<RealReg> = Set::empty();
for &ret in &self.sig.rets {
if let ABIArg::Reg(r, _) = ret {
if let ABIArg::Reg(r, ..) = ret {
set.insert(r);
}
}
@@ -897,8 +915,10 @@ impl ABIBody for AArch64ABIBody {
fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Inst {
match &self.sig.args[idx] {
&ABIArg::Reg(r, ty) => Inst::gen_move(into_reg, r.to_reg(), ty),
&ABIArg::Stack(off, ty) => load_stack(
// Extension mode doesn't matter (we're copying out, not in; we
// ignore high bits by convention).
&ABIArg::Reg(r, ty, _) => Inst::gen_move(into_reg, r.to_reg(), ty),
&ABIArg::Stack(off, ty, _) => load_stack(
MemArg::FPOffset(self.fp_to_arg_offset() + off, ty),
into_reg,
ty,
@@ -921,15 +941,10 @@ impl ABIBody for AArch64ABIBody {
}
}
fn gen_copy_reg_to_retval(
&self,
idx: usize,
from_reg: Writable<Reg>,
ext: ArgumentExtension,
) -> Vec<Inst> {
fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable<Reg>) -> Vec<Inst> {
let mut ret = Vec::new();
match &self.sig.rets[idx] {
&ABIArg::Reg(r, ty) => {
&ABIArg::Reg(r, ty, ext) => {
let from_bits = ty_bits(ty) as u8;
let dest_reg = Writable::from_reg(r.to_reg());
match (ext, from_bits) {
@@ -954,7 +969,7 @@ impl ABIBody for AArch64ABIBody {
_ => ret.push(Inst::gen_move(dest_reg, from_reg.to_reg(), ty)),
};
}
&ABIArg::Stack(off, ty) => {
&ABIArg::Stack(off, ty, ext) => {
let from_bits = ty_bits(ty) as u8;
// Trash the from_reg; it should be its last use.
match (ext, from_bits) {
@@ -1364,7 +1379,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec<Reg>, Vec<Writable<Reg>>) {
let mut uses = Vec::new();
for arg in &sig.args {
match arg {
&ABIArg::Reg(reg, _) => uses.push(reg.to_reg()),
&ABIArg::Reg(reg, ..) => uses.push(reg.to_reg()),
_ => {}
}
}
@@ -1373,7 +1388,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec<Reg>, Vec<Writable<Reg>>) {
let mut defs = get_caller_saves(sig.call_conv);
for ret in &sig.rets {
match ret {
&ABIArg::Reg(reg, _) => defs.push(Writable::from_reg(reg.to_reg())),
&ABIArg::Reg(reg, ..) => defs.push(Writable::from_reg(reg.to_reg())),
_ => {}
}
}
@@ -1469,12 +1484,49 @@ impl ABICall for AArch64ABICall {
from_reg: Reg,
) {
match &self.sig.args[idx] {
&ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move(
Writable::from_reg(reg.to_reg()),
from_reg,
ty,
)),
&ABIArg::Stack(off, ty) => {
&ABIArg::Reg(reg, ty, ext)
if ext != ir::ArgumentExtension::None && ty_bits(ty) < 64 =>
{
assert_eq!(RegClass::I64, reg.get_class());
let signed = match ext {
ir::ArgumentExtension::Uext => false,
ir::ArgumentExtension::Sext => true,
_ => unreachable!(),
};
ctx.emit(Inst::Extend {
rd: Writable::from_reg(reg.to_reg()),
rn: from_reg,
signed,
from_bits: ty_bits(ty) as u8,
to_bits: 64,
});
}
&ABIArg::Reg(reg, ty, _) => {
ctx.emit(Inst::gen_move(
Writable::from_reg(reg.to_reg()),
from_reg,
ty,
));
}
&ABIArg::Stack(off, ty, ext) => {
if ext != ir::ArgumentExtension::None && ty_bits(ty) < 64 {
assert_eq!(RegClass::I64, from_reg.get_class());
let signed = match ext {
ir::ArgumentExtension::Uext => false,
ir::ArgumentExtension::Sext => true,
_ => unreachable!(),
};
// Extend in place in the source register. Our convention is to
// treat high bits as undefined for values in registers, so this
// is safe, even for an argument that is nominally read-only.
ctx.emit(Inst::Extend {
rd: Writable::from_reg(from_reg),
rn: from_reg,
signed,
from_bits: ty_bits(ty) as u8,
to_bits: 64,
});
}
ctx.emit(store_stack(MemArg::SPOffset(off, ty), from_reg, ty))
}
}
@@ -1487,8 +1539,10 @@ impl ABICall for AArch64ABICall {
into_reg: Writable<Reg>,
) {
match &self.sig.rets[idx] {
&ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)),
&ABIArg::Stack(off, ty) => {
// Extension mode doesn't matter because we're copying out, not in,
// and we ignore high bits in our own registers by convention.
&ABIArg::Reg(reg, ty, _) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)),
&ABIArg::Stack(off, ty, _) => {
let ret_area_base = self.sig.stack_arg_space;
ctx.emit(load_stack(
MemArg::SPOffset(off + ret_area_base, ty),

133
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -23,8 +23,8 @@ static STACK_ARG_RET_SIZE_LIMIT: u64 = 128 * 1024 * 1024;
#[derive(Clone, Debug)]
enum ABIArg {
Reg(RealReg, ir::Type),
Stack(i64, ir::Type),
Reg(RealReg, ir::Type, ir::ArgumentExtension),
Stack(i64, ir::Type, ir::ArgumentExtension),
}
/// X64 ABI information shared between body (callee) and caller.
@@ -303,7 +303,7 @@ impl ABIBody for X64ABIBody {
fn liveins(&self) -> Set<RealReg> {
let mut set: Set<RealReg> = Set::empty();
for arg in &self.sig.args {
if let &ABIArg::Reg(r, _) = arg {
if let &ABIArg::Reg(r, ..) = arg {
set.insert(r);
}
}
@@ -313,7 +313,7 @@ impl ABIBody for X64ABIBody {
fn liveouts(&self) -> Set<RealReg> {
let mut set: Set<RealReg> = Set::empty();
for ret in &self.sig.rets {
if let &ABIArg::Reg(r, _) = ret {
if let &ABIArg::Reg(r, ..) = ret {
set.insert(r);
}
}
@@ -322,8 +322,8 @@ impl ABIBody for X64ABIBody {
fn gen_copy_arg_to_reg(&self, idx: usize, to_reg: Writable<Reg>) -> Inst {
match &self.sig.args[idx] {
ABIArg::Reg(from_reg, ty) => Inst::gen_move(to_reg, from_reg.to_reg(), *ty),
&ABIArg::Stack(off, ty) => {
ABIArg::Reg(from_reg, ty, _) => Inst::gen_move(to_reg, from_reg.to_reg(), *ty),
&ABIArg::Stack(off, ty, _) => {
assert!(
self.fp_to_arg_offset() + off <= u32::max_value() as i64,
"large offset nyi"
@@ -352,15 +352,10 @@ impl ABIBody for X64ABIBody {
}
}
fn gen_copy_reg_to_retval(
&self,
idx: usize,
from_reg: Writable<Reg>,
ext: ArgumentExtension,
) -> Vec<Inst> {
fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable<Reg>) -> Vec<Inst> {
let mut ret = Vec::new();
match &self.sig.rets[idx] {
&ABIArg::Reg(r, ty) => {
&ABIArg::Reg(r, ty, ext) => {
let from_bits = ty.bits() as u8;
let ext_mode = match from_bits {
1 | 8 => Some(ExtMode::BQ),
@@ -392,7 +387,7 @@ impl ABIBody for X64ABIBody {
};
}
&ABIArg::Stack(off, ty) => {
&ABIArg::Stack(off, ty, ext) => {
let from_bits = ty.bits() as u8;
let ext_mode = match from_bits {
1 | 8 => Some(ExtMode::BQ),
@@ -759,7 +754,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec<Reg>, Vec<Writable<Reg>>) {
let mut uses = Vec::new();
for arg in &sig.args {
match arg {
&ABIArg::Reg(reg, _) => uses.push(reg.to_reg()),
&ABIArg::Reg(reg, ..) => uses.push(reg.to_reg()),
_ => {}
}
}
@@ -768,7 +763,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec<Reg>, Vec<Writable<Reg>>) {
let mut defs = get_caller_saves(sig.call_conv);
for ret in &sig.rets {
match ret {
&ABIArg::Reg(reg, _) => defs.push(Writable::from_reg(reg.to_reg())),
&ABIArg::Reg(reg, ..) => defs.push(Writable::from_reg(reg.to_reg())),
_ => {}
}
}
@@ -782,11 +777,19 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<A
match &param.purpose {
&ir::ArgumentPurpose::VMContext => {
// This is SpiderMonkey's `WasmTlsReg`.
Some(ABIArg::Reg(regs::r14().to_real_reg(), ir::types::I64))
Some(ABIArg::Reg(
regs::r14().to_real_reg(),
ir::types::I64,
param.extension,
))
}
&ir::ArgumentPurpose::SignatureId => {
// This is SpiderMonkey's `WasmTableCallSigReg`.
Some(ABIArg::Reg(regs::r10().to_real_reg(), ir::types::I64))
Some(ABIArg::Reg(
regs::r10().to_real_reg(),
ir::types::I64,
param.extension,
))
}
_ => None,
}
@@ -873,7 +876,11 @@ fn compute_arg_locs(
assert!(intreg);
ret.push(param);
} else if let Some(reg) = candidate {
ret.push(ABIArg::Reg(reg.to_real_reg(), param.value_type));
ret.push(ABIArg::Reg(
reg.to_real_reg(),
param.value_type,
param.extension,
));
*next_reg += 1;
} else {
// Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte
@@ -883,7 +890,11 @@ fn compute_arg_locs(
// Align.
debug_assert!(size.is_power_of_two());
next_stack = (next_stack + size - 1) & !(size - 1);
ret.push(ABIArg::Stack(next_stack as i64, param.value_type));
ret.push(ABIArg::Stack(
next_stack as i64,
param.value_type,
param.extension,
));
next_stack += size;
}
}
@@ -895,9 +906,17 @@ fn compute_arg_locs(
let extra_arg = if add_ret_area_ptr {
debug_assert!(args_or_rets == ArgsOrRets::Args);
if let Some(reg) = get_intreg_for_arg_systemv(&call_conv, next_gpr) {
ret.push(ABIArg::Reg(reg.to_real_reg(), ir::types::I64));
ret.push(ABIArg::Reg(
reg.to_real_reg(),
ir::types::I64,
ir::ArgumentExtension::None,
));
} else {
ret.push(ABIArg::Stack(next_stack as i64, ir::types::I64));
ret.push(ABIArg::Stack(
next_stack as i64,
ir::types::I64,
ir::ArgumentExtension::None,
));
next_stack += 8;
}
Some(ret.len() - 1)
@@ -1126,12 +1145,74 @@ impl ABICall for X64ABICall {
from_reg: Reg,
) {
match &self.sig.args[idx] {
&ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move(
&ABIArg::Reg(reg, ty, ext) if ext != ir::ArgumentExtension::None && ty.bits() < 64 => {
assert_eq!(RegClass::I64, reg.get_class());
let dest_reg = Writable::from_reg(reg.to_reg());
let ext_mode = match ty.bits() {
1 | 8 => ExtMode::BQ,
16 => ExtMode::WQ,
32 => ExtMode::LQ,
_ => unreachable!(),
};
match ext {
ir::ArgumentExtension::Uext => {
ctx.emit(Inst::movzx_rm_r(
ext_mode,
RegMem::reg(from_reg),
dest_reg,
/* infallible load */ None,
));
}
ir::ArgumentExtension::Sext => {
ctx.emit(Inst::movsx_rm_r(
ext_mode,
RegMem::reg(from_reg),
dest_reg,
/* infallible load */ None,
));
}
_ => unreachable!(),
};
}
&ABIArg::Reg(reg, ty, _) => ctx.emit(Inst::gen_move(
Writable::from_reg(reg.to_reg()),
from_reg,
ty,
)),
&ABIArg::Stack(off, ty) => {
&ABIArg::Stack(off, ty, ext) => {
if ext != ir::ArgumentExtension::None && ty.bits() < 64 {
assert_eq!(RegClass::I64, from_reg.get_class());
let dest_reg = Writable::from_reg(from_reg);
let ext_mode = match ty.bits() {
1 | 8 => ExtMode::BQ,
16 => ExtMode::WQ,
32 => ExtMode::LQ,
_ => unreachable!(),
};
// Extend in place in the source register. Our convention is to
// treat high bits as undefined for values in registers, so this
// is safe, even for an argument that is nominally read-only.
match ext {
ir::ArgumentExtension::Uext => {
ctx.emit(Inst::movzx_rm_r(
ext_mode,
RegMem::reg(from_reg),
dest_reg,
/* infallible load */ None,
));
}
ir::ArgumentExtension::Sext => {
ctx.emit(Inst::movsx_rm_r(
ext_mode,
RegMem::reg(from_reg),
dest_reg,
/* infallible load */ None,
));
}
_ => unreachable!(),
};
}
debug_assert!(off <= u32::max_value() as i64);
debug_assert!(off >= 0);
ctx.emit(store_stack(
@@ -1150,8 +1231,8 @@ impl ABICall for X64ABICall {
into_reg: Writable<Reg>,
) {
match &self.sig.rets[idx] {
&ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)),
&ABIArg::Stack(off, ty) => {
&ABIArg::Reg(reg, ty, _) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)),
&ABIArg::Stack(off, ty, _) => {
let ret_area_base = self.sig.stack_arg_space;
let sp_offset = off + ret_area_base;
// TODO handle offsets bigger than u32::max

9
cranelift/codegen/src/machinst/abi.rs
View File

@@ -1,7 +1,7 @@
//! ABI definitions.
use crate::binemit::Stackmap;
use crate::ir::{ArgumentExtension, StackSlot};
use crate::ir::StackSlot;
use crate::machinst::*;
use crate::settings;
@@ -52,12 +52,7 @@ pub trait ABIBody {
fn gen_retval_area_setup(&self) -> Option<Self::I>;
/// Generate an instruction which copies a source register to a return value slot.
fn gen_copy_reg_to_retval(
&self,
idx: usize,
from_reg: Writable<Reg>,
ext: ArgumentExtension,
) -> Vec<Self::I>;
fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable<Reg>) -> Vec<Self::I>;
/// Generate a return instruction.
fn gen_ret(&self) -> Self::I;

15
cranelift/codegen/src/machinst/lower.rs
View File

@@ -8,9 +8,8 @@ use crate::inst_predicates::{has_side_effect_or_load, is_constant_64bit};
use crate::ir::instructions::BranchInfo;
use crate::ir::types::I64;
use crate::ir::{
ArgumentExtension, ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function,
GlobalValueData, Inst, InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value,
ValueDef,
ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst,
InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
};
use crate::machinst::{
ABIBody, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode, VCodeBuilder,
@@ -232,7 +231,7 @@ pub struct Lower<'func, I: VCodeInst> {
value_regs: SecondaryMap<Value, Reg>,
/// Return-value vregs.
retval_regs: Vec<(Reg, ArgumentExtension)>,
retval_regs: Vec<Reg>,
/// Instruction colors.
inst_colors: SecondaryMap<Inst, InstColor>,
@@ -354,7 +353,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
next_vreg += 1;
let regclass = I::rc_for_type(ret.value_type)?;
let vreg = Reg::new_virtual(regclass, v);
retval_regs.push((vreg, ret.extension));
retval_regs.push(vreg);
vcode.set_vreg_type(vreg.as_virtual_reg().unwrap(), ret.value_type);
}
@@ -427,9 +426,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) {
let retval_regs = self.retval_regs.clone();
for (i, (reg, ext)) in retval_regs.into_iter().enumerate() {
for (i, reg) in retval_regs.into_iter().enumerate() {
let reg = Writable::from_reg(reg);
let insns = self.vcode.abi().gen_copy_reg_to_retval(i, reg, ext);
let insns = self.vcode.abi().gen_copy_reg_to_retval(i, reg);
for insn in insns {
self.emit(insn);
}
@@ -844,7 +843,7 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
}
fn retval(&self, idx: usize) -> Writable<Reg> {
Writable::from_reg(self.retval_regs[idx].0)
Writable::from_reg(self.retval_regs[idx])
}
fn get_vm_context(&self) -> Option<Reg> {

60
cranelift/filetests/filetests/vcode/aarch64/call.clif
View File

@@ -1,7 +1,7 @@
test compile
target aarch64
function %f(i64) -> i64 {
function %f1(i64) -> i64 {
fn0 = %g(i64) -> i64
block0(v0: i64):
@@ -16,3 +16,61 @@ block0(v0: i64):
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f2(i32) -> i64 {
fn0 = %g(i32 uext) -> i64
block0(v0: i32):
v1 = call fn0(v0)
return v1
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: mov w0, w0
; nextln: ldr x16, 8 ; b 12 ; data
; nextln: blr x16
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f3(i32) -> i32 uext {
block0(v0: i32):
return v0
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: mov w0, w0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f4(i32) -> i64 {
fn0 = %g(i32 sext) -> i64
block0(v0: i32):
v1 = call fn0(v0)
return v1
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: sxtw x0, w0
; nextln: ldr x16, 8 ; b 12 ; data
; nextln: blr x16
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret
function %f3(i32) -> i32 sext {
block0(v0: i32):
return v0
}
; check: stp fp, lr, [sp, #-16]!
; nextln: mov fp, sp
; nextln: sxtw x0, w0
; nextln: mov sp, fp
; nextln: ldp fp, lr, [sp], #16
; nextln: ret