diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index f642326172..06f0514721 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -113,9 +113,9 @@ use log::{debug, trace}; #[derive(Clone, Copy, Debug)] enum ABIArg { /// In a real register. - Reg(RealReg, ir::Type), + Reg(RealReg, ir::Type, ir::ArgumentExtension), /// Arguments only: on stack, at given offset from SP at entry. - Stack(i64, ir::Type), + Stack(i64, ir::Type, ir::ArgumentExtension), } /// AArch64 ABI information shared between body (callee) and caller. @@ -187,6 +187,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt Some(ABIArg::Reg( xreg(BALDRDASH_TLS_REG).to_real_reg(), ir::types::I64, + param.extension, )) } &ir::ArgumentPurpose::SignatureId => { @@ -194,6 +195,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt Some(ABIArg::Reg( xreg(BALDRDASH_SIG_REG).to_real_reg(), ir::types::I64, + param.extension, )) } _ => None, @@ -279,7 +281,11 @@ fn compute_arg_locs( } else { vreg(*next_reg) }; - ret.push(ABIArg::Reg(reg.to_real_reg(), param.value_type)); + ret.push(ABIArg::Reg( + reg.to_real_reg(), + param.value_type, + param.extension, + )); *next_reg += 1; } else { // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte @@ -289,7 +295,11 @@ fn compute_arg_locs( // Align. debug_assert!(size.is_power_of_two()); next_stack = (next_stack + size - 1) & !(size - 1); - ret.push(ABIArg::Stack(next_stack as i64, param.value_type)); + ret.push(ABIArg::Stack( + next_stack as i64, + param.value_type, + param.extension, + )); next_stack += size; } } @@ -301,9 +311,17 @@ fn compute_arg_locs( let extra_arg = if add_ret_area_ptr { debug_assert!(args_or_rets == ArgsOrRets::Args); if next_xreg < max_reg_vals { - ret.push(ABIArg::Reg(xreg(next_xreg).to_real_reg(), I64)); + ret.push(ABIArg::Reg( + xreg(next_xreg).to_real_reg(), + I64, + ir::ArgumentExtension::None, + )); } else { - ret.push(ABIArg::Stack(next_stack as i64, I64)); + ret.push(ABIArg::Stack( + next_stack as i64, + I64, + ir::ArgumentExtension::None, + )); next_stack += 8; } Some(ret.len() - 1) @@ -491,7 +509,7 @@ fn get_special_purpose_param_register( ) -> Option { let idx = f.signature.special_param_index(purpose)?; match abi.args[idx] { - ABIArg::Reg(reg, _) => Some(reg.to_reg()), + ABIArg::Reg(reg, ..) => Some(reg.to_reg()), ABIArg::Stack(..) => None, } } @@ -866,7 +884,7 @@ impl ABIBody for AArch64ABIBody { fn liveins(&self) -> Set { let mut set: Set = Set::empty(); for &arg in &self.sig.args { - if let ABIArg::Reg(r, _) = arg { + if let ABIArg::Reg(r, ..) = arg { set.insert(r); } } @@ -876,7 +894,7 @@ impl ABIBody for AArch64ABIBody { fn liveouts(&self) -> Set { let mut set: Set = Set::empty(); for &ret in &self.sig.rets { - if let ABIArg::Reg(r, _) = ret { + if let ABIArg::Reg(r, ..) = ret { set.insert(r); } } @@ -897,8 +915,10 @@ impl ABIBody for AArch64ABIBody { fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable) -> Inst { match &self.sig.args[idx] { - &ABIArg::Reg(r, ty) => Inst::gen_move(into_reg, r.to_reg(), ty), - &ABIArg::Stack(off, ty) => load_stack( + // Extension mode doesn't matter (we're copying out, not in; we + // ignore high bits by convention). + &ABIArg::Reg(r, ty, _) => Inst::gen_move(into_reg, r.to_reg(), ty), + &ABIArg::Stack(off, ty, _) => load_stack( MemArg::FPOffset(self.fp_to_arg_offset() + off, ty), into_reg, ty, @@ -921,15 +941,10 @@ impl ABIBody for AArch64ABIBody { } } - fn gen_copy_reg_to_retval( - &self, - idx: usize, - from_reg: Writable, - ext: ArgumentExtension, - ) -> Vec { + fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable) -> Vec { let mut ret = Vec::new(); match &self.sig.rets[idx] { - &ABIArg::Reg(r, ty) => { + &ABIArg::Reg(r, ty, ext) => { let from_bits = ty_bits(ty) as u8; let dest_reg = Writable::from_reg(r.to_reg()); match (ext, from_bits) { @@ -954,7 +969,7 @@ impl ABIBody for AArch64ABIBody { _ => ret.push(Inst::gen_move(dest_reg, from_reg.to_reg(), ty)), }; } - &ABIArg::Stack(off, ty) => { + &ABIArg::Stack(off, ty, ext) => { let from_bits = ty_bits(ty) as u8; // Trash the from_reg; it should be its last use. match (ext, from_bits) { @@ -1364,7 +1379,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec>) { let mut uses = Vec::new(); for arg in &sig.args { match arg { - &ABIArg::Reg(reg, _) => uses.push(reg.to_reg()), + &ABIArg::Reg(reg, ..) => uses.push(reg.to_reg()), _ => {} } } @@ -1373,7 +1388,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec>) { let mut defs = get_caller_saves(sig.call_conv); for ret in &sig.rets { match ret { - &ABIArg::Reg(reg, _) => defs.push(Writable::from_reg(reg.to_reg())), + &ABIArg::Reg(reg, ..) => defs.push(Writable::from_reg(reg.to_reg())), _ => {} } } @@ -1469,12 +1484,49 @@ impl ABICall for AArch64ABICall { from_reg: Reg, ) { match &self.sig.args[idx] { - &ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move( - Writable::from_reg(reg.to_reg()), - from_reg, - ty, - )), - &ABIArg::Stack(off, ty) => { + &ABIArg::Reg(reg, ty, ext) + if ext != ir::ArgumentExtension::None && ty_bits(ty) < 64 => + { + assert_eq!(RegClass::I64, reg.get_class()); + let signed = match ext { + ir::ArgumentExtension::Uext => false, + ir::ArgumentExtension::Sext => true, + _ => unreachable!(), + }; + ctx.emit(Inst::Extend { + rd: Writable::from_reg(reg.to_reg()), + rn: from_reg, + signed, + from_bits: ty_bits(ty) as u8, + to_bits: 64, + }); + } + &ABIArg::Reg(reg, ty, _) => { + ctx.emit(Inst::gen_move( + Writable::from_reg(reg.to_reg()), + from_reg, + ty, + )); + } + &ABIArg::Stack(off, ty, ext) => { + if ext != ir::ArgumentExtension::None && ty_bits(ty) < 64 { + assert_eq!(RegClass::I64, from_reg.get_class()); + let signed = match ext { + ir::ArgumentExtension::Uext => false, + ir::ArgumentExtension::Sext => true, + _ => unreachable!(), + }; + // Extend in place in the source register. Our convention is to + // treat high bits as undefined for values in registers, so this + // is safe, even for an argument that is nominally read-only. + ctx.emit(Inst::Extend { + rd: Writable::from_reg(from_reg), + rn: from_reg, + signed, + from_bits: ty_bits(ty) as u8, + to_bits: 64, + }); + } ctx.emit(store_stack(MemArg::SPOffset(off, ty), from_reg, ty)) } } @@ -1487,8 +1539,10 @@ impl ABICall for AArch64ABICall { into_reg: Writable, ) { match &self.sig.rets[idx] { - &ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)), - &ABIArg::Stack(off, ty) => { + // Extension mode doesn't matter because we're copying out, not in, + // and we ignore high bits in our own registers by convention. + &ABIArg::Reg(reg, ty, _) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)), + &ABIArg::Stack(off, ty, _) => { let ret_area_base = self.sig.stack_arg_space; ctx.emit(load_stack( MemArg::SPOffset(off + ret_area_base, ty), diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index a6e9e1db48..0da1a5cd3d 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -23,8 +23,8 @@ static STACK_ARG_RET_SIZE_LIMIT: u64 = 128 * 1024 * 1024; #[derive(Clone, Debug)] enum ABIArg { - Reg(RealReg, ir::Type), - Stack(i64, ir::Type), + Reg(RealReg, ir::Type, ir::ArgumentExtension), + Stack(i64, ir::Type, ir::ArgumentExtension), } /// X64 ABI information shared between body (callee) and caller. @@ -303,7 +303,7 @@ impl ABIBody for X64ABIBody { fn liveins(&self) -> Set { let mut set: Set = Set::empty(); for arg in &self.sig.args { - if let &ABIArg::Reg(r, _) = arg { + if let &ABIArg::Reg(r, ..) = arg { set.insert(r); } } @@ -313,7 +313,7 @@ impl ABIBody for X64ABIBody { fn liveouts(&self) -> Set { let mut set: Set = Set::empty(); for ret in &self.sig.rets { - if let &ABIArg::Reg(r, _) = ret { + if let &ABIArg::Reg(r, ..) = ret { set.insert(r); } } @@ -322,8 +322,8 @@ impl ABIBody for X64ABIBody { fn gen_copy_arg_to_reg(&self, idx: usize, to_reg: Writable) -> Inst { match &self.sig.args[idx] { - ABIArg::Reg(from_reg, ty) => Inst::gen_move(to_reg, from_reg.to_reg(), *ty), - &ABIArg::Stack(off, ty) => { + ABIArg::Reg(from_reg, ty, _) => Inst::gen_move(to_reg, from_reg.to_reg(), *ty), + &ABIArg::Stack(off, ty, _) => { assert!( self.fp_to_arg_offset() + off <= u32::max_value() as i64, "large offset nyi" @@ -352,15 +352,10 @@ impl ABIBody for X64ABIBody { } } - fn gen_copy_reg_to_retval( - &self, - idx: usize, - from_reg: Writable, - ext: ArgumentExtension, - ) -> Vec { + fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable) -> Vec { let mut ret = Vec::new(); match &self.sig.rets[idx] { - &ABIArg::Reg(r, ty) => { + &ABIArg::Reg(r, ty, ext) => { let from_bits = ty.bits() as u8; let ext_mode = match from_bits { 1 | 8 => Some(ExtMode::BQ), @@ -392,7 +387,7 @@ impl ABIBody for X64ABIBody { }; } - &ABIArg::Stack(off, ty) => { + &ABIArg::Stack(off, ty, ext) => { let from_bits = ty.bits() as u8; let ext_mode = match from_bits { 1 | 8 => Some(ExtMode::BQ), @@ -759,7 +754,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec>) { let mut uses = Vec::new(); for arg in &sig.args { match arg { - &ABIArg::Reg(reg, _) => uses.push(reg.to_reg()), + &ABIArg::Reg(reg, ..) => uses.push(reg.to_reg()), _ => {} } } @@ -768,7 +763,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec>) { let mut defs = get_caller_saves(sig.call_conv); for ret in &sig.rets { match ret { - &ABIArg::Reg(reg, _) => defs.push(Writable::from_reg(reg.to_reg())), + &ABIArg::Reg(reg, ..) => defs.push(Writable::from_reg(reg.to_reg())), _ => {} } } @@ -782,11 +777,19 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option { // This is SpiderMonkey's `WasmTlsReg`. - Some(ABIArg::Reg(regs::r14().to_real_reg(), ir::types::I64)) + Some(ABIArg::Reg( + regs::r14().to_real_reg(), + ir::types::I64, + param.extension, + )) } &ir::ArgumentPurpose::SignatureId => { // This is SpiderMonkey's `WasmTableCallSigReg`. - Some(ABIArg::Reg(regs::r10().to_real_reg(), ir::types::I64)) + Some(ABIArg::Reg( + regs::r10().to_real_reg(), + ir::types::I64, + param.extension, + )) } _ => None, } @@ -873,7 +876,11 @@ fn compute_arg_locs( assert!(intreg); ret.push(param); } else if let Some(reg) = candidate { - ret.push(ABIArg::Reg(reg.to_real_reg(), param.value_type)); + ret.push(ABIArg::Reg( + reg.to_real_reg(), + param.value_type, + param.extension, + )); *next_reg += 1; } else { // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte @@ -883,7 +890,11 @@ fn compute_arg_locs( // Align. debug_assert!(size.is_power_of_two()); next_stack = (next_stack + size - 1) & !(size - 1); - ret.push(ABIArg::Stack(next_stack as i64, param.value_type)); + ret.push(ABIArg::Stack( + next_stack as i64, + param.value_type, + param.extension, + )); next_stack += size; } } @@ -895,9 +906,17 @@ fn compute_arg_locs( let extra_arg = if add_ret_area_ptr { debug_assert!(args_or_rets == ArgsOrRets::Args); if let Some(reg) = get_intreg_for_arg_systemv(&call_conv, next_gpr) { - ret.push(ABIArg::Reg(reg.to_real_reg(), ir::types::I64)); + ret.push(ABIArg::Reg( + reg.to_real_reg(), + ir::types::I64, + ir::ArgumentExtension::None, + )); } else { - ret.push(ABIArg::Stack(next_stack as i64, ir::types::I64)); + ret.push(ABIArg::Stack( + next_stack as i64, + ir::types::I64, + ir::ArgumentExtension::None, + )); next_stack += 8; } Some(ret.len() - 1) @@ -1126,12 +1145,74 @@ impl ABICall for X64ABICall { from_reg: Reg, ) { match &self.sig.args[idx] { - &ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move( + &ABIArg::Reg(reg, ty, ext) if ext != ir::ArgumentExtension::None && ty.bits() < 64 => { + assert_eq!(RegClass::I64, reg.get_class()); + let dest_reg = Writable::from_reg(reg.to_reg()); + let ext_mode = match ty.bits() { + 1 | 8 => ExtMode::BQ, + 16 => ExtMode::WQ, + 32 => ExtMode::LQ, + _ => unreachable!(), + }; + match ext { + ir::ArgumentExtension::Uext => { + ctx.emit(Inst::movzx_rm_r( + ext_mode, + RegMem::reg(from_reg), + dest_reg, + /* infallible load */ None, + )); + } + ir::ArgumentExtension::Sext => { + ctx.emit(Inst::movsx_rm_r( + ext_mode, + RegMem::reg(from_reg), + dest_reg, + /* infallible load */ None, + )); + } + _ => unreachable!(), + }; + } + &ABIArg::Reg(reg, ty, _) => ctx.emit(Inst::gen_move( Writable::from_reg(reg.to_reg()), from_reg, ty, )), - &ABIArg::Stack(off, ty) => { + &ABIArg::Stack(off, ty, ext) => { + if ext != ir::ArgumentExtension::None && ty.bits() < 64 { + assert_eq!(RegClass::I64, from_reg.get_class()); + let dest_reg = Writable::from_reg(from_reg); + let ext_mode = match ty.bits() { + 1 | 8 => ExtMode::BQ, + 16 => ExtMode::WQ, + 32 => ExtMode::LQ, + _ => unreachable!(), + }; + // Extend in place in the source register. Our convention is to + // treat high bits as undefined for values in registers, so this + // is safe, even for an argument that is nominally read-only. + match ext { + ir::ArgumentExtension::Uext => { + ctx.emit(Inst::movzx_rm_r( + ext_mode, + RegMem::reg(from_reg), + dest_reg, + /* infallible load */ None, + )); + } + ir::ArgumentExtension::Sext => { + ctx.emit(Inst::movsx_rm_r( + ext_mode, + RegMem::reg(from_reg), + dest_reg, + /* infallible load */ None, + )); + } + _ => unreachable!(), + }; + } + debug_assert!(off <= u32::max_value() as i64); debug_assert!(off >= 0); ctx.emit(store_stack( @@ -1150,8 +1231,8 @@ impl ABICall for X64ABICall { into_reg: Writable, ) { match &self.sig.rets[idx] { - &ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)), - &ABIArg::Stack(off, ty) => { + &ABIArg::Reg(reg, ty, _) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)), + &ABIArg::Stack(off, ty, _) => { let ret_area_base = self.sig.stack_arg_space; let sp_offset = off + ret_area_base; // TODO handle offsets bigger than u32::max diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index 84f574317b..39b9084b5a 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -1,7 +1,7 @@ //! ABI definitions. use crate::binemit::Stackmap; -use crate::ir::{ArgumentExtension, StackSlot}; +use crate::ir::StackSlot; use crate::machinst::*; use crate::settings; @@ -52,12 +52,7 @@ pub trait ABIBody { fn gen_retval_area_setup(&self) -> Option; /// Generate an instruction which copies a source register to a return value slot. - fn gen_copy_reg_to_retval( - &self, - idx: usize, - from_reg: Writable, - ext: ArgumentExtension, - ) -> Vec; + fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable) -> Vec; /// Generate a return instruction. fn gen_ret(&self) -> Self::I; diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 6f235f0216..c9cb27ba35 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -8,9 +8,8 @@ use crate::inst_predicates::{has_side_effect_or_load, is_constant_64bit}; use crate::ir::instructions::BranchInfo; use crate::ir::types::I64; use crate::ir::{ - ArgumentExtension, ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, - GlobalValueData, Inst, InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, - ValueDef, + ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst, + InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef, }; use crate::machinst::{ ABIBody, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode, VCodeBuilder, @@ -232,7 +231,7 @@ pub struct Lower<'func, I: VCodeInst> { value_regs: SecondaryMap, /// Return-value vregs. - retval_regs: Vec<(Reg, ArgumentExtension)>, + retval_regs: Vec, /// Instruction colors. inst_colors: SecondaryMap, @@ -354,7 +353,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { next_vreg += 1; let regclass = I::rc_for_type(ret.value_type)?; let vreg = Reg::new_virtual(regclass, v); - retval_regs.push((vreg, ret.extension)); + retval_regs.push(vreg); vcode.set_vreg_type(vreg.as_virtual_reg().unwrap(), ret.value_type); } @@ -427,9 +426,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> { fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) { let retval_regs = self.retval_regs.clone(); - for (i, (reg, ext)) in retval_regs.into_iter().enumerate() { + for (i, reg) in retval_regs.into_iter().enumerate() { let reg = Writable::from_reg(reg); - let insns = self.vcode.abi().gen_copy_reg_to_retval(i, reg, ext); + let insns = self.vcode.abi().gen_copy_reg_to_retval(i, reg); for insn in insns { self.emit(insn); } @@ -844,7 +843,7 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } fn retval(&self, idx: usize) -> Writable { - Writable::from_reg(self.retval_regs[idx].0) + Writable::from_reg(self.retval_regs[idx]) } fn get_vm_context(&self) -> Option { diff --git a/cranelift/filetests/filetests/vcode/aarch64/call.clif b/cranelift/filetests/filetests/vcode/aarch64/call.clif index 40cbd21383..28d75cb36a 100644 --- a/cranelift/filetests/filetests/vcode/aarch64/call.clif +++ b/cranelift/filetests/filetests/vcode/aarch64/call.clif @@ -1,7 +1,7 @@ test compile target aarch64 -function %f(i64) -> i64 { +function %f1(i64) -> i64 { fn0 = %g(i64) -> i64 block0(v0: i64): @@ -16,3 +16,61 @@ block0(v0: i64): ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret + +function %f2(i32) -> i64 { + fn0 = %g(i32 uext) -> i64 + +block0(v0: i32): + v1 = call fn0(v0) + return v1 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: mov w0, w0 +; nextln: ldr x16, 8 ; b 12 ; data +; nextln: blr x16 +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f3(i32) -> i32 uext { +block0(v0: i32): + return v0 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: mov w0, w0 +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f4(i32) -> i64 { + fn0 = %g(i32 sext) -> i64 + +block0(v0: i32): + v1 = call fn0(v0) + return v1 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: sxtw x0, w0 +; nextln: ldr x16, 8 ; b 12 ; data +; nextln: blr x16 +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret + +function %f3(i32) -> i32 sext { +block0(v0: i32): + return v0 +} + +; check: stp fp, lr, [sp, #-16]! +; nextln: mov fp, sp +; nextln: sxtw x0, w0 +; nextln: mov sp, fp +; nextln: ldp fp, lr, [sp], #16 +; nextln: ret