T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2189 from bnjbvr/x64-refactor-sub

Browse Source 
machinst x64: a few small refactorings/renamings
This commit is contained in:
Chris Fallin
2020-09-09 12:40:59 -07:00
committed by GitHub
parent b189321d61 b4a2dd37a4
commit bd3ba0a774
7 changed files with 239 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
cranelift/codegen/src/ir/instructions.rs
View File

@@ -18,6 +18,7 @@ use crate::isa;
use crate::bitset::BitSet; use crate::bitset::BitSet;
use crate::entity; use crate::entity;
use ir::condcodes::{FloatCC, IntCC};
/// Some instructions use an external list of argument values because there is not enough space in /// Some instructions use an external list of argument values because there is not enough space in
/// the 16-byte `InstructionData` struct. These value lists are stored in a memory pool in /// the 16-byte `InstructionData` struct. These value lists are stored in a memory pool in
@@ -295,6 +296,33 @@ impl InstructionData {
} }
} }
/// If this is a control-flow instruction depending on an integer condition, gets its
/// condition. Otherwise, return `None`.
pub fn cond_code(&self) -> Option<IntCC> {
match self {
&InstructionData::IntCond { cond, .. }
| &InstructionData::BranchIcmp { cond, .. }
| &InstructionData::IntCompare { cond, .. }
| &InstructionData::IntCondTrap { cond, .. }
| &InstructionData::BranchInt { cond, .. }
| &InstructionData::IntSelect { cond, .. }
| &InstructionData::IntCompareImm { cond, .. } => Some(cond),
_ => None,
}
}
/// If this is a control-flow instruction depending on a floating-point condition, gets its
/// condition. Otherwise, return `None`.
pub fn fp_cond_code(&self) -> Option<FloatCC> {
match self {
&InstructionData::BranchFloat { cond, .. }
| &InstructionData::FloatCompare { cond, .. }
| &InstructionData::FloatCond { cond, .. }
| &InstructionData::FloatCondTrap { cond, .. } => Some(cond),
_ => None,
}
}
/// If this is a trapping instruction, get an exclusive reference to its /// If this is a trapping instruction, get an exclusive reference to its
/// trap code. Otherwise, return `None`. /// trap code. Otherwise, return `None`.
pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode> { pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode> {
@@ -307,6 +335,27 @@ impl InstructionData {
} }
} }
/// If this is an atomic read/modify/write instruction, return its subopcode.
pub fn atomic_rmw_op(&self) -> Option<ir::AtomicRmwOp> {
match self {
&InstructionData::AtomicRmw { op, .. } => Some(op),
_ => None,
}
}
/// If this is a load/store instruction, returns its immediate offset.
pub fn load_store_offset(&self) -> Option<i32> {
match self {
&InstructionData::Load { offset, .. }
| &InstructionData::StackLoad { offset, .. }
| &InstructionData::LoadComplex { offset, .. }
| &InstructionData::Store { offset, .. }
| &InstructionData::StackStore { offset, .. }
| &InstructionData::StoreComplex { offset, .. } => Some(offset.into()),
_ => None,
}
}
/// Return information about a call instruction. /// Return information about a call instruction.
/// ///
/// Any instruction that can call another function reveals its call signature here. /// Any instruction that can call another function reveals its call signature here.

80
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -7,11 +7,10 @@
//! //!
//! - Floating-point immediates (FIMM instruction). //! - Floating-point immediates (FIMM instruction).
use crate::ir;
use crate::ir::condcodes::{FloatCC, IntCC}; use crate::ir::condcodes::{FloatCC, IntCC};
use crate::ir::types::*; use crate::ir::types::*;
use crate::ir::Inst as IRInst; use crate::ir::Inst as IRInst;
use crate::ir::{InstructionData, Opcode, TrapCode, Type}; use crate::ir::{InstructionData, Opcode, Type};
use crate::machinst::lower::*; use crate::machinst::lower::*;
use crate::machinst::*; use crate::machinst::*;
use crate::CodegenResult; use crate::CodegenResult;
@@ -106,26 +105,6 @@ pub(crate) enum ResultRegImmShift {
ImmShift(ImmShift), ImmShift(ImmShift),
} }
//============================================================================
// Instruction input "slots".
//
// We use these types to refer to operand numbers, and result numbers, together
// with the associated instruction, in a type-safe way.
/// Identifier for a particular input of an instruction.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct InsnInput {
pub(crate) insn: IRInst,
pub(crate) input: usize,
}
/// Identifier for a particular output of an instruction.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct InsnOutput {
pub(crate) insn: IRInst,
pub(crate) output: usize,
}
//============================================================================ //============================================================================
// Lowering: convert instruction inputs to forms that we can use. // Lowering: convert instruction inputs to forms that we can use.
@@ -191,11 +170,6 @@ impl NarrowValueMode {
} }
} }
/// Allocate a register for an instruction output and return it.
pub(crate) fn get_output_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, out: InsnOutput) -> Writable<Reg> {
ctx.get_output(out.insn, out.output)
}
/// Lower an instruction input to a reg. /// Lower an instruction input to a reg.
/// ///
/// The given register will be extended appropriately, according to /// The given register will be extended appropriately, according to
@@ -1023,58 +997,6 @@ pub(crate) fn choose_32_64<T: Copy>(ty: Type, op32: T, op64: T) -> T {
} }
} }
pub(crate) fn ldst_offset(data: &InstructionData) -> Option<i32> {
match data {
&InstructionData::Load { offset, .. }
| &InstructionData::StackLoad { offset, .. }
| &InstructionData::LoadComplex { offset, .. }
| &InstructionData::Store { offset, .. }
| &InstructionData::StackStore { offset, .. }
| &InstructionData::StoreComplex { offset, .. } => Some(offset.into()),
_ => None,
}
}
pub(crate) fn inst_condcode(data: &InstructionData) -> Option<IntCC> {
match data {
&InstructionData::IntCond { cond, .. }
| &InstructionData::BranchIcmp { cond, .. }
| &InstructionData::IntCompare { cond, .. }
| &InstructionData::IntCondTrap { cond, .. }
| &InstructionData::BranchInt { cond, .. }
| &InstructionData::IntSelect { cond, .. }
| &InstructionData::IntCompareImm { cond, .. } => Some(cond),
_ => None,
}
}
pub(crate) fn inst_fp_condcode(data: &InstructionData) -> Option<FloatCC> {
match data {
&InstructionData::BranchFloat { cond, .. }
| &InstructionData::FloatCompare { cond, .. }
| &InstructionData::FloatCond { cond, .. }
| &InstructionData::FloatCondTrap { cond, .. } => Some(cond),
_ => None,
}
}
pub(crate) fn inst_trapcode(data: &InstructionData) -> Option<TrapCode> {
match data {
&InstructionData::Trap { code, .. }
| &InstructionData::CondTrap { code, .. }
| &InstructionData::IntCondTrap { code, .. }
| &InstructionData::FloatCondTrap { code, .. } => Some(code),
_ => None,
}
}
pub(crate) fn inst_atomic_rmw_op(data: &InstructionData) -> Option<ir::AtomicRmwOp> {
match data {
&InstructionData::AtomicRmw { op, .. } => Some(op),
_ => None,
}
}
/// Checks for an instance of `op` feeding the given input. /// Checks for an instance of `op` feeding the given input.
pub(crate) fn maybe_input_insn<C: LowerCtx<I = Inst>>( pub(crate) fn maybe_input_insn<C: LowerCtx<I = Inst>>(
c: &mut C, c: &mut C,

40
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -1092,7 +1092,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::Uload16x4 | Opcode::Uload16x4
| Opcode::Sload32x2 | Opcode::Sload32x2
| Opcode::Uload32x2 => { | Opcode::Uload32x2 => {
let off = ldst_offset(ctx.data(insn)).unwrap(); let off = ctx.data(insn).load_store_offset().unwrap();
let elem_ty = match op { let elem_ty = match op {
Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => { Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => {
I8 I8
@@ -1177,7 +1177,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::Istore8Complex | Opcode::Istore8Complex
| Opcode::Istore16Complex | Opcode::Istore16Complex
| Opcode::Istore32Complex => { | Opcode::Istore32Complex => {
let off = ldst_offset(ctx.data(insn)).unwrap(); let off = ctx.data(insn).load_store_offset().unwrap();
let elem_ty = match op { let elem_ty = match op {
Opcode::Istore8 | Opcode::Istore8Complex => I8, Opcode::Istore8 | Opcode::Istore8Complex => I8,
Opcode::Istore16 | Opcode::Istore16Complex => I16, Opcode::Istore16 | Opcode::Istore16Complex => I16,
@@ -1247,7 +1247,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::gen_move(Writable::from_reg(xreg(25)), r_addr, I64)); ctx.emit(Inst::gen_move(Writable::from_reg(xreg(25)), r_addr, I64));
ctx.emit(Inst::gen_move(Writable::from_reg(xreg(26)), r_arg2, I64)); ctx.emit(Inst::gen_move(Writable::from_reg(xreg(26)), r_arg2, I64));
// Now the AtomicRMW insn itself // Now the AtomicRMW insn itself
let op = inst_common::AtomicRmwOp::from(inst_atomic_rmw_op(ctx.data(insn)).unwrap()); let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
ctx.emit(Inst::AtomicRMW { ctx.emit(Inst::AtomicRMW {
ty: ty_access, ty: ty_access,
op, op,
@@ -1366,7 +1366,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let cond = if let Some(icmp_insn) = let cond = if let Some(icmp_insn) =
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint) maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint)
{ {
let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap(); let condcode = ctx.data(icmp_insn).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let is_signed = condcode_is_signed(condcode); let is_signed = condcode_is_signed(condcode);
lower_icmp_or_ifcmp_to_flags(ctx, icmp_insn, is_signed); lower_icmp_or_ifcmp_to_flags(ctx, icmp_insn, is_signed);
@@ -1374,7 +1374,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else if let Some(fcmp_insn) = } else if let Some(fcmp_insn) =
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint) maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint)
{ {
let condcode = inst_fp_condcode(ctx.data(fcmp_insn)).unwrap(); let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode); let cond = lower_fp_condcode(condcode);
lower_fcmp_or_ffcmp_to_flags(ctx, fcmp_insn); lower_fcmp_or_ffcmp_to_flags(ctx, fcmp_insn);
cond cond
@@ -1413,7 +1413,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Selectif | Opcode::SelectifSpectreGuard => { Opcode::Selectif | Opcode::SelectifSpectreGuard => {
let condcode = inst_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let is_signed = condcode_is_signed(condcode); let is_signed = condcode_is_signed(condcode);
// Verification ensures that the input is always a // Verification ensures that the input is always a
@@ -1485,7 +1485,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Trueif => { Opcode::Trueif => {
let condcode = inst_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let is_signed = condcode_is_signed(condcode); let is_signed = condcode_is_signed(condcode);
// Verification ensures that the input is always a // Verification ensures that the input is always a
@@ -1498,7 +1498,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Trueff => { Opcode::Trueff => {
let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode); let cond = lower_fp_condcode(condcode);
let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap(); let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap();
lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn); lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn);
@@ -1688,7 +1688,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Icmp => { Opcode::Icmp => {
let condcode = inst_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let is_signed = condcode_is_signed(condcode); let is_signed = condcode_is_signed(condcode);
let rd = get_output_reg(ctx, outputs[0]); let rd = get_output_reg(ctx, outputs[0]);
@@ -1715,7 +1715,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Fcmp => { Opcode::Fcmp => {
let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode); let cond = lower_fp_condcode(condcode);
let ty = ctx.input_ty(insn, 0); let ty = ctx.input_ty(insn, 0);
let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
@@ -1748,15 +1748,15 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Trap | Opcode::ResumableTrap => { Opcode::Trap | Opcode::ResumableTrap => {
let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap());
ctx.emit_safepoint(Inst::Udf { trap_info }); ctx.emit_safepoint(Inst::Udf { trap_info });
} }
Opcode::Trapif | Opcode::Trapff => { Opcode::Trapif | Opcode::Trapff => {
let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap());
let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() { let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() {
let condcode = inst_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
// The flags must not have been clobbered by any other // The flags must not have been clobbered by any other
// instruction between the iadd_ifcout and this instruction, as // instruction between the iadd_ifcout and this instruction, as
@@ -1764,7 +1764,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// flags here. // flags here.
cond cond
} else if op == Opcode::Trapif { } else if op == Opcode::Trapif {
let condcode = inst_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let is_signed = condcode_is_signed(condcode); let is_signed = condcode_is_signed(condcode);
@@ -1773,7 +1773,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed); lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed);
cond cond
} else { } else {
let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); let condcode = ctx.data(insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode); let cond = lower_fp_condcode(condcode);
// Verification ensures that the input is always a // Verification ensures that the input is always a
@@ -2826,7 +2826,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
if let Some(icmp_insn) = if let Some(icmp_insn) =
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint) maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint)
{ {
let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap(); let condcode = ctx.data(icmp_insn).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let is_signed = condcode_is_signed(condcode); let is_signed = condcode_is_signed(condcode);
let negated = op0 == Opcode::Brz; let negated = op0 == Opcode::Brz;
@@ -2841,7 +2841,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
} else if let Some(fcmp_insn) = } else if let Some(fcmp_insn) =
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint) maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint)
{ {
let condcode = inst_fp_condcode(ctx.data(fcmp_insn)).unwrap(); let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode); let cond = lower_fp_condcode(condcode);
let negated = op0 == Opcode::Brz; let negated = op0 == Opcode::Brz;
let cond = if negated { cond.invert() } else { cond }; let cond = if negated { cond.invert() } else { cond };
@@ -2874,7 +2874,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
} }
} }
Opcode::BrIcmp => { Opcode::BrIcmp => {
let condcode = inst_condcode(ctx.data(branches[0])).unwrap(); let condcode = ctx.data(branches[0]).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let kind = CondBrKind::Cond(cond); let kind = CondBrKind::Cond(cond);
@@ -2915,7 +2915,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
} }
Opcode::Brif => { Opcode::Brif => {
let condcode = inst_condcode(ctx.data(branches[0])).unwrap(); let condcode = ctx.data(branches[0]).cond_code().unwrap();
let cond = lower_condcode(condcode); let cond = lower_condcode(condcode);
let kind = CondBrKind::Cond(cond); let kind = CondBrKind::Cond(cond);
@@ -2945,7 +2945,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
} }
Opcode::Brff => { Opcode::Brff => {
let condcode = inst_fp_condcode(ctx.data(branches[0])).unwrap(); let condcode = ctx.data(branches[0]).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode); let cond = lower_fp_condcode(condcode);
let kind = CondBrKind::Cond(cond); let kind = CondBrKind::Cond(cond);
let flag_input = InsnInput { let flag_input = InsnInput {

3
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -1,5 +1,4 @@
//! This module defines x86_64-specific machine instruction types.an explanation of what it's //! This module defines x86_64-specific machine instruction types.
//! doing.
#![allow(dead_code)] #![allow(dead_code)]
#![allow(non_snake_case)] #![allow(non_snake_case)]
#![allow(non_camel_case_types)] #![allow(non_camel_case_types)]

344
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -2,10 +2,9 @@
#![allow(non_snake_case)] #![allow(non_snake_case)]
use crate::ir;
use crate::ir::{ use crate::ir::{
condcodes::FloatCC, condcodes::IntCC, types, AbiParam, ArgumentPurpose, ExternalName, condcodes::FloatCC, types, AbiParam, ArgumentPurpose, ExternalName, Inst as IRInst,
Inst as IRInst, InstructionData, LibCall, Opcode, Signature, TrapCode, Type, InstructionData, LibCall, Opcode, Signature, Type,
}; };
use crate::isa::x64::abi::*; use crate::isa::x64::abi::*;
use crate::isa::x64::inst::args::*; use crate::isa::x64::inst::args::*;
@@ -54,76 +53,6 @@ fn is_valid_atomic_transaction_ty(ty: Type) -> bool {
} }
} }
fn iri_to_u64_imm(ctx: Ctx, inst: IRInst) -> Option<u64> {
ctx.get_constant(inst)
}
fn inst_trapcode(data: &InstructionData) -> Option<TrapCode> {
match data {
&InstructionData::Trap { code, .. }
| &InstructionData::CondTrap { code, .. }
| &InstructionData::IntCondTrap { code, .. }
| &InstructionData::FloatCondTrap { code, .. } => Some(code),
_ => None,
}
}
fn inst_condcode(data: &InstructionData) -> IntCC {
match data {
&InstructionData::IntCond { cond, .. }
| &InstructionData::BranchIcmp { cond, .. }
| &InstructionData::IntCompare { cond, .. }
| &InstructionData::IntCondTrap { cond, .. }
| &InstructionData::BranchInt { cond, .. }
| &InstructionData::IntSelect { cond, .. }
| &InstructionData::IntCompareImm { cond, .. } => cond,
_ => panic!("inst_condcode(x64): unhandled: {:?}", data),
}
}
fn inst_fp_condcode(data: &InstructionData) -> FloatCC {
match data {
&InstructionData::BranchFloat { cond, .. }
| &InstructionData::FloatCompare { cond, .. }
| &InstructionData::FloatCond { cond, .. }
| &InstructionData::FloatCondTrap { cond, .. } => cond,
_ => panic!("inst_fp_condcode(x64): unhandled: {:?}", data),
}
}
fn inst_atomic_rmw_op(data: &InstructionData) -> Option<ir::AtomicRmwOp> {
match data {
&InstructionData::AtomicRmw { op, .. } => Some(op),
_ => None,
}
}
fn ldst_offset(data: &InstructionData) -> Option<i32> {
match data {
&InstructionData::Load { offset, .. }
| &InstructionData::StackLoad { offset, .. }
| &InstructionData::LoadComplex { offset, .. }
| &InstructionData::Store { offset, .. }
| &InstructionData::StackStore { offset, .. }
| &InstructionData::StoreComplex { offset, .. } => Some(offset.into()),
_ => None,
}
}
/// Identifier for a particular input of an instruction.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct InsnInput {
insn: IRInst,
input: usize,
}
/// Identifier for a particular output of an instruction.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct InsnOutput {
insn: IRInst,
output: usize,
}
/// Returns whether the given specified `input` is a result produced by an instruction with Opcode /// Returns whether the given specified `input` is a result produced by an instruction with Opcode
/// `op`. /// `op`.
// TODO investigate failures with checking against the result index. // TODO investigate failures with checking against the result index.
@@ -148,7 +77,7 @@ fn lowerinput_to_reg(ctx: Ctx, input: LowerInput) -> Reg {
} }
/// Put the given input into a register, and mark it as used (side-effect). /// Put the given input into a register, and mark it as used (side-effect).
fn input_to_reg(ctx: Ctx, spec: InsnInput) -> Reg { fn put_input_in_reg(ctx: Ctx, spec: InsnInput) -> Reg {
let input = ctx.get_input(spec.insn, spec.input); let input = ctx.get_input(spec.insn, spec.input);
lowerinput_to_reg(ctx, input) lowerinput_to_reg(ctx, input)
} }
@@ -178,8 +107,8 @@ fn extend_input_to_reg(ctx: Ctx, spec: InsnInput, ext_spec: ExtSpec) -> Reg {
}; };
let ext_mode = match (input_size, requested_size) { let ext_mode = match (input_size, requested_size) {
(a, b) if a == b => return input_to_reg(ctx, spec), (a, b) if a == b => return put_input_in_reg(ctx, spec),
(1, 8) => return input_to_reg(ctx, spec), (1, 8) => return put_input_in_reg(ctx, spec),
(a, 16) | (a, 32) if a == 1 || a == 8 => ExtMode::BL, (a, 16) | (a, 32) if a == 1 || a == 8 => ExtMode::BL,
(a, 64) if a == 1 || a == 8 => ExtMode::BQ, (a, 64) if a == 1 || a == 8 => ExtMode::BQ,
(16, 32) => ExtMode::WL, (16, 32) => ExtMode::WL,
@@ -256,17 +185,13 @@ fn input_to_reg_mem_imm(ctx: Ctx, spec: InsnInput) -> RegMemImm {
} }
} }
fn output_to_reg(ctx: Ctx, spec: InsnOutput) -> Writable<Reg> {
ctx.get_output(spec.insn, spec.output)
}
fn emit_cmp(ctx: Ctx, insn: IRInst) { fn emit_cmp(ctx: Ctx, insn: IRInst) {
let ty = ctx.input_ty(insn, 0); let ty = ctx.input_ty(insn, 0);
let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }]; let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }];
// TODO Try to commute the operands (and invert the condition) if one is an immediate. // TODO Try to commute the operands (and invert the condition) if one is an immediate.
let lhs = input_to_reg(ctx, inputs[0]); let lhs = put_input_in_reg(ctx, inputs[0]);
let rhs = input_to_reg_mem_imm(ctx, inputs[1]); let rhs = input_to_reg_mem_imm(ctx, inputs[1]);
// Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives
@@ -342,7 +267,7 @@ fn emit_fcmp(ctx: Ctx, insn: IRInst, mut cond_code: FloatCC, spec: FcmpSpec) ->
} else { } else {
(inputs[0], inputs[1]) (inputs[0], inputs[1])
}; };
let lhs = input_to_reg(ctx, lhs_input); let lhs = put_input_in_reg(ctx, lhs_input);
let rhs = input_to_reg_mem(ctx, rhs_input); let rhs = input_to_reg_mem(ctx, rhs_input);
ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs)); ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs));
@@ -404,7 +329,7 @@ fn emit_vm_call<C: LowerCtx<I = Inst>>(
assert_eq!(inputs.len() + vm_context, abi.num_args()); assert_eq!(inputs.len() + vm_context, abi.num_args());
for (i, input) in inputs.iter().enumerate() { for (i, input) in inputs.iter().enumerate() {
let arg_reg = input_to_reg(ctx, *input); let arg_reg = put_input_in_reg(ctx, *input);
abi.emit_copy_reg_to_arg(ctx, i, arg_reg); abi.emit_copy_reg_to_arg(ctx, i, arg_reg);
} }
if call_conv.extends_baldrdash() { if call_conv.extends_baldrdash() {
@@ -416,7 +341,7 @@ fn emit_vm_call<C: LowerCtx<I = Inst>>(
abi.emit_call(ctx); abi.emit_call(ctx);
for (i, output) in outputs.iter().enumerate() { for (i, output) in outputs.iter().enumerate() {
let retval_reg = output_to_reg(ctx, *output); let retval_reg = get_output_reg(ctx, *output);
abi.emit_copy_retval_to_reg(ctx, i, retval_reg); abi.emit_copy_retval_to_reg(ctx, i, retval_reg);
} }
abi.emit_stack_post_adjust(ctx); abi.emit_stack_post_adjust(ctx);
@@ -471,22 +396,22 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: u
matches_small_constant_shift(ctx, add_inputs[0]) matches_small_constant_shift(ctx, add_inputs[0])
{ {
( (
input_to_reg(ctx, add_inputs[1]), put_input_in_reg(ctx, add_inputs[1]),
input_to_reg(ctx, shift_input), put_input_in_reg(ctx, shift_input),
shift_amt, shift_amt,
) )
} else if let Some((shift_input, shift_amt)) = } else if let Some((shift_input, shift_amt)) =
matches_small_constant_shift(ctx, add_inputs[1]) matches_small_constant_shift(ctx, add_inputs[1])
{ {
( (
input_to_reg(ctx, add_inputs[0]), put_input_in_reg(ctx, add_inputs[0]),
input_to_reg(ctx, shift_input), put_input_in_reg(ctx, shift_input),
shift_amt, shift_amt,
) )
} else { } else {
( (
input_to_reg(ctx, add_inputs[0]), put_input_in_reg(ctx, add_inputs[0]),
input_to_reg(ctx, add_inputs[1]), put_input_in_reg(ctx, add_inputs[1]),
0, 0,
) )
}; };
@@ -494,7 +419,7 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: u
return Amode::imm_reg_reg_shift(offset, base, index, shift); return Amode::imm_reg_reg_shift(offset, base, index, shift);
} }
let input = input_to_reg(ctx, spec); let input = put_input_in_reg(ctx, spec);
Amode::imm_reg(offset, input) Amode::imm_reg(offset, input)
} }
@@ -525,13 +450,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
match op { match op {
Opcode::Iconst | Opcode::Bconst | Opcode::Null => { Opcode::Iconst | Opcode::Bconst | Opcode::Null => {
if let Some(w64) = iri_to_u64_imm(ctx, insn) { let w64 = ctx
.get_constant(insn)
.expect("constant value for iconst et al");
let dst_is_64 = w64 > 0x7fffffff; let dst_is_64 = w64 > 0x7fffffff;
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::imm_r(dst_is_64, w64, dst)); ctx.emit(Inst::imm_r(dst_is_64, w64, dst));
} else {
unimplemented!();
}
} }
Opcode::Iadd Opcode::Iadd
@@ -565,9 +489,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}, },
_ => panic!("Unsupported packed instruction"), _ => panic!("Unsupported packed instruction"),
}; };
let lhs = input_to_reg(ctx, inputs[0]); let lhs = put_input_in_reg(ctx, inputs[0]);
let rhs = input_to_reg_mem(ctx, inputs[1]); let rhs = input_to_reg_mem(ctx, inputs[1]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
// Move the `lhs` to the same register as `dst`. // Move the `lhs` to the same register as `dst`.
ctx.emit(Inst::gen_move(dst, lhs, ty)); ctx.emit(Inst::gen_move(dst, lhs, ty));
@@ -594,22 +518,22 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// For commutative operations, try to commute operands if one is an // For commutative operations, try to commute operands if one is an
// immediate. // immediate.
if let Some(imm) = input_to_sext_imm(ctx, inputs[0]) { if let Some(imm) = input_to_sext_imm(ctx, inputs[0]) {
(input_to_reg(ctx, inputs[1]), RegMemImm::imm(imm)) (put_input_in_reg(ctx, inputs[1]), RegMemImm::imm(imm))
} else { } else {
( (
input_to_reg(ctx, inputs[0]), put_input_in_reg(ctx, inputs[0]),
input_to_reg_mem_imm(ctx, inputs[1]), input_to_reg_mem_imm(ctx, inputs[1]),
) )
} }
} }
Opcode::Isub => ( Opcode::Isub => (
input_to_reg(ctx, inputs[0]), put_input_in_reg(ctx, inputs[0]),
input_to_reg_mem_imm(ctx, inputs[1]), input_to_reg_mem_imm(ctx, inputs[1]),
), ),
_ => unreachable!(), _ => unreachable!(),
}; };
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::mov_r_r(true, lhs, dst)); ctx.emit(Inst::mov_r_r(true, lhs, dst));
ctx.emit(Inst::alu_rmi_r(is_64, alu_op, rhs, dst)); ctx.emit(Inst::alu_rmi_r(is_64, alu_op, rhs, dst));
} }
@@ -623,8 +547,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
unimplemented!("bool bnot") unimplemented!("bool bnot")
} else { } else {
let size = ty.bytes() as u8; let size = ty.bytes() as u8;
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::gen_move(dst, src, ty)); ctx.emit(Inst::gen_move(dst, src, ty));
ctx.emit(Inst::not(size, dst)); ctx.emit(Inst::not(size, dst));
} }
@@ -636,7 +560,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let (size, lhs) = match dst_ty { let (size, lhs) = match dst_ty {
types::I8 | types::I16 => match op { types::I8 | types::I16 => match op {
Opcode::Ishl => (4, input_to_reg(ctx, inputs[0])), Opcode::Ishl => (4, put_input_in_reg(ctx, inputs[0])),
Opcode::Ushr => ( Opcode::Ushr => (
4, 4,
extend_input_to_reg(ctx, inputs[0], ExtSpec::ZeroExtendTo32), extend_input_to_reg(ctx, inputs[0], ExtSpec::ZeroExtendTo32),
@@ -646,27 +570,23 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
extend_input_to_reg(ctx, inputs[0], ExtSpec::SignExtendTo32), extend_input_to_reg(ctx, inputs[0], ExtSpec::SignExtendTo32),
), ),
Opcode::Rotl | Opcode::Rotr => { Opcode::Rotl | Opcode::Rotr => {
(dst_ty.bytes() as u8, input_to_reg(ctx, inputs[0])) (dst_ty.bytes() as u8, put_input_in_reg(ctx, inputs[0]))
} }
_ => unreachable!(), _ => unreachable!(),
}, },
types::I32 | types::I64 => (dst_ty.bytes() as u8, input_to_reg(ctx, inputs[0])), types::I32 | types::I64 => (dst_ty.bytes() as u8, put_input_in_reg(ctx, inputs[0])),
_ => unreachable!("unhandled output type for shift/rotates: {}", dst_ty), _ => unreachable!("unhandled output type for shift/rotates: {}", dst_ty),
}; };
let (count, rhs) = if let Some(cst) = ctx.get_input(insn, 1).constant { let (count, rhs) = if let Some(cst) = ctx.get_input(insn, 1).constant {
let cst = if op == Opcode::Rotl || op == Opcode::Rotr { // Mask count, according to Cranelift's semantics.
// Mask rotation count, according to Cranelift's semantics. let cst = (cst as u8) & (dst_ty.bits() as u8 - 1);
(cst as u8) & (dst_ty.bits() as u8 - 1)
} else {
cst as u8
};
(Some(cst), None) (Some(cst), None)
} else { } else {
(None, Some(input_to_reg(ctx, inputs[1]))) (None, Some(put_input_in_reg(ctx, inputs[1])))
}; };
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let shift_kind = match op { let shift_kind = match op {
Opcode::Ishl => ShiftKind::ShiftLeft, Opcode::Ishl => ShiftKind::ShiftLeft,
@@ -686,7 +606,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Ineg => { Opcode::Ineg => {
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ty.unwrap(); let ty = ty.unwrap();
if ty.is_vector() { if ty.is_vector() {
@@ -719,7 +639,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
} else { } else {
let size = ty.bytes() as u8; let size = ty.bytes() as u8;
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
ctx.emit(Inst::gen_move(dst, src, ty)); ctx.emit(Inst::gen_move(dst, src, ty));
ctx.emit(Inst::neg(size, dst)); ctx.emit(Inst::neg(size, dst));
} }
@@ -746,7 +666,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else { } else {
input_to_reg_mem(ctx, inputs[0]) input_to_reg_mem(ctx, inputs[0])
}; };
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let tmp = ctx.alloc_tmp(RegClass::I64, ty); let tmp = ctx.alloc_tmp(RegClass::I64, ty);
ctx.emit(Inst::imm_r(ty == types::I64, u64::max_value(), dst)); ctx.emit(Inst::imm_r(ty == types::I64, u64::max_value(), dst));
@@ -787,7 +707,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
debug_assert!(ty == types::I32 || ty == types::I64); debug_assert!(ty == types::I32 || ty == types::I64);
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let tmp = ctx.alloc_tmp(RegClass::I64, ty); let tmp = ctx.alloc_tmp(RegClass::I64, ty);
ctx.emit(Inst::imm_r(false /* 64 bits */, ty.bits() as u64, tmp)); ctx.emit(Inst::imm_r(false /* 64 bits */, ty.bits() as u64, tmp));
@@ -821,7 +741,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else { } else {
input_to_reg_mem(ctx, inputs[0]) input_to_reg_mem(ctx, inputs[0])
}; };
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
if ty == types::I64 { if ty == types::I64 {
let is_64 = true; let is_64 = true;
@@ -1083,8 +1003,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// Null references are represented by the constant value 0; invalid references are // Null references are represented by the constant value 0; invalid references are
// represented by the constant value -1. See `define_reftypes()` in // represented by the constant value -1. See `define_reftypes()` in
// `meta/src/isa/x86/encodings.rs` to confirm. // `meta/src/isa/x86/encodings.rs` to confirm.
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ctx.input_ty(insn, 0); let ty = ctx.input_ty(insn, 0);
let imm = match op { let imm = match op {
Opcode::IsNull => { Opcode::IsNull => {
@@ -1112,7 +1032,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let dst_ty = ctx.output_ty(insn, 0); let dst_ty = ctx.output_ty(insn, 0);
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ext_mode = match (src_ty.bits(), dst_ty.bits()) { let ext_mode = match (src_ty.bits(), dst_ty.bits()) {
(1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL), (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),
@@ -1157,14 +1077,14 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::Icmp => { Opcode::Icmp => {
emit_cmp(ctx, insn); emit_cmp(ctx, insn);
let condcode = inst_condcode(ctx.data(insn)); let condcode = ctx.data(insn).cond_code().unwrap();
let cc = CC::from_intcc(condcode); let cc = CC::from_intcc(condcode);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::setcc(cc, dst)); ctx.emit(Inst::setcc(cc, dst));
} }
Opcode::Fcmp => { Opcode::Fcmp => {
let cond_code = inst_fp_condcode(ctx.data(insn)); let cond_code = ctx.data(insn).fp_cond_code().unwrap();
let input_ty = ctx.input_ty(insn, 0); let input_ty = ctx.input_ty(insn, 0);
if !input_ty.is_vector() { if !input_ty.is_vector() {
// Unordered is returned by setting ZF, PF, CF <- 111 // Unordered is returned by setting ZF, PF, CF <- 111
@@ -1182,7 +1102,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// set, then both the ZF and CF flag bits must also be set we can get away with using // set, then both the ZF and CF flag bits must also be set we can get away with using
// one setcc for most condition codes. // one setcc for most condition codes.
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
match emit_fcmp(ctx, insn, cond_code, FcmpSpec::Normal) { match emit_fcmp(ctx, insn, cond_code, FcmpSpec::Normal) {
FcmpCondResult::Condition(cc) => { FcmpCondResult::Condition(cc) => {
@@ -1237,12 +1157,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// Determine the operands of the comparison, possibly by flipping them. // Determine the operands of the comparison, possibly by flipping them.
let (lhs, rhs) = if flip { let (lhs, rhs) = if flip {
( (
input_to_reg(ctx, inputs[1]), put_input_in_reg(ctx, inputs[1]),
input_to_reg_mem(ctx, inputs[0]), input_to_reg_mem(ctx, inputs[0]),
) )
} else { } else {
( (
input_to_reg(ctx, inputs[0]), put_input_in_reg(ctx, inputs[0]),
input_to_reg_mem(ctx, inputs[1]), input_to_reg_mem(ctx, inputs[1]),
) )
}; };
@@ -1250,7 +1170,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// Move the `lhs` to the same register as `dst`; this may not emit an actual move // Move the `lhs` to the same register as `dst`; this may not emit an actual move
// but ensures that the registers are the same to match x86's read-write operand // but ensures that the registers are the same to match x86's read-write operand
// encoding. // encoding.
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::gen_move(dst, lhs, input_ty)); ctx.emit(Inst::gen_move(dst, lhs, input_ty));
// Emit the comparison. // Emit the comparison.
@@ -1260,7 +1180,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::FallthroughReturn | Opcode::Return => { Opcode::FallthroughReturn | Opcode::Return => {
for i in 0..ctx.num_inputs(insn) { for i in 0..ctx.num_inputs(insn) {
let src_reg = input_to_reg(ctx, inputs[i]); let src_reg = put_input_in_reg(ctx, inputs[i]);
let retval_reg = ctx.retval(i); let retval_reg = ctx.retval(i);
let ty = ctx.input_ty(insn, i); let ty = ctx.input_ty(insn, i);
ctx.emit(Inst::gen_move(retval_reg, src_reg, ty)); ctx.emit(Inst::gen_move(retval_reg, src_reg, ty));
@@ -1283,7 +1203,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::CallIndirect => { Opcode::CallIndirect => {
let ptr = input_to_reg(ctx, inputs[0]); let ptr = put_input_in_reg(ctx, inputs[0]);
let sig = ctx.call_sig(insn).unwrap(); let sig = ctx.call_sig(insn).unwrap();
assert_eq!(inputs.len() - 1, sig.params.len()); assert_eq!(inputs.len() - 1, sig.params.len());
assert_eq!(outputs.len(), sig.returns.len()); assert_eq!(outputs.len(), sig.returns.len());
@@ -1296,12 +1216,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
abi.emit_stack_pre_adjust(ctx); abi.emit_stack_pre_adjust(ctx);
assert_eq!(inputs.len(), abi.num_args()); assert_eq!(inputs.len(), abi.num_args());
for (i, input) in inputs.iter().enumerate() { for (i, input) in inputs.iter().enumerate() {
let arg_reg = input_to_reg(ctx, *input); let arg_reg = put_input_in_reg(ctx, *input);
abi.emit_copy_reg_to_arg(ctx, i, arg_reg); abi.emit_copy_reg_to_arg(ctx, i, arg_reg);
} }
abi.emit_call(ctx); abi.emit_call(ctx);
for (i, output) in outputs.iter().enumerate() { for (i, output) in outputs.iter().enumerate() {
let retval_reg = output_to_reg(ctx, *output); let retval_reg = get_output_reg(ctx, *output);
abi.emit_copy_retval_to_reg(ctx, i, retval_reg); abi.emit_copy_retval_to_reg(ctx, i, retval_reg);
} }
abi.emit_stack_post_adjust(ctx); abi.emit_stack_post_adjust(ctx);
@@ -1312,16 +1232,16 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Trap | Opcode::ResumableTrap => { Opcode::Trap | Opcode::ResumableTrap => {
let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap());
ctx.emit_safepoint(Inst::Ud2 { trap_info }); ctx.emit_safepoint(Inst::Ud2 { trap_info });
} }
Opcode::Trapif | Opcode::Trapff => { Opcode::Trapif | Opcode::Trapff => {
let srcloc = ctx.srcloc(insn); let srcloc = ctx.srcloc(insn);
let trap_code = inst_trapcode(ctx.data(insn)).unwrap(); let trap_code = ctx.data(insn).trap_code().unwrap();
if matches_input(ctx, inputs[0], Opcode::IaddIfcout).is_some() { if matches_input(ctx, inputs[0], Opcode::IaddIfcout).is_some() {
let cond_code = inst_condcode(ctx.data(insn)); let cond_code = ctx.data(insn).cond_code().unwrap();
// The flags must not have been clobbered by any other instruction between the // The flags must not have been clobbered by any other instruction between the
// iadd_ifcout and this instruction, as verified by the CLIF validator; so we can // iadd_ifcout and this instruction, as verified by the CLIF validator; so we can
// simply use the flags here. // simply use the flags here.
@@ -1333,7 +1253,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
cc, cc,
}); });
} else if op == Opcode::Trapif { } else if op == Opcode::Trapif {
let cond_code = inst_condcode(ctx.data(insn)); let cond_code = ctx.data(insn).cond_code().unwrap();
let cc = CC::from_intcc(cond_code); let cc = CC::from_intcc(cond_code);
// Verification ensures that the input is always a single-def ifcmp. // Verification ensures that the input is always a single-def ifcmp.
@@ -1346,7 +1266,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
cc, cc,
}); });
} else { } else {
let cond_code = inst_fp_condcode(ctx.data(insn)); let cond_code = ctx.data(insn).fp_cond_code().unwrap();
// Verification ensures that the input is always a single-def ffcmp. // Verification ensures that the input is always a single-def ffcmp.
let ffcmp = matches_input(ctx, inputs[0], Opcode::Ffcmp).unwrap(); let ffcmp = matches_input(ctx, inputs[0], Opcode::Ffcmp).unwrap();
@@ -1396,7 +1316,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::F64const => { Opcode::F64const => {
// TODO use cmpeqpd for all 1s. // TODO use cmpeqpd for all 1s.
let value = ctx.get_constant(insn).unwrap(); let value = ctx.get_constant(insn).unwrap();
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
for inst in Inst::gen_constant(dst, value, types::F64, |reg_class, ty| { for inst in Inst::gen_constant(dst, value, types::F64, |reg_class, ty| {
ctx.alloc_tmp(reg_class, ty) ctx.alloc_tmp(reg_class, ty)
}) { }) {
@@ -1407,7 +1327,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::F32const => { Opcode::F32const => {
// TODO use cmpeqps for all 1s. // TODO use cmpeqps for all 1s.
let value = ctx.get_constant(insn).unwrap(); let value = ctx.get_constant(insn).unwrap();
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
for inst in Inst::gen_constant(dst, value, types::F32, |reg_class, ty| { for inst in Inst::gen_constant(dst, value, types::F32, |reg_class, ty| {
ctx.alloc_tmp(reg_class, ty) ctx.alloc_tmp(reg_class, ty)
}) { }) {
@@ -1416,9 +1336,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => { Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => {
let lhs = input_to_reg(ctx, inputs[0]); let lhs = put_input_in_reg(ctx, inputs[0]);
let rhs = input_to_reg_mem(ctx, inputs[1]); let rhs = input_to_reg_mem(ctx, inputs[1]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ty.unwrap(); let ty = ty.unwrap();
// Move the `lhs` to the same register as `dst`; this may not emit an actual move // Move the `lhs` to the same register as `dst`; this may not emit an actual move
@@ -1467,9 +1387,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Fmin | Opcode::Fmax => { Opcode::Fmin | Opcode::Fmax => {
let lhs = input_to_reg(ctx, inputs[0]); let lhs = put_input_in_reg(ctx, inputs[0]);
let rhs = input_to_reg(ctx, inputs[1]); let rhs = put_input_in_reg(ctx, inputs[1]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let is_min = op == Opcode::Fmin; let is_min = op == Opcode::Fmin;
let output_ty = ty.unwrap(); let output_ty = ty.unwrap();
ctx.emit(Inst::gen_move(dst, rhs, output_ty)); ctx.emit(Inst::gen_move(dst, rhs, output_ty));
@@ -1483,7 +1403,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::Sqrt => { Opcode::Sqrt => {
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ty.unwrap(); let ty = ty.unwrap();
let sse_op = match ty { let sse_op = match ty {
@@ -1502,13 +1422,13 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::Fpromote => { Opcode::Fpromote => {
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, src, dst)); ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, src, dst));
} }
Opcode::Fdemote => { Opcode::Fdemote => {
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, src, dst)); ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, src, dst));
} }
@@ -1533,12 +1453,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
SseOpcode::Cvtsi2sd SseOpcode::Cvtsi2sd
}; };
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::gpr_to_xmm(opcode, src, src_size, dst)); ctx.emit(Inst::gpr_to_xmm(opcode, src, src_size, dst));
} }
Opcode::FcvtFromUint => { Opcode::FcvtFromUint => {
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ty.unwrap(); let ty = ty.unwrap();
let input_ty = ctx.input_ty(insn, 0); let input_ty = ctx.input_ty(insn, 0);
@@ -1559,7 +1479,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
types::I64 => { types::I64 => {
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let src_copy = ctx.alloc_tmp(RegClass::I64, types::I64); let src_copy = ctx.alloc_tmp(RegClass::I64, types::I64);
ctx.emit(Inst::gen_move(src_copy, src, types::I64)); ctx.emit(Inst::gen_move(src_copy, src, types::I64));
@@ -1580,8 +1500,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::FcvtToUint | Opcode::FcvtToUintSat | Opcode::FcvtToSint | Opcode::FcvtToSintSat => { Opcode::FcvtToUint | Opcode::FcvtToUintSat | Opcode::FcvtToSint | Opcode::FcvtToSintSat => {
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let input_ty = ctx.input_ty(insn, 0); let input_ty = ctx.input_ty(insn, 0);
let src_size = if input_ty == types::F32 { let src_size = if input_ty == types::F32 {
@@ -1625,8 +1545,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let output_ty = ctx.output_ty(insn, 0); let output_ty = ctx.output_ty(insn, 0);
match (input_ty, output_ty) { match (input_ty, output_ty) {
(types::F32, types::I32) => { (types::F32, types::I32) => {
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::xmm_to_gpr( ctx.emit(Inst::xmm_to_gpr(
SseOpcode::Movd, SseOpcode::Movd,
src, src,
@@ -1636,7 +1556,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
(types::I32, types::F32) => { (types::I32, types::F32) => {
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::gpr_to_xmm( ctx.emit(Inst::gpr_to_xmm(
SseOpcode::Movd, SseOpcode::Movd,
src, src,
@@ -1645,8 +1565,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
} }
(types::F64, types::I64) => { (types::F64, types::I64) => {
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::xmm_to_gpr( ctx.emit(Inst::xmm_to_gpr(
SseOpcode::Movq, SseOpcode::Movq,
src, src,
@@ -1656,7 +1576,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
(types::I64, types::F64) => { (types::I64, types::F64) => {
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::gpr_to_xmm( ctx.emit(Inst::gpr_to_xmm(
SseOpcode::Movq, SseOpcode::Movq,
src, src,
@@ -1670,7 +1590,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::Fabs | Opcode::Fneg => { Opcode::Fabs | Opcode::Fneg => {
let src = input_to_reg_mem(ctx, inputs[0]); let src = input_to_reg_mem(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
// In both cases, generate a constant and apply a single binary instruction: // In both cases, generate a constant and apply a single binary instruction:
// - to compute the absolute value, set all bits to 1 but the MSB to 0, and bit-AND the // - to compute the absolute value, set all bits to 1 but the MSB to 0, and bit-AND the
@@ -1707,7 +1627,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// Move the `lhs` to the same register as `dst`; this may not emit an actual move // Move the `lhs` to the same register as `dst`; this may not emit an actual move
// but ensures that the registers are the same to match x86's read-write operand // but ensures that the registers are the same to match x86's read-write operand
// encoding. // encoding.
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
ctx.emit(Inst::gen_move(dst, src, output_ty)); ctx.emit(Inst::gen_move(dst, src, output_ty));
// Generate an all 1s constant in an XMM register. This uses CMPPS but could // Generate an all 1s constant in an XMM register. This uses CMPPS but could
@@ -1747,9 +1667,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Fcopysign => { Opcode::Fcopysign => {
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let lhs = input_to_reg(ctx, inputs[0]); let lhs = put_input_in_reg(ctx, inputs[0]);
let rhs = input_to_reg(ctx, inputs[1]); let rhs = put_input_in_reg(ctx, inputs[1]);
let ty = ty.unwrap(); let ty = ty.unwrap();
@@ -1844,7 +1764,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::Sload16Complex | Opcode::Sload16Complex
| Opcode::Uload32Complex | Opcode::Uload32Complex
| Opcode::Sload32Complex => { | Opcode::Sload32Complex => {
let offset = ldst_offset(ctx.data(insn)).unwrap(); let offset = ctx.data(insn).load_store_offset().unwrap();
let elem_ty = match op { let elem_ty = match op {
Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => { Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => {
@@ -1903,8 +1823,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
2, 2,
"can't handle more than two inputs in complex load" "can't handle more than two inputs in complex load"
); );
let base = input_to_reg(ctx, inputs[0]); let base = put_input_in_reg(ctx, inputs[0]);
let index = input_to_reg(ctx, inputs[1]); let index = put_input_in_reg(ctx, inputs[1]);
let shift = 0; let shift = 0;
Amode::imm_reg_reg_shift(offset as u32, base, index, shift) Amode::imm_reg_reg_shift(offset as u32, base, index, shift)
} }
@@ -1914,7 +1834,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let srcloc = Some(ctx.srcloc(insn)); let srcloc = Some(ctx.srcloc(insn));
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let is_xmm = elem_ty.is_float() || elem_ty.is_vector(); let is_xmm = elem_ty.is_float() || elem_ty.is_vector();
match (sign_extend, is_xmm) { match (sign_extend, is_xmm) {
(true, false) => { (true, false) => {
@@ -1966,7 +1886,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::Istore8Complex | Opcode::Istore8Complex
| Opcode::Istore16Complex | Opcode::Istore16Complex
| Opcode::Istore32Complex => { | Opcode::Istore32Complex => {
let offset = ldst_offset(ctx.data(insn)).unwrap(); let offset = ctx.data(insn).load_store_offset().unwrap();
let elem_ty = match op { let elem_ty = match op {
Opcode::Istore8 | Opcode::Istore8Complex => types::I8, Opcode::Istore8 | Opcode::Istore8Complex => types::I8,
@@ -1991,8 +1911,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
3, 3,
"can't handle more than two inputs in complex store" "can't handle more than two inputs in complex store"
); );
let base = input_to_reg(ctx, inputs[1]); let base = put_input_in_reg(ctx, inputs[1]);
let index = input_to_reg(ctx, inputs[2]); let index = put_input_in_reg(ctx, inputs[2]);
let shift = 0; let shift = 0;
Amode::imm_reg_reg_shift(offset as u32, base, index, shift) Amode::imm_reg_reg_shift(offset as u32, base, index, shift)
} }
@@ -2000,7 +1920,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
_ => unreachable!(), _ => unreachable!(),
}; };
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let srcloc = Some(ctx.srcloc(insn)); let srcloc = Some(ctx.srcloc(insn));
@@ -2025,9 +1945,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// use the single instruction `lock xadd`. However, those improvements have been // use the single instruction `lock xadd`. However, those improvements have been
// left for another day. // left for another day.
// TODO: filed as https://github.com/bytecodealliance/wasmtime/issues/2153 // TODO: filed as https://github.com/bytecodealliance/wasmtime/issues/2153
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let mut addr = input_to_reg(ctx, inputs[0]); let mut addr = put_input_in_reg(ctx, inputs[0]);
let mut arg2 = input_to_reg(ctx, inputs[1]); let mut arg2 = put_input_in_reg(ctx, inputs[1]);
let ty_access = ty.unwrap(); let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags"); let memflags = ctx.memflags(insn).expect("memory flags");
@@ -2058,7 +1978,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
// Now the AtomicRmwSeq (pseudo-) instruction itself // Now the AtomicRmwSeq (pseudo-) instruction itself
let op = inst_common::AtomicRmwOp::from(inst_atomic_rmw_op(ctx.data(insn)).unwrap()); let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
ctx.emit(Inst::AtomicRmwSeq { ctx.emit(Inst::AtomicRmwSeq {
ty: ty_access, ty: ty_access,
op, op,
@@ -2072,10 +1992,10 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::AtomicCas => { Opcode::AtomicCas => {
// This is very similar to, but not identical to, the `AtomicRmw` case. As with // This is very similar to, but not identical to, the `AtomicRmw` case. As with
// `AtomicRmw`, there's no need to zero-extend narrow values here. // `AtomicRmw`, there's no need to zero-extend narrow values here.
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let addr = lower_to_amode(ctx, inputs[0], 0); let addr = lower_to_amode(ctx, inputs[0], 0);
let expected = input_to_reg(ctx, inputs[1]); let expected = put_input_in_reg(ctx, inputs[1]);
let replacement = input_to_reg(ctx, inputs[2]); let replacement = put_input_in_reg(ctx, inputs[2]);
let ty_access = ty.unwrap(); let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
let memflags = ctx.memflags(insn).expect("memory flags"); let memflags = ctx.memflags(insn).expect("memory flags");
@@ -2107,7 +2027,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// This is a normal load. The x86-TSO memory model provides sufficient sequencing // This is a normal load. The x86-TSO memory model provides sufficient sequencing
// to satisfy the CLIF synchronisation requirements for `AtomicLoad` without the // to satisfy the CLIF synchronisation requirements for `AtomicLoad` without the
// need for any fence instructions. // need for any fence instructions.
let data = output_to_reg(ctx, outputs[0]); let data = get_output_reg(ctx, outputs[0]);
let addr = lower_to_amode(ctx, inputs[0], 0); let addr = lower_to_amode(ctx, inputs[0], 0);
let ty_access = ty.unwrap(); let ty_access = ty.unwrap();
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
@@ -2134,7 +2054,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::AtomicStore => { Opcode::AtomicStore => {
// This is a normal store, followed by an `mfence` instruction. // This is a normal store, followed by an `mfence` instruction.
let data = input_to_reg(ctx, inputs[0]); let data = put_input_in_reg(ctx, inputs[0]);
let addr = lower_to_amode(ctx, inputs[1], 0); let addr = lower_to_amode(ctx, inputs[1], 0);
let ty_access = ctx.input_ty(insn, 0); let ty_access = ctx.input_ty(insn, 0);
assert!(is_valid_atomic_transaction_ty(ty_access)); assert!(is_valid_atomic_transaction_ty(ty_access));
@@ -2158,7 +2078,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::FuncAddr => { Opcode::FuncAddr => {
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let (extname, _) = ctx.call_target(insn).unwrap(); let (extname, _) = ctx.call_target(insn).unwrap();
let extname = extname.clone(); let extname = extname.clone();
let loc = ctx.srcloc(insn); let loc = ctx.srcloc(insn);
@@ -2171,7 +2091,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::SymbolValue => { Opcode::SymbolValue => {
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let (extname, _, offset) = ctx.symbol_value(insn).unwrap(); let (extname, _, offset) = ctx.symbol_value(insn).unwrap();
let extname = extname.clone(); let extname = extname.clone();
let loc = ctx.srcloc(insn); let loc = ctx.srcloc(insn);
@@ -2192,7 +2112,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} => (stack_slot, offset), } => (stack_slot, offset),
_ => unreachable!(), _ => unreachable!(),
}; };
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let offset: i32 = offset.into(); let offset: i32 = offset.into();
let inst = ctx let inst = ctx
.abi() .abi()
@@ -2203,7 +2123,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::Select => { Opcode::Select => {
let flag_input = inputs[0]; let flag_input = inputs[0];
if let Some(fcmp) = matches_input(ctx, flag_input, Opcode::Fcmp) { if let Some(fcmp) = matches_input(ctx, flag_input, Opcode::Fcmp) {
let cond_code = inst_fp_condcode(ctx.data(fcmp)); let cond_code = ctx.data(fcmp).fp_cond_code().unwrap();
// we request inversion of Equal to NotEqual here: taking LHS if equal would mean // we request inversion of Equal to NotEqual here: taking LHS if equal would mean
// take it if both CC::NP and CC::Z are set, the conjunction of which can't be // take it if both CC::NP and CC::Z are set, the conjunction of which can't be
@@ -2219,14 +2139,14 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}; };
let ty = ctx.output_ty(insn, 0); let ty = ctx.output_ty(insn, 0);
let rhs = input_to_reg(ctx, rhs_input); let rhs = put_input_in_reg(ctx, rhs_input);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let lhs = if is_int_ty(ty) && ty.bytes() < 4 { let lhs = if is_int_ty(ty) && ty.bytes() < 4 {
// Special case: since the higher bits are undefined per CLIF semantics, we // Special case: since the higher bits are undefined per CLIF semantics, we
// can just apply a 32-bit cmove here. Force inputs into registers, to // can just apply a 32-bit cmove here. Force inputs into registers, to
// avoid partial spilling out-of-bounds with memory accesses, though. // avoid partial spilling out-of-bounds with memory accesses, though.
// Sign-extend operands to 32, then do a cmove of size 4. // Sign-extend operands to 32, then do a cmove of size 4.
RegMem::reg(input_to_reg(ctx, lhs_input)) RegMem::reg(put_input_in_reg(ctx, lhs_input))
} else { } else {
input_to_reg_mem(ctx, lhs_input) input_to_reg_mem(ctx, lhs_input)
}; };
@@ -2262,18 +2182,18 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else { } else {
let cc = if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { let cc = if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) {
emit_cmp(ctx, icmp); emit_cmp(ctx, icmp);
let cond_code = inst_condcode(ctx.data(icmp)); let cond_code = ctx.data(icmp).cond_code().unwrap();
CC::from_intcc(cond_code) CC::from_intcc(cond_code)
} else { } else {
// The input is a boolean value, compare it against zero. // The input is a boolean value, compare it against zero.
let size = ctx.input_ty(insn, 0).bytes() as u8; let size = ctx.input_ty(insn, 0).bytes() as u8;
let test = input_to_reg(ctx, inputs[0]); let test = put_input_in_reg(ctx, inputs[0]);
ctx.emit(Inst::cmp_rmi_r(size, RegMemImm::imm(0), test)); ctx.emit(Inst::cmp_rmi_r(size, RegMemImm::imm(0), test));
CC::NZ CC::NZ
}; };
let rhs = input_to_reg(ctx, inputs[2]); let rhs = put_input_in_reg(ctx, inputs[2]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ctx.output_ty(insn, 0); let ty = ctx.output_ty(insn, 0);
ctx.emit(Inst::gen_move(dst, rhs, ty)); ctx.emit(Inst::gen_move(dst, rhs, ty));
@@ -2285,7 +2205,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// can just apply a 32-bit cmove here. Force inputs into registers, to // can just apply a 32-bit cmove here. Force inputs into registers, to
// avoid partial spilling out-of-bounds with memory accesses, though. // avoid partial spilling out-of-bounds with memory accesses, though.
size = 4; size = 4;
RegMem::reg(input_to_reg(ctx, inputs[1])) RegMem::reg(put_input_in_reg(ctx, inputs[1]))
} else { } else {
input_to_reg_mem(ctx, inputs[1]) input_to_reg_mem(ctx, inputs[1])
}; };
@@ -2308,11 +2228,11 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp); debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
emit_cmp(ctx, cmp_insn); emit_cmp(ctx, cmp_insn);
let cc = CC::from_intcc(inst_condcode(ctx.data(insn))); let cc = CC::from_intcc(ctx.data(insn).cond_code().unwrap());
let lhs = input_to_reg_mem(ctx, inputs[1]); let lhs = input_to_reg_mem(ctx, inputs[1]);
let rhs = input_to_reg(ctx, inputs[2]); let rhs = put_input_in_reg(ctx, inputs[2]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ctx.output_ty(insn, 0); let ty = ctx.output_ty(insn, 0);
@@ -2348,8 +2268,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let input_ty = ctx.input_ty(insn, 0); let input_ty = ctx.input_ty(insn, 0);
let size = input_ty.bytes() as u8; let size = input_ty.bytes() as u8;
let dividend = input_to_reg(ctx, inputs[0]); let dividend = put_input_in_reg(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let srcloc = ctx.srcloc(insn); let srcloc = ctx.srcloc(insn);
ctx.emit(Inst::gen_move( ctx.emit(Inst::gen_move(
@@ -2366,7 +2286,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// Note it keeps the result in $rax (for divide) or $rdx (for rem), so that // Note it keeps the result in $rax (for divide) or $rdx (for rem), so that
// regalloc is aware of the coalescing opportunity between rax/rdx and the // regalloc is aware of the coalescing opportunity between rax/rdx and the
// destination register. // destination register.
let divisor = input_to_reg(ctx, inputs[1]); let divisor = put_input_in_reg(ctx, inputs[1]);
let divisor_copy = ctx.alloc_tmp(RegClass::I64, types::I64); let divisor_copy = ctx.alloc_tmp(RegClass::I64, types::I64);
ctx.emit(Inst::gen_move(divisor_copy, divisor, types::I64)); ctx.emit(Inst::gen_move(divisor_copy, divisor, types::I64));
@@ -2432,9 +2352,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let input_ty = ctx.input_ty(insn, 0); let input_ty = ctx.input_ty(insn, 0);
let size = input_ty.bytes() as u8; let size = input_ty.bytes() as u8;
let lhs = input_to_reg(ctx, inputs[0]); let lhs = put_input_in_reg(ctx, inputs[0]);
let rhs = input_to_reg_mem(ctx, inputs[1]); let rhs = input_to_reg_mem(ctx, inputs[1]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
// Move lhs in %rax. // Move lhs in %rax.
ctx.emit(Inst::gen_move( ctx.emit(Inst::gen_move(
@@ -2452,12 +2372,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::GetPinnedReg => { Opcode::GetPinnedReg => {
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
ctx.emit(Inst::gen_move(dst, regs::pinned_reg(), types::I64)); ctx.emit(Inst::gen_move(dst, regs::pinned_reg(), types::I64));
} }
Opcode::SetPinnedReg => { Opcode::SetPinnedReg => {
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
ctx.emit(Inst::gen_move( ctx.emit(Inst::gen_move(
Writable::from_reg(regs::pinned_reg()), Writable::from_reg(regs::pinned_reg()),
src, src,
@@ -2474,7 +2394,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} else { } else {
unreachable!("vconst should always have unary_const format") unreachable!("vconst should always have unary_const format")
}; };
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ty.unwrap(); let ty = ty.unwrap();
ctx.emit(Inst::xmm_load_const_seq(val, dst, ty)); ctx.emit(Inst::xmm_load_const_seq(val, dst, ty));
} }
@@ -2484,8 +2404,8 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// https://github.com/bytecodealliance/wasmtime/issues/1147). As such, this IR // https://github.com/bytecodealliance/wasmtime/issues/1147). As such, this IR
// instruction should emit no machine code but a move is necessary to give the register // instruction should emit no machine code but a move is necessary to give the register
// allocator a definition for the output virtual register. // allocator a definition for the output virtual register.
let src = input_to_reg(ctx, inputs[0]); let src = put_input_in_reg(ctx, inputs[0]);
let dst = output_to_reg(ctx, outputs[0]); let dst = get_output_reg(ctx, outputs[0]);
let ty = ty.unwrap(); let ty = ty.unwrap();
ctx.emit(Inst::gen_move(dst, src, ty)); ctx.emit(Inst::gen_move(dst, src, ty));
} }
@@ -2580,7 +2500,7 @@ impl LowerBackend for X64Backend {
if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) {
emit_cmp(ctx, icmp); emit_cmp(ctx, icmp);
let cond_code = inst_condcode(ctx.data(icmp)); let cond_code = ctx.data(icmp).cond_code().unwrap();
let cond_code = if op0 == Opcode::Brz { let cond_code = if op0 == Opcode::Brz {
cond_code.inverse() cond_code.inverse()
} else { } else {
@@ -2590,7 +2510,7 @@ impl LowerBackend for X64Backend {
let cc = CC::from_intcc(cond_code); let cc = CC::from_intcc(cond_code);
ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); ctx.emit(Inst::jmp_cond(cc, taken, not_taken));
} else if let Some(fcmp) = matches_input(ctx, flag_input, Opcode::Fcmp) { } else if let Some(fcmp) = matches_input(ctx, flag_input, Opcode::Fcmp) {
let cond_code = inst_fp_condcode(ctx.data(fcmp)); let cond_code = ctx.data(fcmp).fp_cond_code().unwrap();
let cond_code = if op0 == Opcode::Brz { let cond_code = if op0 == Opcode::Brz {
cond_code.inverse() cond_code.inverse()
} else { } else {
@@ -2611,7 +2531,7 @@ impl LowerBackend for X64Backend {
FcmpCondResult::InvertedEqualOrConditions(_, _) => unreachable!(), FcmpCondResult::InvertedEqualOrConditions(_, _) => unreachable!(),
} }
} else if is_int_ty(src_ty) || is_bool_ty(src_ty) { } else if is_int_ty(src_ty) || is_bool_ty(src_ty) {
let src = input_to_reg( let src = put_input_in_reg(
ctx, ctx,
InsnInput { InsnInput {
insn: branches[0], insn: branches[0],
@@ -2634,7 +2554,7 @@ impl LowerBackend for X64Backend {
Opcode::BrIcmp => { Opcode::BrIcmp => {
let src_ty = ctx.input_ty(branches[0], 0); let src_ty = ctx.input_ty(branches[0], 0);
if is_int_ty(src_ty) || is_bool_ty(src_ty) { if is_int_ty(src_ty) || is_bool_ty(src_ty) {
let lhs = input_to_reg( let lhs = put_input_in_reg(
ctx, ctx,
InsnInput { InsnInput {
insn: branches[0], insn: branches[0],
@@ -2648,7 +2568,7 @@ impl LowerBackend for X64Backend {
input: 1, input: 1,
}, },
); );
let cc = CC::from_intcc(inst_condcode(ctx.data(branches[0]))); let cc = CC::from_intcc(ctx.data(branches[0]).cond_code().unwrap());
let byte_size = src_ty.bytes() as u8; let byte_size = src_ty.bytes() as u8;
// Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives
// us dst - src at the machine instruction level, so invert operands. // us dst - src at the machine instruction level, so invert operands.

10
cranelift/codegen/src/machinst/helpers.rs
View File

@@ -1,6 +1,8 @@
//! Miscellaneous helpers for machine backends. //! Miscellaneous helpers for machine backends.
use super::{InsnOutput, LowerCtx, VCodeInst};
use crate::ir::Type; use crate::ir::Type;
use regalloc::{Reg, Writable};
/// Returns the size (in bits) of a given type. /// Returns the size (in bits) of a given type.
pub fn ty_bits(ty: Type) -> usize { pub fn ty_bits(ty: Type) -> usize {
@@ -16,3 +18,11 @@ pub(crate) fn ty_has_int_representation(ty: Type) -> bool {
pub(crate) fn ty_has_float_or_vec_representation(ty: Type) -> bool { pub(crate) fn ty_has_float_or_vec_representation(ty: Type) -> bool {
ty.is_vector() || ty.is_float() ty.is_vector() || ty.is_float()
} }
/// Allocate a register for an instruction output and return it.
pub(crate) fn get_output_reg<I: VCodeInst, C: LowerCtx<I = I>>(
ctx: &mut C,
spec: InsnOutput,
) -> Writable<Reg> {
ctx.get_output(spec.insn, spec.output)
}

25
cranelift/codegen/src/machinst/inst_common.rs
View File

@@ -1,6 +1,29 @@
//! A place to park MachInst::Inst fragments which are common across multiple architectures. //! A place to park MachInst::Inst fragments which are common across multiple architectures.
use crate::ir; use crate::ir::{self, Inst as IRInst};
//============================================================================
// Instruction input "slots".
//
// We use these types to refer to operand numbers, and result numbers, together
// with the associated instruction, in a type-safe way.
/// Identifier for a particular input of an instruction.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct InsnInput {
pub(crate) insn: IRInst,
pub(crate) input: usize,
}
/// Identifier for a particular output of an instruction.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct InsnOutput {
pub(crate) insn: IRInst,
pub(crate) output: usize,
}
//============================================================================
// Atomic instructions.
/// Atomic memory update operations. As of 21 Aug 2020 these are used for the aarch64 and x64 /// Atomic memory update operations. As of 21 Aug 2020 these are used for the aarch64 and x64
/// targets. /// targets.