diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs index 6c71212020..48334b1837 100644 --- a/cranelift/codegen/src/ir/instructions.rs +++ b/cranelift/codegen/src/ir/instructions.rs @@ -18,6 +18,7 @@ use crate::isa; use crate::bitset::BitSet; use crate::entity; +use ir::condcodes::{FloatCC, IntCC}; /// Some instructions use an external list of argument values because there is not enough space in /// the 16-byte `InstructionData` struct. These value lists are stored in a memory pool in @@ -295,6 +296,33 @@ impl InstructionData { } } + /// If this is a control-flow instruction depending on an integer condition, gets its + /// condition. Otherwise, return `None`. + pub fn cond_code(&self) -> Option { + match self { + &InstructionData::IntCond { cond, .. } + | &InstructionData::BranchIcmp { cond, .. } + | &InstructionData::IntCompare { cond, .. } + | &InstructionData::IntCondTrap { cond, .. } + | &InstructionData::BranchInt { cond, .. } + | &InstructionData::IntSelect { cond, .. } + | &InstructionData::IntCompareImm { cond, .. } => Some(cond), + _ => None, + } + } + + /// If this is a control-flow instruction depending on a floating-point condition, gets its + /// condition. Otherwise, return `None`. + pub fn fp_cond_code(&self) -> Option { + match self { + &InstructionData::BranchFloat { cond, .. } + | &InstructionData::FloatCompare { cond, .. } + | &InstructionData::FloatCond { cond, .. } + | &InstructionData::FloatCondTrap { cond, .. } => Some(cond), + _ => None, + } + } + /// If this is a trapping instruction, get an exclusive reference to its /// trap code. Otherwise, return `None`. pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode> { @@ -307,6 +335,27 @@ impl InstructionData { } } + /// If this is an atomic read/modify/write instruction, return its subopcode. + pub fn atomic_rmw_op(&self) -> Option { + match self { + &InstructionData::AtomicRmw { op, .. } => Some(op), + _ => None, + } + } + + /// If this is a load/store instruction, returns its immediate offset. + pub fn load_store_offset(&self) -> Option { + match self { + &InstructionData::Load { offset, .. } + | &InstructionData::StackLoad { offset, .. } + | &InstructionData::LoadComplex { offset, .. } + | &InstructionData::Store { offset, .. } + | &InstructionData::StackStore { offset, .. } + | &InstructionData::StoreComplex { offset, .. } => Some(offset.into()), + _ => None, + } + } + /// Return information about a call instruction. /// /// Any instruction that can call another function reveals its call signature here. diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 70ac715e6e..e7f4b9a62f 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -7,11 +7,10 @@ //! //! - Floating-point immediates (FIMM instruction). -use crate::ir; use crate::ir::condcodes::{FloatCC, IntCC}; use crate::ir::types::*; use crate::ir::Inst as IRInst; -use crate::ir::{InstructionData, Opcode, TrapCode, Type}; +use crate::ir::{InstructionData, Opcode, Type}; use crate::machinst::lower::*; use crate::machinst::*; use crate::CodegenResult; @@ -998,58 +997,6 @@ pub(crate) fn choose_32_64(ty: Type, op32: T, op64: T) -> T { } } -pub(crate) fn ldst_offset(data: &InstructionData) -> Option { - match data { - &InstructionData::Load { offset, .. } - | &InstructionData::StackLoad { offset, .. } - | &InstructionData::LoadComplex { offset, .. } - | &InstructionData::Store { offset, .. } - | &InstructionData::StackStore { offset, .. } - | &InstructionData::StoreComplex { offset, .. } => Some(offset.into()), - _ => None, - } -} - -pub(crate) fn inst_condcode(data: &InstructionData) -> Option { - match data { - &InstructionData::IntCond { cond, .. } - | &InstructionData::BranchIcmp { cond, .. } - | &InstructionData::IntCompare { cond, .. } - | &InstructionData::IntCondTrap { cond, .. } - | &InstructionData::BranchInt { cond, .. } - | &InstructionData::IntSelect { cond, .. } - | &InstructionData::IntCompareImm { cond, .. } => Some(cond), - _ => None, - } -} - -pub(crate) fn inst_fp_condcode(data: &InstructionData) -> Option { - match data { - &InstructionData::BranchFloat { cond, .. } - | &InstructionData::FloatCompare { cond, .. } - | &InstructionData::FloatCond { cond, .. } - | &InstructionData::FloatCondTrap { cond, .. } => Some(cond), - _ => None, - } -} - -pub(crate) fn inst_trapcode(data: &InstructionData) -> Option { - match data { - &InstructionData::Trap { code, .. } - | &InstructionData::CondTrap { code, .. } - | &InstructionData::IntCondTrap { code, .. } - | &InstructionData::FloatCondTrap { code, .. } => Some(code), - _ => None, - } -} - -pub(crate) fn inst_atomic_rmw_op(data: &InstructionData) -> Option { - match data { - &InstructionData::AtomicRmw { op, .. } => Some(op), - _ => None, - } -} - /// Checks for an instance of `op` feeding the given input. pub(crate) fn maybe_input_insn>( c: &mut C, diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 8888bbcacd..dbabc9d58c 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -1092,7 +1092,7 @@ pub(crate) fn lower_insn_to_regs>( | Opcode::Uload16x4 | Opcode::Sload32x2 | Opcode::Uload32x2 => { - let off = ldst_offset(ctx.data(insn)).unwrap(); + let off = ctx.data(insn).load_store_offset().unwrap(); let elem_ty = match op { Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => { I8 @@ -1177,7 +1177,7 @@ pub(crate) fn lower_insn_to_regs>( | Opcode::Istore8Complex | Opcode::Istore16Complex | Opcode::Istore32Complex => { - let off = ldst_offset(ctx.data(insn)).unwrap(); + let off = ctx.data(insn).load_store_offset().unwrap(); let elem_ty = match op { Opcode::Istore8 | Opcode::Istore8Complex => I8, Opcode::Istore16 | Opcode::Istore16Complex => I16, @@ -1247,7 +1247,7 @@ pub(crate) fn lower_insn_to_regs>( ctx.emit(Inst::gen_move(Writable::from_reg(xreg(25)), r_addr, I64)); ctx.emit(Inst::gen_move(Writable::from_reg(xreg(26)), r_arg2, I64)); // Now the AtomicRMW insn itself - let op = inst_common::AtomicRmwOp::from(inst_atomic_rmw_op(ctx.data(insn)).unwrap()); + let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap()); ctx.emit(Inst::AtomicRMW { ty: ty_access, op, @@ -1366,7 +1366,7 @@ pub(crate) fn lower_insn_to_regs>( let cond = if let Some(icmp_insn) = maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint) { - let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap(); + let condcode = ctx.data(icmp_insn).cond_code().unwrap(); let cond = lower_condcode(condcode); let is_signed = condcode_is_signed(condcode); lower_icmp_or_ifcmp_to_flags(ctx, icmp_insn, is_signed); @@ -1374,7 +1374,7 @@ pub(crate) fn lower_insn_to_regs>( } else if let Some(fcmp_insn) = maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint) { - let condcode = inst_fp_condcode(ctx.data(fcmp_insn)).unwrap(); + let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap(); let cond = lower_fp_condcode(condcode); lower_fcmp_or_ffcmp_to_flags(ctx, fcmp_insn); cond @@ -1413,7 +1413,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Selectif | Opcode::SelectifSpectreGuard => { - let condcode = inst_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).cond_code().unwrap(); let cond = lower_condcode(condcode); let is_signed = condcode_is_signed(condcode); // Verification ensures that the input is always a @@ -1485,7 +1485,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Trueif => { - let condcode = inst_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).cond_code().unwrap(); let cond = lower_condcode(condcode); let is_signed = condcode_is_signed(condcode); // Verification ensures that the input is always a @@ -1498,7 +1498,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Trueff => { - let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).fp_cond_code().unwrap(); let cond = lower_fp_condcode(condcode); let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap(); lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn); @@ -1688,7 +1688,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Icmp => { - let condcode = inst_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).cond_code().unwrap(); let cond = lower_condcode(condcode); let is_signed = condcode_is_signed(condcode); let rd = get_output_reg(ctx, outputs[0]); @@ -1715,7 +1715,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Fcmp => { - let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).fp_cond_code().unwrap(); let cond = lower_fp_condcode(condcode); let ty = ctx.input_ty(insn, 0); let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); @@ -1748,15 +1748,15 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Trap | Opcode::ResumableTrap => { - let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); + let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap()); ctx.emit_safepoint(Inst::Udf { trap_info }); } Opcode::Trapif | Opcode::Trapff => { - let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); + let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap()); let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() { - let condcode = inst_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).cond_code().unwrap(); let cond = lower_condcode(condcode); // The flags must not have been clobbered by any other // instruction between the iadd_ifcout and this instruction, as @@ -1764,7 +1764,7 @@ pub(crate) fn lower_insn_to_regs>( // flags here. cond } else if op == Opcode::Trapif { - let condcode = inst_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).cond_code().unwrap(); let cond = lower_condcode(condcode); let is_signed = condcode_is_signed(condcode); @@ -1773,7 +1773,7 @@ pub(crate) fn lower_insn_to_regs>( lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed); cond } else { - let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); + let condcode = ctx.data(insn).fp_cond_code().unwrap(); let cond = lower_fp_condcode(condcode); // Verification ensures that the input is always a @@ -2826,7 +2826,7 @@ pub(crate) fn lower_branch>( if let Some(icmp_insn) = maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint) { - let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap(); + let condcode = ctx.data(icmp_insn).cond_code().unwrap(); let cond = lower_condcode(condcode); let is_signed = condcode_is_signed(condcode); let negated = op0 == Opcode::Brz; @@ -2841,7 +2841,7 @@ pub(crate) fn lower_branch>( } else if let Some(fcmp_insn) = maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint) { - let condcode = inst_fp_condcode(ctx.data(fcmp_insn)).unwrap(); + let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap(); let cond = lower_fp_condcode(condcode); let negated = op0 == Opcode::Brz; let cond = if negated { cond.invert() } else { cond }; @@ -2874,7 +2874,7 @@ pub(crate) fn lower_branch>( } } Opcode::BrIcmp => { - let condcode = inst_condcode(ctx.data(branches[0])).unwrap(); + let condcode = ctx.data(branches[0]).cond_code().unwrap(); let cond = lower_condcode(condcode); let kind = CondBrKind::Cond(cond); @@ -2915,7 +2915,7 @@ pub(crate) fn lower_branch>( } Opcode::Brif => { - let condcode = inst_condcode(ctx.data(branches[0])).unwrap(); + let condcode = ctx.data(branches[0]).cond_code().unwrap(); let cond = lower_condcode(condcode); let kind = CondBrKind::Cond(cond); @@ -2945,7 +2945,7 @@ pub(crate) fn lower_branch>( } Opcode::Brff => { - let condcode = inst_fp_condcode(ctx.data(branches[0])).unwrap(); + let condcode = ctx.data(branches[0]).fp_cond_code().unwrap(); let cond = lower_fp_condcode(condcode); let kind = CondBrKind::Cond(cond); let flag_input = InsnInput { diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 6576482f62..6735806007 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2,10 +2,9 @@ #![allow(non_snake_case)] -use crate::ir; use crate::ir::{ - condcodes::FloatCC, condcodes::IntCC, types, AbiParam, ArgumentPurpose, ExternalName, - Inst as IRInst, InstructionData, LibCall, Opcode, Signature, TrapCode, Type, + condcodes::FloatCC, types, AbiParam, ArgumentPurpose, ExternalName, Inst as IRInst, + InstructionData, LibCall, Opcode, Signature, Type, }; use crate::isa::x64::abi::*; use crate::isa::x64::inst::args::*; @@ -58,58 +57,6 @@ fn iri_to_u64_imm(ctx: Ctx, inst: IRInst) -> Option { ctx.get_constant(inst) } -fn inst_trapcode(data: &InstructionData) -> Option { - match data { - &InstructionData::Trap { code, .. } - | &InstructionData::CondTrap { code, .. } - | &InstructionData::IntCondTrap { code, .. } - | &InstructionData::FloatCondTrap { code, .. } => Some(code), - _ => None, - } -} - -fn inst_condcode(data: &InstructionData) -> IntCC { - match data { - &InstructionData::IntCond { cond, .. } - | &InstructionData::BranchIcmp { cond, .. } - | &InstructionData::IntCompare { cond, .. } - | &InstructionData::IntCondTrap { cond, .. } - | &InstructionData::BranchInt { cond, .. } - | &InstructionData::IntSelect { cond, .. } - | &InstructionData::IntCompareImm { cond, .. } => cond, - _ => panic!("inst_condcode(x64): unhandled: {:?}", data), - } -} - -fn inst_fp_condcode(data: &InstructionData) -> FloatCC { - match data { - &InstructionData::BranchFloat { cond, .. } - | &InstructionData::FloatCompare { cond, .. } - | &InstructionData::FloatCond { cond, .. } - | &InstructionData::FloatCondTrap { cond, .. } => cond, - _ => panic!("inst_fp_condcode(x64): unhandled: {:?}", data), - } -} - -fn inst_atomic_rmw_op(data: &InstructionData) -> Option { - match data { - &InstructionData::AtomicRmw { op, .. } => Some(op), - _ => None, - } -} - -fn ldst_offset(data: &InstructionData) -> Option { - match data { - &InstructionData::Load { offset, .. } - | &InstructionData::StackLoad { offset, .. } - | &InstructionData::LoadComplex { offset, .. } - | &InstructionData::Store { offset, .. } - | &InstructionData::StackStore { offset, .. } - | &InstructionData::StoreComplex { offset, .. } => Some(offset.into()), - _ => None, - } -} - /// Returns whether the given specified `input` is a result produced by an instruction with Opcode /// `op`. // TODO investigate failures with checking against the result index. @@ -1135,14 +1082,14 @@ fn lower_insn_to_regs>( Opcode::Icmp => { emit_cmp(ctx, insn); - let condcode = inst_condcode(ctx.data(insn)); + let condcode = ctx.data(insn).cond_code().unwrap(); let cc = CC::from_intcc(condcode); let dst = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::setcc(cc, dst)); } Opcode::Fcmp => { - let cond_code = inst_fp_condcode(ctx.data(insn)); + let cond_code = ctx.data(insn).fp_cond_code().unwrap(); let input_ty = ctx.input_ty(insn, 0); if !input_ty.is_vector() { // Unordered is returned by setting ZF, PF, CF <- 111 @@ -1290,16 +1237,16 @@ fn lower_insn_to_regs>( } Opcode::Trap | Opcode::ResumableTrap => { - let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); + let trap_info = (ctx.srcloc(insn), ctx.data(insn).trap_code().unwrap()); ctx.emit_safepoint(Inst::Ud2 { trap_info }); } Opcode::Trapif | Opcode::Trapff => { let srcloc = ctx.srcloc(insn); - let trap_code = inst_trapcode(ctx.data(insn)).unwrap(); + let trap_code = ctx.data(insn).trap_code().unwrap(); if matches_input(ctx, inputs[0], Opcode::IaddIfcout).is_some() { - let cond_code = inst_condcode(ctx.data(insn)); + let cond_code = ctx.data(insn).cond_code().unwrap(); // The flags must not have been clobbered by any other instruction between the // iadd_ifcout and this instruction, as verified by the CLIF validator; so we can // simply use the flags here. @@ -1311,7 +1258,7 @@ fn lower_insn_to_regs>( cc, }); } else if op == Opcode::Trapif { - let cond_code = inst_condcode(ctx.data(insn)); + let cond_code = ctx.data(insn).cond_code().unwrap(); let cc = CC::from_intcc(cond_code); // Verification ensures that the input is always a single-def ifcmp. @@ -1324,7 +1271,7 @@ fn lower_insn_to_regs>( cc, }); } else { - let cond_code = inst_fp_condcode(ctx.data(insn)); + let cond_code = ctx.data(insn).fp_cond_code().unwrap(); // Verification ensures that the input is always a single-def ffcmp. let ffcmp = matches_input(ctx, inputs[0], Opcode::Ffcmp).unwrap(); @@ -1822,7 +1769,7 @@ fn lower_insn_to_regs>( | Opcode::Sload16Complex | Opcode::Uload32Complex | Opcode::Sload32Complex => { - let offset = ldst_offset(ctx.data(insn)).unwrap(); + let offset = ctx.data(insn).load_store_offset().unwrap(); let elem_ty = match op { Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => { @@ -1944,7 +1891,7 @@ fn lower_insn_to_regs>( | Opcode::Istore8Complex | Opcode::Istore16Complex | Opcode::Istore32Complex => { - let offset = ldst_offset(ctx.data(insn)).unwrap(); + let offset = ctx.data(insn).load_store_offset().unwrap(); let elem_ty = match op { Opcode::Istore8 | Opcode::Istore8Complex => types::I8, @@ -2036,7 +1983,7 @@ fn lower_insn_to_regs>( )); // Now the AtomicRmwSeq (pseudo-) instruction itself - let op = inst_common::AtomicRmwOp::from(inst_atomic_rmw_op(ctx.data(insn)).unwrap()); + let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap()); ctx.emit(Inst::AtomicRmwSeq { ty: ty_access, op, @@ -2181,7 +2128,7 @@ fn lower_insn_to_regs>( Opcode::Select => { let flag_input = inputs[0]; if let Some(fcmp) = matches_input(ctx, flag_input, Opcode::Fcmp) { - let cond_code = inst_fp_condcode(ctx.data(fcmp)); + let cond_code = ctx.data(fcmp).fp_cond_code().unwrap(); // we request inversion of Equal to NotEqual here: taking LHS if equal would mean // take it if both CC::NP and CC::Z are set, the conjunction of which can't be @@ -2240,7 +2187,7 @@ fn lower_insn_to_regs>( } else { let cc = if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { emit_cmp(ctx, icmp); - let cond_code = inst_condcode(ctx.data(icmp)); + let cond_code = ctx.data(icmp).cond_code().unwrap(); CC::from_intcc(cond_code) } else { // The input is a boolean value, compare it against zero. @@ -2286,7 +2233,7 @@ fn lower_insn_to_regs>( debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp); emit_cmp(ctx, cmp_insn); - let cc = CC::from_intcc(inst_condcode(ctx.data(insn))); + let cc = CC::from_intcc(ctx.data(insn).cond_code().unwrap()); let lhs = input_to_reg_mem(ctx, inputs[1]); let rhs = input_to_reg(ctx, inputs[2]); @@ -2558,7 +2505,7 @@ impl LowerBackend for X64Backend { if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { emit_cmp(ctx, icmp); - let cond_code = inst_condcode(ctx.data(icmp)); + let cond_code = ctx.data(icmp).cond_code().unwrap(); let cond_code = if op0 == Opcode::Brz { cond_code.inverse() } else { @@ -2568,7 +2515,7 @@ impl LowerBackend for X64Backend { let cc = CC::from_intcc(cond_code); ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); } else if let Some(fcmp) = matches_input(ctx, flag_input, Opcode::Fcmp) { - let cond_code = inst_fp_condcode(ctx.data(fcmp)); + let cond_code = ctx.data(fcmp).fp_cond_code().unwrap(); let cond_code = if op0 == Opcode::Brz { cond_code.inverse() } else { @@ -2626,7 +2573,7 @@ impl LowerBackend for X64Backend { input: 1, }, ); - let cc = CC::from_intcc(inst_condcode(ctx.data(branches[0]))); + let cc = CC::from_intcc(ctx.data(branches[0]).cond_code().unwrap()); let byte_size = src_ty.bytes() as u8; // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives // us dst - src at the machine instruction level, so invert operands.