diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 3b2ef337e2..704895ab01 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -947,6 +947,13 @@ (Size64x2) )) +;; Helper for calculating the `VectorSize` corresponding to a type +(decl vector_size (Type) VectorSize) +(rule (vector_size (multi_lane 8 16)) (VectorSize.Size8x16)) +(rule (vector_size (multi_lane 16 8)) (VectorSize.Size16x8)) +(rule (vector_size (multi_lane 32 4)) (VectorSize.Size32x4)) +(rule (vector_size (multi_lane 64 2)) (VectorSize.Size64x2)) + ;; A floating-point unit (FPU) operation with one arg. (type FPUOp1 (enum @@ -1294,9 +1301,41 @@ (decl imm_logic_from_u64 (ImmLogic) u64) (extern extractor imm_logic_from_u64 imm_logic_from_u64) +(decl imm12_from_u64 (Imm12) u64) +(extern extractor imm12_from_u64 imm12_from_u64) + +(decl imm12_from_negated_u64 (Imm12) u64) +(extern extractor imm12_from_negated_u64 imm12_from_negated_u64) + +(decl lshl_from_imm64 (Type ShiftOpAndAmt) Imm64) +(extern extractor lshl_from_imm64 lshl_from_imm64 (in out)) + (decl integral_ty (Type) Type) (extern extractor integral_ty integral_ty) +;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`. +(decl imm12_from_value (Imm12) Value) +(extractor + (imm12_from_value n) + (def_inst (iconst (u64_from_imm64 (imm12_from_u64 n))))) + +;; Same as `imm12_from_value`, but tries negating the constant value. +(decl imm12_from_negated_value (Imm12) Value) +(extractor + (imm12_from_negated_value n) + (def_inst (iconst (u64_from_imm64 (imm12_from_negated_u64 n))))) + +;; Helper type to represent a value and an extend operation fused together. +(type ExtendedValue extern (enum)) +(decl extended_value_from_value (ExtendedValue) Value) +(extern extractor extended_value_from_value extended_value_from_value) + +;; Constructors used to poke at the fields of an `ExtendedValue`. +(decl put_extended_in_reg (ExtendedValue) Reg) +(extern constructor put_extended_in_reg put_extended_in_reg) +(decl get_extended_op (ExtendedValue) ExtendOp) +(extern constructor get_extended_op get_extended_op) + ;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Emit an instruction. @@ -1331,9 +1370,83 @@ (_ Unit (emit (MInst.AluRRImmLogic op dst src imm)))) (writable_reg_to_reg dst))) -;; Helper for emitting `orr64` instructions. -(decl orr64 (Reg ImmLogic) Reg) -(rule (orr64 src imm) (alu_rr_imm_logic (ALUOp.Orr64) src imm)) +;; Helper for emitting `MInst.AluRRR` instructions. +(decl alu_rrr (ALUOp Reg Reg) Reg) +(rule (alu_rrr op src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRR op dst src1 src2)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.VecRRR` instructions. +(decl vec_rrr (VecALUOp Reg Reg VectorSize) Reg) +(rule (vec_rrr op src1 src2 size) + (let ((dst WritableReg (temp_writable_reg $I8X16)) + (_ Unit (emit (MInst.VecRRR op dst src1 src2 size)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRImm12` instructions. +(decl alu_rr_imm12 (ALUOp Reg Imm12) Reg) +(rule (alu_rr_imm12 op src imm) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRImm12 op dst src imm)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRRShift` instructions. +(decl alu_rrr_shift (ALUOp Reg Reg ShiftOpAndAmt) Reg) +(rule (alu_rrr_shift op src1 src2 shift) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRRShift op dst src1 src2 shift)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRRExtend` instructions. +(decl alu_rrr_extend (ALUOp Reg Reg ExtendOp) Reg) +(rule (alu_rrr_extend op src1 src2 extend) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRRExtend op dst src1 src2 extend)))) + (writable_reg_to_reg dst))) + +;; Same as `alu_rrr_extend`, but takes an `ExtendedValue` packed "pair" instead +;; of a `Reg` and an `ExtendOp`. +(decl alu_rr_extend_reg (ALUOp Reg ExtendedValue) Reg) +(rule (alu_rr_extend_reg op src1 extended_reg) + (let ((src2 Reg (put_extended_in_reg extended_reg)) + (extend ExtendOp (get_extended_op extended_reg))) + (alu_rrr_extend op src1 src2 extend))) + +;; Helper for emitting `MInst.AluRRRR` instructions. +(decl alu_rrrr (ALUOp3 Reg Reg Reg) Reg) +(rule (alu_rrrr op src1 src2 src3) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRRR op dst src1 src2 src3)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `adds` instructions. +(decl add64_with_flags (Reg Reg) ProducesFlags) +(rule (add64_with_flags src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.AddS64) dst src1 src2) + (writable_reg_to_reg dst)))) + +;; Helper for emitting `adc` instructions. +(decl adc64 (Reg Reg) ConsumesFlags) +(rule (adc64 src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Adc64) dst src1 src2) + (writable_reg_to_reg dst)))) + +;; Helper for emitting `subs` instructions. +(decl sub64_with_flags (Reg Reg) ProducesFlags) +(rule (sub64_with_flags src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.SubS64) dst src1 src2) + (writable_reg_to_reg dst)))) + +;; Helper for emitting `sbc` instructions. +(decl sbc64 (Reg Reg) ConsumesFlags) +(rule (sbc64 src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Sbc64) dst src1 src2) + (writable_reg_to_reg dst)))) ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1349,7 +1462,7 @@ ;; Weird logical-instruction immediate in ORI using zero register (rule (imm (integral_ty _ty) (imm_logic_from_u64 n)) - (orr64 (zero_reg) n)) + (alu_rr_imm_logic (ALUOp.Orr64) (zero_reg) n)) (decl load_constant64_full (u64) Reg) (extern constructor load_constant64_full load_constant64_full) diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs index f838186229..1a836f1a71 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/args.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs @@ -62,7 +62,7 @@ impl ShiftOpShiftImm { } /// A shift operator with an amount, guaranteed to be within range. -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug)] pub struct ShiftOpAndAmt { op: ShiftOp, shift: ShiftOpShiftImm, diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rs index 79473375dd..bc3ebac7de 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs @@ -272,7 +272,7 @@ impl UImm12Scaled { /// A shifted immediate value in 'imm12' format: supports 12 bits, shifted /// left by 0 or 12 places. -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug)] pub struct Imm12 { /// The immediate bits. pub bits: u16, diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 7b793dc271..5e0f9076f3 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -21,3 +21,141 @@ (rule (lower (has_type ty (null))) (value_reg (imm ty 0))) + +;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller + +;; Base case, simply adding things in registers. +(rule (lower (has_type (fits_in_64 ty) (iadd x y))) + (value_reg (alu_rrr (iadd_op ty) (put_in_reg x) (put_in_reg y)))) + +;; Special cases for when one operand is an immediate that fits in 12 bits. +(rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_value y)))) + (value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg x) y))) + +(rule (lower (has_type (fits_in_64 ty) (iadd (imm12_from_value x) y))) + (value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg y) x))) + +;; Same as the previous special cases, except we can switch the addition to a +;; subtraction if the negated immediate fits in 12 bits. +(rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_negated_value y)))) + (value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg x) y))) + +(rule (lower (has_type (fits_in_64 ty) (iadd (imm12_from_negated_value x) y))) + (value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg y) x))) + +;; Special cases for when we're adding an extended register where the extending +;; operation can get folded into the add itself. +(rule (lower (has_type (fits_in_64 ty) (iadd x (extended_value_from_value y)))) + (value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg x) y))) + +(rule (lower (has_type (fits_in_64 ty) (iadd (extended_value_from_value x) y))) + (value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg y) x))) + +;; Special cases for when we're adding the shift of a different +;; register by a constant amount and the shift can get folded into the add. +(rule (lower (has_type (fits_in_64 ty) + (iadd x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 >( } /// Lowers an instruction input to multiple regs -fn lower_input_to_regs>( +fn lower_value_to_regs>( ctx: &mut C, - input: InsnInput, + value: Value, ) -> (ValueRegs, Type, bool) { - log::trace!("lower_input_to_regs: input {:?}", input); - let ty = ctx.input_ty(input.insn, input.input); - let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); + log::trace!("lower_value_to_regs: value {:?}", value); + let ty = ctx.value_ty(value); + let inputs = ctx.get_value_as_source_or_const(value); let is_const = inputs.constant.is_some(); let in_regs = if let Some(c) = inputs.constant { // Generate constants fresh at each use to minimize long-range register pressure. generate_constant(ctx, ty, c as u128) } else { - ctx.put_input_in_regs(input.insn, input.input) + ctx.put_value_in_regs(value) }; (in_regs, ty, is_const) @@ -301,7 +301,17 @@ pub(crate) fn put_input_in_reg>( input: InsnInput, narrow_mode: NarrowValueMode, ) -> Reg { - let (in_regs, ty, is_const) = lower_input_to_regs(ctx, input); + let value = ctx.input_as_value(input.insn, input.input); + put_value_in_reg(ctx, value, narrow_mode) +} + +/// Like above, only for values +fn put_value_in_reg>( + ctx: &mut C, + value: Value, + narrow_mode: NarrowValueMode, +) -> Reg { + let (in_regs, ty, is_const) = lower_value_to_regs(ctx, value); let reg = in_regs .only_reg() .expect("Multi-register value not expected"); @@ -314,7 +324,8 @@ pub(crate) fn put_input_in_regs>( ctx: &mut C, input: InsnInput, ) -> ValueRegs { - let (in_regs, _, _) = lower_input_to_regs(ctx, input); + let value = ctx.input_as_value(input.insn, input.input); + let (in_regs, _, _) = lower_value_to_regs(ctx, value); in_regs } @@ -367,89 +378,94 @@ fn put_input_in_rse>( input: InsnInput, narrow_mode: NarrowValueMode, ) -> ResultRSE { - let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); - if let Some((insn, 0)) = inputs.inst { - let op = ctx.data(insn).opcode(); - let out_ty = ctx.output_ty(insn, 0); - let out_bits = ty_bits(out_ty); - - // Is this a zero-extend or sign-extend and can we handle that with a register-mode operator? - if op == Opcode::Uextend || op == Opcode::Sextend { - let sign_extend = op == Opcode::Sextend; - let inner_ty = ctx.input_ty(insn, 0); - let inner_bits = ty_bits(inner_ty); - assert!(inner_bits < out_bits); - if match (sign_extend, narrow_mode) { - // A single zero-extend or sign-extend is equal to itself. - (_, NarrowValueMode::None) => true, - // Two zero-extends or sign-extends in a row is equal to a single zero-extend or sign-extend. - (false, NarrowValueMode::ZeroExtend32) | (false, NarrowValueMode::ZeroExtend64) => { - true - } - (true, NarrowValueMode::SignExtend32) | (true, NarrowValueMode::SignExtend64) => { - true - } - // A zero-extend and a sign-extend in a row is not equal to a single zero-extend or sign-extend - (false, NarrowValueMode::SignExtend32) | (false, NarrowValueMode::SignExtend64) => { - false - } - (true, NarrowValueMode::ZeroExtend32) | (true, NarrowValueMode::ZeroExtend64) => { - false - } - } { - let extendop = match (sign_extend, inner_bits) { - (true, 8) => ExtendOp::SXTB, - (false, 8) => ExtendOp::UXTB, - (true, 16) => ExtendOp::SXTH, - (false, 16) => ExtendOp::UXTH, - (true, 32) => ExtendOp::SXTW, - (false, 32) => ExtendOp::UXTW, - _ => unreachable!(), - }; - let reg = - put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None); - return ResultRSE::RegExtend(reg, extendop); - } - } - - // If `out_ty` is smaller than 32 bits and we need to zero- or sign-extend, - // then get the result into a register and return an Extend-mode operand on - // that register. - if narrow_mode != NarrowValueMode::None - && ((narrow_mode.is_32bit() && out_bits < 32) - || (!narrow_mode.is_32bit() && out_bits < 64)) - { - let reg = put_input_in_reg(ctx, input, NarrowValueMode::None); - let extendop = match (narrow_mode, out_bits) { - (NarrowValueMode::SignExtend32, 1) | (NarrowValueMode::SignExtend64, 1) => { - ExtendOp::SXTB - } - (NarrowValueMode::ZeroExtend32, 1) | (NarrowValueMode::ZeroExtend64, 1) => { - ExtendOp::UXTB - } - (NarrowValueMode::SignExtend32, 8) | (NarrowValueMode::SignExtend64, 8) => { - ExtendOp::SXTB - } - (NarrowValueMode::ZeroExtend32, 8) | (NarrowValueMode::ZeroExtend64, 8) => { - ExtendOp::UXTB - } - (NarrowValueMode::SignExtend32, 16) | (NarrowValueMode::SignExtend64, 16) => { - ExtendOp::SXTH - } - (NarrowValueMode::ZeroExtend32, 16) | (NarrowValueMode::ZeroExtend64, 16) => { - ExtendOp::UXTH - } - (NarrowValueMode::SignExtend64, 32) => ExtendOp::SXTW, - (NarrowValueMode::ZeroExtend64, 32) => ExtendOp::UXTW, - _ => unreachable!(), - }; - return ResultRSE::RegExtend(reg, extendop); - } + let value = ctx.input_as_value(input.insn, input.input); + if let Some((val, extendop)) = get_as_extended_value(ctx, value, narrow_mode) { + let reg = put_value_in_reg(ctx, val, NarrowValueMode::None); + return ResultRSE::RegExtend(reg, extendop); } ResultRSE::from_rs(put_input_in_rs(ctx, input, narrow_mode)) } +fn get_as_extended_value>( + ctx: &mut C, + val: Value, + narrow_mode: NarrowValueMode, +) -> Option<(Value, ExtendOp)> { + let inputs = ctx.get_value_as_source_or_const(val); + let (insn, n) = inputs.inst?; + if n != 0 { + return None; + } + let op = ctx.data(insn).opcode(); + let out_ty = ctx.output_ty(insn, 0); + let out_bits = ty_bits(out_ty); + + // Is this a zero-extend or sign-extend and can we handle that with a register-mode operator? + if op == Opcode::Uextend || op == Opcode::Sextend { + let sign_extend = op == Opcode::Sextend; + let inner_ty = ctx.input_ty(insn, 0); + let inner_bits = ty_bits(inner_ty); + assert!(inner_bits < out_bits); + if match (sign_extend, narrow_mode) { + // A single zero-extend or sign-extend is equal to itself. + (_, NarrowValueMode::None) => true, + // Two zero-extends or sign-extends in a row is equal to a single zero-extend or sign-extend. + (false, NarrowValueMode::ZeroExtend32) | (false, NarrowValueMode::ZeroExtend64) => true, + (true, NarrowValueMode::SignExtend32) | (true, NarrowValueMode::SignExtend64) => true, + // A zero-extend and a sign-extend in a row is not equal to a single zero-extend or sign-extend + (false, NarrowValueMode::SignExtend32) | (false, NarrowValueMode::SignExtend64) => { + false + } + (true, NarrowValueMode::ZeroExtend32) | (true, NarrowValueMode::ZeroExtend64) => false, + } { + let extendop = match (sign_extend, inner_bits) { + (true, 8) => ExtendOp::SXTB, + (false, 8) => ExtendOp::UXTB, + (true, 16) => ExtendOp::SXTH, + (false, 16) => ExtendOp::UXTH, + (true, 32) => ExtendOp::SXTW, + (false, 32) => ExtendOp::UXTW, + _ => unreachable!(), + }; + return Some((ctx.input_as_value(insn, 0), extendop)); + } + } + + // If `out_ty` is smaller than 32 bits and we need to zero- or sign-extend, + // then get the result into a register and return an Extend-mode operand on + // that register. + if narrow_mode != NarrowValueMode::None + && ((narrow_mode.is_32bit() && out_bits < 32) || (!narrow_mode.is_32bit() && out_bits < 64)) + { + let extendop = match (narrow_mode, out_bits) { + (NarrowValueMode::SignExtend32, 1) | (NarrowValueMode::SignExtend64, 1) => { + ExtendOp::SXTB + } + (NarrowValueMode::ZeroExtend32, 1) | (NarrowValueMode::ZeroExtend64, 1) => { + ExtendOp::UXTB + } + (NarrowValueMode::SignExtend32, 8) | (NarrowValueMode::SignExtend64, 8) => { + ExtendOp::SXTB + } + (NarrowValueMode::ZeroExtend32, 8) | (NarrowValueMode::ZeroExtend64, 8) => { + ExtendOp::UXTB + } + (NarrowValueMode::SignExtend32, 16) | (NarrowValueMode::SignExtend64, 16) => { + ExtendOp::SXTH + } + (NarrowValueMode::ZeroExtend32, 16) | (NarrowValueMode::ZeroExtend64, 16) => { + ExtendOp::UXTH + } + (NarrowValueMode::SignExtend64, 32) => ExtendOp::SXTW, + (NarrowValueMode::ZeroExtend64, 32) => ExtendOp::UXTW, + _ => unreachable!(), + }; + return Some((val, extendop)); + } + None +} + pub(crate) fn put_input_in_rse_imm12>( ctx: &mut C, input: InsnInput, @@ -472,35 +488,6 @@ pub(crate) fn put_input_in_rse_imm12>( ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode)) } -/// Like `put_input_in_rse_imm12` above, except is allowed to negate the -/// argument (assuming a two's-complement representation with the given bit -/// width) if this allows use of 12-bit immediate. Used to flip `add`s with -/// negative immediates to `sub`s (and vice-versa). -pub(crate) fn put_input_in_rse_imm12_maybe_negated>( - ctx: &mut C, - input: InsnInput, - twos_complement_bits: usize, - narrow_mode: NarrowValueMode, -) -> (ResultRSEImm12, bool) { - assert!(twos_complement_bits <= 64); - if let Some(imm_value) = input_to_const(ctx, input) { - if let Some(i) = Imm12::maybe_from_u64(imm_value) { - return (ResultRSEImm12::Imm12(i), false); - } - let sign_extended = - ((imm_value as i64) << (64 - twos_complement_bits)) >> (64 - twos_complement_bits); - let inverted = sign_extended.wrapping_neg(); - if let Some(i) = Imm12::maybe_from_u64(inverted as u64) { - return (ResultRSEImm12::Imm12(i), true); - } - } - - ( - ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode)), - false, - ) -} - pub(crate) fn put_input_in_rs_immlogic>( ctx: &mut C, input: InsnInput, diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 6708431eb9..96bd32d5c8 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -7,8 +7,8 @@ pub mod generated_code; use super::{ zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, AtomicRmwOp, BranchTarget, CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, Imm12, ImmLogic, ImmShift, Inst as MInst, - JTSequenceInfo, MachLabel, MoveWideConst, Opcode, OperandSize, PairAMode, Reg, ScalarSize, - ShiftOpAndAmt, UImm5, VectorSize, NZCV, + JTSequenceInfo, MachLabel, MoveWideConst, NarrowValueMode, Opcode, OperandSize, PairAMode, Reg, + ScalarSize, ShiftOpAndAmt, UImm5, VectorSize, NZCV, }; use crate::isa::aarch64::settings as aarch64_settings; use crate::machinst::isle::*; @@ -19,8 +19,9 @@ use crate::{ ValueLabel, ValueList, }, isa::aarch64::inst::aarch64_map_regs, + isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm}, isa::unwind::UnwindInst, - machinst::{get_output_reg, InsnOutput, LowerCtx, RegRenamer}, + machinst::{get_output_reg, ty_bits, InsnOutput, LowerCtx, RegRenamer}, }; use smallvec::SmallVec; use std::boxed::Box; @@ -76,6 +77,11 @@ pub struct IsleContext<'a, C> { emitted_insts: SmallVec<[MInst; 6]>, } +pub struct ExtendedValue { + val: Value, + extend: ExtendOp, +} + impl<'a, C> IsleContext<'a, C> { pub fn new(lower_ctx: &'a mut C, isa_flags: &'a aarch64_settings::Flags) -> Self { IsleContext { @@ -108,6 +114,25 @@ where ImmLogic::maybe_from_u64(n, I64) } + fn imm12_from_u64(&mut self, n: u64) -> Option { + Imm12::maybe_from_u64(n) + } + + fn imm12_from_negated_u64(&mut self, n: u64) -> Option { + Imm12::maybe_from_u64((n as i64).wrapping_neg() as u64) + } + + fn lshl_from_imm64(&mut self, n: Imm64, ty: Type) -> Option { + let shiftimm = ShiftOpShiftImm::maybe_from_shift(n.bits() as u64)?; + let shiftee_bits = ty_bits(ty); + if shiftee_bits <= std::u8::MAX as usize { + let shiftimm = shiftimm.mask(shiftee_bits as u8); + Some(ShiftOpAndAmt::new(ShiftOp::LSL, shiftimm)) + } else { + None + } + } + fn integral_ty(&mut self, ty: Type) -> Option { match ty { I8 | I16 | I32 | I64 | R64 => Some(ty), @@ -181,6 +206,20 @@ where zero_reg() } + fn extended_value_from_value(&mut self, val: Value) -> Option { + let (val, extend) = + super::get_as_extended_value(self.lower_ctx, val, NarrowValueMode::None)?; + Some(ExtendedValue { val, extend }) + } + + fn put_extended_in_reg(&mut self, reg: &ExtendedValue) -> Reg { + self.put_in_reg(reg.val) + } + + fn get_extended_op(&mut self, reg: &ExtendedValue) -> ExtendOp { + reg.extend + } + fn emit(&mut self, inst: &MInst) -> Unit { self.emitted_insts.push(inst.clone()); } diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index ee3279ed8e..040c81e0bb 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462 -src/isa/aarch64/inst.isle 841748c9c5900821b7086a09a41c6dcdb2172eb47a45293b6ef10f2e1f1389620bf6a2c75152af807d8bc8929029a357af5191f5d87bac2c9ec54bf63a9a2a8f -src/isa/aarch64/lower.isle b3cd0834484e543f39d477d47ee66042276e99955c21fb8c9340a5f27ac317936acb2907a30f758bf596066e36db801a179fda6dbcecaee758a0187a5a5f1412 +src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c +src/isa/aarch64/inst.isle 67a43022bb2e0b8ae06b71c7c49f9b9997a9c6ca109e35f5018b9cd64105a0fe8b103943fb34ca7da45cea9db7327e00954e88606845d5ebc370bc6c3045a04f +src/isa/aarch64/lower.isle 56c3c3f709d2ca18d91e14d8c4cec804990cfa3d85f9ce4fac4eef4861c4c159acee1b5fe6deff7d89337092b35ecf67418da7ce29e6217fa1ab47b0deaf1759 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 4ab9919db5..ace3b1ecd5 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -31,6 +31,7 @@ pub trait Context { fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u16; + fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); @@ -49,12 +50,30 @@ pub trait Context { fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option; fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option; fn imm_logic_from_u64(&mut self, arg0: u64) -> Option; + fn imm12_from_u64(&mut self, arg0: u64) -> Option; + fn imm12_from_negated_u64(&mut self, arg0: u64) -> Option; + fn lshl_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option; fn integral_ty(&mut self, arg0: Type) -> Option; + fn extended_value_from_value(&mut self, arg0: Value) -> Option; + fn put_extended_in_reg(&mut self, arg0: &ExtendedValue) -> Reg; + fn get_extended_op(&mut self, arg0: &ExtendedValue) -> ExtendOp; fn emit(&mut self, arg0: &MInst) -> Unit; fn zero_reg(&mut self) -> Reg; fn load_constant64_full(&mut self, arg0: u64) -> Reg; } +/// Internal type ProducesFlags: defined at src/prelude.isle line 230. +#[derive(Clone, Debug)] +pub enum ProducesFlags { + ProducesFlags { inst: MInst, result: Reg }, +} + +/// Internal type ConsumesFlags: defined at src/prelude.isle line 233. +#[derive(Clone, Debug)] +pub enum ConsumesFlags { + ConsumesFlags { inst: MInst, result: Reg }, +} + /// Internal type MInst: defined at src/isa/aarch64/inst.isle line 2. #[derive(Clone, Debug)] pub enum MInst { @@ -685,7 +704,7 @@ pub enum BitOp { Cls64, } -/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 951. +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 958. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FPUOp1 { Abs32, @@ -698,7 +717,7 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 964. +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 971. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FPUOp2 { Add32, @@ -719,14 +738,14 @@ pub enum FPUOp2 { Uqsub64, } -/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 989. +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 996. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FPUOp3 { MAdd32, MAdd64, } -/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 996. +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1003. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FpuToIntOp { F32ToU32, @@ -739,7 +758,7 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1009. +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1016. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum IntToFpuOp { U32ToF32, @@ -752,7 +771,7 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1023. +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1030. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FpuRoundMode { Minus32, @@ -765,7 +784,7 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1036. +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1043. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecExtendOp { Sxtl8, @@ -776,7 +795,7 @@ pub enum VecExtendOp { Uxtl32, } -/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1053. +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1060. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecALUOp { Sqadd, @@ -818,7 +837,7 @@ pub enum VecALUOp { Sqrdmulh, } -/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1132. +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1139. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecMisc2 { Not, @@ -840,7 +859,7 @@ pub enum VecMisc2 { Cmeq0, } -/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1171. +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1178. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRLongOp { Fcvtl16, @@ -850,7 +869,7 @@ pub enum VecRRLongOp { Shll32, } -/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1186. +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1193. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRNarrowOp { Xtn16, @@ -869,7 +888,7 @@ pub enum VecRRNarrowOp { Fcvtn64, } -/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1218. +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1225. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRRLongOp { Smull8, @@ -883,13 +902,13 @@ pub enum VecRRRLongOp { Umlal32, } -/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1235. +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1242. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecPairOp { Addp, } -/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1243. +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1250. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRPairLongOp { Saddlp8, @@ -898,14 +917,14 @@ pub enum VecRRPairLongOp { Uaddlp16, } -/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1254. +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1261. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecLanesOp { Addv, Uminv, } -/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1263. +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1270. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecShiftImmOp { Shl, @@ -913,7 +932,7 @@ pub enum VecShiftImmOp { Sshr, } -/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1274. +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1281. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum AtomicRMWOp { Add, @@ -945,6 +964,134 @@ pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { return Some(expr2_0); } +// Generated as internal constructor for term with_flags. +pub fn constructor_with_flags( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + // Rule at src/prelude.isle line 243. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); + return Some(expr2_0); + } + } + return None; +} + +// Generated as internal constructor for term with_flags_1. +pub fn constructor_with_flags_1( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + // Rule at src/prelude.isle line 251. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + return Some(pattern3_1); + } + } + return None; +} + +// Generated as internal constructor for term with_flags_2. +pub fn constructor_with_flags_2( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, + arg2: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + let pattern4_0 = arg2; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern5_0, + result: pattern5_1, + } = pattern4_0 + { + // Rule at src/prelude.isle line 261. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + let expr2_0 = C::emit(ctx, &pattern5_0); + let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1); + return Some(expr3_0); + } + } + } + return None; +} + +// Generated as internal constructor for term vector_size. +pub fn constructor_vector_size(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { + if pattern1_0 == 8 { + if pattern1_1 == 16 { + // Rule at src/isa/aarch64/inst.isle line 952. + let expr0_0 = VectorSize::Size8x16; + return Some(expr0_0); + } + } + if pattern1_0 == 16 { + if pattern1_1 == 8 { + // Rule at src/isa/aarch64/inst.isle line 953. + let expr0_0 = VectorSize::Size16x8; + return Some(expr0_0); + } + } + if pattern1_0 == 32 { + if pattern1_1 == 4 { + // Rule at src/isa/aarch64/inst.isle line 954. + let expr0_0 = VectorSize::Size32x4; + return Some(expr0_0); + } + } + if pattern1_0 == 64 { + if pattern1_1 == 2 { + // Rule at src/isa/aarch64/inst.isle line 955. + let expr0_0 = VectorSize::Size64x2; + return Some(expr0_0); + } + } + } + return None; +} + // Generated as internal constructor for term movz. pub fn constructor_movz( ctx: &mut C, @@ -953,7 +1100,7 @@ pub fn constructor_movz( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1315. + // Rule at src/isa/aarch64/inst.isle line 1354. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -974,7 +1121,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1322. + // Rule at src/isa/aarch64/inst.isle line 1361. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -997,7 +1144,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1329. + // Rule at src/isa/aarch64/inst.isle line 1368. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1011,14 +1158,273 @@ pub fn constructor_alu_rr_imm_logic( return Some(expr4_0); } -// Generated as internal constructor for term orr64. -pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { +// Generated as internal constructor for term alu_rrr. +pub fn constructor_alu_rrr( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Reg, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1336. - let expr0_0 = ALUOp::Orr64; - let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1375. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRR { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term vec_rrr. +pub fn constructor_vec_rrr( + ctx: &mut C, + arg0: &VecALUOp, + arg1: Reg, + arg2: Reg, + arg3: &VectorSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1382. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::VecRRR { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + size: pattern3_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rr_imm12. +pub fn constructor_alu_rr_imm12( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Imm12, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1389. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRImm12 { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + imm12: pattern2_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rrr_shift. +pub fn constructor_alu_rrr_shift( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Reg, + arg3: ShiftOpAndAmt, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1396. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRRShift { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + shiftop: pattern3_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rrr_extend. +pub fn constructor_alu_rrr_extend( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Reg, + arg3: &ExtendOp, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1403. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRRExtend { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + extendop: pattern3_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rr_extend_reg. +pub fn constructor_alu_rr_extend_reg( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: &ExtendedValue, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1411. + let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); + let expr1_0 = C::get_extended_op(ctx, pattern2_0); + let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term alu_rrrr. +pub fn constructor_alu_rrrr( + ctx: &mut C, + arg0: &ALUOp3, + arg1: Reg, + arg2: Reg, + arg3: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1418. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRRR { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + ra: pattern3_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term add64_with_flags. +pub fn constructor_add64_with_flags( + ctx: &mut C, + arg0: Reg, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1425. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::AddS64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ProducesFlags::ProducesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term adc64. +pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1432. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::Adc64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ConsumesFlags::ConsumesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term sub64_with_flags. +pub fn constructor_sub64_with_flags( + ctx: &mut C, + arg0: Reg, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1439. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::SubS64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ProducesFlags::ProducesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term sbc64. +pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1446. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::Sbc64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ConsumesFlags::ConsumesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); } // Generated as internal constructor for term imm. @@ -1027,24 +1433,25 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) { let pattern2_0 = arg1; if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1351. - let expr0_0 = C::zero_reg(ctx); - let expr1_0 = constructor_orr64(ctx, expr0_0, pattern3_0)?; - return Some(expr1_0); + // Rule at src/isa/aarch64/inst.isle line 1464. + let expr0_0 = ALUOp::Orr64; + let expr1_0 = C::zero_reg(ctx); + let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern3_0)?; + return Some(expr2_0); } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1343. + // Rule at src/isa/aarch64/inst.isle line 1456. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1347. + // Rule at src/isa/aarch64/inst.isle line 1460. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1358. + // Rule at src/isa/aarch64/inst.isle line 1471. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -1056,6 +1463,54 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 94. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0 = constructor_add64_with_flags(ctx, expr2_0, expr7_0)?; + let expr11_0 = constructor_adc64(ctx, expr4_0, expr9_0)?; + let expr12_0 = constructor_with_flags(ctx, &expr10_0, &expr11_0)?; + return Some(expr12_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 145. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0 = constructor_sub64_with_flags(ctx, expr2_0, expr7_0)?; + let expr11_0 = constructor_sbc64(ctx, expr4_0, expr9_0)?; + let expr12_0 = constructor_with_flags(ctx, &expr10_0, &expr11_0)?; + return Some(expr12_0); + } + _ => {} + } + } + } let pattern3_0 = C::inst_data(ctx, pattern0_0); match &pattern3_0 { &InstructionData::NullAry { @@ -1104,6 +1559,464 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } + if let Some((pattern3_0, pattern3_1)) = C::multi_lane(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Iadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 90. + let expr0_0 = VecALUOp::Add; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern2_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 141. + let expr0_0 = VecALUOp::Sub; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern2_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + _ => {} + } + } + } + if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Iadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_0) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + match &pattern9_0 { + &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } => { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + if let Some(pattern13_0) = + C::imm12_from_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 37. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern13_0) = + C::imm12_from_negated_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 45. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + &InstructionData::Binary { + opcode: ref pattern10_0, + args: ref pattern10_1, + } => { + match &pattern10_0 { + &Opcode::Imul => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + // Rule at src/isa/aarch64/lower.isle line 70. + let expr0_0 = constructor_madd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern12_0); + let expr2_0 = C::put_in_reg(ctx, pattern12_1); + let expr3_0 = C::put_in_reg(ctx, pattern7_1); + let expr4_0 = constructor_alu_rrrr( + ctx, &expr0_0, expr1_0, expr2_0, expr3_0, + )?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Ishl => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + if let Some(pattern13_0) = C::def_inst(ctx, pattern12_1) + { + let pattern14_0 = C::inst_data(ctx, pattern13_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern15_0, + imm: pattern15_1, + } = &pattern14_0 + { + if let &Opcode::Iconst = &pattern15_0 { + let closure17 = || { + return Some(pattern3_0); + }; + if let Some(pattern17_0) = closure17() { + if let Some(pattern18_0) = + C::lshl_from_imm64( + ctx, + pattern15_1, + pattern17_0, + ) + { + // Rule at src/isa/aarch64/lower.isle line 62. + let expr0_0 = constructor_iadd_op( + ctx, pattern3_0, + )?; + let expr1_0 = + C::put_in_reg(ctx, pattern7_1); + let expr2_0 = + C::put_in_reg(ctx, pattern12_0); + let expr3_0 = + constructor_alu_rrr_shift( + ctx, + &expr0_0, + expr1_0, + expr2_0, + pattern18_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + } + } + } + _ => {} + } + } + _ => {} + } + } + if let Some(pattern8_0) = C::extended_value_from_value(ctx, pattern7_0) { + // Rule at src/isa/aarch64/lower.isle line 53. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = + constructor_alu_rr_extend_reg(ctx, &expr0_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + match &pattern9_0 { + &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } => { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + if let Some(pattern13_0) = + C::imm12_from_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 34. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern13_0) = + C::imm12_from_negated_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 42. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + &InstructionData::Binary { + opcode: ref pattern10_0, + args: ref pattern10_1, + } => { + match &pattern10_0 { + &Opcode::Imul => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + // Rule at src/isa/aarch64/lower.isle line 67. + let expr0_0 = constructor_madd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern12_0); + let expr2_0 = C::put_in_reg(ctx, pattern12_1); + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = constructor_alu_rrrr( + ctx, &expr0_0, expr1_0, expr2_0, expr3_0, + )?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Ishl => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + if let Some(pattern13_0) = C::def_inst(ctx, pattern12_1) + { + let pattern14_0 = C::inst_data(ctx, pattern13_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern15_0, + imm: pattern15_1, + } = &pattern14_0 + { + if let &Opcode::Iconst = &pattern15_0 { + let closure17 = || { + return Some(pattern3_0); + }; + if let Some(pattern17_0) = closure17() { + if let Some(pattern18_0) = + C::lshl_from_imm64( + ctx, + pattern15_1, + pattern17_0, + ) + { + // Rule at src/isa/aarch64/lower.isle line 58. + let expr0_0 = constructor_iadd_op( + ctx, pattern3_0, + )?; + let expr1_0 = + C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + C::put_in_reg(ctx, pattern12_0); + let expr3_0 = + constructor_alu_rrr_shift( + ctx, + &expr0_0, + expr1_0, + expr2_0, + pattern18_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + } + } + } + _ => {} + } + } + _ => {} + } + } + if let Some(pattern8_0) = C::extended_value_from_value(ctx, pattern7_1) { + // Rule at src/isa/aarch64/lower.isle line 50. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + constructor_alu_rr_extend_reg(ctx, &expr0_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/aarch64/lower.isle line 30. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + match &pattern9_0 { + &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } => { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + if let Some(pattern13_0) = + C::imm12_from_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 121. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern13_0) = + C::imm12_from_negated_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 126. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + &InstructionData::Binary { + opcode: ref pattern10_0, + args: ref pattern10_1, + } => { + if let &Opcode::Ishl = &pattern10_0 { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + if let Some(pattern13_0) = C::def_inst(ctx, pattern12_1) { + let pattern14_0 = C::inst_data(ctx, pattern13_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern15_0, + imm: pattern15_1, + } = &pattern14_0 + { + if let &Opcode::Iconst = &pattern15_0 { + let closure17 = || { + return Some(pattern3_0); + }; + if let Some(pattern17_0) = closure17() { + if let Some(pattern18_0) = + C::lshl_from_imm64( + ctx, + pattern15_1, + pattern17_0, + ) + { + // Rule at src/isa/aarch64/lower.isle line 136. + let expr0_0 = constructor_isub_op( + ctx, pattern3_0, + )?; + let expr1_0 = + C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + C::put_in_reg(ctx, pattern12_0); + let expr3_0 = + constructor_alu_rrr_shift( + ctx, + &expr0_0, + expr1_0, + expr2_0, + pattern18_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + } + } + } + } + _ => {} + } + } + if let Some(pattern8_0) = C::extended_value_from_value(ctx, pattern7_1) { + // Rule at src/isa/aarch64/lower.isle line 131. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + constructor_alu_rr_extend_reg(ctx, &expr0_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/aarch64/lower.isle line 117. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + _ => {} + } + } + } } return None; } + +// Generated as internal constructor for term iadd_op. +pub fn constructor_iadd_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { + // Rule at src/isa/aarch64/lower.isle line 75. + let expr0_0 = ALUOp::Add32; + return Some(expr0_0); + } + // Rule at src/isa/aarch64/lower.isle line 76. + let expr0_0 = ALUOp::Add64; + return Some(expr0_0); +} + +// Generated as internal constructor for term isub_op. +pub fn constructor_isub_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { + // Rule at src/isa/aarch64/lower.isle line 80. + let expr0_0 = ALUOp::Sub32; + return Some(expr0_0); + } + // Rule at src/isa/aarch64/lower.isle line 81. + let expr0_0 = ALUOp::Sub64; + return Some(expr0_0); +} + +// Generated as internal constructor for term madd_op. +pub fn constructor_madd_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { + // Rule at src/isa/aarch64/lower.isle line 85. + let expr0_0 = ALUOp3::MAdd32; + return Some(expr0_0); + } + // Rule at src/isa/aarch64/lower.isle line 86. + let expr0_0 = ALUOp3::MAdd64; + return Some(expr0_0); +} diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 74d1506c74..b42792bd39 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -42,7 +42,7 @@ pub(crate) fn lower_insn_to_regs>( return Ok(()); } - let implemented_in_isle = || { + let implemented_in_isle = |ctx: &mut C| { unreachable!( "implemented in ISLE: inst = `{}`, type = `{:?}`", ctx.dfg().display_inst(insn), @@ -51,7 +51,7 @@ pub(crate) fn lower_insn_to_regs>( }; match op { - Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(), + Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(ctx), Opcode::F32const => { let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32); @@ -63,143 +63,8 @@ pub(crate) fn lower_insn_to_regs>( let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); lower_constant_f64(ctx, rd, value); } - Opcode::Iadd => { - match ty.unwrap() { - ty if ty.is_vector() => { - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - ctx.emit(Inst::VecRRR { - rd, - rn, - rm, - alu_op: VecALUOp::Add, - size: VectorSize::from_ty(ty), - }); - } - I128 => { - let lhs = put_input_in_regs(ctx, inputs[0]); - let rhs = put_input_in_regs(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]); - assert_eq!(lhs.len(), 2); - assert_eq!(rhs.len(), 2); - assert_eq!(dst.len(), 2); - - // adds x0, x0, x2 - // adc x1, x1, x3 - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::AddS64, - rd: dst.regs()[0], - rn: lhs.regs()[0], - rm: rhs.regs()[0], - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Adc64, - rd: dst.regs()[1], - rn: lhs.regs()[1], - rm: rhs.regs()[1], - }); - } - ty => { - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let mul_insn = if let Some(mul_insn) = - maybe_input_insn(ctx, inputs[1], Opcode::Imul) - { - Some((mul_insn, 0)) - } else if let Some(mul_insn) = maybe_input_insn(ctx, inputs[0], Opcode::Imul) { - Some((mul_insn, 1)) - } else { - None - }; - // If possible combine mul + add into madd. - if let Some((insn, addend_idx)) = mul_insn { - let alu_op = choose_32_64(ty, ALUOp3::MAdd32, ALUOp3::MAdd64); - let rn_input = InsnInput { insn, input: 0 }; - let rm_input = InsnInput { insn, input: 1 }; - - let rn = put_input_in_reg(ctx, rn_input, NarrowValueMode::None); - let rm = put_input_in_reg(ctx, rm_input, NarrowValueMode::None); - let ra = put_input_in_reg(ctx, inputs[addend_idx], NarrowValueMode::None); - - ctx.emit(Inst::AluRRRR { - alu_op, - rd, - rn, - rm, - ra, - }); - } else { - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - let (rm, negated) = put_input_in_rse_imm12_maybe_negated( - ctx, - inputs[1], - ty_bits(ty), - NarrowValueMode::None, - ); - let alu_op = if !negated { - choose_32_64(ty, ALUOp::Add32, ALUOp::Add64) - } else { - choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64) - }; - ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); - } - } - } - } - Opcode::Isub => { - let ty = ty.unwrap(); - if ty == I128 { - let lhs = put_input_in_regs(ctx, inputs[0]); - let rhs = put_input_in_regs(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]); - assert_eq!(lhs.len(), 2); - assert_eq!(rhs.len(), 2); - assert_eq!(dst.len(), 2); - - // subs x0, x0, x2 - // sbc x1, x1, x3 - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::SubS64, - rd: dst.regs()[0], - rn: lhs.regs()[0], - rm: rhs.regs()[0], - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Sbc64, - rd: dst.regs()[1], - rn: lhs.regs()[1], - rm: rhs.regs()[1], - }); - } else { - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - if !ty.is_vector() { - let (rm, negated) = put_input_in_rse_imm12_maybe_negated( - ctx, - inputs[1], - ty_bits(ty), - NarrowValueMode::None, - ); - let alu_op = if !negated { - choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64) - } else { - choose_32_64(ty, ALUOp::Add32, ALUOp::Add64) - }; - ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); - } else { - let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - ctx.emit(Inst::VecRRR { - rd, - rn, - rm, - alu_op: VecALUOp::Sub, - size: VectorSize::from_ty(ty), - }); - } - } - } + Opcode::Iadd => implemented_in_isle(ctx), + Opcode::Isub => implemented_in_isle(ctx), Opcode::UaddSat | Opcode::SaddSat | Opcode::UsubSat | Opcode::SsubSat => { let ty = ty.unwrap(); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 33ac1fc785..8fdb09a1d3 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -385,49 +385,6 @@ (decl sink_load (SinkableLoad) RegMemImm) (extern constructor sink_load sink_load) -;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;; Newtype wrapper around `MInst` for instructions that are used for their -;; effect on flags. -(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg)))) - -;; Newtype wrapper around `MInst` for instructions that consume flags. -(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg)))) - -;; Combine flags-producing and -consuming instructions together, ensuring that -;; they are emitted back-to-back and no other instructions can be emitted -;; between them and potentially clobber the flags. -;; -;; Returns a `ValueRegs` where the first register is the result of the -;; `ProducesFlags` instruction and the second is the result of the -;; `ConsumesFlags` instruction. -(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs) -(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result) - (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) - (let ((_x Unit (emit producer_inst)) - (_y Unit (emit consumer_inst))) - (value_regs producer_result consumer_result))) - -;; Like `with_flags` but returns only the result of the consumer operation. -(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg) -(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result) - (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) - (let ((_x Unit (emit producer_inst)) - (_y Unit (emit consumer_inst))) - consumer_result)) - -;; Like `with_flags` but allows two consumers of the same flags. The result is a -;; `ValueRegs` containing the first consumer's result and then the second -;; consumer's result. -(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs) -(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result) - (ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1) - (ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2)) - (let ((_x Unit (emit producer_inst)) - (_y Unit (emit consumer_inst_1)) - (_z Unit (emit consumer_inst_2))) - (value_regs consumer_result_1 consumer_result_2))) - ;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (type ExtendKind (enum Sign Zero)) diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 174aa529fd..cacbaa17fb 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462 -src/isa/x64/inst.isle 6065d3b9e0fa3361d179d9b87d09568ff474f8bac7eeabd29b328ace723041f96045bc82cfa2d7feda4490ce7e5d4be1a1c7ebe25c99916564d43a51550cd093 +src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c +src/isa/x64/inst.isle e4a0657406056a4cf116fe125e91d16377b602e0b41edd6628cbb7259b0fc2aa6b0482ffd33f00d63d68cf3546f188705877309d43eba5e75abd0f38a52a79b2 src/isa/x64/lower.isle e51b7a67343dba342a43b3c9e4b9ed7df9b2c66a677018acf7054ba48c27e4e93a4421fd892b9bf7c0e5b790bcfafab7cb3e93ce2b8206c04d456918d2ad0b5a diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index a62050787b..c147f914db 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -31,6 +31,7 @@ pub trait Context { fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u16; + fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); @@ -62,19 +63,19 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/isa/x64/inst.isle line 392. +/// Internal type ProducesFlags: defined at src/prelude.isle line 230. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/isa/x64/inst.isle line 395. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 233. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 433. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 390. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -100,35 +101,6 @@ pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { return Some(expr2_0); } -// Generated as internal constructor for term operand_size_bits. -pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize) -> Option { - let pattern0_0 = arg0; - match pattern0_0 { - &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 80. - let expr0_0: u16 = 8; - return Some(expr0_0); - } - &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 81. - let expr0_0: u16 = 16; - return Some(expr0_0); - } - &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 82. - let expr0_0: u16 = 32; - return Some(expr0_0); - } - &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 83. - let expr0_0: u16 = 64; - return Some(expr0_0); - } - _ => {} - } - return None; -} - // Generated as internal constructor for term with_flags. pub fn constructor_with_flags( ctx: &mut C, @@ -147,7 +119,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/isa/x64/inst.isle line 405. + // Rule at src/prelude.isle line 243. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -175,7 +147,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/isa/x64/inst.isle line 413. + // Rule at src/prelude.isle line 251. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -209,7 +181,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/isa/x64/inst.isle line 423. + // Rule at src/prelude.isle line 261. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -221,6 +193,35 @@ pub fn constructor_with_flags_2( return None; } +// Generated as internal constructor for term operand_size_bits. +pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &OperandSize::Size8 => { + // Rule at src/isa/x64/inst.isle line 80. + let expr0_0: u16 = 8; + return Some(expr0_0); + } + &OperandSize::Size16 => { + // Rule at src/isa/x64/inst.isle line 81. + let expr0_0: u16 = 16; + return Some(expr0_0); + } + &OperandSize::Size32 => { + // Rule at src/isa/x64/inst.isle line 82. + let expr0_0: u16 = 32; + return Some(expr0_0); + } + &OperandSize::Size64 => { + // Rule at src/isa/x64/inst.isle line 83. + let expr0_0: u16 = 64; + return Some(expr0_0); + } + _ => {} + } + return None; +} + // Generated as internal constructor for term extend_to_reg. pub fn constructor_extend_to_reg( ctx: &mut C, @@ -233,12 +234,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 445. + // Rule at src/isa/x64/inst.isle line 402. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 448. + // Rule at src/isa/x64/inst.isle line 405. let expr0_0 = C::ty_bits(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -262,7 +263,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 468. + // Rule at src/isa/x64/inst.isle line 425. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -270,7 +271,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 464. + // Rule at src/isa/x64/inst.isle line 421. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -283,17 +284,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 475. + // Rule at src/isa/x64/inst.isle line 432. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 433. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 477. + // Rule at src/isa/x64/inst.isle line 434. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -310,7 +311,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 481. + // Rule at src/isa/x64/inst.isle line 438. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -320,40 +321,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 490. + // Rule at src/isa/x64/inst.isle line 447. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 491. + // Rule at src/isa/x64/inst.isle line 448. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 486. + // Rule at src/isa/x64/inst.isle line 443. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 487. + // Rule at src/isa/x64/inst.isle line 444. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 488. + // Rule at src/isa/x64/inst.isle line 445. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 489. + // Rule at src/isa/x64/inst.isle line 446. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -365,7 +366,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 505. + // Rule at src/isa/x64/inst.isle line 462. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -393,7 +394,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 529. + // Rule at src/isa/x64/inst.isle line 486. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -418,7 +419,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 537. + // Rule at src/isa/x64/inst.isle line 494. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -434,7 +435,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 545. + // Rule at src/isa/x64/inst.isle line 502. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -463,7 +464,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 556. + // Rule at src/isa/x64/inst.isle line 513. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -492,7 +493,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 567. + // Rule at src/isa/x64/inst.isle line 524. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -508,7 +509,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 575. + // Rule at src/isa/x64/inst.isle line 532. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -537,7 +538,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 586. + // Rule at src/isa/x64/inst.isle line 543. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -566,7 +567,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 597. + // Rule at src/isa/x64/inst.isle line 554. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -582,7 +583,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 608. + // Rule at src/isa/x64/inst.isle line 565. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -598,7 +599,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 616. + // Rule at src/isa/x64/inst.isle line 573. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -614,7 +615,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 624. + // Rule at src/isa/x64/inst.isle line 581. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -626,7 +627,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 653. + // Rule at src/isa/x64/inst.isle line 610. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -643,7 +644,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 682. + // Rule at src/isa/x64/inst.isle line 639. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -658,7 +659,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 641. + // Rule at src/isa/x64/inst.isle line 598. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -671,7 +672,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 694. + // Rule at src/isa/x64/inst.isle line 651. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -686,7 +687,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 645. + // Rule at src/isa/x64/inst.isle line 602. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -699,7 +700,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 672. + // Rule at src/isa/x64/inst.isle line 629. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -716,7 +717,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } let pattern1_0 = arg1; if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 659. + // Rule at src/isa/x64/inst.isle line 616. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -732,7 +733,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 634. + // Rule at src/isa/x64/inst.isle line 591. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Imm { @@ -757,7 +758,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 707. + // Rule at src/isa/x64/inst.isle line 664. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -782,7 +783,7 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 716. + // Rule at src/isa/x64/inst.isle line 673. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -798,7 +799,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 721. + // Rule at src/isa/x64/inst.isle line 678. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -814,7 +815,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 726. + // Rule at src/isa/x64/inst.isle line 683. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -830,7 +831,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 731. + // Rule at src/isa/x64/inst.isle line 688. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -848,7 +849,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 736. + // Rule at src/isa/x64/inst.isle line 693. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -873,7 +874,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 745. + // Rule at src/isa/x64/inst.isle line 702. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -889,7 +890,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 750. + // Rule at src/isa/x64/inst.isle line 707. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -907,7 +908,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 755. + // Rule at src/isa/x64/inst.isle line 712. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -935,7 +936,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 763. + // Rule at src/isa/x64/inst.isle line 720. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -957,7 +958,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 770. + // Rule at src/isa/x64/inst.isle line 727. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -981,7 +982,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 777. + // Rule at src/isa/x64/inst.isle line 734. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -998,7 +999,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 784. + // Rule at src/isa/x64/inst.isle line 741. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1009,7 +1010,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 789. + // Rule at src/isa/x64/inst.isle line 746. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1020,7 +1021,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 794. + // Rule at src/isa/x64/inst.isle line 751. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1031,7 +1032,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 799. + // Rule at src/isa/x64/inst.isle line 756. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1042,7 +1043,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 804. + // Rule at src/isa/x64/inst.isle line 761. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1053,7 +1054,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 809. + // Rule at src/isa/x64/inst.isle line 766. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1064,7 +1065,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 814. + // Rule at src/isa/x64/inst.isle line 771. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1075,7 +1076,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 819. + // Rule at src/isa/x64/inst.isle line 776. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1086,7 +1087,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 824. + // Rule at src/isa/x64/inst.isle line 781. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1097,7 +1098,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 829. + // Rule at src/isa/x64/inst.isle line 786. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1108,7 +1109,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 834. + // Rule at src/isa/x64/inst.isle line 791. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1119,7 +1120,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 839. + // Rule at src/isa/x64/inst.isle line 796. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1130,7 +1131,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 844. + // Rule at src/isa/x64/inst.isle line 801. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1141,7 +1142,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 849. + // Rule at src/isa/x64/inst.isle line 806. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1152,7 +1153,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 854. + // Rule at src/isa/x64/inst.isle line 811. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1163,7 +1164,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 859. + // Rule at src/isa/x64/inst.isle line 816. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1174,7 +1175,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 864. + // Rule at src/isa/x64/inst.isle line 821. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1185,7 +1186,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 869. + // Rule at src/isa/x64/inst.isle line 826. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1196,7 +1197,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 831. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1207,7 +1208,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 879. + // Rule at src/isa/x64/inst.isle line 836. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1218,7 +1219,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 884. + // Rule at src/isa/x64/inst.isle line 841. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1229,7 +1230,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 846. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1240,7 +1241,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 851. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1251,7 +1252,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 856. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1262,7 +1263,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 861. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1273,7 +1274,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 866. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1284,7 +1285,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 871. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1295,7 +1296,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 876. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1306,7 +1307,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 924. + // Rule at src/isa/x64/inst.isle line 881. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1317,7 +1318,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 929. + // Rule at src/isa/x64/inst.isle line 886. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1328,7 +1329,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 891. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1339,7 +1340,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 939. + // Rule at src/isa/x64/inst.isle line 896. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1350,7 +1351,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 944. + // Rule at src/isa/x64/inst.isle line 901. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1361,7 +1362,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 949. + // Rule at src/isa/x64/inst.isle line 906. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1372,7 +1373,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 954. + // Rule at src/isa/x64/inst.isle line 911. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1383,7 +1384,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 959. + // Rule at src/isa/x64/inst.isle line 916. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1394,7 +1395,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 964. + // Rule at src/isa/x64/inst.isle line 921. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1405,7 +1406,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 969. + // Rule at src/isa/x64/inst.isle line 926. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1426,7 +1427,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 974. + // Rule at src/isa/x64/inst.isle line 931. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1454,7 +1455,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 986. + // Rule at src/isa/x64/inst.isle line 943. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1472,7 +1473,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 995. + // Rule at src/isa/x64/inst.isle line 952. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1497,7 +1498,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1008. + // Rule at src/isa/x64/inst.isle line 965. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1513,7 +1514,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1015. + // Rule at src/isa/x64/inst.isle line 972. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1522,7 +1523,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1020. + // Rule at src/isa/x64/inst.isle line 977. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1538,7 +1539,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1025. + // Rule at src/isa/x64/inst.isle line 982. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -1556,7 +1557,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1037. + // Rule at src/isa/x64/inst.isle line 994. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1572,7 +1573,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1044. + // Rule at src/isa/x64/inst.isle line 1001. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -1590,7 +1591,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1054. + // Rule at src/isa/x64/inst.isle line 1011. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1600,7 +1601,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1059. + // Rule at src/isa/x64/inst.isle line 1016. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1610,7 +1611,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1064. + // Rule at src/isa/x64/inst.isle line 1021. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1628,7 +1629,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1071. + // Rule at src/isa/x64/inst.isle line 1028. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -1657,7 +1658,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1087. + // Rule at src/isa/x64/inst.isle line 1044. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1673,7 +1674,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1092. + // Rule at src/isa/x64/inst.isle line 1049. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1692,7 +1693,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1105. + // Rule at src/isa/x64/inst.isle line 1062. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1713,7 +1714,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1114. + // Rule at src/isa/x64/inst.isle line 1071. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -1736,7 +1737,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1121. + // Rule at src/isa/x64/inst.isle line 1078. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -1754,7 +1755,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1126. + // Rule at src/isa/x64/inst.isle line 1083. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -1774,7 +1775,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1131. + // Rule at src/isa/x64/inst.isle line 1088. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1792,7 +1793,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1136. + // Rule at src/isa/x64/inst.isle line 1093. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -1804,7 +1805,7 @@ pub fn constructor_insertps( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1141. + // Rule at src/isa/x64/inst.isle line 1098. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1815,7 +1816,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1146. + // Rule at src/isa/x64/inst.isle line 1103. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index ddc579a3b4..dfab0f35d1 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -90,6 +90,15 @@ macro_rules! isle_prelude_methods { ty.bits() } + #[inline] + fn fits_in_32(&mut self, ty: Type) -> Option { + if ty.bits() <= 32 { + Some(ty) + } else { + None + } + } + #[inline] fn fits_in_64(&mut self, ty: Type) -> Option { if ty.bits() <= 64 { diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 8f65449cde..fbc1efbc49 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -105,6 +105,8 @@ pub trait LowerCtx { fn num_outputs(&self, ir_inst: Inst) -> usize; /// Get the type for an instruction's input. fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type; + /// Get the type for a value. + fn value_ty(&self, val: Value) -> Type; /// Get the type for an instruction's output. fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type; /// Get the value of a constant instruction (`iconst`, etc.) as a 64-bit @@ -129,6 +131,9 @@ pub trait LowerCtx { fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput; /// Like `get_input_as_source_or_const` but with a `Value`. fn get_value_as_source_or_const(&self, value: Value) -> NonRegInput; + /// Resolves a particular input of an instruction to the `Value` that it is + /// represented with. + fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value; /// Put the `idx`th input into register(s) and return the assigned register. fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs; /// Put the given value into register(s) and return the assigned register. @@ -1109,8 +1114,10 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type { - let val = self.f.dfg.inst_args(ir_inst)[idx]; - let val = self.f.dfg.resolve_aliases(val); + self.value_ty(self.input_as_value(ir_inst, idx)) + } + + fn value_ty(&self, val: Value) -> Type { self.f.dfg.value_type(val) } @@ -1122,9 +1129,13 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { self.inst_constants.get(&ir_inst).cloned() } - fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput { + fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value { let val = self.f.dfg.inst_args(ir_inst)[idx]; - let val = self.f.dfg.resolve_aliases(val); + self.f.dfg.resolve_aliases(val) + } + + fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput { + let val = self.input_as_value(ir_inst, idx); self.get_value_as_source_or_const(val) } diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 4951c0bd7b..2a1d0b8227 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -144,6 +144,10 @@ ;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; An extractor that only matches types that can fit in 32 bits. +(decl fits_in_32 (Type) Type) +(extern extractor fits_in_32 fits_in_32) + ;; An extractor that only matches types that can fit in 64 bits. (decl fits_in_64 (Type) Type) (extern extractor fits_in_64 fits_in_64) @@ -218,3 +222,47 @@ ;; Match the instruction that defines the given value, if any. (decl def_inst (Inst) Value) (extern extractor def_inst def_inst) + +;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Newtype wrapper around `MInst` for instructions that are used for their +;; effect on flags. +(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg)))) + +;; Newtype wrapper around `MInst` for instructions that consume flags. +(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg)))) + +;; Combine flags-producing and -consuming instructions together, ensuring that +;; they are emitted back-to-back and no other instructions can be emitted +;; between them and potentially clobber the flags. +;; +;; Returns a `ValueRegs` where the first register is the result of the +;; `ProducesFlags` instruction and the second is the result of the +;; `ConsumesFlags` instruction. +(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs) +(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst))) + (value_regs producer_result consumer_result))) + +;; Like `with_flags` but returns only the result of the consumer operation. +(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg) +(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst))) + consumer_result)) + +;; Like `with_flags` but allows two consumers of the same flags. The result is a +;; `ValueRegs` containing the first consumer's result and then the second +;; consumer's result. +(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs) +(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1) + (ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst_1)) + (_z Unit (emit consumer_inst_2))) + (value_regs consumer_result_1 consumer_result_2))) + diff --git a/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif b/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif index 5f27d9f034..6d90ba7e48 100644 --- a/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif +++ b/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif @@ -369,3 +369,23 @@ block0(v0: i128, v1: i128): ; nextln: madd x0, x0, x2, xzr ; nextln: ret +function %add_mul_1(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = imul v1, v2 + v4 = iadd v0, v3 + return v4 +} + +; check: madd w0, w1, w2, w0 +; nextln: ret + +function %add_mul_2(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = imul v1, v2 + v4 = iadd v3, v0 + return v4 +} + +; check: madd w0, w1, w2, w0 +; nextln: ret + diff --git a/cranelift/filetests/filetests/isa/aarch64/extend-op.clif b/cranelift/filetests/filetests/isa/aarch64/extend-op.clif index 58735d5bf8..f8e25d7814 100644 --- a/cranelift/filetests/filetests/isa/aarch64/extend-op.clif +++ b/cranelift/filetests/filetests/isa/aarch64/extend-op.clif @@ -10,8 +10,19 @@ block0(v0: i8): return v3 } -; check: movz x1, #42 -; nextln: add x0, x1, x0, SXTB +; check: sxtb x0, w0 +; nextln: add x0, x0, #42 +; nextln: ret + + +function %f2(i8, i64) -> i64 { +block0(v0: i8, v1: i64): + v2 = sextend.i64 v0 + v3 = iadd.i64 v2, v1 + return v3 +} + +; check: add x0, x1, x0, SXTB ; nextln: ret