From 7d0f6ab90fcc218b8cf4347b408d51e1660e13ad Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 17 Nov 2021 13:19:47 -0800 Subject: [PATCH 1/5] aarch64: Migrate `iadd` and `isub` to ISLE This commit is the first "meaty" instruction added to ISLE for the AArch64 backend. I chose to pick the first two in the current lowering's `match` statement, `isub` and `iadd`. These two turned out to be particularly interesting for a few reasons: * Both had clearly migratable-to-ISLE behavior along the lines of special-casing per type. For example 128-bit and vector arithmetic were both easily translateable. * The `iadd` instruction has special cases for fusing with a multiplication to generate `madd` which is expressed pretty easily in ISLE. * Otherwise both instructions had a number of forms where they attempted to interpret the RHS as various forms of constants, extends, or shifts. There's a bit of a design space of how best to represent this in ISLE and what I settled on was to have a special case for each form of instruction, and the special cases are somewhat duplicated between `iadd` and `isub`. There's custom "extractors" for the special cases and instructions that support these special cases will have an `rule`-per-case. Overall I think the ISLE transitioned pretty well. I don't think that the aarch64 backend is going to follow the x64 backend super closely, though. For example the x64 backend is having a helper-per-instruction at the moment but with AArch64 it seems to make more sense to only have a helper-per-enum-variant-of-`MInst`. This is because the same instruction (e.g. `ALUOp::Sub32`) can be expressed with multiple different forms depending on the payload. It's worth noting that the ISLE looks like it's a good deal larger than the code actually being removed from lowering as part of this commit. I think this is deceptive though because a lot of the logic in `put_input_in_rse_imm12_maybe_negated` and `alu_inst_imm12` is being inlined into the ISLE definitions for each instruction instead of having it all packed into the helper functions. Some of the "boilerplate" here is the addition of various ISLE utilities as well. --- cranelift/codegen/src/isa/aarch64/inst.isle | 93 ++- .../codegen/src/isa/aarch64/inst/args.rs | 2 +- .../codegen/src/isa/aarch64/inst/imms.rs | 2 +- cranelift/codegen/src/isa/aarch64/lower.isle | 125 ++++ cranelift/codegen/src/isa/aarch64/lower.rs | 219 +++--- .../codegen/src/isa/aarch64/lower/isle.rs | 45 +- .../lower/isle/generated_code.manifest | 6 + .../isa/aarch64/lower/isle/generated_code.rs | 683 +++++++++++++++++- .../codegen/src/isa/aarch64/lower_inst.rs | 143 +--- .../x64/lower/isle/generated_code.manifest | 6 + .../src/isa/x64/lower/isle/generated_code.rs | 1 + cranelift/codegen/src/machinst/isle.rs | 9 + cranelift/codegen/src/machinst/lower.rs | 19 +- cranelift/codegen/src/prelude.isle | 4 + .../filetests/isa/aarch64/arithmetic.clif | 20 + 15 files changed, 1076 insertions(+), 301 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 3b2ef337e2..9ed6356da8 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -947,6 +947,13 @@ (Size64x2) )) +;; Helper for calculating the `VectorSize` corresponding to a type +(decl vector_size (Type) VectorSize) +(rule (vector_size (multi_lane 8 16)) (VectorSize.Size8x16)) +(rule (vector_size (multi_lane 16 8)) (VectorSize.Size16x8)) +(rule (vector_size (multi_lane 32 4)) (VectorSize.Size32x4)) +(rule (vector_size (multi_lane 64 2)) (VectorSize.Size64x2)) + ;; A floating-point unit (FPU) operation with one arg. (type FPUOp1 (enum @@ -1294,9 +1301,41 @@ (decl imm_logic_from_u64 (ImmLogic) u64) (extern extractor imm_logic_from_u64 imm_logic_from_u64) +(decl imm12_from_u64 (Imm12) u64) +(extern extractor imm12_from_u64 imm12_from_u64) + +(decl imm12_from_negated_u64 (Imm12) u64) +(extern extractor imm12_from_negated_u64 imm12_from_negated_u64) + +(decl lshl_from_imm64 (Type ShiftOpAndAmt) Imm64) +(extern extractor lshl_from_imm64 lshl_from_imm64 (in out)) + (decl integral_ty (Type) Type) (extern extractor integral_ty integral_ty) +;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`. +(decl imm12_from_value (Imm12) Value) +(extractor + (imm12_from_value n) + (def_inst (iconst (u64_from_imm64 (imm12_from_u64 n))))) + +;; Same as `imm12_from_value`, but tries negating the constant value. +(decl imm12_from_negated_value (Imm12) Value) +(extractor + (imm12_from_negated_value n) + (def_inst (iconst (u64_from_imm64 (imm12_from_negated_u64 n))))) + +;; Helper type to represent a value and an extend operation fused together. +(type ExtendedValue extern (enum)) +(decl extended_value_from_value (ExtendedValue) Value) +(extern extractor extended_value_from_value extended_value_from_value) + +;; Constructors used to poke at the fields of an `ExtendedValue`. +(decl put_extended_in_reg (ExtendedValue) Reg) +(extern constructor put_extended_in_reg put_extended_in_reg) +(decl get_extended_op (ExtendedValue) ExtendOp) +(extern constructor get_extended_op get_extended_op) + ;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Emit an instruction. @@ -1331,9 +1370,55 @@ (_ Unit (emit (MInst.AluRRImmLogic op dst src imm)))) (writable_reg_to_reg dst))) -;; Helper for emitting `orr64` instructions. -(decl orr64 (Reg ImmLogic) Reg) -(rule (orr64 src imm) (alu_rr_imm_logic (ALUOp.Orr64) src imm)) +;; Helper for emitting `MInst.AluRRR` instructions. +(decl alu_rrr (ALUOp Reg Reg) Reg) +(rule (alu_rrr op src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRR op dst src1 src2)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.VecRRR` instructions. +(decl vec_rrr (VecALUOp Reg Reg VectorSize) Reg) +(rule (vec_rrr op src1 src2 size) + (let ((dst WritableReg (temp_writable_reg $I8X16)) + (_ Unit (emit (MInst.VecRRR op dst src1 src2 size)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRImm12` instructions. +(decl alu_rr_imm12 (ALUOp Reg Imm12) Reg) +(rule (alu_rr_imm12 op src imm) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRImm12 op dst src imm)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRRShift` instructions. +(decl alu_rrr_shift (ALUOp Reg Reg ShiftOpAndAmt) Reg) +(rule (alu_rrr_shift op src1 src2 shift) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRRShift op dst src1 src2 shift)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRRExtend` instructions. +(decl alu_rrr_extend (ALUOp Reg Reg ExtendOp) Reg) +(rule (alu_rrr_extend op src1 src2 extend) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRRExtend op dst src1 src2 extend)))) + (writable_reg_to_reg dst))) + +;; Same as `alu_rrr_extend`, but takes an `ExtendedValue` packed "pair" instead +;; of a `Reg` and an `ExtendOp`. +(decl alu_rr_extend_reg (ALUOp Reg ExtendedValue) Reg) +(rule (alu_rr_extend_reg op src1 extended_reg) + (let ((src2 Reg (put_extended_in_reg extended_reg)) + (extend ExtendOp (get_extended_op extended_reg))) + (alu_rrr_extend op src1 src2 extend))) + +;; Helper for emitting `MInst.AluRRRR` instructions. +(decl alu_rrrr (ALUOp3 Reg Reg Reg) Reg) +(rule (alu_rrrr op src1 src2 src3) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRRR op dst src1 src2 src3)))) + (writable_reg_to_reg dst))) ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1349,7 +1434,7 @@ ;; Weird logical-instruction immediate in ORI using zero register (rule (imm (integral_ty _ty) (imm_logic_from_u64 n)) - (orr64 (zero_reg) n)) + (alu_rr_imm_logic (ALUOp.Orr64) (zero_reg) n)) (decl load_constant64_full (u64) Reg) (extern constructor load_constant64_full load_constant64_full) diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs index f838186229..1a836f1a71 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/args.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs @@ -62,7 +62,7 @@ impl ShiftOpShiftImm { } /// A shift operator with an amount, guaranteed to be within range. -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug)] pub struct ShiftOpAndAmt { op: ShiftOp, shift: ShiftOpShiftImm, diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rs index 79473375dd..bc3ebac7de 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs @@ -272,7 +272,7 @@ impl UImm12Scaled { /// A shifted immediate value in 'imm12' format: supports 12 bits, shifted /// left by 0 or 12 places. -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug)] pub struct Imm12 { /// The immediate bits. pub bits: u16, diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 7b793dc271..67d07f9388 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -21,3 +21,128 @@ (rule (lower (has_type ty (null))) (value_reg (imm ty 0))) + +;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller + +;; base case, simply adding things in registers +(rule (lower (has_type (fits_in_64 ty) (iadd x y))) + (value_reg (alu_rrr (iadd_op ty) (put_in_reg x) (put_in_reg y)))) + +;; special case for when one operand is an immediate that fits in 12 bits +(rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_value y)))) + (value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg x) y))) + +;; same as the previous special case, except we can switch the addition to a +;; subtraction if the negated immediate fits in 12 bits. +(rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_negated_value y)))) + (value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg x) y))) + +;; special case for when we're adding an extended register where the extending +;; operation can get folded into the add itself. +(rule (lower (has_type (fits_in_64 ty) (iadd x (extended_value_from_value y)))) + (value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg x) y))) + +;; special case for when we're adding the shift of a different +;; register by a constant amount and the shift can get folded into the add. +(rule (lower (has_type (fits_in_64 ty) + (iadd x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 >( } /// Lowers an instruction input to multiple regs -fn lower_input_to_regs>( +fn lower_value_to_regs>( ctx: &mut C, - input: InsnInput, + value: Value, ) -> (ValueRegs, Type, bool) { - log::trace!("lower_input_to_regs: input {:?}", input); - let ty = ctx.input_ty(input.insn, input.input); - let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); + log::trace!("lower_value_to_regs: value {:?}", value); + let ty = ctx.value_ty(value); + let inputs = ctx.get_value_as_source_or_const(value); let is_const = inputs.constant.is_some(); let in_regs = if let Some(c) = inputs.constant { // Generate constants fresh at each use to minimize long-range register pressure. generate_constant(ctx, ty, c as u128) } else { - ctx.put_input_in_regs(input.insn, input.input) + ctx.put_value_in_regs(value) }; (in_regs, ty, is_const) @@ -301,7 +301,17 @@ pub(crate) fn put_input_in_reg>( input: InsnInput, narrow_mode: NarrowValueMode, ) -> Reg { - let (in_regs, ty, is_const) = lower_input_to_regs(ctx, input); + let value = ctx.input_as_value(input.insn, input.input); + put_value_in_reg(ctx, value, narrow_mode) +} + +/// Like above, only for values +fn put_value_in_reg>( + ctx: &mut C, + value: Value, + narrow_mode: NarrowValueMode, +) -> Reg { + let (in_regs, ty, is_const) = lower_value_to_regs(ctx, value); let reg = in_regs .only_reg() .expect("Multi-register value not expected"); @@ -314,7 +324,8 @@ pub(crate) fn put_input_in_regs>( ctx: &mut C, input: InsnInput, ) -> ValueRegs { - let (in_regs, _, _) = lower_input_to_regs(ctx, input); + let value = ctx.input_as_value(input.insn, input.input); + let (in_regs, _, _) = lower_value_to_regs(ctx, value); in_regs } @@ -367,89 +378,94 @@ fn put_input_in_rse>( input: InsnInput, narrow_mode: NarrowValueMode, ) -> ResultRSE { - let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); - if let Some((insn, 0)) = inputs.inst { - let op = ctx.data(insn).opcode(); - let out_ty = ctx.output_ty(insn, 0); - let out_bits = ty_bits(out_ty); - - // Is this a zero-extend or sign-extend and can we handle that with a register-mode operator? - if op == Opcode::Uextend || op == Opcode::Sextend { - let sign_extend = op == Opcode::Sextend; - let inner_ty = ctx.input_ty(insn, 0); - let inner_bits = ty_bits(inner_ty); - assert!(inner_bits < out_bits); - if match (sign_extend, narrow_mode) { - // A single zero-extend or sign-extend is equal to itself. - (_, NarrowValueMode::None) => true, - // Two zero-extends or sign-extends in a row is equal to a single zero-extend or sign-extend. - (false, NarrowValueMode::ZeroExtend32) | (false, NarrowValueMode::ZeroExtend64) => { - true - } - (true, NarrowValueMode::SignExtend32) | (true, NarrowValueMode::SignExtend64) => { - true - } - // A zero-extend and a sign-extend in a row is not equal to a single zero-extend or sign-extend - (false, NarrowValueMode::SignExtend32) | (false, NarrowValueMode::SignExtend64) => { - false - } - (true, NarrowValueMode::ZeroExtend32) | (true, NarrowValueMode::ZeroExtend64) => { - false - } - } { - let extendop = match (sign_extend, inner_bits) { - (true, 8) => ExtendOp::SXTB, - (false, 8) => ExtendOp::UXTB, - (true, 16) => ExtendOp::SXTH, - (false, 16) => ExtendOp::UXTH, - (true, 32) => ExtendOp::SXTW, - (false, 32) => ExtendOp::UXTW, - _ => unreachable!(), - }; - let reg = - put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None); - return ResultRSE::RegExtend(reg, extendop); - } - } - - // If `out_ty` is smaller than 32 bits and we need to zero- or sign-extend, - // then get the result into a register and return an Extend-mode operand on - // that register. - if narrow_mode != NarrowValueMode::None - && ((narrow_mode.is_32bit() && out_bits < 32) - || (!narrow_mode.is_32bit() && out_bits < 64)) - { - let reg = put_input_in_reg(ctx, input, NarrowValueMode::None); - let extendop = match (narrow_mode, out_bits) { - (NarrowValueMode::SignExtend32, 1) | (NarrowValueMode::SignExtend64, 1) => { - ExtendOp::SXTB - } - (NarrowValueMode::ZeroExtend32, 1) | (NarrowValueMode::ZeroExtend64, 1) => { - ExtendOp::UXTB - } - (NarrowValueMode::SignExtend32, 8) | (NarrowValueMode::SignExtend64, 8) => { - ExtendOp::SXTB - } - (NarrowValueMode::ZeroExtend32, 8) | (NarrowValueMode::ZeroExtend64, 8) => { - ExtendOp::UXTB - } - (NarrowValueMode::SignExtend32, 16) | (NarrowValueMode::SignExtend64, 16) => { - ExtendOp::SXTH - } - (NarrowValueMode::ZeroExtend32, 16) | (NarrowValueMode::ZeroExtend64, 16) => { - ExtendOp::UXTH - } - (NarrowValueMode::SignExtend64, 32) => ExtendOp::SXTW, - (NarrowValueMode::ZeroExtend64, 32) => ExtendOp::UXTW, - _ => unreachable!(), - }; - return ResultRSE::RegExtend(reg, extendop); - } + let value = ctx.input_as_value(input.insn, input.input); + if let Some((val, extendop)) = get_as_extended_value(ctx, value, narrow_mode) { + let reg = put_value_in_reg(ctx, val, NarrowValueMode::None); + return ResultRSE::RegExtend(reg, extendop); } ResultRSE::from_rs(put_input_in_rs(ctx, input, narrow_mode)) } +fn get_as_extended_value>( + ctx: &mut C, + val: Value, + narrow_mode: NarrowValueMode, +) -> Option<(Value, ExtendOp)> { + let inputs = ctx.get_value_as_source_or_const(val); + let (insn, n) = inputs.inst?; + if n != 0 { + return None; + } + let op = ctx.data(insn).opcode(); + let out_ty = ctx.output_ty(insn, 0); + let out_bits = ty_bits(out_ty); + + // Is this a zero-extend or sign-extend and can we handle that with a register-mode operator? + if op == Opcode::Uextend || op == Opcode::Sextend { + let sign_extend = op == Opcode::Sextend; + let inner_ty = ctx.input_ty(insn, 0); + let inner_bits = ty_bits(inner_ty); + assert!(inner_bits < out_bits); + if match (sign_extend, narrow_mode) { + // A single zero-extend or sign-extend is equal to itself. + (_, NarrowValueMode::None) => true, + // Two zero-extends or sign-extends in a row is equal to a single zero-extend or sign-extend. + (false, NarrowValueMode::ZeroExtend32) | (false, NarrowValueMode::ZeroExtend64) => true, + (true, NarrowValueMode::SignExtend32) | (true, NarrowValueMode::SignExtend64) => true, + // A zero-extend and a sign-extend in a row is not equal to a single zero-extend or sign-extend + (false, NarrowValueMode::SignExtend32) | (false, NarrowValueMode::SignExtend64) => { + false + } + (true, NarrowValueMode::ZeroExtend32) | (true, NarrowValueMode::ZeroExtend64) => false, + } { + let extendop = match (sign_extend, inner_bits) { + (true, 8) => ExtendOp::SXTB, + (false, 8) => ExtendOp::UXTB, + (true, 16) => ExtendOp::SXTH, + (false, 16) => ExtendOp::UXTH, + (true, 32) => ExtendOp::SXTW, + (false, 32) => ExtendOp::UXTW, + _ => unreachable!(), + }; + return Some((ctx.input_as_value(insn, 0), extendop)); + } + } + + // If `out_ty` is smaller than 32 bits and we need to zero- or sign-extend, + // then get the result into a register and return an Extend-mode operand on + // that register. + if narrow_mode != NarrowValueMode::None + && ((narrow_mode.is_32bit() && out_bits < 32) || (!narrow_mode.is_32bit() && out_bits < 64)) + { + let extendop = match (narrow_mode, out_bits) { + (NarrowValueMode::SignExtend32, 1) | (NarrowValueMode::SignExtend64, 1) => { + ExtendOp::SXTB + } + (NarrowValueMode::ZeroExtend32, 1) | (NarrowValueMode::ZeroExtend64, 1) => { + ExtendOp::UXTB + } + (NarrowValueMode::SignExtend32, 8) | (NarrowValueMode::SignExtend64, 8) => { + ExtendOp::SXTB + } + (NarrowValueMode::ZeroExtend32, 8) | (NarrowValueMode::ZeroExtend64, 8) => { + ExtendOp::UXTB + } + (NarrowValueMode::SignExtend32, 16) | (NarrowValueMode::SignExtend64, 16) => { + ExtendOp::SXTH + } + (NarrowValueMode::ZeroExtend32, 16) | (NarrowValueMode::ZeroExtend64, 16) => { + ExtendOp::UXTH + } + (NarrowValueMode::SignExtend64, 32) => ExtendOp::SXTW, + (NarrowValueMode::ZeroExtend64, 32) => ExtendOp::UXTW, + _ => unreachable!(), + }; + return Some((val, extendop)); + } + None +} + pub(crate) fn put_input_in_rse_imm12>( ctx: &mut C, input: InsnInput, @@ -472,35 +488,6 @@ pub(crate) fn put_input_in_rse_imm12>( ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode)) } -/// Like `put_input_in_rse_imm12` above, except is allowed to negate the -/// argument (assuming a two's-complement representation with the given bit -/// width) if this allows use of 12-bit immediate. Used to flip `add`s with -/// negative immediates to `sub`s (and vice-versa). -pub(crate) fn put_input_in_rse_imm12_maybe_negated>( - ctx: &mut C, - input: InsnInput, - twos_complement_bits: usize, - narrow_mode: NarrowValueMode, -) -> (ResultRSEImm12, bool) { - assert!(twos_complement_bits <= 64); - if let Some(imm_value) = input_to_const(ctx, input) { - if let Some(i) = Imm12::maybe_from_u64(imm_value) { - return (ResultRSEImm12::Imm12(i), false); - } - let sign_extended = - ((imm_value as i64) << (64 - twos_complement_bits)) >> (64 - twos_complement_bits); - let inverted = sign_extended.wrapping_neg(); - if let Some(i) = Imm12::maybe_from_u64(inverted as u64) { - return (ResultRSEImm12::Imm12(i), true); - } - } - - ( - ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode)), - false, - ) -} - pub(crate) fn put_input_in_rs_immlogic>( ctx: &mut C, input: InsnInput, diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 6708431eb9..96bd32d5c8 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -7,8 +7,8 @@ pub mod generated_code; use super::{ zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, AtomicRmwOp, BranchTarget, CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, Imm12, ImmLogic, ImmShift, Inst as MInst, - JTSequenceInfo, MachLabel, MoveWideConst, Opcode, OperandSize, PairAMode, Reg, ScalarSize, - ShiftOpAndAmt, UImm5, VectorSize, NZCV, + JTSequenceInfo, MachLabel, MoveWideConst, NarrowValueMode, Opcode, OperandSize, PairAMode, Reg, + ScalarSize, ShiftOpAndAmt, UImm5, VectorSize, NZCV, }; use crate::isa::aarch64::settings as aarch64_settings; use crate::machinst::isle::*; @@ -19,8 +19,9 @@ use crate::{ ValueLabel, ValueList, }, isa::aarch64::inst::aarch64_map_regs, + isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm}, isa::unwind::UnwindInst, - machinst::{get_output_reg, InsnOutput, LowerCtx, RegRenamer}, + machinst::{get_output_reg, ty_bits, InsnOutput, LowerCtx, RegRenamer}, }; use smallvec::SmallVec; use std::boxed::Box; @@ -76,6 +77,11 @@ pub struct IsleContext<'a, C> { emitted_insts: SmallVec<[MInst; 6]>, } +pub struct ExtendedValue { + val: Value, + extend: ExtendOp, +} + impl<'a, C> IsleContext<'a, C> { pub fn new(lower_ctx: &'a mut C, isa_flags: &'a aarch64_settings::Flags) -> Self { IsleContext { @@ -108,6 +114,25 @@ where ImmLogic::maybe_from_u64(n, I64) } + fn imm12_from_u64(&mut self, n: u64) -> Option { + Imm12::maybe_from_u64(n) + } + + fn imm12_from_negated_u64(&mut self, n: u64) -> Option { + Imm12::maybe_from_u64((n as i64).wrapping_neg() as u64) + } + + fn lshl_from_imm64(&mut self, n: Imm64, ty: Type) -> Option { + let shiftimm = ShiftOpShiftImm::maybe_from_shift(n.bits() as u64)?; + let shiftee_bits = ty_bits(ty); + if shiftee_bits <= std::u8::MAX as usize { + let shiftimm = shiftimm.mask(shiftee_bits as u8); + Some(ShiftOpAndAmt::new(ShiftOp::LSL, shiftimm)) + } else { + None + } + } + fn integral_ty(&mut self, ty: Type) -> Option { match ty { I8 | I16 | I32 | I64 | R64 => Some(ty), @@ -181,6 +206,20 @@ where zero_reg() } + fn extended_value_from_value(&mut self, val: Value) -> Option { + let (val, extend) = + super::get_as_extended_value(self.lower_ctx, val, NarrowValueMode::None)?; + Some(ExtendedValue { val, extend }) + } + + fn put_extended_in_reg(&mut self, reg: &ExtendedValue) -> Reg { + self.put_in_reg(reg.val) + } + + fn get_extended_op(&mut self, reg: &ExtendedValue) -> ExtendOp { + reg.extend + } + fn emit(&mut self, inst: &MInst) -> Unit { self.emitted_insts.push(inst.clone()); } diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index ee3279ed8e..9c35ce7a90 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,10 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb +<<<<<<< HEAD src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462 src/isa/aarch64/inst.isle 841748c9c5900821b7086a09a41c6dcdb2172eb47a45293b6ef10f2e1f1389620bf6a2c75152af807d8bc8929029a357af5191f5d87bac2c9ec54bf63a9a2a8f src/isa/aarch64/lower.isle b3cd0834484e543f39d477d47ee66042276e99955c21fb8c9340a5f27ac317936acb2907a30f758bf596066e36db801a179fda6dbcecaee758a0187a5a5f1412 +======= +src/prelude.isle bed4d567d548a0df8c975ac88275b75547494cf761c63b9775d117396382eb1302777de53889804ddc5dee96d04a6764cff59d43a4d4a614fb50f95dc1e44749 +src/isa/aarch64/inst.isle 48d55da36f739bca3a748ee39a1c52a87ae8e0b4900e0e05e90f5f79174ffee1a913a85f258b73122e9e31c5d8a5d96d6ae4baae9f7eb40e4f0b47b37a3bc875 +src/isa/aarch64/lower.isle 25146b36406baa91f81b44b0f54a5948e688af72946f0b70495c22396e275f60d038c52632e5a9fa173cbbd2b901959721679e95f8f6177ec89a7ec04b5f2a30 +>>>>>>> aarch64: Migrate `iadd` and `isub` to ISLE diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 4ab9919db5..d6262d3492 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -31,6 +31,7 @@ pub trait Context { fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u16; + fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); @@ -49,7 +50,13 @@ pub trait Context { fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option; fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option; fn imm_logic_from_u64(&mut self, arg0: u64) -> Option; + fn imm12_from_u64(&mut self, arg0: u64) -> Option; + fn imm12_from_negated_u64(&mut self, arg0: u64) -> Option; + fn lshl_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option; fn integral_ty(&mut self, arg0: Type) -> Option; + fn extended_value_from_value(&mut self, arg0: Value) -> Option; + fn put_extended_in_reg(&mut self, arg0: &ExtendedValue) -> Reg; + fn get_extended_op(&mut self, arg0: &ExtendedValue) -> ExtendOp; fn emit(&mut self, arg0: &MInst) -> Unit; fn zero_reg(&mut self) -> Reg; fn load_constant64_full(&mut self, arg0: u64) -> Reg; @@ -685,7 +692,7 @@ pub enum BitOp { Cls64, } -/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 951. +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 958. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FPUOp1 { Abs32, @@ -698,7 +705,7 @@ pub enum FPUOp1 { Cvt64To32, } -/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 964. +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 971. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FPUOp2 { Add32, @@ -719,14 +726,14 @@ pub enum FPUOp2 { Uqsub64, } -/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 989. +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 996. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FPUOp3 { MAdd32, MAdd64, } -/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 996. +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 1003. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FpuToIntOp { F32ToU32, @@ -739,7 +746,7 @@ pub enum FpuToIntOp { F64ToI64, } -/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1009. +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1016. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum IntToFpuOp { U32ToF32, @@ -752,7 +759,7 @@ pub enum IntToFpuOp { I64ToF64, } -/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1023. +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1030. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum FpuRoundMode { Minus32, @@ -765,7 +772,7 @@ pub enum FpuRoundMode { Nearest64, } -/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1036. +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1043. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecExtendOp { Sxtl8, @@ -776,7 +783,7 @@ pub enum VecExtendOp { Uxtl32, } -/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1053. +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1060. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecALUOp { Sqadd, @@ -818,7 +825,7 @@ pub enum VecALUOp { Sqrdmulh, } -/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1132. +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1139. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecMisc2 { Not, @@ -840,7 +847,7 @@ pub enum VecMisc2 { Cmeq0, } -/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1171. +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1178. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRLongOp { Fcvtl16, @@ -850,7 +857,7 @@ pub enum VecRRLongOp { Shll32, } -/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1186. +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1193. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRNarrowOp { Xtn16, @@ -869,7 +876,7 @@ pub enum VecRRNarrowOp { Fcvtn64, } -/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1218. +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1225. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRRLongOp { Smull8, @@ -883,13 +890,13 @@ pub enum VecRRRLongOp { Umlal32, } -/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1235. +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1242. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecPairOp { Addp, } -/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1243. +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1250. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecRRPairLongOp { Saddlp8, @@ -898,14 +905,14 @@ pub enum VecRRPairLongOp { Uaddlp16, } -/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1254. +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1261. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecLanesOp { Addv, Uminv, } -/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1263. +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1270. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum VecShiftImmOp { Shl, @@ -913,7 +920,7 @@ pub enum VecShiftImmOp { Sshr, } -/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1274. +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1281. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum AtomicRMWOp { Add, @@ -945,6 +952,42 @@ pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { return Some(expr2_0); } +// Generated as internal constructor for term vector_size. +pub fn constructor_vector_size(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { + if pattern1_0 == 8 { + if pattern1_1 == 16 { + // Rule at src/isa/aarch64/inst.isle line 952. + let expr0_0 = VectorSize::Size8x16; + return Some(expr0_0); + } + } + if pattern1_0 == 16 { + if pattern1_1 == 8 { + // Rule at src/isa/aarch64/inst.isle line 953. + let expr0_0 = VectorSize::Size16x8; + return Some(expr0_0); + } + } + if pattern1_0 == 32 { + if pattern1_1 == 4 { + // Rule at src/isa/aarch64/inst.isle line 954. + let expr0_0 = VectorSize::Size32x4; + return Some(expr0_0); + } + } + if pattern1_0 == 64 { + if pattern1_1 == 2 { + // Rule at src/isa/aarch64/inst.isle line 955. + let expr0_0 = VectorSize::Size64x2; + return Some(expr0_0); + } + } + } + return None; +} + // Generated as internal constructor for term movz. pub fn constructor_movz( ctx: &mut C, @@ -953,7 +996,7 @@ pub fn constructor_movz( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1315. + // Rule at src/isa/aarch64/inst.isle line 1354. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -974,7 +1017,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1322. + // Rule at src/isa/aarch64/inst.isle line 1361. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -997,7 +1040,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1329. + // Rule at src/isa/aarch64/inst.isle line 1368. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1011,14 +1054,177 @@ pub fn constructor_alu_rr_imm_logic( return Some(expr4_0); } -// Generated as internal constructor for term orr64. -pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { +// Generated as internal constructor for term alu_rrr. +pub fn constructor_alu_rrr( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Reg, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1336. - let expr0_0 = ALUOp::Orr64; - let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; - return Some(expr1_0); + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1375. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRR { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term vec_rrr. +pub fn constructor_vec_rrr( + ctx: &mut C, + arg0: &VecALUOp, + arg1: Reg, + arg2: Reg, + arg3: &VectorSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1382. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::VecRRR { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + size: pattern3_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rr_imm12. +pub fn constructor_alu_rr_imm12( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Imm12, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1389. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRImm12 { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + imm12: pattern2_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rrr_shift. +pub fn constructor_alu_rrr_shift( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Reg, + arg3: ShiftOpAndAmt, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1396. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRRShift { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + shiftop: pattern3_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rrr_extend. +pub fn constructor_alu_rrr_extend( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: Reg, + arg3: &ExtendOp, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1403. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRRExtend { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + extendop: pattern3_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rr_extend_reg. +pub fn constructor_alu_rr_extend_reg( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: &ExtendedValue, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1409. + let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); + let expr1_0 = C::get_extended_op(ctx, pattern2_0); + let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term alu_rrrr. +pub fn constructor_alu_rrrr( + ctx: &mut C, + arg0: &ALUOp3, + arg1: Reg, + arg2: Reg, + arg3: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/aarch64/inst.isle line 1416. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRRR { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + rm: pattern2_0, + ra: pattern3_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); } // Generated as internal constructor for term imm. @@ -1027,24 +1233,25 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) { let pattern2_0 = arg1; if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1351. - let expr0_0 = C::zero_reg(ctx); - let expr1_0 = constructor_orr64(ctx, expr0_0, pattern3_0)?; - return Some(expr1_0); + // Rule at src/isa/aarch64/inst.isle line 1434. + let expr0_0 = ALUOp::Orr64; + let expr1_0 = C::zero_reg(ctx); + let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern3_0)?; + return Some(expr2_0); } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1343. + // Rule at src/isa/aarch64/inst.isle line 1426. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1347. + // Rule at src/isa/aarch64/inst.isle line 1430. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1358. + // Rule at src/isa/aarch64/inst.isle line 1441. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -1056,6 +1263,58 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 81. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0 = ALUOp::AddS64; + let expr11_0 = constructor_alu_rrr(ctx, &expr10_0, expr2_0, expr7_0)?; + let expr12_0 = ALUOp::Adc64; + let expr13_0 = constructor_alu_rrr(ctx, &expr12_0, expr4_0, expr9_0)?; + let expr14_0 = C::value_regs(ctx, expr11_0, expr13_0); + return Some(expr14_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 132. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0 = ALUOp::SubS64; + let expr11_0 = constructor_alu_rrr(ctx, &expr10_0, expr2_0, expr7_0)?; + let expr12_0 = ALUOp::Sbc64; + let expr13_0 = constructor_alu_rrr(ctx, &expr12_0, expr4_0, expr9_0)?; + let expr14_0 = C::value_regs(ctx, expr11_0, expr13_0); + return Some(expr14_0); + } + _ => {} + } + } + } let pattern3_0 = C::inst_data(ctx, pattern0_0); match &pattern3_0 { &InstructionData::NullAry { @@ -1104,6 +1363,364 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } + if let Some((pattern3_0, pattern3_1)) = C::multi_lane(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Iadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 77. + let expr0_0 = VecALUOp::Add; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern2_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 128. + let expr0_0 = VecALUOp::Sub; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern2_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + _ => {} + } + } + } + if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Iadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_0) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::Binary { + opcode: ref pattern10_0, + args: ref pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Imul = &pattern10_0 { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + // Rule at src/isa/aarch64/lower.isle line 57. + let expr0_0 = constructor_madd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern12_0); + let expr2_0 = C::put_in_reg(ctx, pattern12_1); + let expr3_0 = C::put_in_reg(ctx, pattern7_1); + let expr4_0 = constructor_alu_rrrr( + ctx, &expr0_0, expr1_0, expr2_0, expr3_0, + )?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + } + } + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + match &pattern9_0 { + &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } => { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + if let Some(pattern13_0) = + C::imm12_from_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 34. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern13_0) = + C::imm12_from_negated_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 39. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + &InstructionData::Binary { + opcode: ref pattern10_0, + args: ref pattern10_1, + } => { + match &pattern10_0 { + &Opcode::Imul => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + // Rule at src/isa/aarch64/lower.isle line 54. + let expr0_0 = constructor_madd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern12_0); + let expr2_0 = C::put_in_reg(ctx, pattern12_1); + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = constructor_alu_rrrr( + ctx, &expr0_0, expr1_0, expr2_0, expr3_0, + )?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Ishl => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + if let Some(pattern13_0) = C::def_inst(ctx, pattern12_1) + { + let pattern14_0 = C::inst_data(ctx, pattern13_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern15_0, + imm: pattern15_1, + } = &pattern14_0 + { + if let &Opcode::Iconst = &pattern15_0 { + let closure17 = || { + return Some(pattern3_0); + }; + if let Some(pattern17_0) = closure17() { + if let Some(pattern18_0) = + C::lshl_from_imm64( + ctx, + pattern15_1, + pattern17_0, + ) + { + // Rule at src/isa/aarch64/lower.isle line 49. + let expr0_0 = constructor_iadd_op( + ctx, pattern3_0, + )?; + let expr1_0 = + C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + C::put_in_reg(ctx, pattern12_0); + let expr3_0 = + constructor_alu_rrr_shift( + ctx, + &expr0_0, + expr1_0, + expr2_0, + pattern18_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + } + } + } + _ => {} + } + } + _ => {} + } + } + if let Some(pattern8_0) = C::extended_value_from_value(ctx, pattern7_1) { + // Rule at src/isa/aarch64/lower.isle line 44. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + constructor_alu_rr_extend_reg(ctx, &expr0_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/aarch64/lower.isle line 30. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + match &pattern9_0 { + &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } => { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + if let Some(pattern13_0) = + C::imm12_from_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 108. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern13_0) = + C::imm12_from_negated_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 113. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + &InstructionData::Binary { + opcode: ref pattern10_0, + args: ref pattern10_1, + } => { + if let &Opcode::Ishl = &pattern10_0 { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + if let Some(pattern13_0) = C::def_inst(ctx, pattern12_1) { + let pattern14_0 = C::inst_data(ctx, pattern13_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern15_0, + imm: pattern15_1, + } = &pattern14_0 + { + if let &Opcode::Iconst = &pattern15_0 { + let closure17 = || { + return Some(pattern3_0); + }; + if let Some(pattern17_0) = closure17() { + if let Some(pattern18_0) = + C::lshl_from_imm64( + ctx, + pattern15_1, + pattern17_0, + ) + { + // Rule at src/isa/aarch64/lower.isle line 123. + let expr0_0 = constructor_isub_op( + ctx, pattern3_0, + )?; + let expr1_0 = + C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + C::put_in_reg(ctx, pattern12_0); + let expr3_0 = + constructor_alu_rrr_shift( + ctx, + &expr0_0, + expr1_0, + expr2_0, + pattern18_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + } + } + } + } + _ => {} + } + } + if let Some(pattern8_0) = C::extended_value_from_value(ctx, pattern7_1) { + // Rule at src/isa/aarch64/lower.isle line 118. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + constructor_alu_rr_extend_reg(ctx, &expr0_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/aarch64/lower.isle line 104. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + _ => {} + } + } + } } return None; } + +// Generated as internal constructor for term iadd_op. +pub fn constructor_iadd_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { + // Rule at src/isa/aarch64/lower.isle line 62. + let expr0_0 = ALUOp::Add32; + return Some(expr0_0); + } + // Rule at src/isa/aarch64/lower.isle line 63. + let expr0_0 = ALUOp::Add64; + return Some(expr0_0); +} + +// Generated as internal constructor for term isub_op. +pub fn constructor_isub_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { + // Rule at src/isa/aarch64/lower.isle line 67. + let expr0_0 = ALUOp::Sub32; + return Some(expr0_0); + } + // Rule at src/isa/aarch64/lower.isle line 68. + let expr0_0 = ALUOp::Sub64; + return Some(expr0_0); +} + +// Generated as internal constructor for term madd_op. +pub fn constructor_madd_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { + // Rule at src/isa/aarch64/lower.isle line 72. + let expr0_0 = ALUOp3::MAdd32; + return Some(expr0_0); + } + // Rule at src/isa/aarch64/lower.isle line 73. + let expr0_0 = ALUOp3::MAdd64; + return Some(expr0_0); +} diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 74d1506c74..b42792bd39 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -42,7 +42,7 @@ pub(crate) fn lower_insn_to_regs>( return Ok(()); } - let implemented_in_isle = || { + let implemented_in_isle = |ctx: &mut C| { unreachable!( "implemented in ISLE: inst = `{}`, type = `{:?}`", ctx.dfg().display_inst(insn), @@ -51,7 +51,7 @@ pub(crate) fn lower_insn_to_regs>( }; match op { - Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(), + Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(ctx), Opcode::F32const => { let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32); @@ -63,143 +63,8 @@ pub(crate) fn lower_insn_to_regs>( let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); lower_constant_f64(ctx, rd, value); } - Opcode::Iadd => { - match ty.unwrap() { - ty if ty.is_vector() => { - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - ctx.emit(Inst::VecRRR { - rd, - rn, - rm, - alu_op: VecALUOp::Add, - size: VectorSize::from_ty(ty), - }); - } - I128 => { - let lhs = put_input_in_regs(ctx, inputs[0]); - let rhs = put_input_in_regs(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]); - assert_eq!(lhs.len(), 2); - assert_eq!(rhs.len(), 2); - assert_eq!(dst.len(), 2); - - // adds x0, x0, x2 - // adc x1, x1, x3 - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::AddS64, - rd: dst.regs()[0], - rn: lhs.regs()[0], - rm: rhs.regs()[0], - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Adc64, - rd: dst.regs()[1], - rn: lhs.regs()[1], - rm: rhs.regs()[1], - }); - } - ty => { - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let mul_insn = if let Some(mul_insn) = - maybe_input_insn(ctx, inputs[1], Opcode::Imul) - { - Some((mul_insn, 0)) - } else if let Some(mul_insn) = maybe_input_insn(ctx, inputs[0], Opcode::Imul) { - Some((mul_insn, 1)) - } else { - None - }; - // If possible combine mul + add into madd. - if let Some((insn, addend_idx)) = mul_insn { - let alu_op = choose_32_64(ty, ALUOp3::MAdd32, ALUOp3::MAdd64); - let rn_input = InsnInput { insn, input: 0 }; - let rm_input = InsnInput { insn, input: 1 }; - - let rn = put_input_in_reg(ctx, rn_input, NarrowValueMode::None); - let rm = put_input_in_reg(ctx, rm_input, NarrowValueMode::None); - let ra = put_input_in_reg(ctx, inputs[addend_idx], NarrowValueMode::None); - - ctx.emit(Inst::AluRRRR { - alu_op, - rd, - rn, - rm, - ra, - }); - } else { - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - let (rm, negated) = put_input_in_rse_imm12_maybe_negated( - ctx, - inputs[1], - ty_bits(ty), - NarrowValueMode::None, - ); - let alu_op = if !negated { - choose_32_64(ty, ALUOp::Add32, ALUOp::Add64) - } else { - choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64) - }; - ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); - } - } - } - } - Opcode::Isub => { - let ty = ty.unwrap(); - if ty == I128 { - let lhs = put_input_in_regs(ctx, inputs[0]); - let rhs = put_input_in_regs(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]); - assert_eq!(lhs.len(), 2); - assert_eq!(rhs.len(), 2); - assert_eq!(dst.len(), 2); - - // subs x0, x0, x2 - // sbc x1, x1, x3 - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::SubS64, - rd: dst.regs()[0], - rn: lhs.regs()[0], - rm: rhs.regs()[0], - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Sbc64, - rd: dst.regs()[1], - rn: lhs.regs()[1], - rm: rhs.regs()[1], - }); - } else { - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - if !ty.is_vector() { - let (rm, negated) = put_input_in_rse_imm12_maybe_negated( - ctx, - inputs[1], - ty_bits(ty), - NarrowValueMode::None, - ); - let alu_op = if !negated { - choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64) - } else { - choose_32_64(ty, ALUOp::Add32, ALUOp::Add64) - }; - ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); - } else { - let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - ctx.emit(Inst::VecRRR { - rd, - rn, - rm, - alu_op: VecALUOp::Sub, - size: VectorSize::from_ty(ty), - }); - } - } - } + Opcode::Iadd => implemented_in_isle(ctx), + Opcode::Isub => implemented_in_isle(ctx), Opcode::UaddSat | Opcode::SaddSat | Opcode::UsubSat | Opcode::SsubSat => { let ty = ty.unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 174aa529fd..c8dcb64810 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,10 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb +<<<<<<< HEAD src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462 src/isa/x64/inst.isle 6065d3b9e0fa3361d179d9b87d09568ff474f8bac7eeabd29b328ace723041f96045bc82cfa2d7feda4490ce7e5d4be1a1c7ebe25c99916564d43a51550cd093 src/isa/x64/lower.isle e51b7a67343dba342a43b3c9e4b9ed7df9b2c66a677018acf7054ba48c27e4e93a4421fd892b9bf7c0e5b790bcfafab7cb3e93ce2b8206c04d456918d2ad0b5a +======= +src/prelude.isle bed4d567d548a0df8c975ac88275b75547494cf761c63b9775d117396382eb1302777de53889804ddc5dee96d04a6764cff59d43a4d4a614fb50f95dc1e44749 +src/isa/x64/inst.isle fdfbfc6dfad1fc5ed252e0a14ccc69baba51d0538e05cfb9916f6213e5a6fcfc9d22605a29bd684d6a66f6d5e1c8ec36a963660d52c2e8b3fb6e0758f7adb7b5 +src/isa/x64/lower.isle 8555abdae385431c96aaabc392b7b3a8b1bbe733be08b007ef776850860cb77e85a140db02f427586c155c0b0129f9ffd531abd2e4a772c72667535cc015e609 +>>>>>>> aarch64: Migrate `iadd` and `isub` to ISLE diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index a62050787b..c32ee7feb2 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -31,6 +31,7 @@ pub trait Context { fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u16; + fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index ddc579a3b4..dfab0f35d1 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -90,6 +90,15 @@ macro_rules! isle_prelude_methods { ty.bits() } + #[inline] + fn fits_in_32(&mut self, ty: Type) -> Option { + if ty.bits() <= 32 { + Some(ty) + } else { + None + } + } + #[inline] fn fits_in_64(&mut self, ty: Type) -> Option { if ty.bits() <= 64 { diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 8f65449cde..fbc1efbc49 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -105,6 +105,8 @@ pub trait LowerCtx { fn num_outputs(&self, ir_inst: Inst) -> usize; /// Get the type for an instruction's input. fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type; + /// Get the type for a value. + fn value_ty(&self, val: Value) -> Type; /// Get the type for an instruction's output. fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type; /// Get the value of a constant instruction (`iconst`, etc.) as a 64-bit @@ -129,6 +131,9 @@ pub trait LowerCtx { fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput; /// Like `get_input_as_source_or_const` but with a `Value`. fn get_value_as_source_or_const(&self, value: Value) -> NonRegInput; + /// Resolves a particular input of an instruction to the `Value` that it is + /// represented with. + fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value; /// Put the `idx`th input into register(s) and return the assigned register. fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs; /// Put the given value into register(s) and return the assigned register. @@ -1109,8 +1114,10 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type { - let val = self.f.dfg.inst_args(ir_inst)[idx]; - let val = self.f.dfg.resolve_aliases(val); + self.value_ty(self.input_as_value(ir_inst, idx)) + } + + fn value_ty(&self, val: Value) -> Type { self.f.dfg.value_type(val) } @@ -1122,9 +1129,13 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { self.inst_constants.get(&ir_inst).cloned() } - fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput { + fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value { let val = self.f.dfg.inst_args(ir_inst)[idx]; - let val = self.f.dfg.resolve_aliases(val); + self.f.dfg.resolve_aliases(val) + } + + fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput { + let val = self.input_as_value(ir_inst, idx); self.get_value_as_source_or_const(val) } diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 4951c0bd7b..b53ce9c579 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -144,6 +144,10 @@ ;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; An extractor that only matches types that can fit in 32 bits. +(decl fits_in_32 (Type) Type) +(extern extractor fits_in_32 fits_in_32) + ;; An extractor that only matches types that can fit in 64 bits. (decl fits_in_64 (Type) Type) (extern extractor fits_in_64 fits_in_64) diff --git a/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif b/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif index 5f27d9f034..6d90ba7e48 100644 --- a/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif +++ b/cranelift/filetests/filetests/isa/aarch64/arithmetic.clif @@ -369,3 +369,23 @@ block0(v0: i128, v1: i128): ; nextln: madd x0, x0, x2, xzr ; nextln: ret +function %add_mul_1(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = imul v1, v2 + v4 = iadd v0, v3 + return v4 +} + +; check: madd w0, w1, w2, w0 +; nextln: ret + +function %add_mul_2(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = imul v1, v2 + v4 = iadd v3, v0 + return v4 +} + +; check: madd w0, w1, w2, w0 +; nextln: ret + From 15d25429393d5b9c15eec4a2f1f88bd6101ac9e0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 18 Nov 2021 09:32:27 -0800 Subject: [PATCH 2/5] Rebuild ISLE --- .../isa/aarch64/lower/isle/generated_code.manifest | 10 ++-------- .../src/isa/aarch64/lower/isle/generated_code.rs | 12 ++++++------ .../src/isa/x64/lower/isle/generated_code.manifest | 8 +------- 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 9c35ce7a90..dbcb3b8fd2 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,10 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -<<<<<<< HEAD -src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462 -src/isa/aarch64/inst.isle 841748c9c5900821b7086a09a41c6dcdb2172eb47a45293b6ef10f2e1f1389620bf6a2c75152af807d8bc8929029a357af5191f5d87bac2c9ec54bf63a9a2a8f -src/isa/aarch64/lower.isle b3cd0834484e543f39d477d47ee66042276e99955c21fb8c9340a5f27ac317936acb2907a30f758bf596066e36db801a179fda6dbcecaee758a0187a5a5f1412 -======= -src/prelude.isle bed4d567d548a0df8c975ac88275b75547494cf761c63b9775d117396382eb1302777de53889804ddc5dee96d04a6764cff59d43a4d4a614fb50f95dc1e44749 -src/isa/aarch64/inst.isle 48d55da36f739bca3a748ee39a1c52a87ae8e0b4900e0e05e90f5f79174ffee1a913a85f258b73122e9e31c5d8a5d96d6ae4baae9f7eb40e4f0b47b37a3bc875 +src/prelude.isle 6ae74a46b3b1f8099397cb8472d75359bc100a05a9bf38532cc41b423f7e600618d83bdb1f5565e0167696e94df2a18e489e6faabb6cc1b3e3af301ab8de55b8 +src/isa/aarch64/inst.isle 11e77cbb9a0f9c7779dea64db47aaa0b2a90bc1fac851dd0a7f0a37b4234c22b6543469f2a8d477116b2feab563fbf499eb71e414c6e2615e4cc8719e8c0ab1c src/isa/aarch64/lower.isle 25146b36406baa91f81b44b0f54a5948e688af72946f0b70495c22396e275f60d038c52632e5a9fa173cbbd2b901959721679e95f8f6177ec89a7ec04b5f2a30 ->>>>>>> aarch64: Migrate `iadd` and `isub` to ISLE diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index d6262d3492..7ea221f990 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -1193,7 +1193,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1409. + // Rule at src/isa/aarch64/inst.isle line 1411. let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); let expr1_0 = C::get_extended_op(ctx, pattern2_0); let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; @@ -1212,7 +1212,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1416. + // Rule at src/isa/aarch64/inst.isle line 1418. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1233,25 +1233,25 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) { let pattern2_0 = arg1; if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1434. + // Rule at src/isa/aarch64/inst.isle line 1436. let expr0_0 = ALUOp::Orr64; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern3_0)?; return Some(expr2_0); } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1426. + // Rule at src/isa/aarch64/inst.isle line 1428. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1430. + // Rule at src/isa/aarch64/inst.isle line 1432. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1441. + // Rule at src/isa/aarch64/inst.isle line 1443. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index c8dcb64810..2b1e1b6be7 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,10 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -<<<<<<< HEAD -src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462 +src/prelude.isle 6ae74a46b3b1f8099397cb8472d75359bc100a05a9bf38532cc41b423f7e600618d83bdb1f5565e0167696e94df2a18e489e6faabb6cc1b3e3af301ab8de55b8 src/isa/x64/inst.isle 6065d3b9e0fa3361d179d9b87d09568ff474f8bac7eeabd29b328ace723041f96045bc82cfa2d7feda4490ce7e5d4be1a1c7ebe25c99916564d43a51550cd093 src/isa/x64/lower.isle e51b7a67343dba342a43b3c9e4b9ed7df9b2c66a677018acf7054ba48c27e4e93a4421fd892b9bf7c0e5b790bcfafab7cb3e93ce2b8206c04d456918d2ad0b5a -======= -src/prelude.isle bed4d567d548a0df8c975ac88275b75547494cf761c63b9775d117396382eb1302777de53889804ddc5dee96d04a6764cff59d43a4d4a614fb50f95dc1e44749 -src/isa/x64/inst.isle fdfbfc6dfad1fc5ed252e0a14ccc69baba51d0538e05cfb9916f6213e5a6fcfc9d22605a29bd684d6a66f6d5e1c8ec36a963660d52c2e8b3fb6e0758f7adb7b5 -src/isa/x64/lower.isle 8555abdae385431c96aaabc392b7b3a8b1bbe733be08b007ef776850860cb77e85a140db02f427586c155c0b0129f9ffd531abd2e4a772c72667535cc015e609 ->>>>>>> aarch64: Migrate `iadd` and `isub` to ISLE From 7412461b7b70ad2ee383d1354f20dd4aeeffc151 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 19 Nov 2021 06:53:31 -0800 Subject: [PATCH 3/5] Spelling --- cranelift/codegen/src/isa/aarch64/lower.isle | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 67d07f9388..aa90a3546e 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -26,48 +26,48 @@ ;; `i64` and smaller -;; base case, simply adding things in registers +;; Base case, simply adding things in registers. (rule (lower (has_type (fits_in_64 ty) (iadd x y))) (value_reg (alu_rrr (iadd_op ty) (put_in_reg x) (put_in_reg y)))) -;; special case for when one operand is an immediate that fits in 12 bits +;; Special case for when one operand is an immediate that fits in 12 bits. (rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_value y)))) (value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg x) y))) -;; same as the previous special case, except we can switch the addition to a +;; Same as the previous special case, except we can switch the addition to a ;; subtraction if the negated immediate fits in 12 bits. (rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_negated_value y)))) (value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg x) y))) -;; special case for when we're adding an extended register where the extending +;; Special case for when we're adding an extended register where the extending ;; operation can get folded into the add itself. (rule (lower (has_type (fits_in_64 ty) (iadd x (extended_value_from_value y)))) (value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg x) y))) -;; special case for when we're adding the shift of a different +;; Special case for when we're adding the shift of a different ;; register by a constant amount and the shift can get folded into the add. (rule (lower (has_type (fits_in_64 ty) (iadd x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 Date: Fri, 19 Nov 2021 07:00:04 -0800 Subject: [PATCH 4/5] Add commutative addition cases --- cranelift/codegen/src/isa/aarch64/lower.isle | 21 ++- .../lower/isle/generated_code.manifest | 2 +- .../isa/aarch64/lower/isle/generated_code.rs | 174 ++++++++++++++---- .../filetests/isa/aarch64/extend-op.clif | 15 +- 4 files changed, 168 insertions(+), 44 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index aa90a3546e..01335205e0 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -30,26 +30,39 @@ (rule (lower (has_type (fits_in_64 ty) (iadd x y))) (value_reg (alu_rrr (iadd_op ty) (put_in_reg x) (put_in_reg y)))) -;; Special case for when one operand is an immediate that fits in 12 bits. +;; Special cases for when one operand is an immediate that fits in 12 bits. (rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_value y)))) (value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg x) y))) -;; Same as the previous special case, except we can switch the addition to a +(rule (lower (has_type (fits_in_64 ty) (iadd (imm12_from_value x) y))) + (value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg y) x))) + +;; Same as the previous special cases, except we can switch the addition to a ;; subtraction if the negated immediate fits in 12 bits. (rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_negated_value y)))) (value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg x) y))) -;; Special case for when we're adding an extended register where the extending +(rule (lower (has_type (fits_in_64 ty) (iadd (imm12_from_negated_value x) y))) + (value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg y) x))) + +;; Special cases for when we're adding an extended register where the extending ;; operation can get folded into the add itself. (rule (lower (has_type (fits_in_64 ty) (iadd x (extended_value_from_value y)))) (value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg x) y))) -;; Special case for when we're adding the shift of a different +(rule (lower (has_type (fits_in_64 ty) (iadd (extended_value_from_value x) y))) + (value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg y) x))) + +;; Special cases for when we're adding the shift of a different ;; register by a constant amount and the shift can get folded into the add. (rule (lower (has_type (fits_in_64 ty) (iadd x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 (ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/aarch64/lower.isle line 81. + // Rule at src/isa/aarch64/lower.isle line 94. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1293,7 +1293,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/aarch64/lower.isle line 132. + // Rule at src/isa/aarch64/lower.isle line 145. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1373,7 +1373,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/aarch64/lower.isle line 77. + // Rule at src/isa/aarch64/lower.isle line 90. let expr0_0 = VecALUOp::Add; let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = C::put_in_reg(ctx, pattern7_1); @@ -1385,7 +1385,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/aarch64/lower.isle line 128. + // Rule at src/isa/aarch64/lower.isle line 141. let expr0_0 = VecALUOp::Sub; let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = C::put_in_reg(ctx, pattern7_1); @@ -1411,27 +1411,127 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + if let Some(pattern13_0) = + C::imm12_from_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 37. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern13_0) = + C::imm12_from_negated_u64(ctx, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 45. + let expr0_0 = constructor_isub_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_alu_rr_imm12( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } } + &InstructionData::Binary { + opcode: ref pattern10_0, + args: ref pattern10_1, + } => { + match &pattern10_0 { + &Opcode::Imul => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + // Rule at src/isa/aarch64/lower.isle line 70. + let expr0_0 = constructor_madd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern12_0); + let expr2_0 = C::put_in_reg(ctx, pattern12_1); + let expr3_0 = C::put_in_reg(ctx, pattern7_1); + let expr4_0 = constructor_alu_rrrr( + ctx, &expr0_0, expr1_0, expr2_0, expr3_0, + )?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Ishl => { + let (pattern12_0, pattern12_1) = + C::unpack_value_array_2(ctx, &pattern10_1); + if let Some(pattern13_0) = C::def_inst(ctx, pattern12_1) + { + let pattern14_0 = C::inst_data(ctx, pattern13_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern15_0, + imm: pattern15_1, + } = &pattern14_0 + { + if let &Opcode::Iconst = &pattern15_0 { + let closure17 = || { + return Some(pattern3_0); + }; + if let Some(pattern17_0) = closure17() { + if let Some(pattern18_0) = + C::lshl_from_imm64( + ctx, + pattern15_1, + pattern17_0, + ) + { + // Rule at src/isa/aarch64/lower.isle line 62. + let expr0_0 = constructor_iadd_op( + ctx, pattern3_0, + )?; + let expr1_0 = + C::put_in_reg(ctx, pattern7_1); + let expr2_0 = + C::put_in_reg(ctx, pattern12_0); + let expr3_0 = + constructor_alu_rrr_shift( + ctx, + &expr0_0, + expr1_0, + expr2_0, + pattern18_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + } + } + } + _ => {} + } + } + _ => {} } } + if let Some(pattern8_0) = C::extended_value_from_value(ctx, pattern7_0) { + // Rule at src/isa/aarch64/lower.isle line 53. + let expr0_0 = constructor_iadd_op(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = + constructor_alu_rr_extend_reg(ctx, &expr0_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { let pattern9_0 = C::inst_data(ctx, pattern8_0); match &pattern9_0 { @@ -1459,7 +1559,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern12_0, pattern12_1) = C::unpack_value_array_2(ctx, &pattern10_1); - // Rule at src/isa/aarch64/lower.isle line 54. + // Rule at src/isa/aarch64/lower.isle line 67. let expr0_0 = constructor_madd_op(ctx, pattern3_0)?; let expr1_0 = C::put_in_reg(ctx, pattern12_0); let expr2_0 = C::put_in_reg(ctx, pattern12_1); @@ -1515,7 +1615,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/lower.isle line 62. + // Rule at src/isa/aarch64/lower.isle line 75. let expr0_0 = ALUOp::Add32; return Some(expr0_0); } - // Rule at src/isa/aarch64/lower.isle line 63. + // Rule at src/isa/aarch64/lower.isle line 76. let expr0_0 = ALUOp::Add64; return Some(expr0_0); } @@ -1703,11 +1803,11 @@ pub fn constructor_iadd_op(ctx: &mut C, arg0: Type) -> Option pub fn constructor_isub_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/lower.isle line 67. + // Rule at src/isa/aarch64/lower.isle line 80. let expr0_0 = ALUOp::Sub32; return Some(expr0_0); } - // Rule at src/isa/aarch64/lower.isle line 68. + // Rule at src/isa/aarch64/lower.isle line 81. let expr0_0 = ALUOp::Sub64; return Some(expr0_0); } @@ -1716,11 +1816,11 @@ pub fn constructor_isub_op(ctx: &mut C, arg0: Type) -> Option pub fn constructor_madd_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/lower.isle line 72. + // Rule at src/isa/aarch64/lower.isle line 85. let expr0_0 = ALUOp3::MAdd32; return Some(expr0_0); } - // Rule at src/isa/aarch64/lower.isle line 73. + // Rule at src/isa/aarch64/lower.isle line 86. let expr0_0 = ALUOp3::MAdd64; return Some(expr0_0); } diff --git a/cranelift/filetests/filetests/isa/aarch64/extend-op.clif b/cranelift/filetests/filetests/isa/aarch64/extend-op.clif index 58735d5bf8..f8e25d7814 100644 --- a/cranelift/filetests/filetests/isa/aarch64/extend-op.clif +++ b/cranelift/filetests/filetests/isa/aarch64/extend-op.clif @@ -10,8 +10,19 @@ block0(v0: i8): return v3 } -; check: movz x1, #42 -; nextln: add x0, x1, x0, SXTB +; check: sxtb x0, w0 +; nextln: add x0, x0, #42 +; nextln: ret + + +function %f2(i8, i64) -> i64 { +block0(v0: i8, v1: i64): + v2 = sextend.i64 v0 + v3 = iadd.i64 v2, v1 + return v3 +} + +; check: add x0, x1, x0, SXTB ; nextln: ret From cbf539abb88c90a80c410d2abebf334af67c909d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 19 Nov 2021 07:08:08 -0800 Subject: [PATCH 5/5] Use `with_flags` for 128-bith arith in aarch64 Also move the `with_flags` bits and pieces to `prelude.isle` so it can be shared between backends if necessary. --- cranelift/codegen/src/isa/aarch64/inst.isle | 28 ++ cranelift/codegen/src/isa/aarch64/lower.isle | 12 +- .../lower/isle/generated_code.manifest | 6 +- .../isa/aarch64/lower/isle/generated_code.rs | 228 +++++++++++++- cranelift/codegen/src/isa/x64/inst.isle | 43 --- .../x64/lower/isle/generated_code.manifest | 4 +- .../src/isa/x64/lower/isle/generated_code.rs | 284 +++++++++--------- cranelift/codegen/src/prelude.isle | 44 +++ 8 files changed, 437 insertions(+), 212 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 9ed6356da8..704895ab01 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -1420,6 +1420,34 @@ (_ Unit (emit (MInst.AluRRRR op dst src1 src2 src3)))) (writable_reg_to_reg dst))) +;; Helper for emitting `adds` instructions. +(decl add64_with_flags (Reg Reg) ProducesFlags) +(rule (add64_with_flags src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.AddS64) dst src1 src2) + (writable_reg_to_reg dst)))) + +;; Helper for emitting `adc` instructions. +(decl adc64 (Reg Reg) ConsumesFlags) +(rule (adc64 src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Adc64) dst src1 src2) + (writable_reg_to_reg dst)))) + +;; Helper for emitting `subs` instructions. +(decl sub64_with_flags (Reg Reg) ProducesFlags) +(rule (sub64_with_flags src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.SubS64) dst src1 src2) + (writable_reg_to_reg dst)))) + +;; Helper for emitting `sbc` instructions. +(decl sbc64 (Reg Reg) ConsumesFlags) +(rule (sbc64 src1 src2) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Sbc64) dst src1 src2) + (writable_reg_to_reg dst)))) + ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl imm (Type u64) Reg) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 01335205e0..5e0f9076f3 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -105,9 +105,9 @@ ) ;; the actual addition is `adds` followed by `adc` which comprises the ;; low/high bits of the result - (value_regs - (alu_rrr (ALUOp.AddS64) x_lo y_lo) - (alu_rrr (ALUOp.Adc64) x_hi y_hi)))) + (with_flags + (add64_with_flags x_lo y_lo) + (adc64 x_hi y_hi)))) ;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -156,6 +156,6 @@ ) ;; the actual subtraction is `subs` followed by `sbc` which comprises ;; the low/high bits of the result - (value_regs - (alu_rrr (ALUOp.SubS64) x_lo y_lo) - (alu_rrr (ALUOp.Sbc64) x_hi y_hi)))) + (with_flags + (sub64_with_flags x_lo y_lo) + (sbc64 x_hi y_hi)))) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 6b7b95685f..040c81e0bb 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle 6ae74a46b3b1f8099397cb8472d75359bc100a05a9bf38532cc41b423f7e600618d83bdb1f5565e0167696e94df2a18e489e6faabb6cc1b3e3af301ab8de55b8 -src/isa/aarch64/inst.isle 11e77cbb9a0f9c7779dea64db47aaa0b2a90bc1fac851dd0a7f0a37b4234c22b6543469f2a8d477116b2feab563fbf499eb71e414c6e2615e4cc8719e8c0ab1c -src/isa/aarch64/lower.isle 5a36aa28fabb75cb281e67cf23313feaec5e7e08471af9be03253903c7e33531cf3b42c2ed57c7a1864d7dfb923adcd47971d8ad88d66a1df35ad8edb06a1f9a +src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c +src/isa/aarch64/inst.isle 67a43022bb2e0b8ae06b71c7c49f9b9997a9c6ca109e35f5018b9cd64105a0fe8b103943fb34ca7da45cea9db7327e00954e88606845d5ebc370bc6c3045a04f +src/isa/aarch64/lower.isle 56c3c3f709d2ca18d91e14d8c4cec804990cfa3d85f9ce4fac4eef4861c4c159acee1b5fe6deff7d89337092b35ecf67418da7ce29e6217fa1ab47b0deaf1759 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index f3eb9067c9..ace3b1ecd5 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -62,6 +62,18 @@ pub trait Context { fn load_constant64_full(&mut self, arg0: u64) -> Reg; } +/// Internal type ProducesFlags: defined at src/prelude.isle line 230. +#[derive(Clone, Debug)] +pub enum ProducesFlags { + ProducesFlags { inst: MInst, result: Reg }, +} + +/// Internal type ConsumesFlags: defined at src/prelude.isle line 233. +#[derive(Clone, Debug)] +pub enum ConsumesFlags { + ConsumesFlags { inst: MInst, result: Reg }, +} + /// Internal type MInst: defined at src/isa/aarch64/inst.isle line 2. #[derive(Clone, Debug)] pub enum MInst { @@ -952,6 +964,98 @@ pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { return Some(expr2_0); } +// Generated as internal constructor for term with_flags. +pub fn constructor_with_flags( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + // Rule at src/prelude.isle line 243. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); + return Some(expr2_0); + } + } + return None; +} + +// Generated as internal constructor for term with_flags_1. +pub fn constructor_with_flags_1( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + // Rule at src/prelude.isle line 251. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + return Some(pattern3_1); + } + } + return None; +} + +// Generated as internal constructor for term with_flags_2. +pub fn constructor_with_flags_2( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, + arg2: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + let pattern4_0 = arg2; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern5_0, + result: pattern5_1, + } = pattern4_0 + { + // Rule at src/prelude.isle line 261. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + let expr2_0 = C::emit(ctx, &pattern5_0); + let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1); + return Some(expr3_0); + } + } + } + return None; +} + // Generated as internal constructor for term vector_size. pub fn constructor_vector_size(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; @@ -1227,31 +1331,127 @@ pub fn constructor_alu_rrrr( return Some(expr4_0); } +// Generated as internal constructor for term add64_with_flags. +pub fn constructor_add64_with_flags( + ctx: &mut C, + arg0: Reg, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1425. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::AddS64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ProducesFlags::ProducesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term adc64. +pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1432. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::Adc64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ConsumesFlags::ConsumesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term sub64_with_flags. +pub fn constructor_sub64_with_flags( + ctx: &mut C, + arg0: Reg, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1439. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::SubS64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ProducesFlags::ProducesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term sbc64. +pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1446. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = ALUOp::Sbc64; + let expr3_0 = MInst::AluRRR { + alu_op: expr2_0, + rd: expr1_0, + rn: pattern0_0, + rm: pattern1_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr5_0 = ConsumesFlags::ConsumesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + // Generated as internal constructor for term imm. pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) { let pattern2_0 = arg1; if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1436. + // Rule at src/isa/aarch64/inst.isle line 1464. let expr0_0 = ALUOp::Orr64; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern3_0)?; return Some(expr2_0); } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1428. + // Rule at src/isa/aarch64/inst.isle line 1456. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1432. + // Rule at src/isa/aarch64/inst.isle line 1460. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1443. + // Rule at src/isa/aarch64/inst.isle line 1471. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -1284,12 +1484,10 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); @@ -1304,12 +1502,10 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 33ac1fc785..8fdb09a1d3 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -385,49 +385,6 @@ (decl sink_load (SinkableLoad) RegMemImm) (extern constructor sink_load sink_load) -;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;; Newtype wrapper around `MInst` for instructions that are used for their -;; effect on flags. -(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg)))) - -;; Newtype wrapper around `MInst` for instructions that consume flags. -(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg)))) - -;; Combine flags-producing and -consuming instructions together, ensuring that -;; they are emitted back-to-back and no other instructions can be emitted -;; between them and potentially clobber the flags. -;; -;; Returns a `ValueRegs` where the first register is the result of the -;; `ProducesFlags` instruction and the second is the result of the -;; `ConsumesFlags` instruction. -(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs) -(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result) - (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) - (let ((_x Unit (emit producer_inst)) - (_y Unit (emit consumer_inst))) - (value_regs producer_result consumer_result))) - -;; Like `with_flags` but returns only the result of the consumer operation. -(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg) -(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result) - (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) - (let ((_x Unit (emit producer_inst)) - (_y Unit (emit consumer_inst))) - consumer_result)) - -;; Like `with_flags` but allows two consumers of the same flags. The result is a -;; `ValueRegs` containing the first consumer's result and then the second -;; consumer's result. -(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs) -(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result) - (ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1) - (ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2)) - (let ((_x Unit (emit producer_inst)) - (_y Unit (emit consumer_inst_1)) - (_z Unit (emit consumer_inst_2))) - (value_regs consumer_result_1 consumer_result_2))) - ;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (type ExtendKind (enum Sign Zero)) diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 2b1e1b6be7..cacbaa17fb 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle 6ae74a46b3b1f8099397cb8472d75359bc100a05a9bf38532cc41b423f7e600618d83bdb1f5565e0167696e94df2a18e489e6faabb6cc1b3e3af301ab8de55b8 -src/isa/x64/inst.isle 6065d3b9e0fa3361d179d9b87d09568ff474f8bac7eeabd29b328ace723041f96045bc82cfa2d7feda4490ce7e5d4be1a1c7ebe25c99916564d43a51550cd093 +src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c +src/isa/x64/inst.isle e4a0657406056a4cf116fe125e91d16377b602e0b41edd6628cbb7259b0fc2aa6b0482ffd33f00d63d68cf3546f188705877309d43eba5e75abd0f38a52a79b2 src/isa/x64/lower.isle e51b7a67343dba342a43b3c9e4b9ed7df9b2c66a677018acf7054ba48c27e4e93a4421fd892b9bf7c0e5b790bcfafab7cb3e93ce2b8206c04d456918d2ad0b5a diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index c32ee7feb2..c147f914db 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -63,19 +63,19 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/isa/x64/inst.isle line 392. +/// Internal type ProducesFlags: defined at src/prelude.isle line 230. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/isa/x64/inst.isle line 395. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 233. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 433. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 390. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -101,35 +101,6 @@ pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { return Some(expr2_0); } -// Generated as internal constructor for term operand_size_bits. -pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize) -> Option { - let pattern0_0 = arg0; - match pattern0_0 { - &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 80. - let expr0_0: u16 = 8; - return Some(expr0_0); - } - &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 81. - let expr0_0: u16 = 16; - return Some(expr0_0); - } - &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 82. - let expr0_0: u16 = 32; - return Some(expr0_0); - } - &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 83. - let expr0_0: u16 = 64; - return Some(expr0_0); - } - _ => {} - } - return None; -} - // Generated as internal constructor for term with_flags. pub fn constructor_with_flags( ctx: &mut C, @@ -148,7 +119,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/isa/x64/inst.isle line 405. + // Rule at src/prelude.isle line 243. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -176,7 +147,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/isa/x64/inst.isle line 413. + // Rule at src/prelude.isle line 251. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -210,7 +181,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/isa/x64/inst.isle line 423. + // Rule at src/prelude.isle line 261. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -222,6 +193,35 @@ pub fn constructor_with_flags_2( return None; } +// Generated as internal constructor for term operand_size_bits. +pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &OperandSize::Size8 => { + // Rule at src/isa/x64/inst.isle line 80. + let expr0_0: u16 = 8; + return Some(expr0_0); + } + &OperandSize::Size16 => { + // Rule at src/isa/x64/inst.isle line 81. + let expr0_0: u16 = 16; + return Some(expr0_0); + } + &OperandSize::Size32 => { + // Rule at src/isa/x64/inst.isle line 82. + let expr0_0: u16 = 32; + return Some(expr0_0); + } + &OperandSize::Size64 => { + // Rule at src/isa/x64/inst.isle line 83. + let expr0_0: u16 = 64; + return Some(expr0_0); + } + _ => {} + } + return None; +} + // Generated as internal constructor for term extend_to_reg. pub fn constructor_extend_to_reg( ctx: &mut C, @@ -234,12 +234,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 445. + // Rule at src/isa/x64/inst.isle line 402. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 448. + // Rule at src/isa/x64/inst.isle line 405. let expr0_0 = C::ty_bits(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -263,7 +263,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 468. + // Rule at src/isa/x64/inst.isle line 425. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -271,7 +271,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 464. + // Rule at src/isa/x64/inst.isle line 421. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -284,17 +284,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 475. + // Rule at src/isa/x64/inst.isle line 432. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 433. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 477. + // Rule at src/isa/x64/inst.isle line 434. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -311,7 +311,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 481. + // Rule at src/isa/x64/inst.isle line 438. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -321,40 +321,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 490. + // Rule at src/isa/x64/inst.isle line 447. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 491. + // Rule at src/isa/x64/inst.isle line 448. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 486. + // Rule at src/isa/x64/inst.isle line 443. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 487. + // Rule at src/isa/x64/inst.isle line 444. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 488. + // Rule at src/isa/x64/inst.isle line 445. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 489. + // Rule at src/isa/x64/inst.isle line 446. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -366,7 +366,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 505. + // Rule at src/isa/x64/inst.isle line 462. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -394,7 +394,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 529. + // Rule at src/isa/x64/inst.isle line 486. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -419,7 +419,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 537. + // Rule at src/isa/x64/inst.isle line 494. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -435,7 +435,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 545. + // Rule at src/isa/x64/inst.isle line 502. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -464,7 +464,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 556. + // Rule at src/isa/x64/inst.isle line 513. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -493,7 +493,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 567. + // Rule at src/isa/x64/inst.isle line 524. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -509,7 +509,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 575. + // Rule at src/isa/x64/inst.isle line 532. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -538,7 +538,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 586. + // Rule at src/isa/x64/inst.isle line 543. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -567,7 +567,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 597. + // Rule at src/isa/x64/inst.isle line 554. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -583,7 +583,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 608. + // Rule at src/isa/x64/inst.isle line 565. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -599,7 +599,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 616. + // Rule at src/isa/x64/inst.isle line 573. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -615,7 +615,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 624. + // Rule at src/isa/x64/inst.isle line 581. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -627,7 +627,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 653. + // Rule at src/isa/x64/inst.isle line 610. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -644,7 +644,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 682. + // Rule at src/isa/x64/inst.isle line 639. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -659,7 +659,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 641. + // Rule at src/isa/x64/inst.isle line 598. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -672,7 +672,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 694. + // Rule at src/isa/x64/inst.isle line 651. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -687,7 +687,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 645. + // Rule at src/isa/x64/inst.isle line 602. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -700,7 +700,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 672. + // Rule at src/isa/x64/inst.isle line 629. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -717,7 +717,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } let pattern1_0 = arg1; if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 659. + // Rule at src/isa/x64/inst.isle line 616. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -733,7 +733,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 634. + // Rule at src/isa/x64/inst.isle line 591. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Imm { @@ -758,7 +758,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 707. + // Rule at src/isa/x64/inst.isle line 664. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -783,7 +783,7 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 716. + // Rule at src/isa/x64/inst.isle line 673. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -799,7 +799,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 721. + // Rule at src/isa/x64/inst.isle line 678. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -815,7 +815,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 726. + // Rule at src/isa/x64/inst.isle line 683. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -831,7 +831,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 731. + // Rule at src/isa/x64/inst.isle line 688. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -849,7 +849,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 736. + // Rule at src/isa/x64/inst.isle line 693. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -874,7 +874,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 745. + // Rule at src/isa/x64/inst.isle line 702. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -890,7 +890,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 750. + // Rule at src/isa/x64/inst.isle line 707. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -908,7 +908,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 755. + // Rule at src/isa/x64/inst.isle line 712. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -936,7 +936,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 763. + // Rule at src/isa/x64/inst.isle line 720. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -958,7 +958,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 770. + // Rule at src/isa/x64/inst.isle line 727. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -982,7 +982,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 777. + // Rule at src/isa/x64/inst.isle line 734. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -999,7 +999,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 784. + // Rule at src/isa/x64/inst.isle line 741. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1010,7 +1010,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 789. + // Rule at src/isa/x64/inst.isle line 746. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1021,7 +1021,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 794. + // Rule at src/isa/x64/inst.isle line 751. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1032,7 +1032,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 799. + // Rule at src/isa/x64/inst.isle line 756. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1043,7 +1043,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 804. + // Rule at src/isa/x64/inst.isle line 761. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1054,7 +1054,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 809. + // Rule at src/isa/x64/inst.isle line 766. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1065,7 +1065,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 814. + // Rule at src/isa/x64/inst.isle line 771. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1076,7 +1076,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 819. + // Rule at src/isa/x64/inst.isle line 776. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1087,7 +1087,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 824. + // Rule at src/isa/x64/inst.isle line 781. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1098,7 +1098,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 829. + // Rule at src/isa/x64/inst.isle line 786. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1109,7 +1109,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 834. + // Rule at src/isa/x64/inst.isle line 791. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1120,7 +1120,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 839. + // Rule at src/isa/x64/inst.isle line 796. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1131,7 +1131,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 844. + // Rule at src/isa/x64/inst.isle line 801. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1142,7 +1142,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 849. + // Rule at src/isa/x64/inst.isle line 806. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1153,7 +1153,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 854. + // Rule at src/isa/x64/inst.isle line 811. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1164,7 +1164,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 859. + // Rule at src/isa/x64/inst.isle line 816. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1175,7 +1175,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 864. + // Rule at src/isa/x64/inst.isle line 821. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1186,7 +1186,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 869. + // Rule at src/isa/x64/inst.isle line 826. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1197,7 +1197,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 831. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1208,7 +1208,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 879. + // Rule at src/isa/x64/inst.isle line 836. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1219,7 +1219,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 884. + // Rule at src/isa/x64/inst.isle line 841. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1230,7 +1230,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 846. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1241,7 +1241,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 851. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1252,7 +1252,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 856. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1263,7 +1263,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 861. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1274,7 +1274,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 866. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1285,7 +1285,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 871. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1296,7 +1296,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 876. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1307,7 +1307,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 924. + // Rule at src/isa/x64/inst.isle line 881. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1318,7 +1318,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 929. + // Rule at src/isa/x64/inst.isle line 886. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1329,7 +1329,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 891. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1340,7 +1340,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 939. + // Rule at src/isa/x64/inst.isle line 896. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1351,7 +1351,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 944. + // Rule at src/isa/x64/inst.isle line 901. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1362,7 +1362,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 949. + // Rule at src/isa/x64/inst.isle line 906. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1373,7 +1373,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 954. + // Rule at src/isa/x64/inst.isle line 911. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1384,7 +1384,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 959. + // Rule at src/isa/x64/inst.isle line 916. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1395,7 +1395,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 964. + // Rule at src/isa/x64/inst.isle line 921. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1406,7 +1406,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 969. + // Rule at src/isa/x64/inst.isle line 926. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1427,7 +1427,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 974. + // Rule at src/isa/x64/inst.isle line 931. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1455,7 +1455,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 986. + // Rule at src/isa/x64/inst.isle line 943. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1473,7 +1473,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 995. + // Rule at src/isa/x64/inst.isle line 952. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1498,7 +1498,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1008. + // Rule at src/isa/x64/inst.isle line 965. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1514,7 +1514,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1015. + // Rule at src/isa/x64/inst.isle line 972. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1523,7 +1523,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1020. + // Rule at src/isa/x64/inst.isle line 977. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1539,7 +1539,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1025. + // Rule at src/isa/x64/inst.isle line 982. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -1557,7 +1557,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1037. + // Rule at src/isa/x64/inst.isle line 994. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1573,7 +1573,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1044. + // Rule at src/isa/x64/inst.isle line 1001. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -1591,7 +1591,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1054. + // Rule at src/isa/x64/inst.isle line 1011. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1601,7 +1601,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1059. + // Rule at src/isa/x64/inst.isle line 1016. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1611,7 +1611,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1064. + // Rule at src/isa/x64/inst.isle line 1021. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1629,7 +1629,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1071. + // Rule at src/isa/x64/inst.isle line 1028. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -1658,7 +1658,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1087. + // Rule at src/isa/x64/inst.isle line 1044. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1674,7 +1674,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1092. + // Rule at src/isa/x64/inst.isle line 1049. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1693,7 +1693,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1105. + // Rule at src/isa/x64/inst.isle line 1062. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1714,7 +1714,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1114. + // Rule at src/isa/x64/inst.isle line 1071. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -1737,7 +1737,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1121. + // Rule at src/isa/x64/inst.isle line 1078. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -1755,7 +1755,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1126. + // Rule at src/isa/x64/inst.isle line 1083. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -1775,7 +1775,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1131. + // Rule at src/isa/x64/inst.isle line 1088. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1793,7 +1793,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1136. + // Rule at src/isa/x64/inst.isle line 1093. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -1805,7 +1805,7 @@ pub fn constructor_insertps( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1141. + // Rule at src/isa/x64/inst.isle line 1098. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1816,7 +1816,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1146. + // Rule at src/isa/x64/inst.isle line 1103. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index b53ce9c579..2a1d0b8227 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -222,3 +222,47 @@ ;; Match the instruction that defines the given value, if any. (decl def_inst (Inst) Value) (extern extractor def_inst def_inst) + +;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Newtype wrapper around `MInst` for instructions that are used for their +;; effect on flags. +(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg)))) + +;; Newtype wrapper around `MInst` for instructions that consume flags. +(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg)))) + +;; Combine flags-producing and -consuming instructions together, ensuring that +;; they are emitted back-to-back and no other instructions can be emitted +;; between them and potentially clobber the flags. +;; +;; Returns a `ValueRegs` where the first register is the result of the +;; `ProducesFlags` instruction and the second is the result of the +;; `ConsumesFlags` instruction. +(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs) +(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst))) + (value_regs producer_result consumer_result))) + +;; Like `with_flags` but returns only the result of the consumer operation. +(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg) +(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst))) + consumer_result)) + +;; Like `with_flags` but allows two consumers of the same flags. The result is a +;; `ValueRegs` containing the first consumer's result and then the second +;; consumer's result. +(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs) +(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1) + (ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst_1)) + (_z Unit (emit consumer_inst_2))) + (value_regs consumer_result_1 consumer_result_2))) +