T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3545 from alexcrichton/arm64-iadd-isub-isle

Browse Source 
aarch64: Migrate `iadd` and `isub` to ISLE
This commit is contained in:
Nick Fitzgerald
2021-11-19 10:08:19 -08:00
committed by GitHub
parent a947c89055 cbf539abb8
commit f84b30bb59
17 changed files with 1605 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
cranelift/codegen/src/isa/aarch64/inst.isle
View File

@@ -947,6 +947,13 @@
(Size64x2)
))
;; Helper for calculating the `VectorSize` corresponding to a type
(decl vector_size (Type) VectorSize)
(rule (vector_size (multi_lane 8 16)) (VectorSize.Size8x16))
(rule (vector_size (multi_lane 16 8)) (VectorSize.Size16x8))
(rule (vector_size (multi_lane 32 4)) (VectorSize.Size32x4))
(rule (vector_size (multi_lane 64 2)) (VectorSize.Size64x2))
;; A floating-point unit (FPU) operation with one arg.
(type FPUOp1
(enum
@@ -1294,9 +1301,41 @@
(decl imm_logic_from_u64 (ImmLogic) u64)
(extern extractor imm_logic_from_u64 imm_logic_from_u64)
(decl imm12_from_u64 (Imm12) u64)
(extern extractor imm12_from_u64 imm12_from_u64)
(decl imm12_from_negated_u64 (Imm12) u64)
(extern extractor imm12_from_negated_u64 imm12_from_negated_u64)
(decl lshl_from_imm64 (Type ShiftOpAndAmt) Imm64)
(extern extractor lshl_from_imm64 lshl_from_imm64 (in out))
(decl integral_ty (Type) Type)
(extern extractor integral_ty integral_ty)
;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
(decl imm12_from_value (Imm12) Value)
(extractor
(imm12_from_value n)
(def_inst (iconst (u64_from_imm64 (imm12_from_u64 n)))))
;; Same as `imm12_from_value`, but tries negating the constant value.
(decl imm12_from_negated_value (Imm12) Value)
(extractor
(imm12_from_negated_value n)
(def_inst (iconst (u64_from_imm64 (imm12_from_negated_u64 n)))))
;; Helper type to represent a value and an extend operation fused together.
(type ExtendedValue extern (enum))
(decl extended_value_from_value (ExtendedValue) Value)
(extern extractor extended_value_from_value extended_value_from_value)
;; Constructors used to poke at the fields of an `ExtendedValue`.
(decl put_extended_in_reg (ExtendedValue) Reg)
(extern constructor put_extended_in_reg put_extended_in_reg)
(decl get_extended_op (ExtendedValue) ExtendOp)
(extern constructor get_extended_op get_extended_op)
;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Emit an instruction.
@@ -1331,9 +1370,83 @@
(_ Unit (emit (MInst.AluRRImmLogic op dst src imm))))
(writable_reg_to_reg dst)))
;; Helper for emitting `orr64` instructions.
(decl orr64 (Reg ImmLogic) Reg)
(rule (orr64 src imm) (alu_rr_imm_logic (ALUOp.Orr64) src imm))
;; Helper for emitting `MInst.AluRRR` instructions.
(decl alu_rrr (ALUOp Reg Reg) Reg)
(rule (alu_rrr op src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRR op dst src1 src2))))
(writable_reg_to_reg dst)))
;; Helper for emitting `MInst.VecRRR` instructions.
(decl vec_rrr (VecALUOp Reg Reg VectorSize) Reg)
(rule (vec_rrr op src1 src2 size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRR op dst src1 src2 size))))
(writable_reg_to_reg dst)))
;; Helper for emitting `MInst.AluRRImm12` instructions.
(decl alu_rr_imm12 (ALUOp Reg Imm12) Reg)
(rule (alu_rr_imm12 op src imm)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRImm12 op dst src imm))))
(writable_reg_to_reg dst)))
;; Helper for emitting `MInst.AluRRRShift` instructions.
(decl alu_rrr_shift (ALUOp Reg Reg ShiftOpAndAmt) Reg)
(rule (alu_rrr_shift op src1 src2 shift)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRRShift op dst src1 src2 shift))))
(writable_reg_to_reg dst)))
;; Helper for emitting `MInst.AluRRRExtend` instructions.
(decl alu_rrr_extend (ALUOp Reg Reg ExtendOp) Reg)
(rule (alu_rrr_extend op src1 src2 extend)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRRExtend op dst src1 src2 extend))))
(writable_reg_to_reg dst)))
;; Same as `alu_rrr_extend`, but takes an `ExtendedValue` packed "pair" instead
;; of a `Reg` and an `ExtendOp`.
(decl alu_rr_extend_reg (ALUOp Reg ExtendedValue) Reg)
(rule (alu_rr_extend_reg op src1 extended_reg)
(let ((src2 Reg (put_extended_in_reg extended_reg))
(extend ExtendOp (get_extended_op extended_reg)))
(alu_rrr_extend op src1 src2 extend)))
;; Helper for emitting `MInst.AluRRRR` instructions.
(decl alu_rrrr (ALUOp3 Reg Reg Reg) Reg)
(rule (alu_rrrr op src1 src2 src3)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRRR op dst src1 src2 src3))))
(writable_reg_to_reg dst)))
;; Helper for emitting `adds` instructions.
(decl add64_with_flags (Reg Reg) ProducesFlags)
(rule (add64_with_flags src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.AddS64) dst src1 src2)
(writable_reg_to_reg dst))))
;; Helper for emitting `adc` instructions.
(decl adc64 (Reg Reg) ConsumesFlags)
(rule (adc64 src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Adc64) dst src1 src2)
(writable_reg_to_reg dst))))
;; Helper for emitting `subs` instructions.
(decl sub64_with_flags (Reg Reg) ProducesFlags)
(rule (sub64_with_flags src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ProducesFlags.ProducesFlags (MInst.AluRRR (ALUOp.SubS64) dst src1 src2)
(writable_reg_to_reg dst))))
;; Helper for emitting `sbc` instructions.
(decl sbc64 (Reg Reg) ConsumesFlags)
(rule (sbc64 src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlags (MInst.AluRRR (ALUOp.Sbc64) dst src1 src2)
(writable_reg_to_reg dst))))
;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1349,7 +1462,7 @@
;; Weird logical-instruction immediate in ORI using zero register
(rule (imm (integral_ty _ty) (imm_logic_from_u64 n))
(orr64 (zero_reg) n))
(alu_rr_imm_logic (ALUOp.Orr64) (zero_reg) n))
(decl load_constant64_full (u64) Reg)
(extern constructor load_constant64_full load_constant64_full)

2
cranelift/codegen/src/isa/aarch64/inst/args.rs
View File

@@ -62,7 +62,7 @@ impl ShiftOpShiftImm {
}
/// A shift operator with an amount, guaranteed to be within range.
#[derive(Clone, Debug)]
#[derive(Copy, Clone, Debug)]
pub struct ShiftOpAndAmt {
op: ShiftOp,
shift: ShiftOpShiftImm,

2
cranelift/codegen/src/isa/aarch64/inst/imms.rs
View File

@@ -272,7 +272,7 @@ impl UImm12Scaled {
/// A shifted immediate value in 'imm12' format: supports 12 bits, shifted
/// left by 0 or 12 places.
#[derive(Clone, Debug)]
#[derive(Copy, Clone, Debug)]
pub struct Imm12 {
/// The immediate bits.
pub bits: u16,

138
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -21,3 +21,141 @@
(rule (lower (has_type ty (null)))
(value_reg (imm ty 0)))
;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `i64` and smaller
;; Base case, simply adding things in registers.
(rule (lower (has_type (fits_in_64 ty) (iadd x y)))
(value_reg (alu_rrr (iadd_op ty) (put_in_reg x) (put_in_reg y))))
;; Special cases for when one operand is an immediate that fits in 12 bits.
(rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_value y))))
(value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg x) y)))
(rule (lower (has_type (fits_in_64 ty) (iadd (imm12_from_value x) y)))
(value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg y) x)))
;; Same as the previous special cases, except we can switch the addition to a
;; subtraction if the negated immediate fits in 12 bits.
(rule (lower (has_type (fits_in_64 ty) (iadd x (imm12_from_negated_value y))))
(value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg x) y)))
(rule (lower (has_type (fits_in_64 ty) (iadd (imm12_from_negated_value x) y)))
(value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg y) x)))
;; Special cases for when we're adding an extended register where the extending
;; operation can get folded into the add itself.
(rule (lower (has_type (fits_in_64 ty) (iadd x (extended_value_from_value y))))
(value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg x) y)))
(rule (lower (has_type (fits_in_64 ty) (iadd (extended_value_from_value x) y)))
(value_reg (alu_rr_extend_reg (iadd_op ty) (put_in_reg y) x)))
;; Special cases for when we're adding the shift of a different
;; register by a constant amount and the shift can get folded into the add.
(rule (lower (has_type (fits_in_64 ty)
(iadd x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 <ty amt))))))))
(value_reg (alu_rrr_shift (iadd_op ty) (put_in_reg x) (put_in_reg y) amt)))
(rule (lower (has_type (fits_in_64 ty)
(iadd (def_inst (ishl x (def_inst (iconst (lshl_from_imm64 <ty amt))))) y)))
(value_reg (alu_rrr_shift (iadd_op ty) (put_in_reg y) (put_in_reg x) amt)))
;; Fold an `iadd` and `imul` combination into a `madd` instruction.
(rule (lower (has_type (fits_in_64 ty) (iadd x (def_inst (imul y z)))))
(value_reg (alu_rrrr (madd_op ty) (put_in_reg y) (put_in_reg z) (put_in_reg x))))
(rule (lower (has_type (fits_in_64 ty) (iadd (def_inst (imul x y)) z)))
(value_reg (alu_rrrr (madd_op ty) (put_in_reg x) (put_in_reg y) (put_in_reg z))))
;; Helper to use either a 32 or 64-bit add depending on the input type.
(decl iadd_op (Type) ALUOp)
(rule (iadd_op (fits_in_32 _ty)) (ALUOp.Add32))
(rule (iadd_op _ty) (ALUOp.Add64))
;; Helper to use either a 32 or 64-bit sub depending on the input type.
(decl isub_op (Type) ALUOp)
(rule (isub_op (fits_in_32 _ty)) (ALUOp.Sub32))
(rule (isub_op _ty) (ALUOp.Sub64))
;; Helper to use either a 32 or 64-bit madd depending on the input type.
(decl madd_op (Type) ALUOp3)
(rule (madd_op (fits_in_32 _ty)) (ALUOp3.MAdd32))
(rule (madd_op _ty) (ALUOp3.MAdd64))
;; vectors
(rule (lower (has_type ty @ (multi_lane _ _) (iadd x y)))
(value_reg (vec_rrr (VecALUOp.Add) (put_in_reg x) (put_in_reg y) (vector_size ty))))
;; `i128`
(rule (lower (has_type $I128 (iadd x y)))
(let (
;; Get the high/low registers for `x`.
(x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
;; Get the high/low registers for `y`.
(y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1))
)
;; the actual addition is `adds` followed by `adc` which comprises the
;; low/high bits of the result
(with_flags
(add64_with_flags x_lo y_lo)
(adc64 x_hi y_hi))))
;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `i64` and smaller
;; Base case, simply subtracting things in registers.
(rule (lower (has_type (fits_in_64 ty) (isub x y)))
(value_reg (alu_rrr (isub_op ty) (put_in_reg x) (put_in_reg y))))
;; Special case for when one operand is an immediate that fits in 12 bits.
(rule (lower (has_type (fits_in_64 ty) (isub x (imm12_from_value y))))
(value_reg (alu_rr_imm12 (isub_op ty) (put_in_reg x) y)))
;; Same as the previous special case, except we can switch the subtraction to an
;; addition if the negated immediate fits in 12 bits.
(rule (lower (has_type (fits_in_64 ty) (isub x (imm12_from_negated_value y))))
(value_reg (alu_rr_imm12 (iadd_op ty) (put_in_reg x) y)))
;; Special cases for when we're subtracting an extended register where the
;; extending operation can get folded into the sub itself.
(rule (lower (has_type (fits_in_64 ty) (isub x (extended_value_from_value y))))
(value_reg (alu_rr_extend_reg (isub_op ty) (put_in_reg x) y)))
;; Finally a special case for when we're subtracting the shift of a different
;; register by a constant amount and the shift can get folded into the sub.
(rule (lower (has_type (fits_in_64 ty)
(isub x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 <ty amt))))))))
(value_reg (alu_rrr_shift (isub_op ty) (put_in_reg x) (put_in_reg y) amt)))
;; vectors
(rule (lower (has_type ty @ (multi_lane _ _) (isub x y)))
(value_reg (vec_rrr (VecALUOp.Sub) (put_in_reg x) (put_in_reg y) (vector_size ty))))
;; `i128`
(rule (lower (has_type $I128 (isub x y)))
(let (
;; Get the high/low registers for `x`.
(x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
;; Get the high/low registers for `y`.
(y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1))
)
;; the actual subtraction is `subs` followed by `sbc` which comprises
;; the low/high bits of the result
(with_flags
(sub64_with_flags x_lo y_lo)
(sbc64 x_hi y_hi))))

219
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -10,7 +10,7 @@
use crate::ir::condcodes::{FloatCC, IntCC};
use crate::ir::types::*;
use crate::ir::Inst as IRInst;
use crate::ir::{Opcode, Type};
use crate::ir::{Opcode, Type, Value};
use crate::machinst::lower::*;
use crate::machinst::*;
use crate::{CodegenError, CodegenResult};
@@ -272,20 +272,20 @@ fn extend_reg<C: LowerCtx<I = Inst>>(
}
/// Lowers an instruction input to multiple regs
fn lower_input_to_regs<C: LowerCtx<I = Inst>>(
fn lower_value_to_regs<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
value: Value,
) -> (ValueRegs<Reg>, Type, bool) {
log::trace!("lower_input_to_regs: input {:?}", input);
let ty = ctx.input_ty(input.insn, input.input);
let inputs = ctx.get_input_as_source_or_const(input.insn, input.input);
log::trace!("lower_value_to_regs: value {:?}", value);
let ty = ctx.value_ty(value);
let inputs = ctx.get_value_as_source_or_const(value);
let is_const = inputs.constant.is_some();
let in_regs = if let Some(c) = inputs.constant {
// Generate constants fresh at each use to minimize long-range register pressure.
generate_constant(ctx, ty, c as u128)
} else {
ctx.put_input_in_regs(input.insn, input.input)
ctx.put_value_in_regs(value)
};
(in_regs, ty, is_const)
@@ -301,7 +301,17 @@ pub(crate) fn put_input_in_reg<C: LowerCtx<I = Inst>>(
input: InsnInput,
narrow_mode: NarrowValueMode,
) -> Reg {
let (in_regs, ty, is_const) = lower_input_to_regs(ctx, input);
let value = ctx.input_as_value(input.insn, input.input);
put_value_in_reg(ctx, value, narrow_mode)
}
/// Like above, only for values
fn put_value_in_reg<C: LowerCtx<I = Inst>>(
ctx: &mut C,
value: Value,
narrow_mode: NarrowValueMode,
) -> Reg {
let (in_regs, ty, is_const) = lower_value_to_regs(ctx, value);
let reg = in_regs
.only_reg()
.expect("Multi-register value not expected");
@@ -314,7 +324,8 @@ pub(crate) fn put_input_in_regs<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> ValueRegs<Reg> {
let (in_regs, _, _) = lower_input_to_regs(ctx, input);
let value = ctx.input_as_value(input.insn, input.input);
let (in_regs, _, _) = lower_value_to_regs(ctx, value);
in_regs
}
@@ -367,89 +378,94 @@ fn put_input_in_rse<C: LowerCtx<I = Inst>>(
input: InsnInput,
narrow_mode: NarrowValueMode,
) -> ResultRSE {
let inputs = ctx.get_input_as_source_or_const(input.insn, input.input);
if let Some((insn, 0)) = inputs.inst {
let op = ctx.data(insn).opcode();
let out_ty = ctx.output_ty(insn, 0);
let out_bits = ty_bits(out_ty);
// Is this a zero-extend or sign-extend and can we handle that with a register-mode operator?
if op == Opcode::Uextend || op == Opcode::Sextend {
let sign_extend = op == Opcode::Sextend;
let inner_ty = ctx.input_ty(insn, 0);
let inner_bits = ty_bits(inner_ty);
assert!(inner_bits < out_bits);
if match (sign_extend, narrow_mode) {
// A single zero-extend or sign-extend is equal to itself.
(_, NarrowValueMode::None) => true,
// Two zero-extends or sign-extends in a row is equal to a single zero-extend or sign-extend.
(false, NarrowValueMode::ZeroExtend32) | (false, NarrowValueMode::ZeroExtend64) => {
true
}
(true, NarrowValueMode::SignExtend32) | (true, NarrowValueMode::SignExtend64) => {
true
}
// A zero-extend and a sign-extend in a row is not equal to a single zero-extend or sign-extend
(false, NarrowValueMode::SignExtend32) | (false, NarrowValueMode::SignExtend64) => {
false
}
(true, NarrowValueMode::ZeroExtend32) | (true, NarrowValueMode::ZeroExtend64) => {
false
}
} {
let extendop = match (sign_extend, inner_bits) {
(true, 8) => ExtendOp::SXTB,
(false, 8) => ExtendOp::UXTB,
(true, 16) => ExtendOp::SXTH,
(false, 16) => ExtendOp::UXTH,
(true, 32) => ExtendOp::SXTW,
(false, 32) => ExtendOp::UXTW,
_ => unreachable!(),
};
let reg =
put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None);
return ResultRSE::RegExtend(reg, extendop);
}
}
// If `out_ty` is smaller than 32 bits and we need to zero- or sign-extend,
// then get the result into a register and return an Extend-mode operand on
// that register.
if narrow_mode != NarrowValueMode::None
&& ((narrow_mode.is_32bit() && out_bits < 32)
|| (!narrow_mode.is_32bit() && out_bits < 64))
{
let reg = put_input_in_reg(ctx, input, NarrowValueMode::None);
let extendop = match (narrow_mode, out_bits) {
(NarrowValueMode::SignExtend32, 1) | (NarrowValueMode::SignExtend64, 1) => {
ExtendOp::SXTB
}
(NarrowValueMode::ZeroExtend32, 1) | (NarrowValueMode::ZeroExtend64, 1) => {
ExtendOp::UXTB
}
(NarrowValueMode::SignExtend32, 8) | (NarrowValueMode::SignExtend64, 8) => {
ExtendOp::SXTB
}
(NarrowValueMode::ZeroExtend32, 8) | (NarrowValueMode::ZeroExtend64, 8) => {
ExtendOp::UXTB
}
(NarrowValueMode::SignExtend32, 16) | (NarrowValueMode::SignExtend64, 16) => {
ExtendOp::SXTH
}
(NarrowValueMode::ZeroExtend32, 16) | (NarrowValueMode::ZeroExtend64, 16) => {
ExtendOp::UXTH
}
(NarrowValueMode::SignExtend64, 32) => ExtendOp::SXTW,
(NarrowValueMode::ZeroExtend64, 32) => ExtendOp::UXTW,
_ => unreachable!(),
};
return ResultRSE::RegExtend(reg, extendop);
}
let value = ctx.input_as_value(input.insn, input.input);
if let Some((val, extendop)) = get_as_extended_value(ctx, value, narrow_mode) {
let reg = put_value_in_reg(ctx, val, NarrowValueMode::None);
return ResultRSE::RegExtend(reg, extendop);
}
ResultRSE::from_rs(put_input_in_rs(ctx, input, narrow_mode))
}
fn get_as_extended_value<C: LowerCtx<I = Inst>>(
ctx: &mut C,
val: Value,
narrow_mode: NarrowValueMode,
) -> Option<(Value, ExtendOp)> {
let inputs = ctx.get_value_as_source_or_const(val);
let (insn, n) = inputs.inst?;
if n != 0 {
return None;
}
let op = ctx.data(insn).opcode();
let out_ty = ctx.output_ty(insn, 0);
let out_bits = ty_bits(out_ty);
// Is this a zero-extend or sign-extend and can we handle that with a register-mode operator?
if op == Opcode::Uextend || op == Opcode::Sextend {
let sign_extend = op == Opcode::Sextend;
let inner_ty = ctx.input_ty(insn, 0);
let inner_bits = ty_bits(inner_ty);
assert!(inner_bits < out_bits);
if match (sign_extend, narrow_mode) {
// A single zero-extend or sign-extend is equal to itself.
(_, NarrowValueMode::None) => true,
// Two zero-extends or sign-extends in a row is equal to a single zero-extend or sign-extend.
(false, NarrowValueMode::ZeroExtend32) | (false, NarrowValueMode::ZeroExtend64) => true,
(true, NarrowValueMode::SignExtend32) | (true, NarrowValueMode::SignExtend64) => true,
// A zero-extend and a sign-extend in a row is not equal to a single zero-extend or sign-extend
(false, NarrowValueMode::SignExtend32) | (false, NarrowValueMode::SignExtend64) => {
false
}
(true, NarrowValueMode::ZeroExtend32) | (true, NarrowValueMode::ZeroExtend64) => false,
} {
let extendop = match (sign_extend, inner_bits) {
(true, 8) => ExtendOp::SXTB,
(false, 8) => ExtendOp::UXTB,
(true, 16) => ExtendOp::SXTH,
(false, 16) => ExtendOp::UXTH,
(true, 32) => ExtendOp::SXTW,
(false, 32) => ExtendOp::UXTW,
_ => unreachable!(),
};
return Some((ctx.input_as_value(insn, 0), extendop));
}
}
// If `out_ty` is smaller than 32 bits and we need to zero- or sign-extend,
// then get the result into a register and return an Extend-mode operand on
// that register.
if narrow_mode != NarrowValueMode::None
&& ((narrow_mode.is_32bit() && out_bits < 32) || (!narrow_mode.is_32bit() && out_bits < 64))
{
let extendop = match (narrow_mode, out_bits) {
(NarrowValueMode::SignExtend32, 1) | (NarrowValueMode::SignExtend64, 1) => {
ExtendOp::SXTB
}
(NarrowValueMode::ZeroExtend32, 1) | (NarrowValueMode::ZeroExtend64, 1) => {
ExtendOp::UXTB
}
(NarrowValueMode::SignExtend32, 8) | (NarrowValueMode::SignExtend64, 8) => {
ExtendOp::SXTB
}
(NarrowValueMode::ZeroExtend32, 8) | (NarrowValueMode::ZeroExtend64, 8) => {
ExtendOp::UXTB
}
(NarrowValueMode::SignExtend32, 16) | (NarrowValueMode::SignExtend64, 16) => {
ExtendOp::SXTH
}
(NarrowValueMode::ZeroExtend32, 16) | (NarrowValueMode::ZeroExtend64, 16) => {
ExtendOp::UXTH
}
(NarrowValueMode::SignExtend64, 32) => ExtendOp::SXTW,
(NarrowValueMode::ZeroExtend64, 32) => ExtendOp::UXTW,
_ => unreachable!(),
};
return Some((val, extendop));
}
None
}
pub(crate) fn put_input_in_rse_imm12<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
@@ -472,35 +488,6 @@ pub(crate) fn put_input_in_rse_imm12<C: LowerCtx<I = Inst>>(
ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode))
}
/// Like `put_input_in_rse_imm12` above, except is allowed to negate the
/// argument (assuming a two's-complement representation with the given bit
/// width) if this allows use of 12-bit immediate. Used to flip `add`s with
/// negative immediates to `sub`s (and vice-versa).
pub(crate) fn put_input_in_rse_imm12_maybe_negated<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
twos_complement_bits: usize,
narrow_mode: NarrowValueMode,
) -> (ResultRSEImm12, bool) {
assert!(twos_complement_bits <= 64);
if let Some(imm_value) = input_to_const(ctx, input) {
if let Some(i) = Imm12::maybe_from_u64(imm_value) {
return (ResultRSEImm12::Imm12(i), false);
}
let sign_extended =
((imm_value as i64) << (64 - twos_complement_bits)) >> (64 - twos_complement_bits);
let inverted = sign_extended.wrapping_neg();
if let Some(i) = Imm12::maybe_from_u64(inverted as u64) {
return (ResultRSEImm12::Imm12(i), true);
}
}
(
ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode)),
false,
)
}
pub(crate) fn put_input_in_rs_immlogic<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,

45
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -7,8 +7,8 @@ pub mod generated_code;
use super::{
zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, AtomicRmwOp, BranchTarget, CallIndInfo,
CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, Imm12, ImmLogic, ImmShift, Inst as MInst,
JTSequenceInfo, MachLabel, MoveWideConst, Opcode, OperandSize, PairAMode, Reg, ScalarSize,
ShiftOpAndAmt, UImm5, VectorSize, NZCV,
JTSequenceInfo, MachLabel, MoveWideConst, NarrowValueMode, Opcode, OperandSize, PairAMode, Reg,
ScalarSize, ShiftOpAndAmt, UImm5, VectorSize, NZCV,
};
use crate::isa::aarch64::settings as aarch64_settings;
use crate::machinst::isle::*;
@@ -19,8 +19,9 @@ use crate::{
ValueLabel, ValueList,
},
isa::aarch64::inst::aarch64_map_regs,
isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
isa::unwind::UnwindInst,
machinst::{get_output_reg, InsnOutput, LowerCtx, RegRenamer},
machinst::{get_output_reg, ty_bits, InsnOutput, LowerCtx, RegRenamer},
};
use smallvec::SmallVec;
use std::boxed::Box;
@@ -76,6 +77,11 @@ pub struct IsleContext<'a, C> {
emitted_insts: SmallVec<[MInst; 6]>,
}
pub struct ExtendedValue {
val: Value,
extend: ExtendOp,
}
impl<'a, C> IsleContext<'a, C> {
pub fn new(lower_ctx: &'a mut C, isa_flags: &'a aarch64_settings::Flags) -> Self {
IsleContext {
@@ -108,6 +114,25 @@ where
ImmLogic::maybe_from_u64(n, I64)
}
fn imm12_from_u64(&mut self, n: u64) -> Option<Imm12> {
Imm12::maybe_from_u64(n)
}
fn imm12_from_negated_u64(&mut self, n: u64) -> Option<Imm12> {
Imm12::maybe_from_u64((n as i64).wrapping_neg() as u64)
}
fn lshl_from_imm64(&mut self, n: Imm64, ty: Type) -> Option<ShiftOpAndAmt> {
let shiftimm = ShiftOpShiftImm::maybe_from_shift(n.bits() as u64)?;
let shiftee_bits = ty_bits(ty);
if shiftee_bits <= std::u8::MAX as usize {
let shiftimm = shiftimm.mask(shiftee_bits as u8);
Some(ShiftOpAndAmt::new(ShiftOp::LSL, shiftimm))
} else {
None
}
}
fn integral_ty(&mut self, ty: Type) -> Option<Type> {
match ty {
I8 | I16 | I32 | I64 | R64 => Some(ty),
@@ -181,6 +206,20 @@ where
zero_reg()
}
fn extended_value_from_value(&mut self, val: Value) -> Option<ExtendedValue> {
let (val, extend) =
super::get_as_extended_value(self.lower_ctx, val, NarrowValueMode::None)?;
Some(ExtendedValue { val, extend })
}
fn put_extended_in_reg(&mut self, reg: &ExtendedValue) -> Reg {
self.put_in_reg(reg.val)
}
fn get_extended_op(&mut self, reg: &ExtendedValue) -> ExtendOp {
reg.extend
}
fn emit(&mut self, inst: &MInst) -> Unit {
self.emitted_insts.push(inst.clone());
}

6
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb
src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462
src/isa/aarch64/inst.isle 841748c9c5900821b7086a09a41c6dcdb2172eb47a45293b6ef10f2e1f1389620bf6a2c75152af807d8bc8929029a357af5191f5d87bac2c9ec54bf63a9a2a8f
src/isa/aarch64/lower.isle b3cd0834484e543f39d477d47ee66042276e99955c21fb8c9340a5f27ac317936acb2907a30f758bf596066e36db801a179fda6dbcecaee758a0187a5a5f1412
src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c
src/isa/aarch64/inst.isle 67a43022bb2e0b8ae06b71c7c49f9b9997a9c6ca109e35f5018b9cd64105a0fe8b103943fb34ca7da45cea9db7327e00954e88606845d5ebc370bc6c3045a04f
src/isa/aarch64/lower.isle 56c3c3f709d2ca18d91e14d8c4cec804990cfa3d85f9ce4fac4eef4861c4c159acee1b5fe6deff7d89337092b35ecf67418da7ce29e6217fa1ab47b0deaf1759

979
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

143
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -42,7 +42,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
return Ok(());
}
let implemented_in_isle = || {
let implemented_in_isle = |ctx: &mut C| {
unreachable!(
"implemented in ISLE: inst = `{}`, type = `{:?}`",
ctx.dfg().display_inst(insn),
@@ -51,7 +51,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
};
match op {
Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(),
Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(ctx),
Opcode::F32const => {
let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32);
@@ -63,143 +63,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
lower_constant_f64(ctx, rd, value);
}
Opcode::Iadd => {
match ty.unwrap() {
ty if ty.is_vector() => {
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
ctx.emit(Inst::VecRRR {
rd,
rn,
rm,
alu_op: VecALUOp::Add,
size: VectorSize::from_ty(ty),
});
}
I128 => {
let lhs = put_input_in_regs(ctx, inputs[0]);
let rhs = put_input_in_regs(ctx, inputs[1]);
let dst = get_output_reg(ctx, outputs[0]);
assert_eq!(lhs.len(), 2);
assert_eq!(rhs.len(), 2);
assert_eq!(dst.len(), 2);
// adds x0, x0, x2
// adc x1, x1, x3
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::AddS64,
rd: dst.regs()[0],
rn: lhs.regs()[0],
rm: rhs.regs()[0],
});
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::Adc64,
rd: dst.regs()[1],
rn: lhs.regs()[1],
rm: rhs.regs()[1],
});
}
ty => {
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let mul_insn = if let Some(mul_insn) =
maybe_input_insn(ctx, inputs[1], Opcode::Imul)
{
Some((mul_insn, 0))
} else if let Some(mul_insn) = maybe_input_insn(ctx, inputs[0], Opcode::Imul) {
Some((mul_insn, 1))
} else {
None
};
// If possible combine mul + add into madd.
if let Some((insn, addend_idx)) = mul_insn {
let alu_op = choose_32_64(ty, ALUOp3::MAdd32, ALUOp3::MAdd64);
let rn_input = InsnInput { insn, input: 0 };
let rm_input = InsnInput { insn, input: 1 };
let rn = put_input_in_reg(ctx, rn_input, NarrowValueMode::None);
let rm = put_input_in_reg(ctx, rm_input, NarrowValueMode::None);
let ra = put_input_in_reg(ctx, inputs[addend_idx], NarrowValueMode::None);
ctx.emit(Inst::AluRRRR {
alu_op,
rd,
rn,
rm,
ra,
});
} else {
let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let (rm, negated) = put_input_in_rse_imm12_maybe_negated(
ctx,
inputs[1],
ty_bits(ty),
NarrowValueMode::None,
);
let alu_op = if !negated {
choose_32_64(ty, ALUOp::Add32, ALUOp::Add64)
} else {
choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64)
};
ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm));
}
}
}
}
Opcode::Isub => {
let ty = ty.unwrap();
if ty == I128 {
let lhs = put_input_in_regs(ctx, inputs[0]);
let rhs = put_input_in_regs(ctx, inputs[1]);
let dst = get_output_reg(ctx, outputs[0]);
assert_eq!(lhs.len(), 2);
assert_eq!(rhs.len(), 2);
assert_eq!(dst.len(), 2);
// subs x0, x0, x2
// sbc x1, x1, x3
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::SubS64,
rd: dst.regs()[0],
rn: lhs.regs()[0],
rm: rhs.regs()[0],
});
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::Sbc64,
rd: dst.regs()[1],
rn: lhs.regs()[1],
rm: rhs.regs()[1],
});
} else {
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
if !ty.is_vector() {
let (rm, negated) = put_input_in_rse_imm12_maybe_negated(
ctx,
inputs[1],
ty_bits(ty),
NarrowValueMode::None,
);
let alu_op = if !negated {
choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64)
} else {
choose_32_64(ty, ALUOp::Add32, ALUOp::Add64)
};
ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm));
} else {
let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
ctx.emit(Inst::VecRRR {
rd,
rn,
rm,
alu_op: VecALUOp::Sub,
size: VectorSize::from_ty(ty),
});
}
}
}
Opcode::Iadd => implemented_in_isle(ctx),
Opcode::Isub => implemented_in_isle(ctx),
Opcode::UaddSat | Opcode::SaddSat | Opcode::UsubSat | Opcode::SsubSat => {
let ty = ty.unwrap();

43
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -385,49 +385,6 @@
(decl sink_load (SinkableLoad) RegMemImm)
(extern constructor sink_load sink_load)
;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Newtype wrapper around `MInst` for instructions that are used for their
;; effect on flags.
(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg))))
;; Newtype wrapper around `MInst` for instructions that consume flags.
(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg))))
;; Combine flags-producing and -consuming instructions together, ensuring that
;; they are emitted back-to-back and no other instructions can be emitted
;; between them and potentially clobber the flags.
;;
;; Returns a `ValueRegs` where the first register is the result of the
;; `ProducesFlags` instruction and the second is the result of the
;; `ConsumesFlags` instruction.
(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs)
(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result)
(ConsumesFlags.ConsumesFlags consumer_inst consumer_result))
(let ((_x Unit (emit producer_inst))
(_y Unit (emit consumer_inst)))
(value_regs producer_result consumer_result)))
;; Like `with_flags` but returns only the result of the consumer operation.
(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg)
(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result)
(ConsumesFlags.ConsumesFlags consumer_inst consumer_result))
(let ((_x Unit (emit producer_inst))
(_y Unit (emit consumer_inst)))
consumer_result))
;; Like `with_flags` but allows two consumers of the same flags. The result is a
;; `ValueRegs` containing the first consumer's result and then the second
;; consumer's result.
(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs)
(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result)
(ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1)
(ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2))
(let ((_x Unit (emit producer_inst))
(_y Unit (emit consumer_inst_1))
(_z Unit (emit consumer_inst_2)))
(value_regs consumer_result_1 consumer_result_2)))
;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(type ExtendKind (enum Sign Zero))

4
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb
src/prelude.isle c1391bcd436c23caf46b909ba7b5a352405014f0c393e3886cf1b9ad37f610b0563e8a64daad215f107395e6bb55744d955dd9c6344bb19b96587c2deb703462
src/isa/x64/inst.isle 6065d3b9e0fa3361d179d9b87d09568ff474f8bac7eeabd29b328ace723041f96045bc82cfa2d7feda4490ce7e5d4be1a1c7ebe25c99916564d43a51550cd093
src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c
src/isa/x64/inst.isle e4a0657406056a4cf116fe125e91d16377b602e0b41edd6628cbb7259b0fc2aa6b0482ffd33f00d63d68cf3546f188705877309d43eba5e75abd0f38a52a79b2
src/isa/x64/lower.isle e51b7a67343dba342a43b3c9e4b9ed7df9b2c66a677018acf7054ba48c27e4e93a4421fd892b9bf7c0e5b790bcfafab7cb3e93ce2b8206c04d456918d2ad0b5a

285
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

@@ -31,6 +31,7 @@ pub trait Context {
fn u16_as_u64(&mut self, arg0: u16) -> u64;
fn u32_as_u64(&mut self, arg0: u32) -> u64;
fn ty_bits(&mut self, arg0: Type) -> u16;
fn fits_in_32(&mut self, arg0: Type) -> Option<Type>;
fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice);
@@ -62,19 +63,19 @@ pub trait Context {
fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8;
}
/// Internal type ProducesFlags: defined at src/isa/x64/inst.isle line 392.
/// Internal type ProducesFlags: defined at src/prelude.isle line 230.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlags { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/isa/x64/inst.isle line 395.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 233.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlags { inst: MInst, result: Reg },
}
/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 433.
/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 390.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum ExtendKind {
Sign,
@@ -100,35 +101,6 @@ pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
return Some(expr2_0);
}
// Generated as internal constructor for term operand_size_bits.
pub fn constructor_operand_size_bits<C: Context>(ctx: &mut C, arg0: &OperandSize) -> Option<u16> {
let pattern0_0 = arg0;
match pattern0_0 {
&OperandSize::Size8 => {
// Rule at src/isa/x64/inst.isle line 80.
let expr0_0: u16 = 8;
return Some(expr0_0);
}
&OperandSize::Size16 => {
// Rule at src/isa/x64/inst.isle line 81.
let expr0_0: u16 = 16;
return Some(expr0_0);
}
&OperandSize::Size32 => {
// Rule at src/isa/x64/inst.isle line 82.
let expr0_0: u16 = 32;
return Some(expr0_0);
}
&OperandSize::Size64 => {
// Rule at src/isa/x64/inst.isle line 83.
let expr0_0: u16 = 64;
return Some(expr0_0);
}
_ => {}
}
return None;
}
// Generated as internal constructor for term with_flags.
pub fn constructor_with_flags<C: Context>(
ctx: &mut C,
@@ -147,7 +119,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/isa/x64/inst.isle line 405.
// Rule at src/prelude.isle line 243.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -175,7 +147,7 @@ pub fn constructor_with_flags_1<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/isa/x64/inst.isle line 413.
// Rule at src/prelude.isle line 251.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
return Some(pattern3_1);
@@ -209,7 +181,7 @@ pub fn constructor_with_flags_2<C: Context>(
result: pattern5_1,
} = pattern4_0
{
// Rule at src/isa/x64/inst.isle line 423.
// Rule at src/prelude.isle line 261.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
let expr2_0 = C::emit(ctx, &pattern5_0);
@@ -221,6 +193,35 @@ pub fn constructor_with_flags_2<C: Context>(
return None;
}
// Generated as internal constructor for term operand_size_bits.
pub fn constructor_operand_size_bits<C: Context>(ctx: &mut C, arg0: &OperandSize) -> Option<u16> {
let pattern0_0 = arg0;
match pattern0_0 {
&OperandSize::Size8 => {
// Rule at src/isa/x64/inst.isle line 80.
let expr0_0: u16 = 8;
return Some(expr0_0);
}
&OperandSize::Size16 => {
// Rule at src/isa/x64/inst.isle line 81.
let expr0_0: u16 = 16;
return Some(expr0_0);
}
&OperandSize::Size32 => {
// Rule at src/isa/x64/inst.isle line 82.
let expr0_0: u16 = 32;
return Some(expr0_0);
}
&OperandSize::Size64 => {
// Rule at src/isa/x64/inst.isle line 83.
let expr0_0: u16 = 64;
return Some(expr0_0);
}
_ => {}
}
return None;
}
// Generated as internal constructor for term extend_to_reg.
pub fn constructor_extend_to_reg<C: Context>(
ctx: &mut C,
@@ -233,12 +234,12 @@ pub fn constructor_extend_to_reg<C: Context>(
let pattern2_0 = arg1;
if pattern2_0 == pattern1_0 {
let pattern4_0 = arg2;
// Rule at src/isa/x64/inst.isle line 445.
// Rule at src/isa/x64/inst.isle line 402.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
return Some(expr0_0);
}
let pattern3_0 = arg2;
// Rule at src/isa/x64/inst.isle line 448.
// Rule at src/isa/x64/inst.isle line 405.
let expr0_0 = C::ty_bits(ctx, pattern1_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern2_0);
let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?;
@@ -262,7 +263,7 @@ pub fn constructor_extend<C: Context>(
let pattern2_0 = arg1;
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/x64/inst.isle line 468.
// Rule at src/isa/x64/inst.isle line 425.
let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?;
return Some(expr0_0);
}
@@ -270,7 +271,7 @@ pub fn constructor_extend<C: Context>(
let pattern2_0 = arg1;
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/x64/inst.isle line 464.
// Rule at src/isa/x64/inst.isle line 421.
let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?;
return Some(expr0_0);
}
@@ -283,17 +284,17 @@ pub fn constructor_extend<C: Context>(
pub fn constructor_sse_xor_op<C: Context>(ctx: &mut C, arg0: Type) -> Option<SseOpcode> {
let pattern0_0 = arg0;
if pattern0_0 == F32X4 {
// Rule at src/isa/x64/inst.isle line 475.
// Rule at src/isa/x64/inst.isle line 432.
let expr0_0 = SseOpcode::Xorps;
return Some(expr0_0);
}
if pattern0_0 == F64X2 {
// Rule at src/isa/x64/inst.isle line 476.
// Rule at src/isa/x64/inst.isle line 433.
let expr0_0 = SseOpcode::Xorpd;
return Some(expr0_0);
}
if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) {
// Rule at src/isa/x64/inst.isle line 477.
// Rule at src/isa/x64/inst.isle line 434.
let expr0_0 = SseOpcode::Pxor;
return Some(expr0_0);
}
@@ -310,7 +311,7 @@ pub fn constructor_sse_xor<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 481.
// Rule at src/isa/x64/inst.isle line 438.
let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?;
let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -320,40 +321,40 @@ pub fn constructor_sse_xor<C: Context>(
pub fn constructor_sse_cmp_op<C: Context>(ctx: &mut C, arg0: Type) -> Option<SseOpcode> {
let pattern0_0 = arg0;
if pattern0_0 == F32X4 {
// Rule at src/isa/x64/inst.isle line 490.
// Rule at src/isa/x64/inst.isle line 447.
let expr0_0 = SseOpcode::Cmpps;
return Some(expr0_0);
}
if pattern0_0 == F64X2 {
// Rule at src/isa/x64/inst.isle line 491.
// Rule at src/isa/x64/inst.isle line 448.
let expr0_0 = SseOpcode::Cmppd;
return Some(expr0_0);
}
if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) {
if pattern1_0 == 8 {
if pattern1_1 == 16 {
// Rule at src/isa/x64/inst.isle line 486.
// Rule at src/isa/x64/inst.isle line 443.
let expr0_0 = SseOpcode::Pcmpeqb;
return Some(expr0_0);
}
}
if pattern1_0 == 16 {
if pattern1_1 == 8 {
// Rule at src/isa/x64/inst.isle line 487.
// Rule at src/isa/x64/inst.isle line 444.
let expr0_0 = SseOpcode::Pcmpeqw;
return Some(expr0_0);
}
}
if pattern1_0 == 32 {
if pattern1_1 == 4 {
// Rule at src/isa/x64/inst.isle line 488.
// Rule at src/isa/x64/inst.isle line 445.
let expr0_0 = SseOpcode::Pcmpeqd;
return Some(expr0_0);
}
}
if pattern1_0 == 64 {
if pattern1_1 == 2 {
// Rule at src/isa/x64/inst.isle line 489.
// Rule at src/isa/x64/inst.isle line 446.
let expr0_0 = SseOpcode::Pcmpeqq;
return Some(expr0_0);
}
@@ -365,7 +366,7 @@ pub fn constructor_sse_cmp_op<C: Context>(ctx: &mut C, arg0: Type) -> Option<Sse
// Generated as internal constructor for term vector_all_ones.
pub fn constructor_vector_all_ones<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/isa/x64/inst.isle line 505.
// Rule at src/isa/x64/inst.isle line 462.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
let expr2_0: Type = I32X4;
@@ -393,7 +394,7 @@ pub fn constructor_alu_rmi_r<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 529.
// Rule at src/isa/x64/inst.isle line 486.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = MInst::AluRmiR {
@@ -418,7 +419,7 @@ pub fn constructor_add<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 537.
// Rule at src/isa/x64/inst.isle line 494.
let expr0_0 = AluRmiROpcode::Add;
let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -434,7 +435,7 @@ pub fn constructor_add_with_flags<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 545.
// Rule at src/isa/x64/inst.isle line 502.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = AluRmiROpcode::Add;
@@ -463,7 +464,7 @@ pub fn constructor_adc<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 556.
// Rule at src/isa/x64/inst.isle line 513.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = AluRmiROpcode::Adc;
@@ -492,7 +493,7 @@ pub fn constructor_sub<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 567.
// Rule at src/isa/x64/inst.isle line 524.
let expr0_0 = AluRmiROpcode::Sub;
let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -508,7 +509,7 @@ pub fn constructor_sub_with_flags<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 575.
// Rule at src/isa/x64/inst.isle line 532.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = AluRmiROpcode::Sub;
@@ -537,7 +538,7 @@ pub fn constructor_sbb<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 586.
// Rule at src/isa/x64/inst.isle line 543.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = AluRmiROpcode::Sbb;
@@ -566,7 +567,7 @@ pub fn constructor_mul<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 597.
// Rule at src/isa/x64/inst.isle line 554.
let expr0_0 = AluRmiROpcode::Mul;
let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -582,7 +583,7 @@ pub fn constructor_m_and<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 608.
// Rule at src/isa/x64/inst.isle line 565.
let expr0_0 = AluRmiROpcode::And;
let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -598,7 +599,7 @@ pub fn constructor_or<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 616.
// Rule at src/isa/x64/inst.isle line 573.
let expr0_0 = AluRmiROpcode::Or;
let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -614,7 +615,7 @@ pub fn constructor_xor<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 624.
// Rule at src/isa/x64/inst.isle line 581.
let expr0_0 = AluRmiROpcode::Xor;
let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -626,7 +627,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
if pattern0_0 == I64 {
let pattern2_0 = arg1;
if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) {
// Rule at src/isa/x64/inst.isle line 653.
// Rule at src/isa/x64/inst.isle line 610.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = OperandSize::Size32;
@@ -643,7 +644,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
if pattern0_0 == F32 {
let pattern2_0 = arg1;
if pattern2_0 == 0 {
// Rule at src/isa/x64/inst.isle line 682.
// Rule at src/isa/x64/inst.isle line 639.
let expr0_0: Type = F32;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
@@ -658,7 +659,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
let expr6_0 = C::emit(ctx, &expr5_0);
return Some(expr2_0);
}
// Rule at src/isa/x64/inst.isle line 641.
// Rule at src/isa/x64/inst.isle line 598.
let expr0_0: Type = F32;
let expr1_0 = SseOpcode::Movd;
let expr2_0: Type = I32;
@@ -671,7 +672,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
if pattern0_0 == F64 {
let pattern2_0 = arg1;
if pattern2_0 == 0 {
// Rule at src/isa/x64/inst.isle line 694.
// Rule at src/isa/x64/inst.isle line 651.
let expr0_0: Type = F64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
@@ -686,7 +687,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
let expr6_0 = C::emit(ctx, &expr5_0);
return Some(expr2_0);
}
// Rule at src/isa/x64/inst.isle line 645.
// Rule at src/isa/x64/inst.isle line 602.
let expr0_0: Type = F64;
let expr1_0 = SseOpcode::Movq;
let expr2_0: Type = I64;
@@ -699,7 +700,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) {
let pattern2_0 = arg1;
if pattern2_0 == 0 {
// Rule at src/isa/x64/inst.isle line 672.
// Rule at src/isa/x64/inst.isle line 629.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?;
@@ -716,7 +717,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
}
let pattern1_0 = arg1;
if pattern1_0 == 0 {
// Rule at src/isa/x64/inst.isle line 659.
// Rule at src/isa/x64/inst.isle line 616.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
let expr2_0 = C::operand_size_of_type(ctx, pattern0_0);
@@ -732,7 +733,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
let expr6_0 = C::emit(ctx, &expr5_0);
return Some(expr1_0);
}
// Rule at src/isa/x64/inst.isle line 634.
// Rule at src/isa/x64/inst.isle line 591.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = MInst::Imm {
@@ -757,7 +758,7 @@ pub fn constructor_shift_r<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 707.
// Rule at src/isa/x64/inst.isle line 664.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = MInst::ShiftR {
@@ -782,7 +783,7 @@ pub fn constructor_m_rotl<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 716.
// Rule at src/isa/x64/inst.isle line 673.
let expr0_0 = ShiftKind::RotateLeft;
let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -798,7 +799,7 @@ pub fn constructor_shl<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 721.
// Rule at src/isa/x64/inst.isle line 678.
let expr0_0 = ShiftKind::ShiftLeft;
let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -814,7 +815,7 @@ pub fn constructor_shr<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 726.
// Rule at src/isa/x64/inst.isle line 683.
let expr0_0 = ShiftKind::ShiftRightLogical;
let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -830,7 +831,7 @@ pub fn constructor_sar<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 731.
// Rule at src/isa/x64/inst.isle line 688.
let expr0_0 = ShiftKind::ShiftRightArithmetic;
let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -848,7 +849,7 @@ pub fn constructor_cmp_rmi_r<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 736.
// Rule at src/isa/x64/inst.isle line 693.
let expr0_0 = MInst::CmpRmiR {
size: pattern0_0.clone(),
opcode: pattern1_0.clone(),
@@ -873,7 +874,7 @@ pub fn constructor_cmp<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 745.
// Rule at src/isa/x64/inst.isle line 702.
let expr0_0 = CmpOpcode::Cmp;
let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -889,7 +890,7 @@ pub fn constructor_test<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 750.
// Rule at src/isa/x64/inst.isle line 707.
let expr0_0 = CmpOpcode::Test;
let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -907,7 +908,7 @@ pub fn constructor_cmove<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 755.
// Rule at src/isa/x64/inst.isle line 712.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
let expr2_0 = MInst::Cmove {
@@ -935,7 +936,7 @@ pub fn constructor_movzx<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 763.
// Rule at src/isa/x64/inst.isle line 720.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = MInst::MovzxRmR {
ext_mode: pattern1_0.clone(),
@@ -957,7 +958,7 @@ pub fn constructor_movsx<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 770.
// Rule at src/isa/x64/inst.isle line 727.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = MInst::MovsxRmR {
ext_mode: pattern1_0.clone(),
@@ -981,7 +982,7 @@ pub fn constructor_xmm_rm_r<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 777.
// Rule at src/isa/x64/inst.isle line 734.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = MInst::XmmRmR {
op: pattern1_0.clone(),
@@ -998,7 +999,7 @@ pub fn constructor_xmm_rm_r<C: Context>(
pub fn constructor_paddb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 784.
// Rule at src/isa/x64/inst.isle line 741.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Paddb;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1009,7 +1010,7 @@ pub fn constructor_paddb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_paddw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 789.
// Rule at src/isa/x64/inst.isle line 746.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Paddw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1020,7 +1021,7 @@ pub fn constructor_paddw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_paddd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 794.
// Rule at src/isa/x64/inst.isle line 751.
let expr0_0: Type = I32X4;
let expr1_0 = SseOpcode::Paddd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1031,7 +1032,7 @@ pub fn constructor_paddd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_paddq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 799.
// Rule at src/isa/x64/inst.isle line 756.
let expr0_0: Type = I64X2;
let expr1_0 = SseOpcode::Paddq;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1042,7 +1043,7 @@ pub fn constructor_paddq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_paddsb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 804.
// Rule at src/isa/x64/inst.isle line 761.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Paddsb;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1053,7 +1054,7 @@ pub fn constructor_paddsb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_paddsw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 809.
// Rule at src/isa/x64/inst.isle line 766.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Paddsw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1064,7 +1065,7 @@ pub fn constructor_paddsw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_paddusb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 814.
// Rule at src/isa/x64/inst.isle line 771.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Paddusb;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1075,7 +1076,7 @@ pub fn constructor_paddusb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_paddusw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 819.
// Rule at src/isa/x64/inst.isle line 776.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Paddusw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1086,7 +1087,7 @@ pub fn constructor_paddusw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_psubb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 824.
// Rule at src/isa/x64/inst.isle line 781.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Psubb;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1097,7 +1098,7 @@ pub fn constructor_psubb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_psubw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 829.
// Rule at src/isa/x64/inst.isle line 786.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Psubw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1108,7 +1109,7 @@ pub fn constructor_psubw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_psubd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 834.
// Rule at src/isa/x64/inst.isle line 791.
let expr0_0: Type = I32X4;
let expr1_0 = SseOpcode::Psubd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1119,7 +1120,7 @@ pub fn constructor_psubd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_psubq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 839.
// Rule at src/isa/x64/inst.isle line 796.
let expr0_0: Type = I64X2;
let expr1_0 = SseOpcode::Psubq;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1130,7 +1131,7 @@ pub fn constructor_psubq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_psubsb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 844.
// Rule at src/isa/x64/inst.isle line 801.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Psubsb;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1141,7 +1142,7 @@ pub fn constructor_psubsb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_psubsw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 849.
// Rule at src/isa/x64/inst.isle line 806.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Psubsw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1152,7 +1153,7 @@ pub fn constructor_psubsw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_psubusb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 854.
// Rule at src/isa/x64/inst.isle line 811.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Psubusb;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1163,7 +1164,7 @@ pub fn constructor_psubusb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_psubusw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 859.
// Rule at src/isa/x64/inst.isle line 816.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Psubusw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1174,7 +1175,7 @@ pub fn constructor_psubusw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_pavgb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 864.
// Rule at src/isa/x64/inst.isle line 821.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Pavgb;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1185,7 +1186,7 @@ pub fn constructor_pavgb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_pavgw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 869.
// Rule at src/isa/x64/inst.isle line 826.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Pavgw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1196,7 +1197,7 @@ pub fn constructor_pavgw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_pand<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 874.
// Rule at src/isa/x64/inst.isle line 831.
let expr0_0: Type = F32X4;
let expr1_0 = SseOpcode::Pand;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1207,7 +1208,7 @@ pub fn constructor_pand<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op
pub fn constructor_andps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 879.
// Rule at src/isa/x64/inst.isle line 836.
let expr0_0: Type = F32X4;
let expr1_0 = SseOpcode::Andps;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1218,7 +1219,7 @@ pub fn constructor_andps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_andpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 884.
// Rule at src/isa/x64/inst.isle line 841.
let expr0_0: Type = F64X2;
let expr1_0 = SseOpcode::Andpd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1229,7 +1230,7 @@ pub fn constructor_andpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_por<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 889.
// Rule at src/isa/x64/inst.isle line 846.
let expr0_0: Type = F32X4;
let expr1_0 = SseOpcode::Por;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1240,7 +1241,7 @@ pub fn constructor_por<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt
pub fn constructor_orps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 894.
// Rule at src/isa/x64/inst.isle line 851.
let expr0_0: Type = F32X4;
let expr1_0 = SseOpcode::Orps;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1251,7 +1252,7 @@ pub fn constructor_orps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op
pub fn constructor_orpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 899.
// Rule at src/isa/x64/inst.isle line 856.
let expr0_0: Type = F64X2;
let expr1_0 = SseOpcode::Orpd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1262,7 +1263,7 @@ pub fn constructor_orpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op
pub fn constructor_pxor<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 904.
// Rule at src/isa/x64/inst.isle line 861.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Pxor;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1273,7 +1274,7 @@ pub fn constructor_pxor<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op
pub fn constructor_xorps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 909.
// Rule at src/isa/x64/inst.isle line 866.
let expr0_0: Type = F32X4;
let expr1_0 = SseOpcode::Xorps;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1284,7 +1285,7 @@ pub fn constructor_xorps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_xorpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 914.
// Rule at src/isa/x64/inst.isle line 871.
let expr0_0: Type = F64X2;
let expr1_0 = SseOpcode::Xorpd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1295,7 +1296,7 @@ pub fn constructor_xorpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_pmullw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 919.
// Rule at src/isa/x64/inst.isle line 876.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Pmullw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1306,7 +1307,7 @@ pub fn constructor_pmullw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_pmulld<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 924.
// Rule at src/isa/x64/inst.isle line 881.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Pmulld;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1317,7 +1318,7 @@ pub fn constructor_pmulld<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_pmulhw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 929.
// Rule at src/isa/x64/inst.isle line 886.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Pmulhw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1328,7 +1329,7 @@ pub fn constructor_pmulhw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_pmulhuw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 934.
// Rule at src/isa/x64/inst.isle line 891.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Pmulhuw;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1339,7 +1340,7 @@ pub fn constructor_pmulhuw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_pmuldq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 939.
// Rule at src/isa/x64/inst.isle line 896.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Pmuldq;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1350,7 +1351,7 @@ pub fn constructor_pmuldq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_pmuludq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 944.
// Rule at src/isa/x64/inst.isle line 901.
let expr0_0: Type = I64X2;
let expr1_0 = SseOpcode::Pmuludq;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1361,7 +1362,7 @@ pub fn constructor_pmuludq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_punpckhwd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 949.
// Rule at src/isa/x64/inst.isle line 906.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Punpckhwd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1372,7 +1373,7 @@ pub fn constructor_punpckhwd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem)
pub fn constructor_punpcklwd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 954.
// Rule at src/isa/x64/inst.isle line 911.
let expr0_0: Type = I16X8;
let expr1_0 = SseOpcode::Punpcklwd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1383,7 +1384,7 @@ pub fn constructor_punpcklwd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem)
pub fn constructor_andnps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 959.
// Rule at src/isa/x64/inst.isle line 916.
let expr0_0: Type = F32X4;
let expr1_0 = SseOpcode::Andnps;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1394,7 +1395,7 @@ pub fn constructor_andnps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_andnpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 964.
// Rule at src/isa/x64/inst.isle line 921.
let expr0_0: Type = F64X2;
let expr1_0 = SseOpcode::Andnpd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1405,7 +1406,7 @@ pub fn constructor_andnpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) ->
pub fn constructor_pandn<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 969.
// Rule at src/isa/x64/inst.isle line 926.
let expr0_0: Type = F64X2;
let expr1_0 = SseOpcode::Pandn;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1426,7 +1427,7 @@ pub fn constructor_xmm_rm_r_imm<C: Context>(
let pattern2_0 = arg2;
let pattern3_0 = arg3;
let pattern4_0 = arg4;
// Rule at src/isa/x64/inst.isle line 974.
// Rule at src/isa/x64/inst.isle line 931.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::XmmRmRImm {
@@ -1454,7 +1455,7 @@ pub fn constructor_palignr<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 986.
// Rule at src/isa/x64/inst.isle line 943.
let expr0_0 = SseOpcode::Palignr;
let expr1_0 = constructor_xmm_rm_r_imm(
ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -1472,7 +1473,7 @@ pub fn constructor_pshufd<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 995.
// Rule at src/isa/x64/inst.isle line 952.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
@@ -1497,7 +1498,7 @@ pub fn constructor_xmm_unary_rm_r<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 1008.
// Rule at src/isa/x64/inst.isle line 965.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::XmmUnaryRmR {
@@ -1513,7 +1514,7 @@ pub fn constructor_xmm_unary_rm_r<C: Context>(
// Generated as internal constructor for term pmovsxbw.
pub fn constructor_pmovsxbw<C: Context>(ctx: &mut C, arg0: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/isa/x64/inst.isle line 1015.
// Rule at src/isa/x64/inst.isle line 972.
let expr0_0 = SseOpcode::Pmovsxbw;
let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?;
return Some(expr1_0);
@@ -1522,7 +1523,7 @@ pub fn constructor_pmovsxbw<C: Context>(ctx: &mut C, arg0: &RegMem) -> Option<Re
// Generated as internal constructor for term pmovzxbw.
pub fn constructor_pmovzxbw<C: Context>(ctx: &mut C, arg0: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/isa/x64/inst.isle line 1020.
// Rule at src/isa/x64/inst.isle line 977.
let expr0_0 = SseOpcode::Pmovzxbw;
let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?;
return Some(expr1_0);
@@ -1538,7 +1539,7 @@ pub fn constructor_xmm_rm_r_evex<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1025.
// Rule at src/isa/x64/inst.isle line 982.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::XmmRmREvex {
@@ -1556,7 +1557,7 @@ pub fn constructor_xmm_rm_r_evex<C: Context>(
pub fn constructor_vpmullq<C: Context>(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 1037.
// Rule at src/isa/x64/inst.isle line 994.
let expr0_0 = Avx512Opcode::Vpmullq;
let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?;
return Some(expr1_0);
@@ -1572,7 +1573,7 @@ pub fn constructor_xmm_rmi_reg<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1044.
// Rule at src/isa/x64/inst.isle line 1001.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::XmmRmiReg {
@@ -1590,7 +1591,7 @@ pub fn constructor_xmm_rmi_reg<C: Context>(
pub fn constructor_psllq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 1054.
// Rule at src/isa/x64/inst.isle line 1011.
let expr0_0 = SseOpcode::Psllq;
let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?;
return Some(expr1_0);
@@ -1600,7 +1601,7 @@ pub fn constructor_psllq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -
pub fn constructor_psrld<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 1059.
// Rule at src/isa/x64/inst.isle line 1016.
let expr0_0 = SseOpcode::Psrld;
let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?;
return Some(expr1_0);
@@ -1610,7 +1611,7 @@ pub fn constructor_psrld<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -
pub fn constructor_psrlq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 1064.
// Rule at src/isa/x64/inst.isle line 1021.
let expr0_0 = SseOpcode::Psrlq;
let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?;
return Some(expr1_0);
@@ -1628,7 +1629,7 @@ pub fn constructor_mul_hi<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 1071.
// Rule at src/isa/x64/inst.isle line 1028.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr2_0 = C::operand_size_of_type(ctx, pattern0_0);
@@ -1657,7 +1658,7 @@ pub fn constructor_mulhi_u<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1087.
// Rule at src/isa/x64/inst.isle line 1044.
let expr0_0: bool = false;
let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?;
return Some(expr1_0);
@@ -1673,7 +1674,7 @@ pub fn constructor_cmpps<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1092.
// Rule at src/isa/x64/inst.isle line 1049.
let expr0_0 = SseOpcode::Cmpps;
let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0);
let expr2_0 = OperandSize::Size32;
@@ -1692,7 +1693,7 @@ pub fn constructor_cmppd<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1105.
// Rule at src/isa/x64/inst.isle line 1062.
let expr0_0 = SseOpcode::Cmppd;
let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0);
let expr2_0 = OperandSize::Size32;
@@ -1713,7 +1714,7 @@ pub fn constructor_gpr_to_xmm<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 1114.
// Rule at src/isa/x64/inst.isle line 1071.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = MInst::GprToXmm {
op: pattern1_0.clone(),
@@ -1736,7 +1737,7 @@ pub fn constructor_pinsrb<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1121.
// Rule at src/isa/x64/inst.isle line 1078.
let expr0_0 = SseOpcode::Pinsrb;
let expr1_0 = OperandSize::Size32;
let expr2_0 =
@@ -1754,7 +1755,7 @@ pub fn constructor_pinsrw<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1126.
// Rule at src/isa/x64/inst.isle line 1083.
let expr0_0 = SseOpcode::Pinsrw;
let expr1_0 = OperandSize::Size32;
let expr2_0 =
@@ -1774,7 +1775,7 @@ pub fn constructor_pinsrd<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/x64/inst.isle line 1131.
// Rule at src/isa/x64/inst.isle line 1088.
let expr0_0 = SseOpcode::Pinsrd;
let expr1_0 = constructor_xmm_rm_r_imm(
ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
@@ -1792,7 +1793,7 @@ pub fn constructor_insertps<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/x64/inst.isle line 1136.
// Rule at src/isa/x64/inst.isle line 1093.
let expr0_0 = SseOpcode::Insertps;
let expr1_0 = OperandSize::Size32;
let expr2_0 =
@@ -1804,7 +1805,7 @@ pub fn constructor_insertps<C: Context>(
pub fn constructor_movsd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 1141.
// Rule at src/isa/x64/inst.isle line 1098.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Movsd;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
@@ -1815,7 +1816,7 @@ pub fn constructor_movsd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O
pub fn constructor_movlhps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/inst.isle line 1146.
// Rule at src/isa/x64/inst.isle line 1103.
let expr0_0: Type = I8X16;
let expr1_0 = SseOpcode::Movlhps;
let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;

9
cranelift/codegen/src/machinst/isle.rs
View File

@@ -90,6 +90,15 @@ macro_rules! isle_prelude_methods {
ty.bits()
}
#[inline]
fn fits_in_32(&mut self, ty: Type) -> Option<Type> {
if ty.bits() <= 32 {
Some(ty)
} else {
None
}
}
#[inline]
fn fits_in_64(&mut self, ty: Type) -> Option<Type> {
if ty.bits() <= 64 {

19
cranelift/codegen/src/machinst/lower.rs
View File

@@ -105,6 +105,8 @@ pub trait LowerCtx {
fn num_outputs(&self, ir_inst: Inst) -> usize;
/// Get the type for an instruction's input.
fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type;
/// Get the type for a value.
fn value_ty(&self, val: Value) -> Type;
/// Get the type for an instruction's output.
fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type;
/// Get the value of a constant instruction (`iconst`, etc.) as a 64-bit
@@ -129,6 +131,9 @@ pub trait LowerCtx {
fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput;
/// Like `get_input_as_source_or_const` but with a `Value`.
fn get_value_as_source_or_const(&self, value: Value) -> NonRegInput;
/// Resolves a particular input of an instruction to the `Value` that it is
/// represented with.
fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value;
/// Put the `idx`th input into register(s) and return the assigned register.
fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>;
/// Put the given value into register(s) and return the assigned register.
@@ -1109,8 +1114,10 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
}
fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type {
let val = self.f.dfg.inst_args(ir_inst)[idx];
let val = self.f.dfg.resolve_aliases(val);
self.value_ty(self.input_as_value(ir_inst, idx))
}
fn value_ty(&self, val: Value) -> Type {
self.f.dfg.value_type(val)
}
@@ -1122,9 +1129,13 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
self.inst_constants.get(&ir_inst).cloned()
}
fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput {
fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value {
let val = self.f.dfg.inst_args(ir_inst)[idx];
let val = self.f.dfg.resolve_aliases(val);
self.f.dfg.resolve_aliases(val)
}
fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput {
let val = self.input_as_value(ir_inst, idx);
self.get_value_as_source_or_const(val)
}

48
cranelift/codegen/src/prelude.isle
View File

@@ -144,6 +144,10 @@
;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; An extractor that only matches types that can fit in 32 bits.
(decl fits_in_32 (Type) Type)
(extern extractor fits_in_32 fits_in_32)
;; An extractor that only matches types that can fit in 64 bits.
(decl fits_in_64 (Type) Type)
(extern extractor fits_in_64 fits_in_64)
@@ -218,3 +222,47 @@
;; Match the instruction that defines the given value, if any.
(decl def_inst (Inst) Value)
(extern extractor def_inst def_inst)
;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Newtype wrapper around `MInst` for instructions that are used for their
;; effect on flags.
(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg))))
;; Newtype wrapper around `MInst` for instructions that consume flags.
(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg))))
;; Combine flags-producing and -consuming instructions together, ensuring that
;; they are emitted back-to-back and no other instructions can be emitted
;; between them and potentially clobber the flags.
;;
;; Returns a `ValueRegs` where the first register is the result of the
;; `ProducesFlags` instruction and the second is the result of the
;; `ConsumesFlags` instruction.
(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs)
(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result)
(ConsumesFlags.ConsumesFlags consumer_inst consumer_result))
(let ((_x Unit (emit producer_inst))
(_y Unit (emit consumer_inst)))
(value_regs producer_result consumer_result)))
;; Like `with_flags` but returns only the result of the consumer operation.
(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg)
(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result)
(ConsumesFlags.ConsumesFlags consumer_inst consumer_result))
(let ((_x Unit (emit producer_inst))
(_y Unit (emit consumer_inst)))
consumer_result))
;; Like `with_flags` but allows two consumers of the same flags. The result is a
;; `ValueRegs` containing the first consumer's result and then the second
;; consumer's result.
(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs)
(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result)
(ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1)
(ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2))
(let ((_x Unit (emit producer_inst))
(_y Unit (emit consumer_inst_1))
(_z Unit (emit consumer_inst_2)))
(value_regs consumer_result_1 consumer_result_2)))

20
cranelift/filetests/filetests/isa/aarch64/arithmetic.clif
View File

@@ -369,3 +369,23 @@ block0(v0: i128, v1: i128):
; nextln: madd x0, x0, x2, xzr
; nextln: ret
function %add_mul_1(i32, i32, i32) -> i32 {
block0(v0: i32, v1: i32, v2: i32):
v3 = imul v1, v2
v4 = iadd v0, v3
return v4
}
; check: madd w0, w1, w2, w0
; nextln: ret
function %add_mul_2(i32, i32, i32) -> i32 {
block0(v0: i32, v1: i32, v2: i32):
v3 = imul v1, v2
v4 = iadd v3, v0
return v4
}
; check: madd w0, w1, w2, w0
; nextln: ret

15
cranelift/filetests/filetests/isa/aarch64/extend-op.clif
View File

@@ -10,8 +10,19 @@ block0(v0: i8):
return v3
}
; check: movz x1, #42
; nextln: add x0, x1, x0, SXTB
; check: sxtb x0, w0
; nextln: add x0, x0, #42
; nextln: ret
function %f2(i8, i64) -> i64 {
block0(v0: i8, v1: i64):
v2 = sextend.i64 v0
v3 = iadd.i64 v2, v1
return v3
}
; check: add x0, x1, x0, SXTB
; nextln: ret