diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index dc00a84cc4..54cdd0b574 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -329,6 +329,19 @@ Vpmullq Vpopcntb)) +(type FcmpImm extern + (enum Equal + LessThan + LessThanOrEqual + Unordered + NotEqual + UnorderedOrGreaterThanOrEqual + UnorderedOrGreaterThan + Ordered)) + +(decl encode_fcmp_imm (FcmpImm) u8) +(extern constructor encode_fcmp_imm encode_fcmp_imm) + ;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl avx512vl_enabled () Type) @@ -450,6 +463,49 @@ (rule (extend (ExtendKind.Sign) ty mode src) (movsx ty mode src)) +;;;; Helpers for Working SSE tidbits ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Determine the appropriate operation for xor-ing vectors of the specified type +(decl sse_xor_op (Type) SseOpcode) +(rule (sse_xor_op $F32X4) (SseOpcode.Xorps)) +(rule (sse_xor_op $F64X2) (SseOpcode.Xorpd)) +(rule (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor)) + +;; Performs an xor operation of the two operands specified +(decl sse_xor (Type Reg RegMem) Reg) +(rule (sse_xor ty x y) (xmm_rm_r ty (sse_xor_op ty) x y)) + +;; Determine the appropriate operation to compare two vectors of the specified +;; type. +(decl sse_cmp_op (Type) SseOpcode) +(rule (sse_cmp_op (multi_lane 8 16)) (SseOpcode.Pcmpeqb)) +(rule (sse_cmp_op (multi_lane 16 8)) (SseOpcode.Pcmpeqw)) +(rule (sse_cmp_op (multi_lane 32 4)) (SseOpcode.Pcmpeqd)) +(rule (sse_cmp_op (multi_lane 64 2)) (SseOpcode.Pcmpeqq)) +(rule (sse_cmp_op $F32X4) (SseOpcode.Cmpps)) +(rule (sse_cmp_op $F64X2) (SseOpcode.Cmppd)) + +;; Generates a register value which has an all-ones pattern of the specified +;; type. +;; +;; Note that this is accomplished by comparing a fresh register with itself, +;; which for integers is always true. Also note that the comparison is always +;; done for integers, it doesn't actually take the input `ty` into account. This +;; is because we're comparing a fresh register to itself and we don't know the +;; previous contents of the register. If a floating-point comparison is used +;; then it runs the risk of comparing NaN against NaN and not actually producing +;; an all-ones mask. By using integer comparision operations we're guaranteeed +;; that everything is equal to itself. +(decl vector_all_ones (Type) Reg) +(rule (vector_all_ones ty) + (let ((wr WritableReg (temp_writable_reg ty)) + (r Reg (writable_reg_to_reg wr)) + (_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4) + r + (RegMem.Reg r) + wr)))) + r)) + ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These constructors create SSA-style `MInst`s. It is their responsibility to @@ -596,6 +652,17 @@ wr)))) r)) +;; Special case for zero immediates with vector types, they turn into an xor +;; specific to the vector type. +(rule (imm ty @ (multi_lane _bits _lanes) 0) + (let ((wr WritableReg (temp_writable_reg ty)) + (r Reg (writable_reg_to_reg wr)) + (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty) + r + (RegMem.Reg r) + wr)))) + r)) + ;; Helper for creating `MInst.ShifR` instructions. (decl shift_r (Type ShiftKind Reg Imm8Reg) Reg) (rule (shift_r ty kind src1 src2) @@ -948,6 +1015,11 @@ (rule (psllq src1 src2) (xmm_rmi_reg (SseOpcode.Psllq) src1 src2)) +;; Helper for creating `psrld` instructions. +(decl psrld (Reg RegMemImm) Reg) +(rule (psrld src1 src2) + (xmm_rmi_reg (SseOpcode.Psrld) src1 src2)) + ;; Helper for creating `psrlq` instructions. (decl psrlq (Reg RegMemImm) Reg) (rule (psrlq src1 src2) @@ -975,3 +1047,25 @@ (decl mulhi_u (Type Reg RegMem) ValueRegs) (rule (mulhi_u ty src1 src2) (mul_hi ty $false src1 src2)) + +;; Helper for creating `cmpps` instructions. +(decl cmpps (Reg RegMem FcmpImm) Reg) +(rule (cmpps src1 src2 imm) + (xmm_rm_r_imm (SseOpcode.Cmpps) + src1 + src2 + (encode_fcmp_imm imm) + (OperandSize.Size32))) + +;; Helper for creating `cmppd` instructions. +;; +;; Note that `Size32` is intentional despite this being used for 64-bit +;; operations, since this presumably induces the correct encoding of the +;; instruction. +(decl cmppd (Reg RegMem FcmpImm) Reg) +(rule (cmppd src1 src2 imm) + (xmm_rm_r_imm (SseOpcode.Cmppd) + src1 + src2 + (encode_fcmp_imm imm) + (OperandSize.Size32))) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index e7aaf108fe..74e3373c6c 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -1391,7 +1391,8 @@ impl fmt::Display for CC { /// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`, /// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS /// whereas [FcmpImm] is used as an immediate. -pub(crate) enum FcmpImm { +#[derive(Clone, Copy)] +pub enum FcmpImm { Equal = 0x00, LessThan = 0x01, LessThanOrEqual = 0x02, diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index d011cd2b97..db1b48226f 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1301,31 +1301,6 @@ impl Inst { } } - /// Choose which instruction to use for comparing two values for equality. - pub(crate) fn equals(ty: Type, from: RegMem, to: Writable) -> Inst { - match ty { - types::I8X16 | types::B8X16 => Inst::xmm_rm_r(SseOpcode::Pcmpeqb, from, to), - types::I16X8 | types::B16X8 => Inst::xmm_rm_r(SseOpcode::Pcmpeqw, from, to), - types::I32X4 | types::B32X4 => Inst::xmm_rm_r(SseOpcode::Pcmpeqd, from, to), - types::I64X2 | types::B64X2 => Inst::xmm_rm_r(SseOpcode::Pcmpeqq, from, to), - types::F32X4 => Inst::xmm_rm_r_imm( - SseOpcode::Cmpps, - from, - to, - FcmpImm::Equal.encode(), - OperandSize::Size32, - ), - types::F64X2 => Inst::xmm_rm_r_imm( - SseOpcode::Cmppd, - from, - to, - FcmpImm::Equal.encode(), - OperandSize::Size32, - ), - _ => unimplemented!("unimplemented type for Inst::equals: {}", ty), - } - } - /// Choose which instruction to use for computing a bitwise AND on two values. pub(crate) fn and(ty: Type, from: RegMem, to: Writable) -> Inst { match ty { @@ -1356,16 +1331,6 @@ impl Inst { } } - /// Choose which instruction to use for computing a bitwise XOR on two values. - pub(crate) fn xor(ty: Type, from: RegMem, to: Writable) -> Inst { - match ty { - types::F32X4 => Inst::xmm_rm_r(SseOpcode::Xorps, from, to), - types::F64X2 => Inst::xmm_rm_r(SseOpcode::Xorpd, from, to), - _ if ty.is_vector() && ty.bits() == 128 => Inst::xmm_rm_r(SseOpcode::Pxor, from, to), - _ => unimplemented!("unimplemented type for Inst::xor: {}", ty), - } - } - /// Translate three-operand instructions into a sequence of two-operand /// instructions. /// diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index ee67b81f11..185cb23ce8 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -484,18 +484,8 @@ ;; SSE. -(rule (lower (has_type $F32X4 (bxor x y))) - (value_reg (xorps (put_in_reg x) - (put_in_reg_mem y)))) - -(rule (lower (has_type $F64X2 (bxor x y))) - (value_reg (xorpd (put_in_reg x) - (put_in_reg_mem y)))) - -(rule (lower (has_type (multi_lane _bits _lanes) - (bxor x y))) - (value_reg (pxor (put_in_reg x) - (put_in_reg_mem y)))) +(rule (lower (has_type ty @ (multi_lane _bits _lanes) (bxor x y))) + (value_reg (sse_xor ty (put_in_reg x) (put_in_reg_mem y)))) ;; `{i,b}128`. @@ -945,3 +935,22 @@ (rule (lower (has_type (multi_lane _bits _lanes) (band_not x y))) (value_reg (pandn (put_in_reg y) (put_in_reg_mem x)))) + +;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Special case for `f32x4.abs`. +(rule (lower (has_type $F32X4 (fabs x))) + (value_reg (andps (put_in_reg x) + (RegMem.Reg (psrld (vector_all_ones $F32X4) (RegMemImm.Imm 1)))))) + +;; Special case for `f64x2.abs`. +(rule (lower (has_type $F64X2 (fabs x))) + (value_reg (andpd (put_in_reg x) + (RegMem.Reg (psrlq (vector_all_ones $F64X2) (RegMemImm.Imm 1)))))) + +;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Special case for vector-types where bit-negation is an xor against an +;; all-one value +(rule (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x))) + (value_reg (sse_xor ty (put_in_reg x) (RegMem.Reg (vector_all_ones ty))))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index e7549a0841..77bdfd35b2 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1615,14 +1615,11 @@ fn lower_insn_to_regs>( let ty = ty.unwrap(); if ty.is_vector() { - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, src, ty)); - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - - // Set tmp to all 1s before flipping the bits - ctx.emit(Inst::equals(types::I32X4, RegMem::from(tmp), tmp)); - ctx.emit(Inst::xor(ty, RegMem::from(tmp), dst)); + unreachable!( + "implemented in ISLE: inst = `{}`, type = `{:?}`", + ctx.dfg().display_inst(insn), + ty + ); } else if ty == types::I128 || ty == types::B128 { let src = put_input_in_regs(ctx, inputs[0]); let dst = get_output_reg(ctx, outputs[0]); @@ -4669,8 +4666,13 @@ fn lower_insn_to_regs>( // Shift the all 1s constant to generate the mask. let lane_bits = output_ty.lane_bits(); let (shift_opcode, opcode, shift_by) = match (op, lane_bits) { - (Opcode::Fabs, 32) => (SseOpcode::Psrld, SseOpcode::Andps, 1), - (Opcode::Fabs, 64) => (SseOpcode::Psrlq, SseOpcode::Andpd, 1), + (Opcode::Fabs, _) => { + unreachable!( + "implemented in ISLE: inst = `{}`, type = `{:?}`", + ctx.dfg().display_inst(insn), + ty + ); + } (Opcode::Fneg, 32) => (SseOpcode::Pslld, SseOpcode::Xorps, 31), (Opcode::Fneg, 64) => (SseOpcode::Psllq, SseOpcode::Xorpd, 63), _ => unreachable!( diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 1fe15ea976..a79d699f42 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -13,7 +13,9 @@ use crate::isa::x64::settings as x64_settings; use crate::{ ir::{immediates::*, types::*, Inst, InstructionData, Opcode, Value, ValueList}, isa::x64::inst::{ - args::{Avx512Opcode, CmpOpcode, ExtMode, Imm8Reg, RegMem, ShiftKind, SseOpcode, CC}, + args::{ + Avx512Opcode, CmpOpcode, ExtMode, FcmpImm, Imm8Reg, RegMem, ShiftKind, SseOpcode, CC, + }, x64_map_regs, RegMapper, }, machinst::{get_output_reg, InsnInput, InsnOutput, LowerCtx}, @@ -313,6 +315,11 @@ where RegMem::reg(self.put_in_reg(val)) } + #[inline] + fn encode_fcmp_imm(&mut self, imm: &FcmpImm) -> u8 { + imm.encode() + } + #[inline] fn avx512vl_enabled(&mut self, _: Type) -> Option<()> { if self.isa_flags.use_avx512vl_simd() { diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index da0932027e..04bfcca29a 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -45,6 +45,7 @@ pub trait Context { fn def_inst(&mut self, arg0: Value) -> Option; fn operand_size_of_type(&mut self, arg0: Type) -> OperandSize; fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; + fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn imm8_from_value(&mut self, arg0: Value) -> Option; @@ -57,19 +58,19 @@ pub trait Context { fn nonzero_u64_fits_in_u32(&mut self, arg0: u64) -> Option; } -/// Internal type ProducesFlags: defined at src/isa/x64/inst.isle line 374. +/// Internal type ProducesFlags: defined at src/isa/x64/inst.isle line 387. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/isa/x64/inst.isle line 377. +/// Internal type ConsumesFlags: defined at src/isa/x64/inst.isle line 390. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 415. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 428. #[derive(Clone, Debug)] pub enum ExtendKind { Sign, @@ -142,7 +143,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/isa/x64/inst.isle line 387. + // Rule at src/isa/x64/inst.isle line 400. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -170,7 +171,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/isa/x64/inst.isle line 395. + // Rule at src/isa/x64/inst.isle line 408. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -204,7 +205,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/isa/x64/inst.isle line 405. + // Rule at src/isa/x64/inst.isle line 418. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -228,12 +229,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 427. + // Rule at src/isa/x64/inst.isle line 440. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 430. + // Rule at src/isa/x64/inst.isle line 443. let expr0_0 = C::ty_bits(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -257,7 +258,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 450. + // Rule at src/isa/x64/inst.isle line 463. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -265,7 +266,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 446. + // Rule at src/isa/x64/inst.isle line 459. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -274,6 +275,108 @@ pub fn constructor_extend( return None; } +// Generated as internal constructor for term sse_xor_op. +pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if pattern0_0 == F32X4 { + // Rule at src/isa/x64/inst.isle line 470. + let expr0_0 = SseOpcode::Xorps; + return Some(expr0_0); + } + if pattern0_0 == F64X2 { + // Rule at src/isa/x64/inst.isle line 471. + let expr0_0 = SseOpcode::Xorpd; + return Some(expr0_0); + } + if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { + // Rule at src/isa/x64/inst.isle line 472. + let expr0_0 = SseOpcode::Pxor; + return Some(expr0_0); + } + return None; +} + +// Generated as internal constructor for term sse_xor. +pub fn constructor_sse_xor( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMem, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 476. + let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; + let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term sse_cmp_op. +pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + if pattern0_0 == F32X4 { + // Rule at src/isa/x64/inst.isle line 485. + let expr0_0 = SseOpcode::Cmpps; + return Some(expr0_0); + } + if pattern0_0 == F64X2 { + // Rule at src/isa/x64/inst.isle line 486. + let expr0_0 = SseOpcode::Cmppd; + return Some(expr0_0); + } + if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { + if pattern1_0 == 8 { + if pattern1_1 == 16 { + // Rule at src/isa/x64/inst.isle line 481. + let expr0_0 = SseOpcode::Pcmpeqb; + return Some(expr0_0); + } + } + if pattern1_0 == 16 { + if pattern1_1 == 8 { + // Rule at src/isa/x64/inst.isle line 482. + let expr0_0 = SseOpcode::Pcmpeqw; + return Some(expr0_0); + } + } + if pattern1_0 == 32 { + if pattern1_1 == 4 { + // Rule at src/isa/x64/inst.isle line 483. + let expr0_0 = SseOpcode::Pcmpeqd; + return Some(expr0_0); + } + } + if pattern1_0 == 64 { + if pattern1_1 == 2 { + // Rule at src/isa/x64/inst.isle line 484. + let expr0_0 = SseOpcode::Pcmpeqq; + return Some(expr0_0); + } + } + } + return None; +} + +// Generated as internal constructor for term vector_all_ones. +pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 500. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr2_0: Type = I32X4; + let expr3_0 = constructor_sse_cmp_op(ctx, expr2_0)?; + let expr4_0 = RegMem::Reg { reg: expr1_0 }; + let expr5_0 = MInst::XmmRmR { + op: expr3_0, + src1: expr1_0, + src2: expr4_0, + dst: expr0_0, + }; + let expr6_0 = C::emit(ctx, &expr5_0); + return Some(expr1_0); +} + // Generated as internal constructor for term alu_rmi_r. pub fn constructor_alu_rmi_r( ctx: &mut C, @@ -286,7 +389,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 468. + // Rule at src/isa/x64/inst.isle line 524. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -311,7 +414,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 532. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -327,7 +430,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 484. + // Rule at src/isa/x64/inst.isle line 540. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -356,7 +459,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 495. + // Rule at src/isa/x64/inst.isle line 551. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -385,7 +488,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 506. + // Rule at src/isa/x64/inst.isle line 562. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -401,7 +504,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 514. + // Rule at src/isa/x64/inst.isle line 570. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -430,7 +533,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 525. + // Rule at src/isa/x64/inst.isle line 581. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -459,7 +562,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 536. + // Rule at src/isa/x64/inst.isle line 592. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -475,7 +578,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 547. + // Rule at src/isa/x64/inst.isle line 603. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -491,7 +594,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 555. + // Rule at src/isa/x64/inst.isle line 611. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -507,7 +610,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 563. + // Rule at src/isa/x64/inst.isle line 619. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -519,7 +622,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 582. + // Rule at src/isa/x64/inst.isle line 638. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -533,9 +636,27 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option return Some(expr5_0); } } + if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { + let pattern2_0 = arg1; + if pattern2_0 == 0 { + // Rule at src/isa/x64/inst.isle line 657. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; + let expr3_0 = RegMem::Reg { reg: expr1_0 }; + let expr4_0 = MInst::XmmRmR { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, + }; + let expr5_0 = C::emit(ctx, &expr4_0); + return Some(expr1_0); + } + } let pattern1_0 = arg1; if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 588. + // Rule at src/isa/x64/inst.isle line 644. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -551,7 +672,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 571. + // Rule at src/isa/x64/inst.isle line 627. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Imm { @@ -576,7 +697,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 601. + // Rule at src/isa/x64/inst.isle line 668. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -601,7 +722,7 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 610. + // Rule at src/isa/x64/inst.isle line 677. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -617,7 +738,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 615. + // Rule at src/isa/x64/inst.isle line 682. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -633,7 +754,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 620. + // Rule at src/isa/x64/inst.isle line 687. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -649,7 +770,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 625. + // Rule at src/isa/x64/inst.isle line 692. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -667,7 +788,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 630. + // Rule at src/isa/x64/inst.isle line 697. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -692,7 +813,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 639. + // Rule at src/isa/x64/inst.isle line 706. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -708,7 +829,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 644. + // Rule at src/isa/x64/inst.isle line 711. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -726,7 +847,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 649. + // Rule at src/isa/x64/inst.isle line 716. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -754,7 +875,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 657. + // Rule at src/isa/x64/inst.isle line 724. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -776,7 +897,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 664. + // Rule at src/isa/x64/inst.isle line 731. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -800,7 +921,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 671. + // Rule at src/isa/x64/inst.isle line 738. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -817,7 +938,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 678. + // Rule at src/isa/x64/inst.isle line 745. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -828,7 +949,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 683. + // Rule at src/isa/x64/inst.isle line 750. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -839,7 +960,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 688. + // Rule at src/isa/x64/inst.isle line 755. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -850,7 +971,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 693. + // Rule at src/isa/x64/inst.isle line 760. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -861,7 +982,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 698. + // Rule at src/isa/x64/inst.isle line 765. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -872,7 +993,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 703. + // Rule at src/isa/x64/inst.isle line 770. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -883,7 +1004,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 708. + // Rule at src/isa/x64/inst.isle line 775. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -894,7 +1015,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 713. + // Rule at src/isa/x64/inst.isle line 780. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -905,7 +1026,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 718. + // Rule at src/isa/x64/inst.isle line 785. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -916,7 +1037,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 723. + // Rule at src/isa/x64/inst.isle line 790. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -927,7 +1048,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 728. + // Rule at src/isa/x64/inst.isle line 795. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -938,7 +1059,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 733. + // Rule at src/isa/x64/inst.isle line 800. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -949,7 +1070,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 738. + // Rule at src/isa/x64/inst.isle line 805. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -960,7 +1081,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 743. + // Rule at src/isa/x64/inst.isle line 810. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -971,7 +1092,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 748. + // Rule at src/isa/x64/inst.isle line 815. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -982,7 +1103,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 753. + // Rule at src/isa/x64/inst.isle line 820. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -993,7 +1114,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 758. + // Rule at src/isa/x64/inst.isle line 825. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1004,7 +1125,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 763. + // Rule at src/isa/x64/inst.isle line 830. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1015,7 +1136,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 768. + // Rule at src/isa/x64/inst.isle line 835. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1026,7 +1147,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 773. + // Rule at src/isa/x64/inst.isle line 840. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1037,7 +1158,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 778. + // Rule at src/isa/x64/inst.isle line 845. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1048,7 +1169,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 783. + // Rule at src/isa/x64/inst.isle line 850. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1059,7 +1180,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 788. + // Rule at src/isa/x64/inst.isle line 855. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1070,7 +1191,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 793. + // Rule at src/isa/x64/inst.isle line 860. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1081,7 +1202,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 798. + // Rule at src/isa/x64/inst.isle line 865. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1092,7 +1213,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 803. + // Rule at src/isa/x64/inst.isle line 870. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1103,7 +1224,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 808. + // Rule at src/isa/x64/inst.isle line 875. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1114,7 +1235,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 813. + // Rule at src/isa/x64/inst.isle line 880. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1125,7 +1246,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 818. + // Rule at src/isa/x64/inst.isle line 885. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1136,7 +1257,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 823. + // Rule at src/isa/x64/inst.isle line 890. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1147,7 +1268,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 828. + // Rule at src/isa/x64/inst.isle line 895. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1158,7 +1279,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 833. + // Rule at src/isa/x64/inst.isle line 900. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1169,7 +1290,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 838. + // Rule at src/isa/x64/inst.isle line 905. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1180,7 +1301,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 843. + // Rule at src/isa/x64/inst.isle line 910. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1191,7 +1312,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 848. + // Rule at src/isa/x64/inst.isle line 915. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1202,7 +1323,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 853. + // Rule at src/isa/x64/inst.isle line 920. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1213,7 +1334,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 858. + // Rule at src/isa/x64/inst.isle line 925. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1224,7 +1345,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 863. + // Rule at src/isa/x64/inst.isle line 930. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1245,7 +1366,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 868. + // Rule at src/isa/x64/inst.isle line 935. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1273,7 +1394,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 880. + // Rule at src/isa/x64/inst.isle line 947. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1291,7 +1412,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 956. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1316,7 +1437,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 902. + // Rule at src/isa/x64/inst.isle line 969. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1332,7 +1453,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 976. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1341,7 +1462,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 981. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1357,7 +1478,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 986. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -1375,7 +1496,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 931. + // Rule at src/isa/x64/inst.isle line 998. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1391,7 +1512,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 938. + // Rule at src/isa/x64/inst.isle line 1005. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -1409,17 +1530,27 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 948. + // Rule at src/isa/x64/inst.isle line 1015. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); } +// Generated as internal constructor for term psrld. +pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1020. + let expr0_0 = SseOpcode::Psrld; + let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + // Generated as internal constructor for term psrlq. pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 953. + // Rule at src/isa/x64/inst.isle line 1025. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1437,7 +1568,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 960. + // Rule at src/isa/x64/inst.isle line 1032. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -1466,12 +1597,50 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 976. + // Rule at src/isa/x64/inst.isle line 1048. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } +// Generated as internal constructor for term cmpps. +pub fn constructor_cmpps( + ctx: &mut C, + arg0: Reg, + arg1: &RegMem, + arg2: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1053. + let expr0_0 = SseOpcode::Cmpps; + let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr2_0 = OperandSize::Size32; + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; + return Some(expr3_0); +} + +// Generated as internal constructor for term cmppd. +pub fn constructor_cmppd( + ctx: &mut C, + arg0: Reg, + arg1: &RegMem, + arg2: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1066. + let expr0_0 = SseOpcode::Cmppd; + let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr2_0 = OperandSize::Size32; + let expr3_0 = + constructor_xmm_rm_r_imm(ctx, &expr0_0, pattern0_0, pattern1_0, expr1_0, &expr2_0)?; + return Some(expr3_0); +} + // Generated as internal constructor for term lower. pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern0_0 = arg0; @@ -1549,7 +1718,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 512. + // Rule at src/isa/x64/lower.isle line 502. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1643,7 +1812,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 696. + // Rule at src/isa/x64/lower.isle line 686. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1711,7 +1880,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 502. + // Rule at src/isa/x64/lower.isle line 492. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1734,7 +1903,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 629. + // Rule at src/isa/x64/lower.isle line 619. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -1751,7 +1920,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 562. + // Rule at src/isa/x64/lower.isle line 552. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?; @@ -1760,7 +1929,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 608. + // Rule at src/isa/x64/lower.isle line 598. let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?; @@ -1800,98 +1969,126 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 333. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_andps(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); + match &pattern4_0 { + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match &pattern5_0 { + &Opcode::Band => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 333. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_andps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 409. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_orps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::BandNot => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 930. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); + let expr2_0 = constructor_andnps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} } - &Opcode::Bor => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 409. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_orps(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - &Opcode::Bxor => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 487. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_xorps(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - &Opcode::BandNot => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 940. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); - let expr2_0 = constructor_andnps(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - _ => {} } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + if let &Opcode::Fabs = &pattern5_0 { + // Rule at src/isa/x64/lower.isle line 942. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0: Type = F32X4; + let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; + let expr3_0: u32 = 1; + let expr4_0 = RegMemImm::Imm { simm32: expr3_0 }; + let expr5_0 = constructor_psrld(ctx, expr2_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { reg: expr5_0 }; + let expr7_0 = constructor_andps(ctx, expr0_0, &expr6_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some(expr8_0); + } + } + _ => {} } } if pattern2_0 == F64X2 { let pattern4_0 = C::inst_data(ctx, pattern0_0); - if let &InstructionData::Binary { - opcode: ref pattern5_0, - args: ref pattern5_1, - } = &pattern4_0 - { - match &pattern5_0 { - &Opcode::Band => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 337. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_andpd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); + match &pattern4_0 { + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match &pattern5_0 { + &Opcode::Band => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 337. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_andpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 413. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_orpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::BandNot => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 933. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); + let expr2_0 = constructor_andnpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} } - &Opcode::Bor => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 413. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_orpd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - &Opcode::Bxor => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 491. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_xorpd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - &Opcode::BandNot => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 943. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); - let expr2_0 = constructor_andnpd(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - _ => {} } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + if let &Opcode::Fabs = &pattern5_0 { + // Rule at src/isa/x64/lower.isle line 947. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0: Type = F64X2; + let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; + let expr3_0: u32 = 1; + let expr4_0 = RegMemImm::Imm { simm32: expr3_0 }; + let expr5_0 = constructor_psrlq(ctx, expr2_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { reg: expr5_0 }; + let expr7_0 = constructor_andpd(ctx, expr0_0, &expr6_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some(expr8_0); + } + } + _ => {} } } let pattern3_0 = C::inst_data(ctx, pattern0_0); @@ -1921,7 +2118,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 639. + // Rule at src/isa/x64/lower.isle line 629. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -2031,7 +2228,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 643. + // Rule at src/isa/x64/lower.isle line 633. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -2144,7 +2341,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 341. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_pand(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); + match &pattern4_0 { + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match &pattern5_0 { + &Opcode::Band => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 341. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_pand(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 417. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_por(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bxor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 487. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::BandNot => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 936. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); + let expr2_0 = constructor_pandn(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} } - &Opcode::Bor => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 417. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_por(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - &Opcode::Bxor => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 495. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); - let expr2_0 = constructor_pxor(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - &Opcode::BandNot => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 946. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); - let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); - let expr2_0 = constructor_pandn(ctx, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); - return Some(expr3_0); - } - _ => {} } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + if let &Opcode::Bnot = &pattern5_0 { + // Rule at src/isa/x64/lower.isle line 955. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; + let expr2_0 = RegMem::Reg { reg: expr1_0 }; + let expr3_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + _ => {} } } if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) { @@ -3189,7 +3406,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 539. + // Rule at src/isa/x64/lower.isle line 529. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; @@ -3584,7 +3801,7 @@ pub fn constructor_shr_i128( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 586. + // Rule at src/isa/x64/lower.isle line 576. let expr0_0: usize = 0; let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); let expr2_0: usize = 1; diff --git a/tests/misc_testsuite/simd/issue_3327_bnot_lowering.wast b/tests/misc_testsuite/simd/issue_3327_bnot_lowering.wast index 0d30c630d7..4563afe89f 100644 --- a/tests/misc_testsuite/simd/issue_3327_bnot_lowering.wast +++ b/tests/misc_testsuite/simd/issue_3327_bnot_lowering.wast @@ -7,4 +7,50 @@ v128.not) ) -(assert_return (invoke "v128_not") (v128.const i32x4 -1 -1 -1 -1)) \ No newline at end of file +(assert_return (invoke "v128_not") (v128.const i32x4 -1 -1 -1 -1)) + +;; from #3327 +(module + (func (result i32) + v128.const i32x4 0xffffffff 0x80bfffff 0x80bf0a0a 0x80bf0a0a + f64x2.promote_low_f32x4 + v128.not + v128.not + v128.not + v128.not + v128.not + v128.not + v128.not + v128.const i32x4 0 0 0 0 + f64x2.gt + v128.not + i64x2.bitmask) + (export "" (func 0))) +(assert_return (invoke "") (i32.const 0)) + +;; from #3327 +(module + (type (func (param i32) (result i32))) + (func (type 0) (param i32) (result i32) + local.get 0 + i32x4.splat + f64x2.abs + v128.not + i64x2.bitmask) + (export "1" (func 0))) +(assert_return (invoke "1" (i32.const 0)) (i32.const 3)) + +(module + (type (;0;) (func (result v128))) + (func (;0;) (type 0) (result v128) + v128.const i32x4 0x733c3e67 0x3c3e6776 0x3e677673 0x6776733c + i64x2.abs + i64x2.bitmask + i8x16.splat + v128.const i32x4 0x733c3e67 0x3c3e6776 0x3e677673 0x6776733c + i64x2.ge_s + f32x4.floor + v128.not + i16x8.extadd_pairwise_i8x16_u) + (export "x" (func 0))) +(assert_return (invoke "x") (v128.const i32x4 0x01fe01fe 0x01fe01fe 0x01fe01fe 0x01fe01fe))