From 4120e4031820598dcb7163333aeb7b58e69381d3 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 Jan 2022 13:19:51 -0800 Subject: [PATCH 1/7] cranelift: Update assertions to indicate that `rotl` is fully ported to ISLE on x64 --- cranelift/codegen/src/isa/x64/lower.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 606b208c17..576a3fdf38 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1541,17 +1541,14 @@ fn lower_insn_to_regs>( | Opcode::Vselect | Opcode::Ushr | Opcode::Sshr - | Opcode::Ishl => implemented_in_isle(ctx), + | Opcode::Ishl + | Opcode::Rotl => implemented_in_isle(ctx), - Opcode::Rotl | Opcode::Rotr => { + Opcode::Rotr => { let dst_ty = ctx.output_ty(insn, 0); debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty); if !dst_ty.is_vector() && dst_ty.bits() <= 64 { - if op != Opcode::Rotr { - implemented_in_isle(ctx); - } - // Scalar shifts on x86 have various encodings: // - shift by one bit, e.g. `SAL r/m8, 1` (not used here) // - shift by an immediate amount, e.g. `SAL r/m8, imm8` From a41fdb030389c50d40b6cd7d0e7286fad7b46e81 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 Jan 2022 13:22:48 -0800 Subject: [PATCH 2/7] cranelift: Port `rotr` lowering to ISLE on x64 --- cranelift/codegen/src/isa/x64/inst.isle | 6 + cranelift/codegen/src/isa/x64/lower.isle | 40 ++ cranelift/codegen/src/isa/x64/lower.rs | 405 +------------ .../x64/lower/isle/generated_code.manifest | 4 +- .../src/isa/x64/lower/isle/generated_code.rs | 551 +++++++++++------- .../filetests/filetests/isa/x64/i128.clif | 94 ++- 6 files changed, 422 insertions(+), 678 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 3a6f4089f5..0f79daa4bb 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -811,6 +811,12 @@ (rule (m_rotl ty src1 src2) (shift_r ty (ShiftKind.RotateLeft) src1 src2)) +;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach +;; inst", to disambiguate this from clif's `rotr`). +(decl m_rotr (Type Reg Imm8Reg) Reg) +(rule (m_rotr ty src1 src2) + (shift_r ty (ShiftKind.RotateRight) src1 src2)) + ;; Helper for creating `shl` instructions. (decl shl (Type Reg Imm8Reg) Reg) (rule (shl ty src1 src2) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 04513a1c5d..3422b7bc46 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -865,6 +865,46 @@ (or_i128 (shl_i128 src_ amt_) (shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) +;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i16` and `i8`: we need to extend the shift amount, or mask the +;; constant. + +(rule (lower (has_type (ty_8_or_16 ty) (rotr src amt))) + (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) + (value_reg (m_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (ty_8_or_16 ty) + (rotr src (u64_from_iconst amt)))) + (value_reg (m_rotr ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) + +;; `i64` and `i32`: we can rely on x86's rotate-amount masking since +;; we operate on the whole register. + +(rule (lower (has_type (ty_32_or_64 ty) (rotr src amt))) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; shift amount to the value's bit width. + (let ((amt_ Reg (lo_reg amt))) + (value_reg (m_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (ty_32_or_64 ty) + (rotr src (u64_from_iconst amt)))) + (value_reg (m_rotr ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) + +;; `i128`. + +(rule (lower (has_type $I128 (rotr src amt))) + (let ((src_ ValueRegs (put_in_regs src)) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; rotation amount to the value's bit width. + (amt_ Reg (lo_reg amt))) + (or_i128 (shr_i128 src_ amt_) + (shl_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) + ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 576a3fdf38..f6d2a7721b 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -877,316 +877,6 @@ fn emit_bitrev>(ctx: &mut C, src: Reg, dst: Writable, ctx.emit(Inst::gen_move(dst, tmp0.to_reg(), types::I64)); } -fn emit_shl_i128>( - ctx: &mut C, - src: ValueRegs, - dst: ValueRegs>, - amt_src: Reg, -) { - let src_lo = src.regs()[0]; - let src_hi = src.regs()[1]; - let dst_lo = dst.regs()[0]; - let dst_hi = dst.regs()[1]; - - // mov tmp1, src_lo - // shl tmp1, amt_src - // mov tmp2, src_hi - // shl tmp2, amt_src - // mov amt, 64 - // sub amt, amt_src - // mov tmp3, src_lo - // shr tmp3, amt - // xor dst_lo, dst_lo - // test amt_src, 127 - // cmovz tmp3, dst_lo - // or tmp3, tmp2 - // mov amt, amt_src - // and amt, 64 - // cmovz dst_hi, tmp3 - // cmovz dst_lo, tmp1 - // cmovnz dst_hi, tmp1 - - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - - ctx.emit(Inst::gen_move(tmp1, src_lo, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftLeft, - None, - tmp1, - )); - - ctx.emit(Inst::gen_move(tmp2, src_hi, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftLeft, - None, - tmp2, - )); - - ctx.emit(Inst::imm(OperandSize::Size64, 64, amt)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(amt_src), - amt, - )); - - ctx.emit(Inst::gen_move(tmp3, src_lo, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt.to_reg(), - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - None, - tmp3, - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_lo.to_reg()), - dst_lo, - )); - - ctx.emit(Inst::test_rmi_r( - OperandSize::Size64, - RegMemImm::imm(127), - amt_src, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(dst_lo.to_reg()), - tmp3, - )); - - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp2.to_reg()), - tmp3, - )); - - // This isn't semantically necessary, but it keeps the - // register allocator happy, because it cannot otherwise - // infer that cmovz + cmovnz always defines dst_hi. - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_hi.to_reg()), - dst_hi, - )); - - ctx.emit(Inst::gen_move(amt, amt_src, types::I64)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::imm(64), - amt, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp3.to_reg()), - dst_hi, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp1.to_reg()), - dst_lo, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::NZ, - RegMem::reg(tmp1.to_reg()), - dst_hi, - )); -} - -fn emit_shr_i128>( - ctx: &mut C, - src: ValueRegs, - dst: ValueRegs>, - amt_src: Reg, - is_signed: bool, -) { - let src_lo = src.regs()[0]; - let src_hi = src.regs()[1]; - let dst_lo = dst.regs()[0]; - let dst_hi = dst.regs()[1]; - - // mov tmp1, src_hi - // {u,s}shr tmp1, amt_src - // mov tmp2, src_lo - // ushr tmp2, amt_src - // mov amt, 64 - // sub amt, amt_src - // mov tmp3, src_hi - // shl tmp3, amt - // xor dst_lo, dst_lo - // test amt_src, 127 - // cmovz tmp3, dst_lo - // or tmp3, tmp2 - // if is_signed: - // mov dst_hi, src_hi - // sshr dst_hi, 63 // get the sign bit - // else: - // xor dst_hi, dst_hi - // mov amt, amt_src - // and amt, 64 - // cmovz dst_hi, tmp1 - // cmovz dst_lo, tmp3 - // cmovnz dst_lo, tmp1 - - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - - let shift_kind = if is_signed { - ShiftKind::ShiftRightArithmetic - } else { - ShiftKind::ShiftRightLogical - }; - - ctx.emit(Inst::gen_move(tmp1, src_hi, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - ctx.emit(Inst::shift_r(OperandSize::Size64, shift_kind, None, tmp1)); - - ctx.emit(Inst::gen_move(tmp2, src_lo, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt_src, - types::I64, - )); - // N.B.: right-shift of *lower* half is *always* unsigned (its MSB is not a sign bit). - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - None, - tmp2, - )); - - ctx.emit(Inst::imm(OperandSize::Size64, 64, amt)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(amt_src), - amt, - )); - - ctx.emit(Inst::gen_move(tmp3, src_hi, types::I64)); - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::rcx()), - amt.to_reg(), - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftLeft, - None, - tmp3, - )); - - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_lo.to_reg()), - dst_lo, - )); - ctx.emit(Inst::test_rmi_r( - OperandSize::Size64, - RegMemImm::imm(127), - amt_src, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(dst_lo.to_reg()), - tmp3, - )); - - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp2.to_reg()), - tmp3, - )); - - if is_signed { - ctx.emit(Inst::gen_move(dst_hi, src_hi, types::I64)); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightArithmetic, - Some(63), - dst_hi, - )); - } else { - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_hi.to_reg()), - dst_hi, - )); - } - // This isn't semantically necessary, but it keeps the - // register allocator happy, because it cannot otherwise - // infer that cmovz + cmovnz always defines dst_lo. - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst_lo.to_reg()), - dst_lo, - )); - - ctx.emit(Inst::gen_move(amt, amt_src, types::I64)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::imm(64), - amt, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp1.to_reg()), - dst_hi, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp3.to_reg()), - dst_lo, - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::NZ, - RegMem::reg(tmp1.to_reg()), - dst_lo, - )); -} - fn make_libcall_sig>( ctx: &mut C, insn: IRInst, @@ -1542,99 +1232,8 @@ fn lower_insn_to_regs>( | Opcode::Ushr | Opcode::Sshr | Opcode::Ishl - | Opcode::Rotl => implemented_in_isle(ctx), - - Opcode::Rotr => { - let dst_ty = ctx.output_ty(insn, 0); - debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty); - - if !dst_ty.is_vector() && dst_ty.bits() <= 64 { - // Scalar shifts on x86 have various encodings: - // - shift by one bit, e.g. `SAL r/m8, 1` (not used here) - // - shift by an immediate amount, e.g. `SAL r/m8, imm8` - // - shift by a dynamic amount but only from the CL register, e.g. `SAL r/m8, CL`. - // This implementation uses the last two encoding methods. - let (size, lhs) = match dst_ty { - types::I8 | types::I16 => match op { - Opcode::Rotr => ( - OperandSize::from_ty(dst_ty), - put_input_in_reg(ctx, inputs[0]), - ), - _ => unreachable!(), - }, - types::I32 | types::I64 => ( - OperandSize::from_ty(dst_ty), - put_input_in_reg(ctx, inputs[0]), - ), - _ => unreachable!("unhandled output type for shift/rotates: {}", dst_ty), - }; - - let (count, rhs) = - if let Some(cst) = ctx.get_input_as_source_or_const(insn, 1).constant { - // Mask count, according to Cranelift's semantics. - let cst = (cst as u8) & (dst_ty.bits() as u8 - 1); - (Some(cst), None) - } else { - // We can ignore upper registers if shift amount is multi-reg, because we - // are taking the shift amount mod 2^(lhs_width) anyway. - (None, Some(put_input_in_regs(ctx, inputs[1]).regs()[0])) - }; - - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - let shift_kind = match op { - Opcode::Rotr => ShiftKind::RotateRight, - _ => unreachable!(), - }; - - let w_rcx = Writable::from_reg(regs::rcx()); - ctx.emit(Inst::mov_r_r(OperandSize::Size64, lhs, dst)); - if count.is_none() { - ctx.emit(Inst::mov_r_r(OperandSize::Size64, rhs.unwrap(), w_rcx)); - } - ctx.emit(Inst::shift_r(size, shift_kind, count, dst)); - } else if dst_ty == types::I128 { - let amt_src = put_input_in_regs(ctx, inputs[1]).regs()[0]; - let src = put_input_in_regs(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]); - - match op { - Opcode::Rotr => { - // (mov tmp, src) - // (ushr.i128 tmp, amt) - // (mov dst, src) - // (shl.i128 dst, 128-amt) - // (or dst, tmp) - let tmp = ctx.alloc_tmp(types::I128); - emit_shr_i128(ctx, src, tmp, amt_src, /* is_signed = */ false); - let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(amt_src), - inv_amt, - )); - emit_shl_i128(ctx, src, dst, inv_amt.to_reg()); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp.regs()[0].to_reg()), - dst.regs()[0], - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp.regs()[1].to_reg()), - dst.regs()[1], - )); - } - _ => unreachable!(), - } - } else { - implemented_in_isle(ctx); - } - } + | Opcode::Rotl + | Opcode::Rotr => implemented_in_isle(ctx), Opcode::Ineg => { let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 2ed8c1a2e3..c9c3b0595c 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 src/prelude.isle 7b911d3b894ae17 -src/isa/x64/inst.isle 41304d8ef6f7d816 -src/isa/x64/lower.isle 4689585f55f41438 +src/isa/x64/inst.isle 54ffef8c4f373807 +src/isa/x64/lower.isle 28de5d6bf49c8471 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 354ca57192..018eb9b0c8 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -971,6 +971,22 @@ pub fn constructor_m_rotl( return Some(expr1_0); } +// Generated as internal constructor for term m_rotr. +pub fn constructor_m_rotr( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 817. + let expr0_0 = ShiftKind::RotateRight; + let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + // Generated as internal constructor for term shl. pub fn constructor_shl( ctx: &mut C, @@ -981,7 +997,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 816. + // Rule at src/isa/x64/inst.isle line 822. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -997,7 +1013,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 821. + // Rule at src/isa/x64/inst.isle line 827. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1013,7 +1029,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 826. + // Rule at src/isa/x64/inst.isle line 832. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1031,7 +1047,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 831. + // Rule at src/isa/x64/inst.isle line 837. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1056,7 +1072,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 840. + // Rule at src/isa/x64/inst.isle line 846. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1072,7 +1088,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 845. + // Rule at src/isa/x64/inst.isle line 851. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1090,7 +1106,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 850. + // Rule at src/isa/x64/inst.isle line 856. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1118,7 +1134,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 858. + // Rule at src/isa/x64/inst.isle line 864. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1140,7 +1156,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 865. + // Rule at src/isa/x64/inst.isle line 871. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1164,7 +1180,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 872. + // Rule at src/isa/x64/inst.isle line 878. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1181,7 +1197,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 879. + // Rule at src/isa/x64/inst.isle line 885. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1192,7 +1208,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 884. + // Rule at src/isa/x64/inst.isle line 890. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1203,7 +1219,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 895. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1214,7 +1230,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 900. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1225,7 +1241,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 905. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1236,7 +1252,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 910. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1247,7 +1263,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 915. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1258,7 +1274,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 920. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1269,7 +1285,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 925. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1280,7 +1296,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 924. + // Rule at src/isa/x64/inst.isle line 930. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1291,7 +1307,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 929. + // Rule at src/isa/x64/inst.isle line 935. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1302,7 +1318,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 940. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1313,7 +1329,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 939. + // Rule at src/isa/x64/inst.isle line 945. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1324,7 +1340,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 944. + // Rule at src/isa/x64/inst.isle line 950. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1335,7 +1351,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 949. + // Rule at src/isa/x64/inst.isle line 955. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1346,7 +1362,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 954. + // Rule at src/isa/x64/inst.isle line 960. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1357,7 +1373,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 959. + // Rule at src/isa/x64/inst.isle line 965. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1368,7 +1384,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 964. + // Rule at src/isa/x64/inst.isle line 970. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1379,7 +1395,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 969. + // Rule at src/isa/x64/inst.isle line 975. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1390,7 +1406,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 974. + // Rule at src/isa/x64/inst.isle line 980. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1401,7 +1417,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 979. + // Rule at src/isa/x64/inst.isle line 985. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1412,7 +1428,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 984. + // Rule at src/isa/x64/inst.isle line 990. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1423,7 +1439,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 989. + // Rule at src/isa/x64/inst.isle line 995. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1434,7 +1450,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 994. + // Rule at src/isa/x64/inst.isle line 1000. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1445,7 +1461,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 999. + // Rule at src/isa/x64/inst.isle line 1005. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1456,7 +1472,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1004. + // Rule at src/isa/x64/inst.isle line 1010. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1467,7 +1483,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1009. + // Rule at src/isa/x64/inst.isle line 1015. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1478,7 +1494,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1014. + // Rule at src/isa/x64/inst.isle line 1020. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1489,7 +1505,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1019. + // Rule at src/isa/x64/inst.isle line 1025. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1500,7 +1516,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1024. + // Rule at src/isa/x64/inst.isle line 1030. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1511,7 +1527,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1029. + // Rule at src/isa/x64/inst.isle line 1035. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1522,7 +1538,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1034. + // Rule at src/isa/x64/inst.isle line 1040. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1533,7 +1549,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1039. + // Rule at src/isa/x64/inst.isle line 1045. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1544,7 +1560,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1044. + // Rule at src/isa/x64/inst.isle line 1050. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1555,7 +1571,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1049. + // Rule at src/isa/x64/inst.isle line 1055. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1566,7 +1582,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1054. + // Rule at src/isa/x64/inst.isle line 1060. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1577,7 +1593,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1059. + // Rule at src/isa/x64/inst.isle line 1065. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1588,7 +1604,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1064. + // Rule at src/isa/x64/inst.isle line 1070. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1599,17 +1615,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1068. + // Rule at src/isa/x64/inst.isle line 1074. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1069. + // Rule at src/isa/x64/inst.isle line 1075. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1070. + // Rule at src/isa/x64/inst.isle line 1076. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1620,17 +1636,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1073. + // Rule at src/isa/x64/inst.isle line 1079. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1074. + // Rule at src/isa/x64/inst.isle line 1080. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1075. + // Rule at src/isa/x64/inst.isle line 1081. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1649,7 +1665,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1079. + // Rule at src/isa/x64/inst.isle line 1085. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1673,7 +1689,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1091. + // Rule at src/isa/x64/inst.isle line 1097. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = RegMem::Reg { reg: pattern2_0 }; @@ -1693,7 +1709,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1103. + // Rule at src/isa/x64/inst.isle line 1109. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1704,7 +1720,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1108. + // Rule at src/isa/x64/inst.isle line 1114. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1715,7 +1731,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1113. + // Rule at src/isa/x64/inst.isle line 1119. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1726,7 +1742,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1118. + // Rule at src/isa/x64/inst.isle line 1124. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1737,7 +1753,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1123. + // Rule at src/isa/x64/inst.isle line 1129. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1748,7 +1764,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1128. + // Rule at src/isa/x64/inst.isle line 1134. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1759,7 +1775,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1133. + // Rule at src/isa/x64/inst.isle line 1139. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1770,7 +1786,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1138. + // Rule at src/isa/x64/inst.isle line 1144. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1781,7 +1797,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1143. + // Rule at src/isa/x64/inst.isle line 1149. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1792,7 +1808,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1148. + // Rule at src/isa/x64/inst.isle line 1154. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1803,7 +1819,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1153. + // Rule at src/isa/x64/inst.isle line 1159. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1814,7 +1830,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1158. + // Rule at src/isa/x64/inst.isle line 1164. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1825,7 +1841,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1163. + // Rule at src/isa/x64/inst.isle line 1169. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1836,7 +1852,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1168. + // Rule at src/isa/x64/inst.isle line 1174. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1847,7 +1863,7 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1173. + // Rule at src/isa/x64/inst.isle line 1179. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1858,7 +1874,7 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1178. + // Rule at src/isa/x64/inst.isle line 1184. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1869,7 +1885,7 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1183. + // Rule at src/isa/x64/inst.isle line 1189. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1890,7 +1906,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1188. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1918,7 +1934,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1200. + // Rule at src/isa/x64/inst.isle line 1206. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1936,7 +1952,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1209. + // Rule at src/isa/x64/inst.isle line 1215. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1961,7 +1977,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1222. + // Rule at src/isa/x64/inst.isle line 1228. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1977,7 +1993,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1229. + // Rule at src/isa/x64/inst.isle line 1235. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1986,7 +2002,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1234. + // Rule at src/isa/x64/inst.isle line 1240. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1995,7 +2011,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1239. + // Rule at src/isa/x64/inst.isle line 1245. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2004,7 +2020,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1244. + // Rule at src/isa/x64/inst.isle line 1250. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2013,7 +2029,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1249. + // Rule at src/isa/x64/inst.isle line 1255. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2027,7 +2043,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1254. + // Rule at src/isa/x64/inst.isle line 1260. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmREvex { @@ -2043,7 +2059,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1261. + // Rule at src/isa/x64/inst.isle line 1267. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2059,7 +2075,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1266. + // Rule at src/isa/x64/inst.isle line 1272. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -2077,7 +2093,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1278. + // Rule at src/isa/x64/inst.isle line 1284. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2093,7 +2109,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1285. + // Rule at src/isa/x64/inst.isle line 1291. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -2111,7 +2127,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1295. + // Rule at src/isa/x64/inst.isle line 1301. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2121,7 +2137,7 @@ pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1300. + // Rule at src/isa/x64/inst.isle line 1306. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2131,7 +2147,7 @@ pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1305. + // Rule at src/isa/x64/inst.isle line 1311. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2141,7 +2157,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1310. + // Rule at src/isa/x64/inst.isle line 1316. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2151,7 +2167,7 @@ pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1315. + // Rule at src/isa/x64/inst.isle line 1321. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2161,7 +2177,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1320. + // Rule at src/isa/x64/inst.isle line 1326. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2171,7 +2187,7 @@ pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1325. + // Rule at src/isa/x64/inst.isle line 1331. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2181,7 +2197,7 @@ pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1330. + // Rule at src/isa/x64/inst.isle line 1336. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2199,7 +2215,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1337. + // Rule at src/isa/x64/inst.isle line 1343. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2228,7 +2244,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1353. + // Rule at src/isa/x64/inst.isle line 1359. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2244,7 +2260,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1358. + // Rule at src/isa/x64/inst.isle line 1364. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2263,7 +2279,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1371. + // Rule at src/isa/x64/inst.isle line 1377. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2284,7 +2300,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1380. + // Rule at src/isa/x64/inst.isle line 1386. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -2307,7 +2323,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1387. + // Rule at src/isa/x64/inst.isle line 1393. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2325,7 +2341,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1392. + // Rule at src/isa/x64/inst.isle line 1398. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2345,7 +2361,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1397. + // Rule at src/isa/x64/inst.isle line 1403. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2363,7 +2379,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1402. + // Rule at src/isa/x64/inst.isle line 1408. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2376,7 +2392,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1407. + // Rule at src/isa/x64/inst.isle line 1413. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = SseOpcode::Pextrd; @@ -2399,7 +2415,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1420. + // Rule at src/isa/x64/inst.isle line 1426. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -2415,7 +2431,7 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option // Generated as internal constructor for term lea. pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1427. + // Rule at src/isa/x64/inst.isle line 1433. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadEffectiveAddress { @@ -2467,7 +2483,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1239. + // Rule at src/isa/x64/lower.isle line 1279. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2656,7 +2672,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 927. + // Rule at src/isa/x64/lower.isle line 967. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2761,6 +2777,23 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 900. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr2_0 = constructor_shr_i128(ctx, expr0_0, expr1_0)?; + let expr3_0: Type = I64; + let expr4_0: Type = I64; + let expr5_0: u64 = 128; + let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?; + let expr7_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr8_0 = constructor_sub(ctx, expr3_0, expr6_0, &expr7_0)?; + let expr9_0 = constructor_shl_i128(ctx, expr0_0, expr8_0)?; + let expr10_0 = constructor_or_i128(ctx, expr2_0, expr9_0)?; + return Some(expr10_0); + } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); @@ -2796,7 +2829,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1236. + // Rule at src/isa/x64/lower.isle line 1276. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2841,7 +2874,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1340. + // Rule at src/isa/x64/lower.isle line 1380. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2851,7 +2884,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1362. + // Rule at src/isa/x64/lower.isle line 1402. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2861,7 +2894,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1329. + // Rule at src/isa/x64/lower.isle line 1369. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2871,7 +2904,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1351. + // Rule at src/isa/x64/lower.isle line 1391. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -2942,7 +2975,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1182. + // Rule at src/isa/x64/lower.isle line 1222. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2963,7 +2996,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1343. + // Rule at src/isa/x64/lower.isle line 1383. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -2973,7 +3006,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1365. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -2983,7 +3016,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1332. + // Rule at src/isa/x64/lower.isle line 1372. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -2993,7 +3026,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1354. + // Rule at src/isa/x64/lower.isle line 1394. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -3041,7 +3074,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1185. + // Rule at src/isa/x64/lower.isle line 1225. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -3062,7 +3095,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1346. + // Rule at src/isa/x64/lower.isle line 1386. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -3072,7 +3105,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1368. + // Rule at src/isa/x64/lower.isle line 1408. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -3082,7 +3115,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1335. + // Rule at src/isa/x64/lower.isle line 1375. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -3092,7 +3125,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1357. + // Rule at src/isa/x64/lower.isle line 1397. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -3140,7 +3173,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1188. + // Rule at src/isa/x64/lower.isle line 1228. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -3212,7 +3245,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1202. + // Rule at src/isa/x64/lower.isle line 1242. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = I64X2; let expr2_0: u64 = 0; @@ -3236,7 +3269,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1175. + // Rule at src/isa/x64/lower.isle line 1215. let expr0_0 = C::put_in_reg(ctx, pattern6_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; @@ -3315,7 +3348,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 870. + // Rule at src/isa/x64/lower.isle line 910. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -3443,7 +3476,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 874. + // Rule at src/isa/x64/lower.isle line 914. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3556,7 +3589,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1249. + // Rule at src/isa/x64/lower.isle line 1289. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg(ctx, pattern7_1); let expr2_0 = RegMem::Reg { reg: expr0_0 }; @@ -4503,7 +4536,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1263. + // Rule at src/isa/x64/lower.isle line 1303. let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = C::put_in_reg(ctx, pattern7_2); @@ -4521,7 +4554,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1244. + // Rule at src/isa/x64/lower.isle line 1284. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4648,7 +4681,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1223. + // Rule at src/isa/x64/lower.isle line 1263. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -4951,35 +4984,68 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 852. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_m_rotl(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 846. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); } - // Rule at src/isa/x64/lower.isle line 846. - let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; - let expr1_0 = C::put_in_reg(ctx, pattern7_0); - let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; - let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 892. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_m_rotr(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + // Rule at src/isa/x64/lower.isle line 886. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_m_rotr(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + _ => {} } } } @@ -4990,37 +5056,74 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 837. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_m_rotl(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 833. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr5_0 = constructor_m_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); } - // Rule at src/isa/x64/lower.isle line 833. - let expr0_0: Type = I32; - let expr1_0 = ExtendKind::Zero; - let expr2_0 = constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; - let expr3_0 = C::put_in_reg(ctx, pattern7_0); - let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; - let expr5_0 = constructor_m_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; - let expr6_0 = C::value_reg(ctx, expr5_0); - return Some(expr6_0); + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 877. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); + let expr2_0 = + constructor_m_rotr(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + // Rule at src/isa/x64/lower.isle line 873. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr5_0 = constructor_m_rotr(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); + } + _ => {} } } } @@ -5439,21 +5542,21 @@ pub fn constructor_sse_and_not( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1164. + // Rule at src/isa/x64/lower.isle line 1204. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1165. + // Rule at src/isa/x64/lower.isle line 1205. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1166. + // Rule at src/isa/x64/lower.isle line 1206. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5463,7 +5566,7 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1229. + // Rule at src/isa/x64/lower.isle line 1269. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -5490,7 +5593,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1283. + // Rule at src/isa/x64/lower.isle line 1323. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5498,7 +5601,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1286. + // Rule at src/isa/x64/lower.isle line 1326. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5506,7 +5609,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1289. + // Rule at src/isa/x64/lower.isle line 1329. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5515,7 +5618,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1292. + // Rule at src/isa/x64/lower.isle line 1332. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5524,7 +5627,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1295. + // Rule at src/isa/x64/lower.isle line 1335. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -5535,7 +5638,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1316. + // Rule at src/isa/x64/lower.isle line 1356. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -5543,7 +5646,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1317. + // Rule at src/isa/x64/lower.isle line 1357. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -5551,7 +5654,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1325. + // Rule at src/isa/x64/lower.isle line 1365. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index f1d90210b1..20064dac82 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -1270,56 +1270,52 @@ block0(v0: i128, v1: i128): ; Entry block: 0 ; Block 0: ; (original IR block: block0) -; (instruction range: 0 .. 50) +; (instruction range: 0 .. 46) ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp -; Inst 2: movq %rsi, %rax -; Inst 3: movq %rax, %r9 -; Inst 4: movq %rdx, %rcx -; Inst 5: shrq %cl, %r9 -; Inst 6: movq %rdi, %rsi -; Inst 7: movq %rdx, %rcx -; Inst 8: shrq %cl, %rsi -; Inst 9: movl $64, %ecx -; Inst 10: subq %rdx, %rcx -; Inst 11: movq %rax, %r10 -; Inst 12: shlq %cl, %r10 -; Inst 13: xorq %rcx, %rcx -; Inst 14: testq $127, %rdx -; Inst 15: cmovzq %rcx, %r10 -; Inst 16: orq %rsi, %r10 -; Inst 17: xorq %rsi, %rsi -; Inst 18: xorq %r8, %r8 -; Inst 19: movq %rdx, %rcx -; Inst 20: andq $64, %rcx -; Inst 21: cmovzq %r9, %rsi -; Inst 22: cmovzq %r10, %r8 -; Inst 23: cmovnzq %r9, %r8 -; Inst 24: movl $128, %r9d -; Inst 25: subq %rdx, %r9 -; Inst 26: movq %rdi, %rdx -; Inst 27: movq %r9, %rcx -; Inst 28: shlq %cl, %rdx -; Inst 29: movq %r9, %rcx -; Inst 30: shlq %cl, %rax -; Inst 31: movl $64, %ecx -; Inst 32: subq %r9, %rcx -; Inst 33: shrq %cl, %rdi -; Inst 34: xorq %rcx, %rcx -; Inst 35: testq $127, %r9 -; Inst 36: cmovzq %rcx, %rdi -; Inst 37: orq %rax, %rdi -; Inst 38: xorq %rax, %rax -; Inst 39: andq $64, %r9 -; Inst 40: cmovzq %rdi, %rax -; Inst 41: cmovzq %rdx, %rcx -; Inst 42: cmovnzq %rdx, %rax -; Inst 43: orq %r8, %rcx -; Inst 44: orq %rsi, %rax -; Inst 45: movq %rax, %rdx -; Inst 46: movq %rcx, %rax -; Inst 47: movq %rbp, %rsp -; Inst 48: popq %rbp -; Inst 49: ret +; Inst 2: movq %rdi, %rax +; Inst 3: movq %rdx, %rcx +; Inst 4: shrq %cl, %rax +; Inst 5: movq %rsi, %r8 +; Inst 6: movq %rdx, %rcx +; Inst 7: shrq %cl, %r8 +; Inst 8: movl $64, %ecx +; Inst 9: subq %rdx, %rcx +; Inst 10: movq %rsi, %r9 +; Inst 11: shlq %cl, %r9 +; Inst 12: xorq %rcx, %rcx +; Inst 13: testq $127, %rdx +; Inst 14: cmovzq %rcx, %r9 +; Inst 15: movq %r9, %rcx +; Inst 16: orq %rax, %rcx +; Inst 17: xorq %rax, %rax +; Inst 18: testq $64, %rdx +; Inst 19: cmovzq %r8, %rax +; Inst 20: cmovzq %rcx, %r8 +; Inst 21: movl $128, %r9d +; Inst 22: subq %rdx, %r9 +; Inst 23: movq %rdi, %rdx +; Inst 24: movq %r9, %rcx +; Inst 25: shlq %cl, %rdx +; Inst 26: movq %r9, %rcx +; Inst 27: shlq %cl, %rsi +; Inst 28: movl $64, %ecx +; Inst 29: subq %r9, %rcx +; Inst 30: shrq %cl, %rdi +; Inst 31: xorq %rcx, %rcx +; Inst 32: testq $127, %r9 +; Inst 33: cmovzq %rcx, %rdi +; Inst 34: orq %rsi, %rdi +; Inst 35: testq $64, %r9 +; Inst 36: movq %rdx, %rsi +; Inst 37: cmovzq %rdi, %rsi +; Inst 38: cmovzq %rdx, %rcx +; Inst 39: orq %rcx, %r8 +; Inst 40: orq %rsi, %rax +; Inst 41: movq %rax, %rdx +; Inst 42: movq %r8, %rax +; Inst 43: movq %rbp, %rsp +; Inst 44: popq %rbp +; Inst 45: ret ; }} From b78731839b09fce57b3b5ade9ef04a42d7eab1b6 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 Jan 2022 13:34:55 -0800 Subject: [PATCH 3/7] cranelift: Use `x64_` prefix to disambiguate with clif in ISLE Instead of using `m_` like we used to, which was short for "mach inst" but not obvious or clear at all. --- cranelift/codegen/src/isa/x64/inst.isle | 27 +++++----- cranelift/codegen/src/isa/x64/lower.isle | 68 ++++++++++++------------ 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 0f79daa4bb..e3082a92f2 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -73,8 +73,7 @@ (src Reg) (dst WritableReg)) (LoadEffectiveAddress (addr SyntheticAmode) - (dst WritableReg)) -)) + (dst WritableReg)))) (type OperandSize extern (enum Size8 @@ -534,9 +533,9 @@ (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) (_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4) - r - (RegMem.Reg r) - wr)))) + r + (RegMem.Reg r) + wr)))) r)) ;; Helper for creating an SSE register holding an `i64x2` from two `i64` values. @@ -697,8 +696,8 @@ ;; ;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin ;; `and` operator. -(decl m_and (Type Reg RegMemImm) Reg) -(rule (m_and ty src1 src2) +(decl x64_and (Type Reg RegMemImm) Reg) +(rule (x64_and ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.And) src1 @@ -766,9 +765,9 @@ (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) (_ Unit (emit (MInst.XmmRmR (sse_xor_op ty) - r - (RegMem.Reg r) - wr)))) + r + (RegMem.Reg r) + wr)))) r)) ;; Special case for `f32` zero immediates to use `xorps`. @@ -807,14 +806,14 @@ ;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach ;; inst", to disambiguate this from clif's `rotl`). -(decl m_rotl (Type Reg Imm8Reg) Reg) -(rule (m_rotl ty src1 src2) +(decl x64_rotl (Type Reg Imm8Reg) Reg) +(rule (x64_rotl ty src1 src2) (shift_r ty (ShiftKind.RotateLeft) src1 src2)) ;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach ;; inst", to disambiguate this from clif's `rotr`). -(decl m_rotr (Type Reg Imm8Reg) Reg) -(rule (m_rotr ty src1 src2) +(decl x64_rotr (Type Reg Imm8Reg) Reg) +(rule (x64_rotr ty src1 src2) (shift_r ty (ShiftKind.RotateRight) src1 src2)) ;; Helper for creating `shl` instructions. diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 3422b7bc46..0a9b1c04eb 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -326,37 +326,37 @@ ;; And two registers. (rule (lower (has_type (fits_in_64 ty) (band x y))) - (value_reg (m_and ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_reg (x64_and ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) ;; And with a memory operand. (rule (lower (has_type (fits_in_64 ty) (band x (sinkable_load y)))) - (value_reg (m_and ty - (put_in_reg x) - (sink_load y)))) + (value_reg (x64_and ty + (put_in_reg x) + (sink_load y)))) (rule (lower (has_type (fits_in_64 ty) (band (sinkable_load x) y))) - (value_reg (m_and ty - (put_in_reg y) - (sink_load x)))) + (value_reg (x64_and ty + (put_in_reg y) + (sink_load x)))) ;; And with an immediate. (rule (lower (has_type (fits_in_64 ty) (band x (simm32_from_value y)))) - (value_reg (m_and ty - (put_in_reg x) - y))) + (value_reg (x64_and ty + (put_in_reg x) + y))) (rule (lower (has_type (fits_in_64 ty) (band (simm32_from_value x) y))) - (value_reg (m_and ty - (put_in_reg y) - x))) + (value_reg (x64_and ty + (put_in_reg y) + x))) ;; SSE. @@ -378,8 +378,8 @@ (y_regs ValueRegs (put_in_regs y)) (y_lo Reg (value_regs_get y_regs 0)) (y_hi Reg (value_regs_get y_regs 1))) - (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo)) - (m_and $I64 x_hi (RegMemImm.Reg y_hi))))) + (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo)) + (x64_and $I64 x_hi (RegMemImm.Reg y_hi))))) (rule (lower (has_type $B128 (band x y))) ;; Booleans are always `0` or `1`, so we only need to do the `and` on the @@ -389,7 +389,7 @@ (x_lo Reg (value_regs_get x_regs 0)) (x_hi Reg (value_regs_get x_regs 1)) (y_lo Reg (lo_reg y))) - (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo)) + (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo)) x_hi))) ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -832,13 +832,13 @@ (rule (lower (has_type (ty_8_or_16 ty) (rotl src amt))) (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) - (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) (rule (lower (has_type (ty_8_or_16 ty) (rotl src (u64_from_iconst amt)))) - (value_reg (m_rotl ty - (put_in_reg src) - (const_to_type_masked_imm8 amt ty)))) + (value_reg (x64_rotl ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) ;; `i64` and `i32`: we can rely on x86's rotate-amount masking since ;; we operate on the whole register. @@ -847,13 +847,13 @@ ;; NB: Only the low bits of `amt` matter since we logically mask the ;; shift amount to the value's bit width. (let ((amt_ Reg (lo_reg amt))) - (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) (rule (lower (has_type (ty_32_or_64 ty) (rotl src (u64_from_iconst amt)))) - (value_reg (m_rotl ty - (put_in_reg src) - (const_to_type_masked_imm8 amt ty)))) + (value_reg (x64_rotl ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) ;; `i128`. @@ -872,13 +872,13 @@ (rule (lower (has_type (ty_8_or_16 ty) (rotr src amt))) (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) - (value_reg (m_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) (rule (lower (has_type (ty_8_or_16 ty) (rotr src (u64_from_iconst amt)))) - (value_reg (m_rotr ty - (put_in_reg src) - (const_to_type_masked_imm8 amt ty)))) + (value_reg (x64_rotr ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) ;; `i64` and `i32`: we can rely on x86's rotate-amount masking since ;; we operate on the whole register. @@ -887,13 +887,13 @@ ;; NB: Only the low bits of `amt` matter since we logically mask the ;; shift amount to the value's bit width. (let ((amt_ Reg (lo_reg amt))) - (value_reg (m_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) (rule (lower (has_type (ty_32_or_64 ty) (rotr src (u64_from_iconst amt)))) - (value_reg (m_rotr ty - (put_in_reg src) - (const_to_type_masked_imm8 amt ty)))) + (value_reg (x64_rotr ty + (put_in_reg src) + (const_to_type_masked_imm8 amt ty)))) ;; `i128`. From 5917f1d2c20c25f8761f91cfb7754e84823f78e9 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 Jan 2022 15:08:01 -0800 Subject: [PATCH 4/7] cranelift: Port `ineg` scalar lowering to ISLE on x64 --- cranelift/codegen/src/isa/x64/inst.isle | 11 + cranelift/codegen/src/isa/x64/lower.isle | 5 + cranelift/codegen/src/isa/x64/lower.rs | 4 +- .../x64/lower/isle/generated_code.manifest | 4 +- .../src/isa/x64/lower/isle/generated_code.rs | 527 +++++++++--------- 5 files changed, 296 insertions(+), 255 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index e3082a92f2..0fbc95a89f 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -72,6 +72,9 @@ (Not (size OperandSize) (src Reg) (dst WritableReg)) + (Neg (size OperandSize) + (src Reg) + (dst WritableReg)) (LoadEffectiveAddress (addr SyntheticAmode) (dst WritableReg)))) @@ -1428,6 +1431,14 @@ (_ Unit (emit (MInst.Not size src dst)))) (writable_reg_to_reg dst))) +;; Helper for creating `neg` instructions. +(decl neg (Type Reg) Reg) +(rule (neg ty src) + (let ((dst WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.Neg size src dst)))) + (writable_reg_to_reg dst))) + (decl lea (SyntheticAmode) Reg) (rule (lea addr) (let ((dst WritableReg (temp_writable_reg $I64)) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 0a9b1c04eb..a2250a6269 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -905,6 +905,11 @@ (or_i128 (shr_i128 src_ amt_) (shl_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) +;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (fits_in_64 ty) (ineg x))) + (value_reg (neg ty (put_in_reg x)))) + ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index f6d2a7721b..868c889a3d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1268,9 +1268,7 @@ fn lower_insn_to_regs>( dst, )); } else { - let src = put_input_in_reg(ctx, inputs[0]); - ctx.emit(Inst::gen_move(dst, src, ty)); - ctx.emit(Inst::neg(OperandSize::from_ty(ty), dst)); + implemented_in_isle(ctx); } } diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index c9c3b0595c..fd495b9b74 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 src/prelude.isle 7b911d3b894ae17 -src/isa/x64/inst.isle 54ffef8c4f373807 -src/isa/x64/lower.isle 28de5d6bf49c8471 +src/isa/x64/inst.isle 7c0c209f30946919 +src/isa/x64/lower.isle 563fa026ec4ed363 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 018eb9b0c8..efdb8789f4 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -101,7 +101,7 @@ pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 461. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 463. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -224,22 +224,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 95. + // Rule at src/isa/x64/inst.isle line 97. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 96. + // Rule at src/isa/x64/inst.isle line 98. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 97. + // Rule at src/isa/x64/inst.isle line 99. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 98. + // Rule at src/isa/x64/inst.isle line 100. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -260,12 +260,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 473. + // Rule at src/isa/x64/inst.isle line 475. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 476. + // Rule at src/isa/x64/inst.isle line 478. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -289,7 +289,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 496. + // Rule at src/isa/x64/inst.isle line 498. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -297,7 +297,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 492. + // Rule at src/isa/x64/inst.isle line 494. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -310,17 +310,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 503. + // Rule at src/isa/x64/inst.isle line 505. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 504. + // Rule at src/isa/x64/inst.isle line 506. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 505. + // Rule at src/isa/x64/inst.isle line 507. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -337,7 +337,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 509. + // Rule at src/isa/x64/inst.isle line 511. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -347,40 +347,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 518. + // Rule at src/isa/x64/inst.isle line 520. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 519. + // Rule at src/isa/x64/inst.isle line 521. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 514. + // Rule at src/isa/x64/inst.isle line 516. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 515. + // Rule at src/isa/x64/inst.isle line 517. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 516. + // Rule at src/isa/x64/inst.isle line 518. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 517. + // Rule at src/isa/x64/inst.isle line 519. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -392,7 +392,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 533. + // Rule at src/isa/x64/inst.isle line 535. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -416,7 +416,7 @@ pub fn constructor_make_i64x2_from_lanes( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 544. + // Rule at src/isa/x64/inst.isle line 546. let expr0_0: Type = I64X2; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -457,11 +457,11 @@ pub fn constructor_reg_mem_imm_to_xmm( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 565. + // Rule at src/isa/x64/inst.isle line 567. return Some(pattern0_0.clone()); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 566. + // Rule at src/isa/x64/inst.isle line 568. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movd; let expr2_0 = RegMem::Reg { reg: pattern1_0 }; @@ -473,7 +473,7 @@ pub fn constructor_reg_mem_imm_to_xmm( &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 564. + // Rule at src/isa/x64/inst.isle line 566. return Some(pattern0_0.clone()); } _ => {} @@ -492,7 +492,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 582. + // Rule at src/isa/x64/inst.isle line 584. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Mov64MR { @@ -506,7 +506,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 587. + // Rule at src/isa/x64/inst.isle line 589. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -515,7 +515,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 591. + // Rule at src/isa/x64/inst.isle line 593. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -524,7 +524,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 595. + // Rule at src/isa/x64/inst.isle line 597. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -533,7 +533,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 599. + // Rule at src/isa/x64/inst.isle line 601. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -542,7 +542,7 @@ pub fn constructor_x64_load( if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 603. + // Rule at src/isa/x64/inst.isle line 605. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -552,7 +552,7 @@ pub fn constructor_x64_load( let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 577. + // Rule at src/isa/x64/inst.isle line 579. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -576,7 +576,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 622. + // Rule at src/isa/x64/inst.isle line 624. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -601,7 +601,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 630. + // Rule at src/isa/x64/inst.isle line 632. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -617,7 +617,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 638. + // Rule at src/isa/x64/inst.isle line 640. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -646,7 +646,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 649. + // Rule at src/isa/x64/inst.isle line 651. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -675,7 +675,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 660. + // Rule at src/isa/x64/inst.isle line 662. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -691,7 +691,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 668. + // Rule at src/isa/x64/inst.isle line 670. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -720,7 +720,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 679. + // Rule at src/isa/x64/inst.isle line 681. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -749,14 +749,14 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 690. + // Rule at src/isa/x64/inst.isle line 692. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } -// Generated as internal constructor for term m_and. -pub fn constructor_m_and( +// Generated as internal constructor for term x64_and. +pub fn constructor_x64_and( ctx: &mut C, arg0: Type, arg1: Reg, @@ -765,7 +765,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 701. + // Rule at src/isa/x64/inst.isle line 703. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -781,7 +781,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 709. + // Rule at src/isa/x64/inst.isle line 711. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -797,7 +797,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 717. + // Rule at src/isa/x64/inst.isle line 719. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -809,7 +809,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 746. + // Rule at src/isa/x64/inst.isle line 748. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -826,7 +826,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 775. + // Rule at src/isa/x64/inst.isle line 777. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -841,7 +841,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 734. + // Rule at src/isa/x64/inst.isle line 736. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -854,7 +854,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 787. + // Rule at src/isa/x64/inst.isle line 789. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -869,7 +869,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 738. + // Rule at src/isa/x64/inst.isle line 740. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -882,7 +882,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 765. + // Rule at src/isa/x64/inst.isle line 767. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -899,7 +899,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } let pattern1_0 = arg1; if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 752. + // Rule at src/isa/x64/inst.isle line 754. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -915,7 +915,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 727. + // Rule at src/isa/x64/inst.isle line 729. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Imm { @@ -940,7 +940,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 800. + // Rule at src/isa/x64/inst.isle line 802. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -955,8 +955,8 @@ pub fn constructor_shift_r( return Some(expr4_0); } -// Generated as internal constructor for term m_rotl. -pub fn constructor_m_rotl( +// Generated as internal constructor for term x64_rotl. +pub fn constructor_x64_rotl( ctx: &mut C, arg0: Type, arg1: Reg, @@ -965,14 +965,14 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 811. + // Rule at src/isa/x64/inst.isle line 813. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } -// Generated as internal constructor for term m_rotr. -pub fn constructor_m_rotr( +// Generated as internal constructor for term x64_rotr. +pub fn constructor_x64_rotr( ctx: &mut C, arg0: Type, arg1: Reg, @@ -981,7 +981,7 @@ pub fn constructor_m_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 817. + // Rule at src/isa/x64/inst.isle line 819. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -997,7 +997,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 822. + // Rule at src/isa/x64/inst.isle line 824. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1013,7 +1013,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 827. + // Rule at src/isa/x64/inst.isle line 829. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1029,7 +1029,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 832. + // Rule at src/isa/x64/inst.isle line 834. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1047,7 +1047,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 837. + // Rule at src/isa/x64/inst.isle line 839. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1072,7 +1072,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 846. + // Rule at src/isa/x64/inst.isle line 848. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1088,7 +1088,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 851. + // Rule at src/isa/x64/inst.isle line 853. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1106,7 +1106,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 856. + // Rule at src/isa/x64/inst.isle line 858. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1134,7 +1134,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 864. + // Rule at src/isa/x64/inst.isle line 866. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1156,7 +1156,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 871. + // Rule at src/isa/x64/inst.isle line 873. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1180,7 +1180,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 878. + // Rule at src/isa/x64/inst.isle line 880. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1197,7 +1197,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 885. + // Rule at src/isa/x64/inst.isle line 887. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1208,7 +1208,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 890. + // Rule at src/isa/x64/inst.isle line 892. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1219,7 +1219,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 895. + // Rule at src/isa/x64/inst.isle line 897. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1230,7 +1230,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 900. + // Rule at src/isa/x64/inst.isle line 902. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1241,7 +1241,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 905. + // Rule at src/isa/x64/inst.isle line 907. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1252,7 +1252,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 910. + // Rule at src/isa/x64/inst.isle line 912. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1263,7 +1263,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 915. + // Rule at src/isa/x64/inst.isle line 917. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1274,7 +1274,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 920. + // Rule at src/isa/x64/inst.isle line 922. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1285,7 +1285,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 925. + // Rule at src/isa/x64/inst.isle line 927. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1296,7 +1296,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 930. + // Rule at src/isa/x64/inst.isle line 932. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1307,7 +1307,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 935. + // Rule at src/isa/x64/inst.isle line 937. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1318,7 +1318,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 940. + // Rule at src/isa/x64/inst.isle line 942. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1329,7 +1329,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 945. + // Rule at src/isa/x64/inst.isle line 947. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1340,7 +1340,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 950. + // Rule at src/isa/x64/inst.isle line 952. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1351,7 +1351,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 955. + // Rule at src/isa/x64/inst.isle line 957. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1362,7 +1362,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 960. + // Rule at src/isa/x64/inst.isle line 962. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1373,7 +1373,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 965. + // Rule at src/isa/x64/inst.isle line 967. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1384,7 +1384,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 970. + // Rule at src/isa/x64/inst.isle line 972. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1395,7 +1395,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 975. + // Rule at src/isa/x64/inst.isle line 977. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1406,7 +1406,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 980. + // Rule at src/isa/x64/inst.isle line 982. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1417,7 +1417,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 985. + // Rule at src/isa/x64/inst.isle line 987. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1428,7 +1428,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 990. + // Rule at src/isa/x64/inst.isle line 992. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1439,7 +1439,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 995. + // Rule at src/isa/x64/inst.isle line 997. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1450,7 +1450,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1000. + // Rule at src/isa/x64/inst.isle line 1002. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1461,7 +1461,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1005. + // Rule at src/isa/x64/inst.isle line 1007. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1472,7 +1472,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1010. + // Rule at src/isa/x64/inst.isle line 1012. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1483,7 +1483,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1015. + // Rule at src/isa/x64/inst.isle line 1017. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1494,7 +1494,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1020. + // Rule at src/isa/x64/inst.isle line 1022. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1505,7 +1505,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1025. + // Rule at src/isa/x64/inst.isle line 1027. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1516,7 +1516,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1030. + // Rule at src/isa/x64/inst.isle line 1032. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1527,7 +1527,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1035. + // Rule at src/isa/x64/inst.isle line 1037. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1538,7 +1538,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1040. + // Rule at src/isa/x64/inst.isle line 1042. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1549,7 +1549,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1045. + // Rule at src/isa/x64/inst.isle line 1047. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1560,7 +1560,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1050. + // Rule at src/isa/x64/inst.isle line 1052. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1571,7 +1571,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1055. + // Rule at src/isa/x64/inst.isle line 1057. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1582,7 +1582,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1060. + // Rule at src/isa/x64/inst.isle line 1062. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1593,7 +1593,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1065. + // Rule at src/isa/x64/inst.isle line 1067. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1604,7 +1604,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1070. + // Rule at src/isa/x64/inst.isle line 1072. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1615,17 +1615,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1074. + // Rule at src/isa/x64/inst.isle line 1076. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1075. + // Rule at src/isa/x64/inst.isle line 1077. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1076. + // Rule at src/isa/x64/inst.isle line 1078. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1636,17 +1636,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1079. + // Rule at src/isa/x64/inst.isle line 1081. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1080. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1081. + // Rule at src/isa/x64/inst.isle line 1083. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1665,7 +1665,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1085. + // Rule at src/isa/x64/inst.isle line 1087. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1689,7 +1689,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1097. + // Rule at src/isa/x64/inst.isle line 1099. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = RegMem::Reg { reg: pattern2_0 }; @@ -1709,7 +1709,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1109. + // Rule at src/isa/x64/inst.isle line 1111. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1720,7 +1720,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1114. + // Rule at src/isa/x64/inst.isle line 1116. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1731,7 +1731,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1119. + // Rule at src/isa/x64/inst.isle line 1121. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1742,7 +1742,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1124. + // Rule at src/isa/x64/inst.isle line 1126. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1753,7 +1753,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1129. + // Rule at src/isa/x64/inst.isle line 1131. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1764,7 +1764,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1134. + // Rule at src/isa/x64/inst.isle line 1136. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1775,7 +1775,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1139. + // Rule at src/isa/x64/inst.isle line 1141. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1786,7 +1786,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1144. + // Rule at src/isa/x64/inst.isle line 1146. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1797,7 +1797,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1149. + // Rule at src/isa/x64/inst.isle line 1151. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1808,7 +1808,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1154. + // Rule at src/isa/x64/inst.isle line 1156. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1819,7 +1819,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1159. + // Rule at src/isa/x64/inst.isle line 1161. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1830,7 +1830,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1164. + // Rule at src/isa/x64/inst.isle line 1166. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1841,7 +1841,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1169. + // Rule at src/isa/x64/inst.isle line 1171. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1852,7 +1852,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1174. + // Rule at src/isa/x64/inst.isle line 1176. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1863,7 +1863,7 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1179. + // Rule at src/isa/x64/inst.isle line 1181. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1874,7 +1874,7 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1184. + // Rule at src/isa/x64/inst.isle line 1186. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1885,7 +1885,7 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1189. + // Rule at src/isa/x64/inst.isle line 1191. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1906,7 +1906,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1194. + // Rule at src/isa/x64/inst.isle line 1196. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1934,7 +1934,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1206. + // Rule at src/isa/x64/inst.isle line 1208. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1952,7 +1952,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1215. + // Rule at src/isa/x64/inst.isle line 1217. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1977,7 +1977,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1228. + // Rule at src/isa/x64/inst.isle line 1230. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1993,7 +1993,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1235. + // Rule at src/isa/x64/inst.isle line 1237. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2002,7 +2002,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1240. + // Rule at src/isa/x64/inst.isle line 1242. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2011,7 +2011,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1245. + // Rule at src/isa/x64/inst.isle line 1247. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2020,7 +2020,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1250. + // Rule at src/isa/x64/inst.isle line 1252. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2029,7 +2029,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1255. + // Rule at src/isa/x64/inst.isle line 1257. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2043,7 +2043,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1260. + // Rule at src/isa/x64/inst.isle line 1262. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmREvex { @@ -2059,7 +2059,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1267. + // Rule at src/isa/x64/inst.isle line 1269. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2075,7 +2075,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1272. + // Rule at src/isa/x64/inst.isle line 1274. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -2093,7 +2093,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1284. + // Rule at src/isa/x64/inst.isle line 1286. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2109,7 +2109,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1291. + // Rule at src/isa/x64/inst.isle line 1293. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -2127,7 +2127,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1301. + // Rule at src/isa/x64/inst.isle line 1303. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2137,7 +2137,7 @@ pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1306. + // Rule at src/isa/x64/inst.isle line 1308. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2147,7 +2147,7 @@ pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1311. + // Rule at src/isa/x64/inst.isle line 1313. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2157,7 +2157,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1316. + // Rule at src/isa/x64/inst.isle line 1318. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2167,7 +2167,7 @@ pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1321. + // Rule at src/isa/x64/inst.isle line 1323. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2177,7 +2177,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1326. + // Rule at src/isa/x64/inst.isle line 1328. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2187,7 +2187,7 @@ pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1331. + // Rule at src/isa/x64/inst.isle line 1333. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2197,7 +2197,7 @@ pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1336. + // Rule at src/isa/x64/inst.isle line 1338. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2215,7 +2215,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1343. + // Rule at src/isa/x64/inst.isle line 1345. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2244,7 +2244,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1359. + // Rule at src/isa/x64/inst.isle line 1361. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2260,7 +2260,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1364. + // Rule at src/isa/x64/inst.isle line 1366. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2279,7 +2279,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1377. + // Rule at src/isa/x64/inst.isle line 1379. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2300,7 +2300,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1386. + // Rule at src/isa/x64/inst.isle line 1388. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -2323,7 +2323,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1393. + // Rule at src/isa/x64/inst.isle line 1395. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2341,7 +2341,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1398. + // Rule at src/isa/x64/inst.isle line 1400. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2361,7 +2361,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1403. + // Rule at src/isa/x64/inst.isle line 1405. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2379,7 +2379,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1408. + // Rule at src/isa/x64/inst.isle line 1410. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2392,7 +2392,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1413. + // Rule at src/isa/x64/inst.isle line 1415. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = SseOpcode::Pextrd; @@ -2415,7 +2415,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1426. + // Rule at src/isa/x64/inst.isle line 1428. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -2428,10 +2428,27 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option return Some(expr4_0); } +// Generated as internal constructor for term neg. +pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1436. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr2_0 = MInst::Neg { + size: expr1_0, + src: pattern1_0, + dst: expr0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); +} + // Generated as internal constructor for term lea. pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1433. + // Rule at src/isa/x64/inst.isle line 1443. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadEffectiveAddress { @@ -2483,7 +2500,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1279. + // Rule at src/isa/x64/lower.isle line 1284. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2672,7 +2689,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 967. + // Rule at src/isa/x64/lower.isle line 972. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2721,10 +2738,10 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1276. + // Rule at src/isa/x64/lower.isle line 1281. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2874,7 +2891,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1380. + // Rule at src/isa/x64/lower.isle line 1385. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2884,7 +2901,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1402. + // Rule at src/isa/x64/lower.isle line 1407. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2894,7 +2911,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1369. + // Rule at src/isa/x64/lower.isle line 1374. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2904,7 +2921,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1391. + // Rule at src/isa/x64/lower.isle line 1396. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -2975,7 +2992,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1222. + // Rule at src/isa/x64/lower.isle line 1227. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2996,7 +3013,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1383. + // Rule at src/isa/x64/lower.isle line 1388. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -3006,7 +3023,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1405. + // Rule at src/isa/x64/lower.isle line 1410. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -3016,7 +3033,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1372. + // Rule at src/isa/x64/lower.isle line 1377. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -3026,7 +3043,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1394. + // Rule at src/isa/x64/lower.isle line 1399. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -3074,7 +3091,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1225. + // Rule at src/isa/x64/lower.isle line 1230. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -3095,7 +3112,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1386. + // Rule at src/isa/x64/lower.isle line 1391. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -3105,7 +3122,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1408. + // Rule at src/isa/x64/lower.isle line 1413. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -3115,7 +3132,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1375. + // Rule at src/isa/x64/lower.isle line 1380. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -3125,7 +3142,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1397. + // Rule at src/isa/x64/lower.isle line 1402. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -3173,7 +3190,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1228. + // Rule at src/isa/x64/lower.isle line 1233. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -3245,7 +3262,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1242. + // Rule at src/isa/x64/lower.isle line 1247. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = I64X2; let expr2_0: u64 = 0; @@ -3269,7 +3286,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1215. + // Rule at src/isa/x64/lower.isle line 1220. let expr0_0 = C::put_in_reg(ctx, pattern6_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; @@ -3348,7 +3365,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 910. + // Rule at src/isa/x64/lower.isle line 915. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -3476,7 +3493,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 914. + // Rule at src/isa/x64/lower.isle line 919. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3589,7 +3606,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1289. + // Rule at src/isa/x64/lower.isle line 1294. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg(ctx, pattern7_1); let expr2_0 = RegMem::Reg { reg: expr0_0 }; @@ -4536,7 +4553,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1303. + // Rule at src/isa/x64/lower.isle line 1308. let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = C::put_in_reg(ctx, pattern7_2); @@ -4554,7 +4571,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1284. + // Rule at src/isa/x64/lower.isle line 1289. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4681,7 +4698,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1263. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); - let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 910. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + &Opcode::Bnot => { + // Rule at src/isa/x64/lower.isle line 1268. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } &InstructionData::BinaryImm64 { @@ -5001,7 +5028,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1204. + // Rule at src/isa/x64/lower.isle line 1209. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1205. + // Rule at src/isa/x64/lower.isle line 1210. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1206. + // Rule at src/isa/x64/lower.isle line 1211. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5566,7 +5593,7 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1269. + // Rule at src/isa/x64/lower.isle line 1274. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -5593,7 +5620,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1323. + // Rule at src/isa/x64/lower.isle line 1328. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5601,7 +5628,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1326. + // Rule at src/isa/x64/lower.isle line 1331. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5609,7 +5636,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1329. + // Rule at src/isa/x64/lower.isle line 1334. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5618,7 +5645,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1332. + // Rule at src/isa/x64/lower.isle line 1337. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5627,7 +5654,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1335. + // Rule at src/isa/x64/lower.isle line 1340. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -5638,7 +5665,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1356. + // Rule at src/isa/x64/lower.isle line 1361. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -5646,7 +5673,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1357. + // Rule at src/isa/x64/lower.isle line 1362. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -5654,7 +5681,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1365. + // Rule at src/isa/x64/lower.isle line 1370. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } From 5bb3645bd47626d8ec50975b711eaa525dff35e1 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 Jan 2022 15:18:15 -0800 Subject: [PATCH 5/7] cranelift: Port `ineg` SIMD lowering to ISLE on x64 --- cranelift/codegen/src/isa/x64/inst.isle | 4 +- .../codegen/src/isa/x64/inst/emit_tests.rs | 11 + cranelift/codegen/src/isa/x64/inst/mod.rs | 9 - cranelift/codegen/src/isa/x64/lower.isle | 20 ++ cranelift/codegen/src/isa/x64/lower.rs | 40 +-- .../x64/lower/isle/generated_code.manifest | 4 +- .../src/isa/x64/lower/isle/generated_code.rs | 287 +++++++++++------- 7 files changed, 208 insertions(+), 167 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 0fbc95a89f..8cff579673 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -726,7 +726,7 @@ (decl imm (Type u64) Reg) ;; Integer immediates. -(rule (imm ty simm64) +(rule (imm (fits_in_64 ty) simm64) (let ((dst WritableReg (temp_writable_reg ty)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Imm size simm64 dst)))) @@ -751,7 +751,7 @@ (writable_reg_to_reg dst))) ;; Special case for integer zero immediates: turn them into an `xor r, r`. -(rule (imm ty 0) +(rule (imm (fits_in_64 ty) 0) (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) (size OperandSize (operand_size_of_type_32_64 ty)) diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index e8f905bcc5..1c71187fde 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -16,6 +16,17 @@ use super::*; use crate::isa::x64; use alloc::vec::Vec; +impl Inst { + fn neg(size: OperandSize, src: Writable) -> Inst { + debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); + Inst::Neg { + size, + src: src.to_reg(), + dst: src, + } + } +} + #[test] fn test_x64_emit() { let rax = regs::rax(); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index e8d958f738..6f00370ea0 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -668,15 +668,6 @@ impl Inst { } } - pub(crate) fn neg(size: OperandSize, src: Writable) -> Inst { - debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); - Inst::Neg { - size, - src: src.to_reg(), - dst: src, - } - } - pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst { divisor.assert_regclass_is(RegClass::I64); Inst::Div { diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index a2250a6269..41b2214828 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -907,9 +907,29 @@ ;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; `i64` and smaller. + (rule (lower (has_type (fits_in_64 ty) (ineg x))) (value_reg (neg ty (put_in_reg x)))) +;; SSE. + +(rule (lower (has_type $I8X16 (ineg x))) + (value_reg (psubb (imm $I8X16 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I16X8 (ineg x))) + (value_reg (psubw (imm $I16X8 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I32X4 (ineg x))) + (value_reg (psubd (imm $I32X4 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I64X2 (ineg x))) + (value_reg (psubq (imm $I64X2 0) + (put_in_reg_mem x)))) + ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 868c889a3d..242ad13deb 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1233,44 +1233,8 @@ fn lower_insn_to_regs>( | Opcode::Sshr | Opcode::Ishl | Opcode::Rotl - | Opcode::Rotr => implemented_in_isle(ctx), - - Opcode::Ineg => { - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let ty = ty.unwrap(); - - if ty.is_vector() { - // Zero's out a register and then does a packed subtraction - // of the input from the register. - - let src = input_to_reg_mem(ctx, inputs[0]); - let tmp = ctx.alloc_tmp(types::I32X4).only_reg().unwrap(); - - let subtract_opcode = match ty { - types::I8X16 => SseOpcode::Psubb, - types::I16X8 => SseOpcode::Psubw, - types::I32X4 => SseOpcode::Psubd, - types::I64X2 => SseOpcode::Psubq, - _ => panic!("Unsupported type for Ineg instruction, found {}", ty), - }; - - // Note we must zero out a tmp instead of using the destination register since - // the desitnation could be an alias for the source input register - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pxor, - RegMem::reg(tmp.to_reg()), - tmp, - )); - ctx.emit(Inst::xmm_rm_r(subtract_opcode, src, tmp)); - ctx.emit(Inst::xmm_unary_rm_r( - SseOpcode::Movapd, - RegMem::reg(tmp.to_reg()), - dst, - )); - } else { - implemented_in_isle(ctx); - } - } + | Opcode::Rotr + | Opcode::Ineg => implemented_in_isle(ctx), Opcode::Clz => { let orig_ty = ty.unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index fd495b9b74..3bdd76a1b4 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 src/prelude.isle 7b911d3b894ae17 -src/isa/x64/inst.isle 7c0c209f30946919 -src/isa/x64/lower.isle 563fa026ec4ed363 +src/isa/x64/inst.isle 755420ee1a9c7aeb +src/isa/x64/lower.isle 38ddbc7b4a8444cf diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index efdb8789f4..ea1f0e0557 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -897,35 +897,38 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option return Some(expr1_0); } } - let pattern1_0 = arg1; - if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 754. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); - let expr3_0 = AluRmiROpcode::Xor; - let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr5_0 = MInst::AluRmiR { - size: expr2_0, - op: expr3_0, - src1: expr1_0, - src2: expr4_0, + if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { + let pattern2_0 = arg1; + if pattern2_0 == 0 { + // Rule at src/isa/x64/inst.isle line 754. + let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); + let expr3_0 = AluRmiROpcode::Xor; + let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr5_0 = MInst::AluRmiR { + size: expr2_0, + op: expr3_0, + src1: expr1_0, + src2: expr4_0, + dst: expr0_0, + }; + let expr6_0 = C::emit(ctx, &expr5_0); + return Some(expr1_0); + } + // Rule at src/isa/x64/inst.isle line 729. + let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); + let expr2_0 = MInst::Imm { + dst_size: expr1_0, + simm64: pattern2_0, dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); } - // Rule at src/isa/x64/inst.isle line 729. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); - let expr2_0 = MInst::Imm { - dst_size: expr1_0, - simm64: pattern1_0, - dst: expr0_0, - }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr4_0); + return None; } // Generated as internal constructor for term shift_r. @@ -2500,7 +2503,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1284. + // Rule at src/isa/x64/lower.isle line 1304. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2689,7 +2692,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 972. + // Rule at src/isa/x64/lower.isle line 992. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2846,7 +2849,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1281. + // Rule at src/isa/x64/lower.isle line 1301. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2891,7 +2894,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1385. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2901,7 +2904,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1407. + // Rule at src/isa/x64/lower.isle line 1427. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2911,7 +2914,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1374. + // Rule at src/isa/x64/lower.isle line 1394. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2921,7 +2924,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1396. + // Rule at src/isa/x64/lower.isle line 1416. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -2991,12 +2994,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1227. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 917. + let expr0_0: Type = I8X16; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubb(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1247. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3013,7 +3029,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1388. + // Rule at src/isa/x64/lower.isle line 1408. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -3023,7 +3039,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1410. + // Rule at src/isa/x64/lower.isle line 1430. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -3033,7 +3049,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1377. + // Rule at src/isa/x64/lower.isle line 1397. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -3043,7 +3059,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1399. + // Rule at src/isa/x64/lower.isle line 1419. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -3090,12 +3106,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1230. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 921. + let expr0_0: Type = I16X8; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubw(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1250. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3112,7 +3141,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1391. + // Rule at src/isa/x64/lower.isle line 1411. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -3122,7 +3151,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1413. + // Rule at src/isa/x64/lower.isle line 1433. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -3132,7 +3161,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1380. + // Rule at src/isa/x64/lower.isle line 1400. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -3142,7 +3171,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1402. + // Rule at src/isa/x64/lower.isle line 1422. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -3189,12 +3218,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1233. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 925. + let expr0_0: Type = I32X4; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubd(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1253. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3261,18 +3303,31 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1247. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); - let expr1_0: Type = I64X2; - let expr2_0: u64 = 0; - let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr0_0 }; - let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr0_0 }; - let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); - return Some(expr8_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 929. + let expr0_0: Type = I64X2; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubq(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1267. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0: Type = I64X2; + let expr2_0: u64 = 0; + let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; + let expr4_0 = RegMem::Reg { reg: expr0_0 }; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { reg: expr0_0 }; + let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some(expr8_0); + } + _ => {} } } _ => {} @@ -3286,7 +3341,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1220. + // Rule at src/isa/x64/lower.isle line 1240. let expr0_0 = C::put_in_reg(ctx, pattern6_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; @@ -3365,7 +3420,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 915. + // Rule at src/isa/x64/lower.isle line 935. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -3493,7 +3548,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 919. + // Rule at src/isa/x64/lower.isle line 939. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3606,7 +3661,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1294. + // Rule at src/isa/x64/lower.isle line 1314. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg(ctx, pattern7_1); let expr2_0 = RegMem::Reg { reg: expr0_0 }; @@ -4553,7 +4608,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1308. + // Rule at src/isa/x64/lower.isle line 1328. let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = C::put_in_reg(ctx, pattern7_2); @@ -4571,7 +4626,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1289. + // Rule at src/isa/x64/lower.isle line 1309. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4698,7 +4753,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 910. + // Rule at src/isa/x64/lower.isle line 912. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); return Some(expr2_0); } &Opcode::Bnot => { - // Rule at src/isa/x64/lower.isle line 1268. + // Rule at src/isa/x64/lower.isle line 1288. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -5569,21 +5624,21 @@ pub fn constructor_sse_and_not( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1209. + // Rule at src/isa/x64/lower.isle line 1229. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1210. + // Rule at src/isa/x64/lower.isle line 1230. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1211. + // Rule at src/isa/x64/lower.isle line 1231. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5593,7 +5648,7 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1274. + // Rule at src/isa/x64/lower.isle line 1294. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -5620,7 +5675,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1328. + // Rule at src/isa/x64/lower.isle line 1348. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5628,7 +5683,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1331. + // Rule at src/isa/x64/lower.isle line 1351. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5636,7 +5691,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1334. + // Rule at src/isa/x64/lower.isle line 1354. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5645,7 +5700,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1337. + // Rule at src/isa/x64/lower.isle line 1357. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5654,7 +5709,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1340. + // Rule at src/isa/x64/lower.isle line 1360. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -5665,7 +5720,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1361. + // Rule at src/isa/x64/lower.isle line 1381. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -5673,7 +5728,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1362. + // Rule at src/isa/x64/lower.isle line 1382. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -5681,7 +5736,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1370. + // Rule at src/isa/x64/lower.isle line 1390. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } From 658c5d33c10610b8e2a505f63f5a7184ad4cbcd6 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 Jan 2022 15:39:41 -0800 Subject: [PATCH 6/7] cranelift: Port `trap` and `resumable_trap` lowering to ISLE on x64 --- .../lower/isle/generated_code.manifest | 2 +- .../isa/aarch64/lower/isle/generated_code.rs | 39 +- cranelift/codegen/src/isa/x64/inst.isle | 6 + cranelift/codegen/src/isa/x64/lower.isle | 10 + cranelift/codegen/src/isa/x64/lower.rs | 9 +- .../x64/lower/isle/generated_code.manifest | 6 +- .../src/isa/x64/lower/isle/generated_code.rs | 392 ++++++++++-------- cranelift/codegen/src/machinst/isle.rs | 5 + cranelift/codegen/src/prelude.isle | 15 + 9 files changed, 299 insertions(+), 185 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 1e38a6cc63..8d51997f3c 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle 7b911d3b894ae17 +src/prelude.isle 181213982a49a4b8 src/isa/aarch64/inst.isle 5fa80451697b084f src/isa/aarch64/lower.isle 2d2e1e076a0c8a23 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index ca4f68a598..9764b5b808 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -24,6 +24,7 @@ pub trait Context { fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8; fn value_reg(&mut self, arg0: Reg) -> ValueRegs; fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; + fn value_regs_invalid(&mut self) -> ValueRegs; fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; fn invalid_reg(&mut self) -> Reg; fn put_in_reg(&mut self, arg0: Value) -> Reg; @@ -92,13 +93,19 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 277. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 279. +#[derive(Clone, Debug)] +pub enum SideEffectNoResult { + Inst { inst: MInst }, +} + +/// Internal type ProducesFlags: defined at src/prelude.isle line 292. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 280. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 295. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -978,7 +985,7 @@ pub enum AtomicRMWOp { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 66. + // Rule at src/prelude.isle line 70. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -987,13 +994,31 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 101. + // Rule at src/prelude.isle line 105. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); return Some(expr2_0); } +// Generated as internal constructor for term value_regs_none. +pub fn constructor_value_regs_none( + ctx: &mut C, + arg0: &SideEffectNoResult, +) -> Option { + let pattern0_0 = arg0; + if let &SideEffectNoResult::Inst { + inst: ref pattern1_0, + } = pattern0_0 + { + // Rule at src/prelude.isle line 284. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::value_regs_invalid(ctx); + return Some(expr1_0); + } + return None; +} + // Generated as internal constructor for term with_flags. pub fn constructor_with_flags( ctx: &mut C, @@ -1012,7 +1037,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 290. + // Rule at src/prelude.isle line 305. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1040,7 +1065,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 298. + // Rule at src/prelude.isle line 313. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1074,7 +1099,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 323. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 8cff579673..a126750872 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -4,6 +4,7 @@ (type MInst extern (enum (Nop (len u8)) + (Ud2 (trap_code TrapCode)) (AluRmiR (size OperandSize) (op AluRmiROpcode) (src1 Reg) @@ -1444,3 +1445,8 @@ (let ((dst WritableReg (temp_writable_reg $I64)) (_ Unit (emit (MInst.LoadEffectiveAddress addr dst)))) (writable_reg_to_reg dst))) + +;; Helper for creating `ud2` instructions. +(decl ud2 (TrapCode) SideEffectNoResult) +(rule (ud2 code) + (SideEffectNoResult.Inst (MInst.Ud2 code))) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 41b2214828..9b881b5f69 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1432,3 +1432,13 @@ (rule (lower (has_type $I32X4 (umin x y))) (value_reg (pminud (put_in_reg x) (put_in_reg_mem y)))) + +;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (trap code)) + (value_regs_none (ud2 code))) + +;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (resumable_trap code)) + (value_regs_none (ud2 code))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 242ad13deb..a7765de677 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1234,7 +1234,9 @@ fn lower_insn_to_regs>( | Opcode::Ishl | Opcode::Rotl | Opcode::Rotr - | Opcode::Ineg => implemented_in_isle(ctx), + | Opcode::Ineg + | Opcode::Trap + | Opcode::ResumableTrap => implemented_in_isle(ctx), Opcode::Clz => { let orig_ty = ty.unwrap(); @@ -2369,11 +2371,6 @@ fn lower_insn_to_regs>( ctx.emit(Inst::Hlt); } - Opcode::Trap | Opcode::ResumableTrap => { - let trap_code = ctx.data(insn).trap_code().unwrap(); - ctx.emit_safepoint(Inst::Ud2 { trap_code }); - } - Opcode::Trapif | Opcode::Trapff => { let trap_code = ctx.data(insn).trap_code().unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 3bdd76a1b4..820fc40fee 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle 7b911d3b894ae17 -src/isa/x64/inst.isle 755420ee1a9c7aeb -src/isa/x64/lower.isle 38ddbc7b4a8444cf +src/prelude.isle 181213982a49a4b8 +src/isa/x64/inst.isle 61004acbb1289816 +src/isa/x64/lower.isle 82db7f7d47ac7809 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index ea1f0e0557..a31e0583c1 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -24,6 +24,7 @@ pub trait Context { fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8; fn value_reg(&mut self, arg0: Reg) -> ValueRegs; fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; + fn value_regs_invalid(&mut self) -> ValueRegs; fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; fn invalid_reg(&mut self) -> Reg; fn put_in_reg(&mut self, arg0: Value) -> Reg; @@ -89,19 +90,25 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 277. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 279. +#[derive(Clone, Debug)] +pub enum SideEffectNoResult { + Inst { inst: MInst }, +} + +/// Internal type ProducesFlags: defined at src/prelude.isle line 292. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 280. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 295. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 463. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 464. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -111,7 +118,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 66. + // Rule at src/prelude.isle line 70. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -120,13 +127,31 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 101. + // Rule at src/prelude.isle line 105. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); return Some(expr2_0); } +// Generated as internal constructor for term value_regs_none. +pub fn constructor_value_regs_none( + ctx: &mut C, + arg0: &SideEffectNoResult, +) -> Option { + let pattern0_0 = arg0; + if let &SideEffectNoResult::Inst { + inst: ref pattern1_0, + } = pattern0_0 + { + // Rule at src/prelude.isle line 284. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::value_regs_invalid(ctx); + return Some(expr1_0); + } + return None; +} + // Generated as internal constructor for term with_flags. pub fn constructor_with_flags( ctx: &mut C, @@ -145,7 +170,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 290. + // Rule at src/prelude.isle line 305. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -173,7 +198,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 298. + // Rule at src/prelude.isle line 313. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -207,7 +232,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 308. + // Rule at src/prelude.isle line 323. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -224,22 +249,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 97. + // Rule at src/isa/x64/inst.isle line 98. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 98. + // Rule at src/isa/x64/inst.isle line 99. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 99. + // Rule at src/isa/x64/inst.isle line 100. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 100. + // Rule at src/isa/x64/inst.isle line 101. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -260,12 +285,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 475. + // Rule at src/isa/x64/inst.isle line 476. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 478. + // Rule at src/isa/x64/inst.isle line 479. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -289,7 +314,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 498. + // Rule at src/isa/x64/inst.isle line 499. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -297,7 +322,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 494. + // Rule at src/isa/x64/inst.isle line 495. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -310,17 +335,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 505. + // Rule at src/isa/x64/inst.isle line 506. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 506. + // Rule at src/isa/x64/inst.isle line 507. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 507. + // Rule at src/isa/x64/inst.isle line 508. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -337,7 +362,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 511. + // Rule at src/isa/x64/inst.isle line 512. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -347,40 +372,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 521. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 521. + // Rule at src/isa/x64/inst.isle line 522. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 516. + // Rule at src/isa/x64/inst.isle line 517. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 517. + // Rule at src/isa/x64/inst.isle line 518. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 518. + // Rule at src/isa/x64/inst.isle line 519. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 519. + // Rule at src/isa/x64/inst.isle line 520. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -392,7 +417,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 535. + // Rule at src/isa/x64/inst.isle line 536. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -416,7 +441,7 @@ pub fn constructor_make_i64x2_from_lanes( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 546. + // Rule at src/isa/x64/inst.isle line 547. let expr0_0: Type = I64X2; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -457,11 +482,11 @@ pub fn constructor_reg_mem_imm_to_xmm( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 567. + // Rule at src/isa/x64/inst.isle line 568. return Some(pattern0_0.clone()); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 568. + // Rule at src/isa/x64/inst.isle line 569. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movd; let expr2_0 = RegMem::Reg { reg: pattern1_0 }; @@ -473,7 +498,7 @@ pub fn constructor_reg_mem_imm_to_xmm( &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 566. + // Rule at src/isa/x64/inst.isle line 567. return Some(pattern0_0.clone()); } _ => {} @@ -492,7 +517,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 584. + // Rule at src/isa/x64/inst.isle line 585. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Mov64MR { @@ -506,7 +531,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 589. + // Rule at src/isa/x64/inst.isle line 590. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -515,7 +540,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 593. + // Rule at src/isa/x64/inst.isle line 594. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -524,7 +549,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 597. + // Rule at src/isa/x64/inst.isle line 598. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -533,7 +558,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 601. + // Rule at src/isa/x64/inst.isle line 602. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -542,7 +567,7 @@ pub fn constructor_x64_load( if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 605. + // Rule at src/isa/x64/inst.isle line 606. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -552,7 +577,7 @@ pub fn constructor_x64_load( let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 579. + // Rule at src/isa/x64/inst.isle line 580. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -576,7 +601,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 624. + // Rule at src/isa/x64/inst.isle line 625. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -601,7 +626,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 632. + // Rule at src/isa/x64/inst.isle line 633. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -617,7 +642,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 640. + // Rule at src/isa/x64/inst.isle line 641. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -646,7 +671,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 651. + // Rule at src/isa/x64/inst.isle line 652. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -675,7 +700,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 662. + // Rule at src/isa/x64/inst.isle line 663. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -691,7 +716,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 670. + // Rule at src/isa/x64/inst.isle line 671. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -720,7 +745,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 681. + // Rule at src/isa/x64/inst.isle line 682. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -749,7 +774,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 692. + // Rule at src/isa/x64/inst.isle line 693. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -765,7 +790,7 @@ pub fn constructor_x64_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 703. + // Rule at src/isa/x64/inst.isle line 704. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -781,7 +806,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 711. + // Rule at src/isa/x64/inst.isle line 712. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -797,7 +822,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 719. + // Rule at src/isa/x64/inst.isle line 720. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -809,7 +834,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 748. + // Rule at src/isa/x64/inst.isle line 749. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -826,7 +851,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 777. + // Rule at src/isa/x64/inst.isle line 778. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -841,7 +866,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 736. + // Rule at src/isa/x64/inst.isle line 737. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -854,7 +879,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 789. + // Rule at src/isa/x64/inst.isle line 790. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -869,7 +894,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 740. + // Rule at src/isa/x64/inst.isle line 741. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -882,7 +907,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 767. + // Rule at src/isa/x64/inst.isle line 768. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -900,7 +925,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 754. + // Rule at src/isa/x64/inst.isle line 755. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); @@ -916,7 +941,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 729. + // Rule at src/isa/x64/inst.isle line 730. let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { @@ -943,7 +968,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 802. + // Rule at src/isa/x64/inst.isle line 803. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -968,7 +993,7 @@ pub fn constructor_x64_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 813. + // Rule at src/isa/x64/inst.isle line 814. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -984,7 +1009,7 @@ pub fn constructor_x64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 819. + // Rule at src/isa/x64/inst.isle line 820. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1000,7 +1025,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 824. + // Rule at src/isa/x64/inst.isle line 825. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1016,7 +1041,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 829. + // Rule at src/isa/x64/inst.isle line 830. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1032,7 +1057,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 834. + // Rule at src/isa/x64/inst.isle line 835. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1050,7 +1075,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 839. + // Rule at src/isa/x64/inst.isle line 840. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1075,7 +1100,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 848. + // Rule at src/isa/x64/inst.isle line 849. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1091,7 +1116,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 853. + // Rule at src/isa/x64/inst.isle line 854. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1109,7 +1134,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 858. + // Rule at src/isa/x64/inst.isle line 859. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1137,7 +1162,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 866. + // Rule at src/isa/x64/inst.isle line 867. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -1159,7 +1184,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 873. + // Rule at src/isa/x64/inst.isle line 874. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -1183,7 +1208,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 880. + // Rule at src/isa/x64/inst.isle line 881. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1200,7 +1225,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 887. + // Rule at src/isa/x64/inst.isle line 888. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1211,7 +1236,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 892. + // Rule at src/isa/x64/inst.isle line 893. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1222,7 +1247,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 897. + // Rule at src/isa/x64/inst.isle line 898. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1233,7 +1258,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 902. + // Rule at src/isa/x64/inst.isle line 903. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1244,7 +1269,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 907. + // Rule at src/isa/x64/inst.isle line 908. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1255,7 +1280,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 912. + // Rule at src/isa/x64/inst.isle line 913. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1266,7 +1291,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 917. + // Rule at src/isa/x64/inst.isle line 918. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1277,7 +1302,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 922. + // Rule at src/isa/x64/inst.isle line 923. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1288,7 +1313,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 927. + // Rule at src/isa/x64/inst.isle line 928. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1299,7 +1324,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 932. + // Rule at src/isa/x64/inst.isle line 933. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1310,7 +1335,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 937. + // Rule at src/isa/x64/inst.isle line 938. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1321,7 +1346,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 942. + // Rule at src/isa/x64/inst.isle line 943. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1332,7 +1357,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 947. + // Rule at src/isa/x64/inst.isle line 948. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1343,7 +1368,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 952. + // Rule at src/isa/x64/inst.isle line 953. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1354,7 +1379,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 957. + // Rule at src/isa/x64/inst.isle line 958. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1365,7 +1390,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 962. + // Rule at src/isa/x64/inst.isle line 963. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1376,7 +1401,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 967. + // Rule at src/isa/x64/inst.isle line 968. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1387,7 +1412,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 972. + // Rule at src/isa/x64/inst.isle line 973. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1398,7 +1423,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 977. + // Rule at src/isa/x64/inst.isle line 978. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1409,7 +1434,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 982. + // Rule at src/isa/x64/inst.isle line 983. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1420,7 +1445,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 987. + // Rule at src/isa/x64/inst.isle line 988. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1431,7 +1456,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 992. + // Rule at src/isa/x64/inst.isle line 993. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1442,7 +1467,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 997. + // Rule at src/isa/x64/inst.isle line 998. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1453,7 +1478,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1002. + // Rule at src/isa/x64/inst.isle line 1003. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1464,7 +1489,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1007. + // Rule at src/isa/x64/inst.isle line 1008. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1475,7 +1500,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1012. + // Rule at src/isa/x64/inst.isle line 1013. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1486,7 +1511,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1017. + // Rule at src/isa/x64/inst.isle line 1018. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1497,7 +1522,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1022. + // Rule at src/isa/x64/inst.isle line 1023. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1508,7 +1533,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1027. + // Rule at src/isa/x64/inst.isle line 1028. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1519,7 +1544,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1032. + // Rule at src/isa/x64/inst.isle line 1033. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1530,7 +1555,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1037. + // Rule at src/isa/x64/inst.isle line 1038. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1541,7 +1566,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1042. + // Rule at src/isa/x64/inst.isle line 1043. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1552,7 +1577,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1047. + // Rule at src/isa/x64/inst.isle line 1048. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1563,7 +1588,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1052. + // Rule at src/isa/x64/inst.isle line 1053. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1574,7 +1599,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1057. + // Rule at src/isa/x64/inst.isle line 1058. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1585,7 +1610,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1062. + // Rule at src/isa/x64/inst.isle line 1063. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1596,7 +1621,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1067. + // Rule at src/isa/x64/inst.isle line 1068. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1607,7 +1632,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1072. + // Rule at src/isa/x64/inst.isle line 1073. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1618,17 +1643,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1076. + // Rule at src/isa/x64/inst.isle line 1077. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1077. + // Rule at src/isa/x64/inst.isle line 1078. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1078. + // Rule at src/isa/x64/inst.isle line 1079. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1639,17 +1664,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1081. + // Rule at src/isa/x64/inst.isle line 1082. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1082. + // Rule at src/isa/x64/inst.isle line 1083. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1083. + // Rule at src/isa/x64/inst.isle line 1084. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1668,7 +1693,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1087. + // Rule at src/isa/x64/inst.isle line 1088. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1692,7 +1717,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1099. + // Rule at src/isa/x64/inst.isle line 1100. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = RegMem::Reg { reg: pattern2_0 }; @@ -1712,7 +1737,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1111. + // Rule at src/isa/x64/inst.isle line 1112. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1723,7 +1748,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1116. + // Rule at src/isa/x64/inst.isle line 1117. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1734,7 +1759,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1121. + // Rule at src/isa/x64/inst.isle line 1122. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1745,7 +1770,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1126. + // Rule at src/isa/x64/inst.isle line 1127. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1756,7 +1781,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1131. + // Rule at src/isa/x64/inst.isle line 1132. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1767,7 +1792,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1136. + // Rule at src/isa/x64/inst.isle line 1137. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1778,7 +1803,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1141. + // Rule at src/isa/x64/inst.isle line 1142. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1789,7 +1814,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1146. + // Rule at src/isa/x64/inst.isle line 1147. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1800,7 +1825,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1151. + // Rule at src/isa/x64/inst.isle line 1152. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1811,7 +1836,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1156. + // Rule at src/isa/x64/inst.isle line 1157. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1822,7 +1847,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1161. + // Rule at src/isa/x64/inst.isle line 1162. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1833,7 +1858,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1166. + // Rule at src/isa/x64/inst.isle line 1167. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1844,7 +1869,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1171. + // Rule at src/isa/x64/inst.isle line 1172. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1855,7 +1880,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1176. + // Rule at src/isa/x64/inst.isle line 1177. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1866,7 +1891,7 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1181. + // Rule at src/isa/x64/inst.isle line 1182. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1877,7 +1902,7 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1186. + // Rule at src/isa/x64/inst.isle line 1187. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1888,7 +1913,7 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1191. + // Rule at src/isa/x64/inst.isle line 1192. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1909,7 +1934,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1196. + // Rule at src/isa/x64/inst.isle line 1197. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1937,7 +1962,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1208. + // Rule at src/isa/x64/inst.isle line 1209. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1955,7 +1980,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1217. + // Rule at src/isa/x64/inst.isle line 1218. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1980,7 +2005,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1230. + // Rule at src/isa/x64/inst.isle line 1231. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1996,7 +2021,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1237. + // Rule at src/isa/x64/inst.isle line 1238. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2005,7 +2030,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1242. + // Rule at src/isa/x64/inst.isle line 1243. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2014,7 +2039,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1247. + // Rule at src/isa/x64/inst.isle line 1248. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2023,7 +2048,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1252. + // Rule at src/isa/x64/inst.isle line 1253. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2032,7 +2057,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1257. + // Rule at src/isa/x64/inst.isle line 1258. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2046,7 +2071,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1262. + // Rule at src/isa/x64/inst.isle line 1263. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmREvex { @@ -2062,7 +2087,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1269. + // Rule at src/isa/x64/inst.isle line 1270. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2078,7 +2103,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1274. + // Rule at src/isa/x64/inst.isle line 1275. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -2096,7 +2121,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1286. + // Rule at src/isa/x64/inst.isle line 1287. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2112,7 +2137,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1293. + // Rule at src/isa/x64/inst.isle line 1294. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -2130,7 +2155,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1303. + // Rule at src/isa/x64/inst.isle line 1304. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2140,7 +2165,7 @@ pub fn constructor_psllw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1308. + // Rule at src/isa/x64/inst.isle line 1309. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2150,7 +2175,7 @@ pub fn constructor_pslld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1313. + // Rule at src/isa/x64/inst.isle line 1314. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2160,7 +2185,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1318. + // Rule at src/isa/x64/inst.isle line 1319. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2170,7 +2195,7 @@ pub fn constructor_psrlw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1323. + // Rule at src/isa/x64/inst.isle line 1324. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2180,7 +2205,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1328. + // Rule at src/isa/x64/inst.isle line 1329. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2190,7 +2215,7 @@ pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1333. + // Rule at src/isa/x64/inst.isle line 1334. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2200,7 +2225,7 @@ pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1338. + // Rule at src/isa/x64/inst.isle line 1339. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2218,7 +2243,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1345. + // Rule at src/isa/x64/inst.isle line 1346. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2247,7 +2272,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1361. + // Rule at src/isa/x64/inst.isle line 1362. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2263,7 +2288,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1366. + // Rule at src/isa/x64/inst.isle line 1367. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2282,7 +2307,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1379. + // Rule at src/isa/x64/inst.isle line 1380. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2303,7 +2328,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1388. + // Rule at src/isa/x64/inst.isle line 1389. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -2326,7 +2351,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1395. + // Rule at src/isa/x64/inst.isle line 1396. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2344,7 +2369,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1400. + // Rule at src/isa/x64/inst.isle line 1401. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2364,7 +2389,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1405. + // Rule at src/isa/x64/inst.isle line 1406. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2382,7 +2407,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1410. + // Rule at src/isa/x64/inst.isle line 1411. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2395,7 +2420,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1415. + // Rule at src/isa/x64/inst.isle line 1416. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = SseOpcode::Pextrd; @@ -2418,7 +2443,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1428. + // Rule at src/isa/x64/inst.isle line 1429. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -2435,7 +2460,7 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1436. + // Rule at src/isa/x64/inst.isle line 1437. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { @@ -2451,7 +2476,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Reg) -> Option // Generated as internal constructor for term lea. pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1443. + // Rule at src/isa/x64/inst.isle line 1444. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadEffectiveAddress { @@ -2463,6 +2488,17 @@ pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option return Some(expr4_0); } +// Generated as internal constructor for term ud2. +pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 1451. + let expr0_0 = MInst::Ud2 { + trap_code: pattern0_0.clone(), + }; + let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; + return Some(expr1_0); +} + // Generated as internal constructor for term lower. pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern0_0 = arg0; @@ -2494,6 +2530,26 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + match &pattern2_0 { + &Opcode::Trap => { + // Rule at src/isa/x64/lower.isle line 1438. + let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; + let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; + return Some(expr1_0); + } + &Opcode::ResumableTrap => { + // Rule at src/isa/x64/lower.isle line 1443. + let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; + let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?; + return Some(expr1_0); + } + _ => {} + } + } &InstructionData::TernaryImm8 { opcode: ref pattern2_0, args: ref pattern2_1, diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 95541b057d..808954dea5 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -48,6 +48,11 @@ macro_rules! isle_prelude_methods { ValueRegs::two(r1, r2) } + #[inline] + fn value_regs_invalid(&mut self) -> ValueRegs { + ValueRegs::invalid() + } + #[inline] fn temp_writable_reg(&mut self, ty: Type) -> WritableReg { let value_regs = self.lower_ctx.alloc_tmp(ty); diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 96c5740b5d..b3b779f57d 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -57,6 +57,10 @@ (decl value_regs (Reg Reg) ValueRegs) (extern constructor value_regs value_regs) +;; Construct an empty `ValueRegs` containing only invalid register sentinals. +(decl value_regs_invalid () ValueRegs) +(extern constructor value_regs_invalid value_regs_invalid) + ;; Get a temporary register for writing. (decl temp_writable_reg (Type) WritableReg) (extern constructor temp_writable_reg temp_writable_reg) @@ -270,6 +274,17 @@ (extractor (u64_from_iconst x) (def_inst (iconst (u64_from_imm64 x)))) +;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;; + +(type SideEffectNoResult (enum (Inst (inst MInst)))) + +;; Create an empty `ValueRegs`, but do emit the given side-effectful +;; instruction. +(decl value_regs_none (SideEffectNoResult) ValueRegs) +(rule (value_regs_none (SideEffectNoResult.Inst inst)) + (let ((_ Unit (emit inst))) + (value_regs_invalid))) + ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Newtype wrapper around `MInst` for instructions that are used for their From a052285340bde170baa4bdc245c57e1c6b7d0915 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 13 Jan 2022 16:49:42 -0800 Subject: [PATCH 7/7] Fix typo: s/sentinals/sentinels/ --- .../codegen/src/isa/aarch64/lower/isle/generated_code.manifest | 2 +- .../codegen/src/isa/x64/lower/isle/generated_code.manifest | 2 +- cranelift/codegen/src/prelude.isle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 8d51997f3c..f75e4e5adb 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle 181213982a49a4b8 +src/prelude.isle 22dd5ff133398960 src/isa/aarch64/inst.isle 5fa80451697b084f src/isa/aarch64/lower.isle 2d2e1e076a0c8a23 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 820fc40fee..5a897a6008 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle 181213982a49a4b8 +src/prelude.isle 22dd5ff133398960 src/isa/x64/inst.isle 61004acbb1289816 src/isa/x64/lower.isle 82db7f7d47ac7809 diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index b3b779f57d..8ce3d787a5 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -57,7 +57,7 @@ (decl value_regs (Reg Reg) ValueRegs) (extern constructor value_regs value_regs) -;; Construct an empty `ValueRegs` containing only invalid register sentinals. +;; Construct an empty `ValueRegs` containing only invalid register sentinels. (decl value_regs_invalid () ValueRegs) (extern constructor value_regs_invalid value_regs_invalid)