diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 0fbc95a89f..8cff579673 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -726,7 +726,7 @@ (decl imm (Type u64) Reg) ;; Integer immediates. -(rule (imm ty simm64) +(rule (imm (fits_in_64 ty) simm64) (let ((dst WritableReg (temp_writable_reg ty)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Imm size simm64 dst)))) @@ -751,7 +751,7 @@ (writable_reg_to_reg dst))) ;; Special case for integer zero immediates: turn them into an `xor r, r`. -(rule (imm ty 0) +(rule (imm (fits_in_64 ty) 0) (let ((wr WritableReg (temp_writable_reg ty)) (r Reg (writable_reg_to_reg wr)) (size OperandSize (operand_size_of_type_32_64 ty)) diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index e8f905bcc5..1c71187fde 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -16,6 +16,17 @@ use super::*; use crate::isa::x64; use alloc::vec::Vec; +impl Inst { + fn neg(size: OperandSize, src: Writable) -> Inst { + debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); + Inst::Neg { + size, + src: src.to_reg(), + dst: src, + } + } +} + #[test] fn test_x64_emit() { let rax = regs::rax(); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index e8d958f738..6f00370ea0 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -668,15 +668,6 @@ impl Inst { } } - pub(crate) fn neg(size: OperandSize, src: Writable) -> Inst { - debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); - Inst::Neg { - size, - src: src.to_reg(), - dst: src, - } - } - pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst { divisor.assert_regclass_is(RegClass::I64); Inst::Div { diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index a2250a6269..41b2214828 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -907,9 +907,29 @@ ;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; `i64` and smaller. + (rule (lower (has_type (fits_in_64 ty) (ineg x))) (value_reg (neg ty (put_in_reg x)))) +;; SSE. + +(rule (lower (has_type $I8X16 (ineg x))) + (value_reg (psubb (imm $I8X16 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I16X8 (ineg x))) + (value_reg (psubw (imm $I16X8 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I32X4 (ineg x))) + (value_reg (psubd (imm $I32X4 0) + (put_in_reg_mem x)))) + +(rule (lower (has_type $I64X2 (ineg x))) + (value_reg (psubq (imm $I64X2 0) + (put_in_reg_mem x)))) + ;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type (multi_lane 8 16) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 868c889a3d..242ad13deb 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1233,44 +1233,8 @@ fn lower_insn_to_regs>( | Opcode::Sshr | Opcode::Ishl | Opcode::Rotl - | Opcode::Rotr => implemented_in_isle(ctx), - - Opcode::Ineg => { - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let ty = ty.unwrap(); - - if ty.is_vector() { - // Zero's out a register and then does a packed subtraction - // of the input from the register. - - let src = input_to_reg_mem(ctx, inputs[0]); - let tmp = ctx.alloc_tmp(types::I32X4).only_reg().unwrap(); - - let subtract_opcode = match ty { - types::I8X16 => SseOpcode::Psubb, - types::I16X8 => SseOpcode::Psubw, - types::I32X4 => SseOpcode::Psubd, - types::I64X2 => SseOpcode::Psubq, - _ => panic!("Unsupported type for Ineg instruction, found {}", ty), - }; - - // Note we must zero out a tmp instead of using the destination register since - // the desitnation could be an alias for the source input register - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pxor, - RegMem::reg(tmp.to_reg()), - tmp, - )); - ctx.emit(Inst::xmm_rm_r(subtract_opcode, src, tmp)); - ctx.emit(Inst::xmm_unary_rm_r( - SseOpcode::Movapd, - RegMem::reg(tmp.to_reg()), - dst, - )); - } else { - implemented_in_isle(ctx); - } - } + | Opcode::Rotr + | Opcode::Ineg => implemented_in_isle(ctx), Opcode::Clz => { let orig_ty = ty.unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index fd495b9b74..3bdd76a1b4 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 src/prelude.isle 7b911d3b894ae17 -src/isa/x64/inst.isle 7c0c209f30946919 -src/isa/x64/lower.isle 563fa026ec4ed363 +src/isa/x64/inst.isle 755420ee1a9c7aeb +src/isa/x64/lower.isle 38ddbc7b4a8444cf diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index efdb8789f4..ea1f0e0557 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -897,35 +897,38 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option return Some(expr1_0); } } - let pattern1_0 = arg1; - if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 754. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); - let expr3_0 = AluRmiROpcode::Xor; - let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr5_0 = MInst::AluRmiR { - size: expr2_0, - op: expr3_0, - src1: expr1_0, - src2: expr4_0, + if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { + let pattern2_0 = arg1; + if pattern2_0 == 0 { + // Rule at src/isa/x64/inst.isle line 754. + let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); + let expr3_0 = AluRmiROpcode::Xor; + let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr5_0 = MInst::AluRmiR { + size: expr2_0, + op: expr3_0, + src1: expr1_0, + src2: expr4_0, + dst: expr0_0, + }; + let expr6_0 = C::emit(ctx, &expr5_0); + return Some(expr1_0); + } + // Rule at src/isa/x64/inst.isle line 729. + let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); + let expr2_0 = MInst::Imm { + dst_size: expr1_0, + simm64: pattern2_0, dst: expr0_0, }; - let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); } - // Rule at src/isa/x64/inst.isle line 729. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); - let expr2_0 = MInst::Imm { - dst_size: expr1_0, - simm64: pattern1_0, - dst: expr0_0, - }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr4_0); + return None; } // Generated as internal constructor for term shift_r. @@ -2500,7 +2503,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1284. + // Rule at src/isa/x64/lower.isle line 1304. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2689,7 +2692,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 972. + // Rule at src/isa/x64/lower.isle line 992. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2846,7 +2849,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1281. + // Rule at src/isa/x64/lower.isle line 1301. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2891,7 +2894,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1385. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2901,7 +2904,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1407. + // Rule at src/isa/x64/lower.isle line 1427. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2911,7 +2914,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1374. + // Rule at src/isa/x64/lower.isle line 1394. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2921,7 +2924,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1396. + // Rule at src/isa/x64/lower.isle line 1416. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -2991,12 +2994,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1227. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 917. + let expr0_0: Type = I8X16; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubb(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1247. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3013,7 +3029,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1388. + // Rule at src/isa/x64/lower.isle line 1408. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -3023,7 +3039,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1410. + // Rule at src/isa/x64/lower.isle line 1430. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -3033,7 +3049,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1377. + // Rule at src/isa/x64/lower.isle line 1397. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -3043,7 +3059,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1399. + // Rule at src/isa/x64/lower.isle line 1419. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -3090,12 +3106,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1230. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 921. + let expr0_0: Type = I16X8; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubw(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1250. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3112,7 +3141,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1391. + // Rule at src/isa/x64/lower.isle line 1411. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -3122,7 +3151,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1413. + // Rule at src/isa/x64/lower.isle line 1433. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -3132,7 +3161,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1380. + // Rule at src/isa/x64/lower.isle line 1400. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -3142,7 +3171,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1402. + // Rule at src/isa/x64/lower.isle line 1422. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -3189,12 +3218,25 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1233. - let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); - let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 925. + let expr0_0: Type = I32X4; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubd(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1253. + let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + _ => {} } } _ => {} @@ -3261,18 +3303,31 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1247. - let expr0_0 = C::put_in_reg(ctx, pattern5_1); - let expr1_0: Type = I64X2; - let expr2_0: u64 = 0; - let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; - let expr4_0 = RegMem::Reg { reg: expr0_0 }; - let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; - let expr6_0 = RegMem::Reg { reg: expr0_0 }; - let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; - let expr8_0 = C::value_reg(ctx, expr7_0); - return Some(expr8_0); + match &pattern5_0 { + &Opcode::Ineg => { + // Rule at src/isa/x64/lower.isle line 929. + let expr0_0: Type = I64X2; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::put_in_reg_mem(ctx, pattern5_1); + let expr4_0 = constructor_psubq(ctx, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::Iabs => { + // Rule at src/isa/x64/lower.isle line 1267. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0: Type = I64X2; + let expr2_0: u64 = 0; + let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; + let expr4_0 = RegMem::Reg { reg: expr0_0 }; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { reg: expr0_0 }; + let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some(expr8_0); + } + _ => {} } } _ => {} @@ -3286,7 +3341,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1220. + // Rule at src/isa/x64/lower.isle line 1240. let expr0_0 = C::put_in_reg(ctx, pattern6_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; @@ -3365,7 +3420,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 915. + // Rule at src/isa/x64/lower.isle line 935. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -3493,7 +3548,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 919. + // Rule at src/isa/x64/lower.isle line 939. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3606,7 +3661,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1294. + // Rule at src/isa/x64/lower.isle line 1314. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg(ctx, pattern7_1); let expr2_0 = RegMem::Reg { reg: expr0_0 }; @@ -4553,7 +4608,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1308. + // Rule at src/isa/x64/lower.isle line 1328. let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = C::put_in_reg(ctx, pattern7_2); @@ -4571,7 +4626,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1289. + // Rule at src/isa/x64/lower.isle line 1309. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4698,7 +4753,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 910. + // Rule at src/isa/x64/lower.isle line 912. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); return Some(expr2_0); } &Opcode::Bnot => { - // Rule at src/isa/x64/lower.isle line 1268. + // Rule at src/isa/x64/lower.isle line 1288. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -5569,21 +5624,21 @@ pub fn constructor_sse_and_not( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1209. + // Rule at src/isa/x64/lower.isle line 1229. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1210. + // Rule at src/isa/x64/lower.isle line 1230. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1211. + // Rule at src/isa/x64/lower.isle line 1231. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -5593,7 +5648,7 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1274. + // Rule at src/isa/x64/lower.isle line 1294. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -5620,7 +5675,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1328. + // Rule at src/isa/x64/lower.isle line 1348. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5628,7 +5683,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1331. + // Rule at src/isa/x64/lower.isle line 1351. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -5636,7 +5691,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1334. + // Rule at src/isa/x64/lower.isle line 1354. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5645,7 +5700,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1337. + // Rule at src/isa/x64/lower.isle line 1357. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -5654,7 +5709,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1340. + // Rule at src/isa/x64/lower.isle line 1360. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -5665,7 +5720,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1361. + // Rule at src/isa/x64/lower.isle line 1381. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -5673,7 +5728,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1362. + // Rule at src/isa/x64/lower.isle line 1382. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -5681,7 +5736,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1370. + // Rule at src/isa/x64/lower.isle line 1390. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); }