From 5d8dd648d71db22838e376b18ecd03b015a08485 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Tue, 29 Mar 2022 15:41:49 -0700 Subject: [PATCH] x64: port `fcmp` to ISLE (#3967) * x64: port scalar `fcmp` to ISLE Implement the CLIF lowering for the `fcmp` to ISLE. This adds a new type-matcher, `ty_scalar_float`, for detecting uses of `F32` and `F64`. * isle: rename `vec128` to `ty_vec12` This refactoring changes the name of the `vec128` matcher function to follow the `ty_*` convention of the other type matchers. It also makes the helper an inline function call. * x64: port vector `fcmp` to ISLE --- cranelift/codegen/src/isa/aarch64/lower.isle | 30 +- .../lower/isle/generated_code.manifest | 4 +- .../isa/aarch64/lower/isle/generated_code.rs | 33 +- .../s390x/lower/isle/generated_code.manifest | 2 +- .../isa/s390x/lower/isle/generated_code.rs | 31 +- cranelift/codegen/src/isa/x64/inst.isle | 30 +- cranelift/codegen/src/isa/x64/lower.isle | 129 ++- cranelift/codegen/src/isa/x64/lower.rs | 99 +-- .../x64/lower/isle/generated_code.manifest | 6 +- .../src/isa/x64/lower/isle/generated_code.rs | 759 +++++++++++++----- cranelift/codegen/src/machinst/isle.rs | 11 +- cranelift/codegen/src/prelude.isle | 8 +- .../filetests/isa/x64/cmp-mem-bug.clif | 4 +- 13 files changed, 746 insertions(+), 400 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index e7dfb80d41..9b951d80d9 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -145,22 +145,22 @@ ;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(rule (lower (has_type (vec128 ty) (uadd_sat x y))) +(rule (lower (has_type (ty_vec128 ty) (uadd_sat x y))) (uqadd x y (vector_size ty))) ;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(rule (lower (has_type (vec128 ty) (sadd_sat x y))) +(rule (lower (has_type (ty_vec128 ty) (sadd_sat x y))) (sqadd x y (vector_size ty))) ;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(rule (lower (has_type (vec128 ty) (usub_sat x y))) +(rule (lower (has_type (ty_vec128 ty) (usub_sat x y))) (uqsub x y (vector_size ty))) ;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(rule (lower (has_type (vec128 ty) (ssub_sat x y))) +(rule (lower (has_type (ty_vec128 ty) (ssub_sat x y))) (sqsub x y (vector_size ty))) ;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -170,7 +170,7 @@ (sub ty (zero_reg) x)) ;; vectors. -(rule (lower (has_type (vec128 ty) (ineg x))) +(rule (lower (has_type (ty_vec128 ty) (ineg x))) (neg x (vector_size ty))) ;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -208,7 +208,7 @@ (value_regs dst_lo dst_hi))) ;; Case for i8x16, i16x8, and i32x4. -(rule (lower (has_type (vec128 ty @ (not_i64x2)) (imul x y))) +(rule (lower (has_type (ty_vec128 ty @ (not_i64x2)) (imul x y))) (mul x y (vector_size ty))) ;; Special lowering for i64x2. @@ -575,7 +575,7 @@ (value_regs new_lo new_hi))) ;; Implementation of `bnot` for vector types. -(rule (lower (has_type (vec128 ty) (bnot x))) +(rule (lower (has_type (ty_vec128 ty) (bnot x))) (not x (vector_size ty))) ;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -588,7 +588,7 @@ (rule (lower (has_type $I128 (band x y))) (i128_alu_bitop (ALUOp.And) $I64 x y)) -(rule (lower (has_type (vec128 ty) (band x y))) +(rule (lower (has_type (ty_vec128 ty) (band x y))) (and_vec x y (vector_size ty))) ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -601,7 +601,7 @@ (rule (lower (has_type $I128 (bor x y))) (i128_alu_bitop (ALUOp.Orr) $I64 x y)) -(rule (lower (has_type (vec128 ty) (bor x y))) +(rule (lower (has_type (ty_vec128 ty) (bor x y))) (orr_vec x y (vector_size ty))) ;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -614,7 +614,7 @@ (rule (lower (has_type $I128 (bxor x y))) (i128_alu_bitop (ALUOp.Eor) $I64 x y)) -(rule (lower (has_type (vec128 ty) (bxor x y))) +(rule (lower (has_type (ty_vec128 ty) (bxor x y))) (eor_vec x y (vector_size ty))) ;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -627,7 +627,7 @@ (rule (lower (has_type $I128 (band_not x y))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y)) -(rule (lower (has_type (vec128 ty) (band_not x y))) +(rule (lower (has_type (ty_vec128 ty) (band_not x y))) (bic_vec x y (vector_size ty))) ;;;; Rules for `bor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -691,7 +691,7 @@ (csel (Cond.Ne) lo_lshift maybe_hi))))) ;; Shift for vector types. -(rule (lower (has_type (vec128 ty) (ishl x y))) +(rule (lower (has_type (ty_vec128 ty) (ishl x y))) (let ((size VectorSize (vector_size ty)) (shift Reg (vec_dup y size))) (sshl x shift size))) @@ -749,7 +749,7 @@ (lower_ushr128 x (value_regs_get y 0))) ;; Vector shifts. -(rule (lower (has_type (vec128 ty) (ushr x y))) +(rule (lower (has_type (ty_vec128 ty) (ushr x y))) (let ((size VectorSize (vector_size ty)) (shift Reg (vec_dup (sub $I32 (zero_reg) y) size))) (ushl x shift size))) @@ -769,7 +769,7 @@ (src_hi Reg (value_regs_get src 1)) (lo_rshift Reg (lsr $I64 src_lo amt)) (hi_rshift Reg (lsr $I64 src_hi amt)) - + (inv_amt Reg (orr_not $I32 (zero_reg) amt)) (hi_lshift Reg (lsl $I64 (lsl_imm $I64 src_hi (imm_shift_from_u8 1)) inv_amt)) @@ -798,7 +798,7 @@ ;; Vector shifts. ;; ;; Note that right shifts are implemented with a negative left shift. -(rule (lower (has_type (vec128 ty) (sshr x y))) +(rule (lower (has_type (ty_vec128 ty) (sshr x y))) (let ((size VectorSize (vector_size ty)) (shift Reg (vec_dup (sub $I32 (zero_reg) y) size))) (sshl x shift size))) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 3cd53aaf74..92abb1b020 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle b2bc986bcbbbb77 +src/prelude.isle 74d9514ac948e163 src/isa/aarch64/inst.isle 19ccefb6a496d392 -src/isa/aarch64/lower.isle 90ead921762336d2 +src/isa/aarch64/lower.isle d88b62dd6b40622 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 4b866a7cc3..ef5dbf5e25 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -56,7 +56,8 @@ pub trait Context { fn ty_8_or_16(&mut self, arg0: Type) -> Option; fn ty_int_bool_64(&mut self, arg0: Type) -> Option; fn ty_int_bool_128(&mut self, arg0: Type) -> Option; - fn vec128(&mut self, arg0: Type) -> Option; + fn ty_scalar_float(&mut self, arg0: Type) -> Option; + fn ty_vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>; @@ -129,13 +130,13 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 397. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 415. +/// Internal type ProducesFlags: defined at src/prelude.isle line 419. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -143,7 +144,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 426. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 430. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -1085,7 +1086,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 398. + // Rule at src/prelude.isle line 402. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -1103,7 +1104,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 404. + // Rule at src/prelude.isle line 408. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -1122,7 +1123,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 442. + // Rule at src/prelude.isle line 446. return Some(pattern1_1); } return None; @@ -1139,7 +1140,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 447. + // Rule at src/prelude.isle line 451. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1149,7 +1150,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 449. + // Rule at src/prelude.isle line 453. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1178,7 +1179,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 456. + // Rule at src/prelude.isle line 460. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1208,7 +1209,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 481. + // Rule at src/prelude.isle line 485. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1219,7 +1220,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 487. + // Rule at src/prelude.isle line 491. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1232,7 +1233,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 499. + // Rule at src/prelude.isle line 503. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1253,7 +1254,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 475. + // Rule at src/prelude.isle line 479. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1273,7 +1274,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 516. + // Rule at src/prelude.isle line 520. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -6477,7 +6478,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } } - if let Some(pattern3_0) = C::vec128(ctx, pattern2_0) { + if let Some(pattern3_0) = C::ty_vec128(ctx, pattern2_0) { let pattern4_0 = C::inst_data(ctx, pattern0_0); match &pattern4_0 { &InstructionData::Binary { diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index f72035cd1f..4b49b6f0f5 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle b2bc986bcbbbb77 +src/prelude.isle 74d9514ac948e163 src/isa/s390x/inst.isle d91a16074ab186a8 src/isa/s390x/lower.isle 1cc5a12adc8c75f9 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index 09fbd7b4cb..0bd0a602bd 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -56,7 +56,8 @@ pub trait Context { fn ty_8_or_16(&mut self, arg0: Type) -> Option; fn ty_int_bool_64(&mut self, arg0: Type) -> Option; fn ty_int_bool_128(&mut self, arg0: Type) -> Option; - fn vec128(&mut self, arg0: Type) -> Option; + fn ty_scalar_float(&mut self, arg0: Type) -> Option; + fn ty_vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>; @@ -152,13 +153,13 @@ pub trait Context { fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 397. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 415. +/// Internal type ProducesFlags: defined at src/prelude.isle line 419. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -166,7 +167,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 426. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 430. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -956,7 +957,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 398. + // Rule at src/prelude.isle line 402. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -974,7 +975,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 404. + // Rule at src/prelude.isle line 408. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -993,7 +994,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 442. + // Rule at src/prelude.isle line 446. return Some(pattern1_1); } return None; @@ -1010,7 +1011,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 447. + // Rule at src/prelude.isle line 451. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1020,7 +1021,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 449. + // Rule at src/prelude.isle line 453. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1049,7 +1050,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 456. + // Rule at src/prelude.isle line 460. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1079,7 +1080,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 481. + // Rule at src/prelude.isle line 485. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1090,7 +1091,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 487. + // Rule at src/prelude.isle line 491. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1103,7 +1104,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 499. + // Rule at src/prelude.isle line 503. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1124,7 +1125,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 475. + // Rule at src/prelude.isle line 479. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1144,7 +1145,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 516. + // Rule at src/prelude.isle line 520. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 214a75a0fd..b0597f06bb 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -2092,7 +2092,11 @@ imm size)) -;; Helper for creating `cmpps` instructions. +;; Helpers for creating `cmpp*` instructions. +(decl x64_cmpp (Type Xmm XmmMem FcmpImm) Xmm) +(rule (x64_cmpp $F32X4 x y imm) (x64_cmpps x y imm)) +(rule (x64_cmpp $F64X2 x y imm) (x64_cmppd x y imm)) + (decl x64_cmpps (Xmm XmmMem FcmpImm) Xmm) (rule (x64_cmpps src1 src2 imm) (xmm_rm_r_imm (SseOpcode.Cmpps) @@ -2101,6 +2105,17 @@ (encode_fcmp_imm imm) (OperandSize.Size32))) +;; Note that `Size32` is intentional despite this being used for 64-bit +;; operations, since this presumably induces the correct encoding of the +;; instruction. +(decl x64_cmppd (Xmm XmmMem FcmpImm) Xmm) +(rule (x64_cmppd src1 src2 imm) + (xmm_rm_r_imm (SseOpcode.Cmppd) + src1 + src2 + (encode_fcmp_imm imm) + (OperandSize.Size32))) + ;; Helper for creating `pinsrb` instructions. (decl x64_pinsrb (Xmm GprMem u8) Xmm) (rule (x64_pinsrb src1 src2 lane) @@ -2321,19 +2336,6 @@ (operand_size_of_type_32_64 (lane_type ty)))))) dst)) -;; Helper for creating `cmppd` instructions. -;; -;; Note that `Size32` is intentional despite this being used for 64-bit -;; operations, since this presumably induces the correct encoding of the -;; instruction. -(decl x64_cmppd (Xmm XmmMem FcmpImm) Xmm) -(rule (x64_cmppd src1 src2 imm) - (xmm_rm_r_imm (SseOpcode.Cmppd) - src1 - src2 - (encode_fcmp_imm imm) - (OperandSize.Size32))) - ;; Helper for creating `MInst.GprToXmm` instructions. (decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm) (rule (gpr_to_xmm op src size) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index af32bfa91d..84cca94422 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1454,22 +1454,22 @@ ;; lane will be filled with all 1s or all 0s according to the comparison, ;; whereas for GPR-held values, the result will be simply 0 or 1 (upper bits ;; unset). -(rule (lower (icmp (IntCC.Equal) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.Equal) a @ (value_type (ty_vec128 ty)) b)) (x64_pcmpeq ty a b)) ;; To lower a not-equals comparison, we perform an equality comparison ;; (PCMPEQ*) and then invert the bits (PXOR with all 1s). -(rule (lower (icmp (IntCC.NotEqual) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.NotEqual) a @ (value_type (ty_vec128 ty)) b)) (let ((checked Xmm (x64_pcmpeq ty a b)) (all_ones Xmm (vector_all_ones ty))) (x64_pxor checked all_ones))) ;; Signed comparisons have a single-instruction lowering, unlike their unsigned ;; counterparts. These latter instructions use the unsigned min/max ;; (PMINU*/PMAXU*) and negate the result (PXOR with all 1s). -(rule (lower (icmp (IntCC.SignedGreaterThan) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.SignedGreaterThan) a @ (value_type (ty_vec128 ty)) b)) (x64_pcmpgt ty a b)) -(rule (lower (icmp (IntCC.SignedLessThan) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.SignedLessThan) a @ (value_type (ty_vec128 ty)) b)) (x64_pcmpgt ty b a)) -(rule (lower (icmp (IntCC.UnsignedGreaterThan) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.UnsignedGreaterThan) a @ (value_type (ty_vec128 ty)) b)) ;; N.B.: we must manually prevent load coalescing of these operands; the ;; register allocator gets confused otherwise. TODO: ;; https://github.com/bytecodealliance/wasmtime/issues/3953. @@ -1479,7 +1479,7 @@ (eq Xmm (x64_pcmpeq ty max xmm_b)) (all_ones Xmm (vector_all_ones ty))) (x64_pxor eq all_ones))) -(rule (lower (icmp (IntCC.UnsignedLessThan) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.UnsignedLessThan) a @ (value_type (ty_vec128 ty)) b)) ;; N.B.: see note above. (let ((xmm_a Xmm (put_in_xmm a)) (xmm_b Xmm (put_in_xmm b)) @@ -1490,16 +1490,16 @@ ;; To lower signed and unsigned *-or-equals comparisons, we find the minimum ;; number (PMIN[U|S]*) and compare that to one of the terms (PCMPEQ*). Note that ;; there is no 64x2 version of this lowering (see below). -(rule (lower (icmp (IntCC.SignedGreaterThanOrEqual) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.SignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) (let ((max Xmm (x64_pmaxs ty a b))) (x64_pcmpeq ty a max))) -(rule (lower (icmp (IntCC.SignedLessThanOrEqual) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.SignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) (let ((min Xmm (x64_pmins ty a b))) (x64_pcmpeq ty a min))) -(rule (lower (icmp (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) (let ((max Xmm (x64_pmaxu ty a b))) (x64_pcmpeq ty a max))) -(rule (lower (icmp (IntCC.UnsignedLessThanOrEqual) a @ (value_type (vec128 ty)) b)) +(rule (lower (icmp (IntCC.UnsignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) (let ((min Xmm (x64_pminu ty a b))) (x64_pcmpeq ty a min))) ;; The PMIN[S|U]Q instruction is only available in AVX512VL/F so we must instead @@ -1550,6 +1550,115 @@ (cmp Reg (x64_or $I64 cmp_lo cmp_hi))) (with_flags (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) cmp) (x64_setcc (CC.NZ))))) +;;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; CLIF's `fcmp` instruction always operates on XMM registers--both scalar and +;; vector. For the scalar versions, we use the flag-setting behavior of the +;; `UCOMIS*` instruction to `SETcc` a 0 or 1 in a GPR register. Note that CLIF's +;; `select` uses the same kind of flag-setting behavior but chooses values other +;; than 0 or 1. +;; +;; Checking the result of `UCOMIS*` is unfortunately difficult in some cases +;; because we do not have `SETcc` instructions that explicitly check +;; simultaneously for the condition (i.e., `eq`, `le`, `gt`, etc.) *and* +;; orderedness. Instead, we must check the flags multiple times. The UCOMIS* +;; documentation (see Intel's Software Developer's Manual, volume 2, chapter 4) +;; is helpful: +;; - unordered assigns Z = 1, P = 1, C = 1 +;; - greater than assigns Z = 0, P = 0, C = 0 +;; - less than assigns Z = 0, P = 0, C = 1 +;; - equal assigns Z = 1, P = 0, C = 0 + +(rule (lower (fcmp (FloatCC.Equal) a @ (value_type (ty_scalar_float ty)) b)) + (let ((maybe ValueRegs (with_flags (x64_ucomis b a) + (consumes_flags_concat + (x64_setcc (CC.NP)) + (x64_setcc (CC.Z))))) + (maybe_np Gpr (value_regs_get_gpr maybe 0)) + (maybe_z Gpr (value_regs_get_gpr maybe 1))) + (x64_and $I32 maybe_np maybe_z))) + +(rule (lower (fcmp (FloatCC.NotEqual) a @ (value_type (ty_scalar_float ty)) b)) + (let ((maybe ValueRegs (with_flags (x64_ucomis b a) + (consumes_flags_concat + (x64_setcc (CC.P)) + (x64_setcc (CC.NZ))))) + (maybe_p Gpr (value_regs_get_gpr maybe 0)) + (maybe_nz Gpr (value_regs_get_gpr maybe 1))) + (x64_or $I32 maybe_p maybe_nz))) + +;; Some scalar lowerings correspond to one condition code. + +(rule (lower (fcmp (FloatCC.Ordered) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.NP)))) +(rule (lower (fcmp (FloatCC.Unordered) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.P)))) +(rule (lower (fcmp (FloatCC.OrderedNotEqual) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.NZ)))) +(rule (lower (fcmp (FloatCC.UnorderedOrEqual) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.Z)))) +(rule (lower (fcmp (FloatCC.GreaterThan) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.NBE)))) +(rule (lower (fcmp (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.NB)))) +(rule (lower (fcmp (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.B)))) +(rule (lower (fcmp (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)) + (with_flags (x64_ucomis b a) (x64_setcc (CC.BE)))) + +;; Other scalar lowerings are made possible by flipping the operands and +;; reversing the condition code. + +(rule (lower (fcmp (FloatCC.LessThan) a @ (value_type (ty_scalar_float ty)) b)) + ;; Same flags as `GreaterThan`. + (with_flags (x64_ucomis a b) (x64_setcc (CC.NBE)))) +(rule (lower (fcmp (FloatCC.LessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)) + ;; Same flags as `GreaterThanOrEqual`. + (with_flags (x64_ucomis a b) (x64_setcc (CC.NB)))) +(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_scalar_float ty)) b)) + ;; Same flags as `UnorderedOrLessThan`. + (with_flags (x64_ucomis a b) (x64_setcc (CC.B)))) +(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)) + ;; Same flags as `UnorderedOrLessThanOrEqual`. + (with_flags (x64_ucomis a b) (x64_setcc (CC.BE)))) + +;; For vector lowerings, we use `CMPP*` instructions with a 3-bit operand that +;; determines the comparison to make. Note that comparisons that succeed will +;; fill the lane with 1s; comparisons that do not will fill the lane with 0s. + +(rule (lower (fcmp (FloatCC.Equal) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.Equal))) +(rule (lower (fcmp (FloatCC.NotEqual) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.NotEqual))) +(rule (lower (fcmp (FloatCC.LessThan) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.LessThan))) +(rule (lower (fcmp (FloatCC.LessThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.LessThanOrEqual))) +(rule (lower (fcmp (FloatCC.Ordered) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.Ordered))) +(rule (lower (fcmp (FloatCC.Unordered) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.Unordered))) +(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.UnorderedOrGreaterThan))) +(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty a b (FcmpImm.UnorderedOrGreaterThanOrEqual))) + +;; Some vector lowerings rely on flipping the operands and using a reversed +;; comparison code. + +(rule (lower (fcmp (FloatCC.GreaterThan) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty b a (FcmpImm.LessThan))) +(rule (lower (fcmp (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty b a (FcmpImm.LessThanOrEqual))) +(rule (lower (fcmp (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty b a (FcmpImm.UnorderedOrGreaterThan))) +(rule (lower (fcmp (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b)) + (x64_cmpp ty b a (FcmpImm.UnorderedOrGreaterThanOrEqual))) + +;; Some vector lowerings are simply not supported for certain codes: +;; - FloatCC::OrderedNotEqual +;; - FloatCC::UnorderedOrEqual + ;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; CLIF `select` instructions receive a testable argument (i.e. boolean or diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 3b9341420c..207dd1c06a 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -930,104 +930,7 @@ fn lower_insn_to_regs>( } Opcode::Fcmp => { - let cond_code = ctx.data(insn).fp_cond_code().unwrap(); - let input_ty = ctx.input_ty(insn, 0); - if !input_ty.is_vector() { - // Unordered is returned by setting ZF, PF, CF <- 111 - // Greater than by ZF, PF, CF <- 000 - // Less than by ZF, PF, CF <- 001 - // Equal by ZF, PF, CF <- 100 - // - // Checking the result of comiss is somewhat annoying because you don't have setcc - // instructions that explicitly check simultaneously for the condition (i.e. eq, le, - // gt, etc) *and* orderedness. - // - // So that might mean we need more than one setcc check and then a logical "and" or - // "or" to determine both, in some cases. However knowing that if the parity bit is - // set, then the result was considered unordered and knowing that if the parity bit is - // set, then both the ZF and CF flag bits must also be set we can get away with using - // one setcc for most condition codes. - - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - match emit_fcmp(ctx, insn, cond_code, FcmpSpec::Normal) { - FcmpCondResult::Condition(cc) => { - ctx.emit(Inst::setcc(cc, dst)); - } - FcmpCondResult::AndConditions(cc1, cc2) => { - let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap(); - ctx.emit(Inst::setcc(cc1, tmp)); - ctx.emit(Inst::setcc(cc2, dst)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::And, - RegMemImm::reg(tmp.to_reg()), - dst, - )); - } - FcmpCondResult::OrConditions(cc1, cc2) => { - let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap(); - ctx.emit(Inst::setcc(cc1, tmp)); - ctx.emit(Inst::setcc(cc2, dst)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::Or, - RegMemImm::reg(tmp.to_reg()), - dst, - )); - } - FcmpCondResult::InvertedEqualOrConditions(_, _) => unreachable!(), - } - } else { - let op = match input_ty { - types::F32X4 => SseOpcode::Cmpps, - types::F64X2 => SseOpcode::Cmppd, - _ => panic!("Bad input type to fcmp: {}", input_ty), - }; - - // Since some packed comparisons are not available, some of the condition codes - // must be inverted, with a corresponding `flip` of the operands. - let (imm, flip) = match cond_code { - FloatCC::GreaterThan => (FcmpImm::LessThan, true), - FloatCC::GreaterThanOrEqual => (FcmpImm::LessThanOrEqual, true), - FloatCC::UnorderedOrLessThan => (FcmpImm::UnorderedOrGreaterThan, true), - FloatCC::UnorderedOrLessThanOrEqual => { - (FcmpImm::UnorderedOrGreaterThanOrEqual, true) - } - FloatCC::OrderedNotEqual | FloatCC::UnorderedOrEqual => { - panic!("unsupported float condition code: {}", cond_code) - } - _ => (FcmpImm::from(cond_code), false), - }; - - // Determine the operands of the comparison, possibly by flipping them. - let (lhs, rhs) = if flip { - ( - put_input_in_reg(ctx, inputs[1]), - input_to_reg_mem(ctx, inputs[0]), - ) - } else { - ( - put_input_in_reg(ctx, inputs[0]), - input_to_reg_mem(ctx, inputs[1]), - ) - }; - - // Move the `lhs` to the same register as `dst`; this may not emit an actual move - // but ensures that the registers are the same to match x86's read-write operand - // encoding. - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, lhs, input_ty)); - - // Emit the comparison. - ctx.emit(Inst::xmm_rm_r_imm( - op, - rhs, - dst, - imm.encode(), - OperandSize::Size32, - )); - } + implemented_in_isle(ctx); } Opcode::FallthroughReturn | Opcode::Return => { diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 86fd00f3ce..f208a2d444 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle b2bc986bcbbbb77 -src/isa/x64/inst.isle bfb0fb7143d8dc34 -src/isa/x64/lower.isle ccaee2b83bdf73e1 +src/prelude.isle 74d9514ac948e163 +src/isa/x64/inst.isle a002d62dcfce285 +src/isa/x64/lower.isle d8facef52a4e2ac6 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 71cf730fe4..d4727c6fc4 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -56,7 +56,8 @@ pub trait Context { fn ty_8_or_16(&mut self, arg0: Type) -> Option; fn ty_int_bool_64(&mut self, arg0: Type) -> Option; fn ty_int_bool_128(&mut self, arg0: Type) -> Option; - fn vec128(&mut self, arg0: Type) -> Option; + fn ty_scalar_float(&mut self, arg0: Type) -> Option; + fn ty_vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>; @@ -150,13 +151,13 @@ pub trait Context { fn popcount_low_mask(&mut self) -> VCodeConstant; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 397. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 415. +/// Internal type ProducesFlags: defined at src/prelude.isle line 419. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -164,7 +165,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 426. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 430. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -556,7 +557,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 398. + // Rule at src/prelude.isle line 402. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -574,7 +575,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 404. + // Rule at src/prelude.isle line 408. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -593,7 +594,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 442. + // Rule at src/prelude.isle line 446. return Some(pattern1_1); } return None; @@ -610,7 +611,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 447. + // Rule at src/prelude.isle line 451. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -620,7 +621,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 449. + // Rule at src/prelude.isle line 453. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -649,7 +650,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 456. + // Rule at src/prelude.isle line 460. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -679,7 +680,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 481. + // Rule at src/prelude.isle line 485. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -690,7 +691,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 487. + // Rule at src/prelude.isle line 491. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -703,7 +704,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 499. + // Rule at src/prelude.isle line 503. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -724,7 +725,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 475. + // Rule at src/prelude.isle line 479. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -744,7 +745,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 516. + // Rule at src/prelude.isle line 520. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -3363,6 +3364,34 @@ pub fn constructor_x64_palignr( return Some(expr3_0); } +// Generated as internal constructor for term x64_cmpp. +pub fn constructor_x64_cmpp( + ctx: &mut C, + arg0: Type, + arg1: Xmm, + arg2: &XmmMem, + arg3: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + if pattern0_0 == F32X4 { + let pattern2_0 = arg1; + let pattern3_0 = arg2; + let pattern4_0 = arg3; + // Rule at src/isa/x64/inst.isle line 2097. + let expr0_0 = constructor_x64_cmpps(ctx, pattern2_0, pattern3_0, pattern4_0)?; + return Some(expr0_0); + } + if pattern0_0 == F64X2 { + let pattern2_0 = arg1; + let pattern3_0 = arg2; + let pattern4_0 = arg3; + // Rule at src/isa/x64/inst.isle line 2098. + let expr0_0 = constructor_x64_cmppd(ctx, pattern2_0, pattern3_0, pattern4_0)?; + return Some(expr0_0); + } + return None; +} + // Generated as internal constructor for term x64_cmpps. pub fn constructor_x64_cmpps( ctx: &mut C, @@ -3373,7 +3402,7 @@ pub fn constructor_x64_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2097. + // Rule at src/isa/x64/inst.isle line 2101. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3383,6 +3412,26 @@ pub fn constructor_x64_cmpps( return Some(expr5_0); } +// Generated as internal constructor for term x64_cmppd. +pub fn constructor_x64_cmppd( + ctx: &mut C, + arg0: Xmm, + arg1: &XmmMem, + arg2: &FcmpImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 2112. + let expr0_0 = SseOpcode::Cmppd; + let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); + let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); + let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); + let expr4_0 = OperandSize::Size32; + let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; + return Some(expr5_0); +} + // Generated as internal constructor for term x64_pinsrb. pub fn constructor_x64_pinsrb( ctx: &mut C, @@ -3393,7 +3442,7 @@ pub fn constructor_x64_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2106. + // Rule at src/isa/x64/inst.isle line 2121. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3412,7 +3461,7 @@ pub fn constructor_x64_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2115. + // Rule at src/isa/x64/inst.isle line 2130. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3433,7 +3482,7 @@ pub fn constructor_x64_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2124. + // Rule at src/isa/x64/inst.isle line 2139. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3446,7 +3495,7 @@ pub fn constructor_x64_pinsrd( pub fn constructor_x64_pmaddwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2133. + // Rule at src/isa/x64/inst.isle line 2148. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pmaddwd; let expr2_0 = MInst::XmmRmR { @@ -3470,7 +3519,7 @@ pub fn constructor_x64_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2143. + // Rule at src/isa/x64/inst.isle line 2158. let expr0_0 = SseOpcode::Insertps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3489,7 +3538,7 @@ pub fn constructor_x64_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2152. + // Rule at src/isa/x64/inst.isle line 2167. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pshufd; let expr2_0 = constructor_writable_xmm_to_r_reg(ctx, expr0_0)?; @@ -3512,7 +3561,7 @@ pub fn constructor_x64_pshufd( pub fn constructor_x64_pshufb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2164. + // Rule at src/isa/x64/inst.isle line 2179. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pshufb; let expr2_0 = MInst::XmmRmR { @@ -3534,7 +3583,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2174. + // Rule at src/isa/x64/inst.isle line 2189. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), @@ -3549,7 +3598,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term x64_pmovsxbw. pub fn constructor_x64_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2181. + // Rule at src/isa/x64/inst.isle line 2196. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3558,7 +3607,7 @@ pub fn constructor_x64_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Optio // Generated as internal constructor for term x64_pmovzxbw. pub fn constructor_x64_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2186. + // Rule at src/isa/x64/inst.isle line 2201. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3567,7 +3616,7 @@ pub fn constructor_x64_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Optio // Generated as internal constructor for term x64_pabsb. pub fn constructor_x64_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2191. + // Rule at src/isa/x64/inst.isle line 2206. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3576,7 +3625,7 @@ pub fn constructor_x64_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2196. + // Rule at src/isa/x64/inst.isle line 2211. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3585,7 +3634,7 @@ pub fn constructor_x64_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2201. + // Rule at src/isa/x64/inst.isle line 2216. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3599,7 +3648,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2206. + // Rule at src/isa/x64/inst.isle line 2221. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), @@ -3614,7 +3663,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term x64_vpabsq. pub fn constructor_x64_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2213. + // Rule at src/isa/x64/inst.isle line 2228. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3623,7 +3672,7 @@ pub fn constructor_x64_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option< // Generated as internal constructor for term x64_vpopcntb. pub fn constructor_x64_vpopcntb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2218. + // Rule at src/isa/x64/inst.isle line 2233. let expr0_0 = Avx512Opcode::Vpopcntb; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3639,7 +3688,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2223. + // Rule at src/isa/x64/inst.isle line 2238. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), @@ -3656,7 +3705,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_x64_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2235. + // Rule at src/isa/x64/inst.isle line 2250. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3674,7 +3723,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2244. + // Rule at src/isa/x64/inst.isle line 2259. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::temp_writable_gpr(ctx); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -3703,7 +3752,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2259. + // Rule at src/isa/x64/inst.isle line 2274. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3719,7 +3768,7 @@ pub fn constructor_xmm_rmi_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2264. + // Rule at src/isa/x64/inst.isle line 2279. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmiReg { opcode: pattern0_0.clone(), @@ -3736,7 +3785,7 @@ pub fn constructor_xmm_rmi_xmm( pub fn constructor_x64_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2274. + // Rule at src/isa/x64/inst.isle line 2289. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3746,7 +3795,7 @@ pub fn constructor_x64_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2279. + // Rule at src/isa/x64/inst.isle line 2294. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3756,7 +3805,7 @@ pub fn constructor_x64_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2284. + // Rule at src/isa/x64/inst.isle line 2299. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3766,7 +3815,7 @@ pub fn constructor_x64_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2289. + // Rule at src/isa/x64/inst.isle line 2304. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3776,7 +3825,7 @@ pub fn constructor_x64_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2294. + // Rule at src/isa/x64/inst.isle line 2309. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3786,7 +3835,7 @@ pub fn constructor_x64_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2299. + // Rule at src/isa/x64/inst.isle line 2314. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3796,7 +3845,7 @@ pub fn constructor_x64_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2304. + // Rule at src/isa/x64/inst.isle line 2319. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3806,7 +3855,7 @@ pub fn constructor_x64_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2309. + // Rule at src/isa/x64/inst.isle line 2324. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3822,7 +3871,7 @@ pub fn constructor_x64_pextrd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2314. + // Rule at src/isa/x64/inst.isle line 2329. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = SseOpcode::Pextrd; let expr2_0 = constructor_writable_gpr_to_r_reg(ctx, expr0_0)?; @@ -3845,26 +3894,6 @@ pub fn constructor_x64_pextrd( return Some(expr11_0); } -// Generated as internal constructor for term x64_cmppd. -pub fn constructor_x64_cmppd( - ctx: &mut C, - arg0: Xmm, - arg1: &XmmMem, - arg2: &FcmpImm, -) -> Option { - let pattern0_0 = arg0; - let pattern1_0 = arg1; - let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2330. - let expr0_0 = SseOpcode::Cmppd; - let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); - let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); - let expr3_0 = C::encode_fcmp_imm(ctx, pattern2_0); - let expr4_0 = OperandSize::Size32; - let expr5_0 = constructor_xmm_rm_r_imm(ctx, &expr0_0, expr1_0, &expr2_0, expr3_0, &expr4_0)?; - return Some(expr5_0); -} - // Generated as internal constructor for term gpr_to_xmm. pub fn constructor_gpr_to_xmm( ctx: &mut C, @@ -3875,7 +3904,7 @@ pub fn constructor_gpr_to_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2339. + // Rule at src/isa/x64/inst.isle line 2341. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::GprToXmm { op: pattern0_0.clone(), @@ -3892,7 +3921,7 @@ pub fn constructor_gpr_to_xmm( pub fn constructor_x64_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2346. + // Rule at src/isa/x64/inst.isle line 2348. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -3909,7 +3938,7 @@ pub fn constructor_x64_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Op pub fn constructor_x64_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2354. + // Rule at src/isa/x64/inst.isle line 2356. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { @@ -3925,7 +3954,7 @@ pub fn constructor_x64_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Op // Generated as internal constructor for term x64_lea. pub fn constructor_x64_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2361. + // Rule at src/isa/x64/inst.isle line 2363. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), @@ -3939,7 +3968,7 @@ pub fn constructor_x64_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Op // Generated as internal constructor for term x64_ud2. pub fn constructor_x64_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2368. + // Rule at src/isa/x64/inst.isle line 2370. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -3949,7 +3978,7 @@ pub fn constructor_x64_ud2(ctx: &mut C, arg0: &TrapCode) -> Option(ctx: &mut C) -> Option { - // Rule at src/isa/x64/inst.isle line 2373. + // Rule at src/isa/x64/inst.isle line 2375. let expr0_0 = MInst::Hlt; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -3959,7 +3988,7 @@ pub fn constructor_x64_hlt(ctx: &mut C) -> Option(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2378. + // Rule at src/isa/x64/inst.isle line 2380. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Lzcnt; @@ -3979,7 +4008,7 @@ pub fn constructor_x64_lzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_x64_tzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2386. + // Rule at src/isa/x64/inst.isle line 2388. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Tzcnt; @@ -4003,7 +4032,7 @@ pub fn constructor_x64_bsr( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2394. + // Rule at src/isa/x64/inst.isle line 2396. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Bsr; @@ -4032,7 +4061,7 @@ pub fn constructor_bsr_or_else( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2403. + // Rule at src/isa/x64/inst.isle line 2405. let expr0_0 = constructor_x64_bsr(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_produces_flags_get_reg(ctx, &expr0_0)?; let expr2_0 = C::gpr_new(ctx, expr1_0); @@ -4053,7 +4082,7 @@ pub fn constructor_x64_bsf( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2414. + // Rule at src/isa/x64/inst.isle line 2416. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Bsf; @@ -4082,7 +4111,7 @@ pub fn constructor_bsf_or_else( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2423. + // Rule at src/isa/x64/inst.isle line 2425. let expr0_0 = constructor_x64_bsf(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_produces_flags_get_reg(ctx, &expr0_0)?; let expr2_0 = C::gpr_new(ctx, expr1_0); @@ -4099,7 +4128,7 @@ pub fn constructor_bsf_or_else( pub fn constructor_x64_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2434. + // Rule at src/isa/x64/inst.isle line 2436. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Popcnt; @@ -4127,7 +4156,7 @@ pub fn constructor_xmm_min_max_seq( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2442. + // Rule at src/isa/x64/inst.isle line 2444. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::XmmMinMaxSeq { @@ -4146,7 +4175,7 @@ pub fn constructor_xmm_min_max_seq( pub fn constructor_x64_minss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2450. + // Rule at src/isa/x64/inst.isle line 2452. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minss; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4165,7 +4194,7 @@ pub fn constructor_x64_minss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_minsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2457. + // Rule at src/isa/x64/inst.isle line 2459. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minsd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4184,7 +4213,7 @@ pub fn constructor_x64_minsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_minps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2465. + // Rule at src/isa/x64/inst.isle line 2467. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minps; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4203,7 +4232,7 @@ pub fn constructor_x64_minps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_minpd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2472. + // Rule at src/isa/x64/inst.isle line 2474. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minpd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4222,7 +4251,7 @@ pub fn constructor_x64_minpd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2479. + // Rule at src/isa/x64/inst.isle line 2481. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxss; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4241,7 +4270,7 @@ pub fn constructor_x64_maxss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2486. + // Rule at src/isa/x64/inst.isle line 2488. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxsd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4260,7 +4289,7 @@ pub fn constructor_x64_maxsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2493. + // Rule at src/isa/x64/inst.isle line 2495. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxps; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4279,7 +4308,7 @@ pub fn constructor_x64_maxps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxpd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2500. + // Rule at src/isa/x64/inst.isle line 2502. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxpd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4305,28 +4334,28 @@ pub fn constructor_x64_pcmpeq( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2507. + // Rule at src/isa/x64/inst.isle line 2509. let expr0_0 = constructor_x64_pcmpeqb(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2508. + // Rule at src/isa/x64/inst.isle line 2510. let expr0_0 = constructor_x64_pcmpeqw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2509. + // Rule at src/isa/x64/inst.isle line 2511. let expr0_0 = constructor_x64_pcmpeqd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2510. + // Rule at src/isa/x64/inst.isle line 2512. let expr0_0 = constructor_x64_pcmpeqq(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4337,7 +4366,7 @@ pub fn constructor_x64_pcmpeq( pub fn constructor_x64_pcmpeqb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2513. + // Rule at src/isa/x64/inst.isle line 2515. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pcmpeqb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4348,7 +4377,7 @@ pub fn constructor_x64_pcmpeqb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpeqw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2515. + // Rule at src/isa/x64/inst.isle line 2517. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pcmpeqw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4359,7 +4388,7 @@ pub fn constructor_x64_pcmpeqw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpeqd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2517. + // Rule at src/isa/x64/inst.isle line 2519. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Pcmpeqd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4370,7 +4399,7 @@ pub fn constructor_x64_pcmpeqd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpeqq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2519. + // Rule at src/isa/x64/inst.isle line 2521. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pcmpeqq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4388,28 +4417,28 @@ pub fn constructor_x64_pcmpgt( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2523. + // Rule at src/isa/x64/inst.isle line 2525. let expr0_0 = constructor_x64_pcmpgtb(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2524. + // Rule at src/isa/x64/inst.isle line 2526. let expr0_0 = constructor_x64_pcmpgtw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2525. + // Rule at src/isa/x64/inst.isle line 2527. let expr0_0 = constructor_x64_pcmpgtd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2526. + // Rule at src/isa/x64/inst.isle line 2528. let expr0_0 = constructor_x64_pcmpgtq(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4420,7 +4449,7 @@ pub fn constructor_x64_pcmpgt( pub fn constructor_x64_pcmpgtb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2529. + // Rule at src/isa/x64/inst.isle line 2531. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pcmpgtb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4431,7 +4460,7 @@ pub fn constructor_x64_pcmpgtb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpgtw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2531. + // Rule at src/isa/x64/inst.isle line 2533. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pcmpgtw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4442,7 +4471,7 @@ pub fn constructor_x64_pcmpgtw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpgtd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2533. + // Rule at src/isa/x64/inst.isle line 2535. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Pcmpgtd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4453,7 +4482,7 @@ pub fn constructor_x64_pcmpgtd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpgtq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2535. + // Rule at src/isa/x64/inst.isle line 2537. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pcmpgtq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4463,7 +4492,7 @@ pub fn constructor_x64_pcmpgtq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem // Generated as internal constructor for term reg_to_xmm_mem. pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2589. + // Rule at src/isa/x64/inst.isle line 2591. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -4472,7 +4501,7 @@ pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option< // Generated as internal constructor for term xmm_to_reg_mem. pub fn constructor_xmm_to_reg_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2592. + // Rule at src/isa/x64/inst.isle line 2594. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_reg(ctx, expr0_0); let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4486,7 +4515,7 @@ pub fn constructor_writable_gpr_to_r_reg( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2596. + // Rule at src/isa/x64/inst.isle line 2598. let expr0_0 = C::writable_gpr_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -4498,7 +4527,7 @@ pub fn constructor_writable_gpr_to_gpr_mem( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2599. + // Rule at src/isa/x64/inst.isle line 2601. let expr0_0 = C::writable_gpr_to_gpr(ctx, pattern0_0); let expr1_0 = C::gpr_to_gpr_mem(ctx, expr0_0); return Some(expr1_0); @@ -4510,7 +4539,7 @@ pub fn constructor_writable_gpr_to_value_regs( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2602. + // Rule at src/isa/x64/inst.isle line 2604. let expr0_0 = constructor_writable_gpr_to_r_reg(ctx, pattern0_0)?; let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); @@ -4522,7 +4551,7 @@ pub fn constructor_writable_xmm_to_r_reg( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2605. + // Rule at src/isa/x64/inst.isle line 2607. let expr0_0 = C::writable_xmm_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -4534,7 +4563,7 @@ pub fn constructor_writable_xmm_to_xmm_mem( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2608. + // Rule at src/isa/x64/inst.isle line 2610. let expr0_0 = C::writable_xmm_to_xmm(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -4546,7 +4575,7 @@ pub fn constructor_writable_xmm_to_value_regs( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2611. + // Rule at src/isa/x64/inst.isle line 2613. let expr0_0 = constructor_writable_xmm_to_r_reg(ctx, pattern0_0)?; let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); @@ -4558,7 +4587,7 @@ pub fn constructor_synthetic_amode_to_gpr_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2615. + // Rule at src/isa/x64/inst.isle line 2617. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_gpr_mem(ctx, &expr0_0); return Some(expr1_0); @@ -4570,7 +4599,7 @@ pub fn constructor_synthetic_amode_to_xmm_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2618. + // Rule at src/isa/x64/inst.isle line 2620. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); return Some(expr1_0); @@ -4590,7 +4619,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Debugtrap = pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2116. + // Rule at src/isa/x64/lower.isle line 2225. let expr0_0 = constructor_x64_hlt(ctx)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); @@ -4785,6 +4814,294 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + if let &Opcode::Fcmp = pattern2_0 { + let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, pattern2_1); + let pattern5_0 = C::value_type(ctx, pattern4_0); + if let Some(pattern6_0) = C::ty_scalar_float(ctx, pattern5_0) { + match pattern2_2 { + &FloatCC::Equal => { + // Rule at src/isa/x64/lower.isle line 1572. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::NP; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = CC::Z; + let expr4_0 = constructor_x64_setcc(ctx, &expr3_0)?; + let expr5_0 = + constructor_consumes_flags_concat(ctx, &expr2_0, &expr4_0)?; + let expr6_0 = constructor_with_flags(ctx, &expr0_0, &expr5_0)?; + let expr7_0: usize = 0; + let expr8_0 = constructor_value_regs_get_gpr(ctx, expr6_0, expr7_0)?; + let expr9_0: usize = 1; + let expr10_0 = constructor_value_regs_get_gpr(ctx, expr6_0, expr9_0)?; + let expr11_0: Type = I32; + let expr12_0 = C::gpr_to_gpr_mem_imm(ctx, expr10_0); + let expr13_0 = constructor_x64_and(ctx, expr11_0, expr8_0, &expr12_0)?; + let expr14_0 = constructor_output_gpr(ctx, expr13_0)?; + return Some(expr14_0); + } + &FloatCC::GreaterThan => { + // Rule at src/isa/x64/lower.isle line 1600. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::NBE; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::GreaterThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1602. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::NB; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::LessThan => { + // Rule at src/isa/x64/lower.isle line 1612. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; + let expr1_0 = CC::NBE; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::LessThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1615. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; + let expr1_0 = CC::NB; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::NotEqual => { + // Rule at src/isa/x64/lower.isle line 1581. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::P; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = CC::NZ; + let expr4_0 = constructor_x64_setcc(ctx, &expr3_0)?; + let expr5_0 = + constructor_consumes_flags_concat(ctx, &expr2_0, &expr4_0)?; + let expr6_0 = constructor_with_flags(ctx, &expr0_0, &expr5_0)?; + let expr7_0: usize = 0; + let expr8_0 = constructor_value_regs_get_gpr(ctx, expr6_0, expr7_0)?; + let expr9_0: usize = 1; + let expr10_0 = constructor_value_regs_get_gpr(ctx, expr6_0, expr9_0)?; + let expr11_0: Type = I32; + let expr12_0 = C::gpr_to_gpr_mem_imm(ctx, expr10_0); + let expr13_0 = constructor_x64_or(ctx, expr11_0, expr8_0, &expr12_0)?; + let expr14_0 = constructor_output_gpr(ctx, expr13_0)?; + return Some(expr14_0); + } + &FloatCC::Ordered => { + // Rule at src/isa/x64/lower.isle line 1592. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::NP; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::OrderedNotEqual => { + // Rule at src/isa/x64/lower.isle line 1596. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::NZ; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::Unordered => { + // Rule at src/isa/x64/lower.isle line 1594. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::P; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrEqual => { + // Rule at src/isa/x64/lower.isle line 1598. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::Z; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrGreaterThan => { + // Rule at src/isa/x64/lower.isle line 1618. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; + let expr1_0 = CC::B; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrGreaterThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1621. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; + let expr1_0 = CC::BE; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrLessThan => { + // Rule at src/isa/x64/lower.isle line 1604. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::B; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrLessThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1606. + let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; + let expr1_0 = CC::BE; + let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; + let expr3_0 = constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + _ => {} + } + } + if let Some(pattern6_0) = C::ty_vec128(ctx, pattern5_0) { + match pattern2_2 { + &FloatCC::Equal => { + // Rule at src/isa/x64/lower.isle line 1629. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::Equal; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::GreaterThan => { + // Rule at src/isa/x64/lower.isle line 1649. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; + let expr2_0 = FcmpImm::LessThan; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::GreaterThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1651. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; + let expr2_0 = FcmpImm::LessThanOrEqual; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::LessThan => { + // Rule at src/isa/x64/lower.isle line 1633. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::LessThan; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::LessThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1635. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::LessThanOrEqual; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::NotEqual => { + // Rule at src/isa/x64/lower.isle line 1631. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::NotEqual; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::Ordered => { + // Rule at src/isa/x64/lower.isle line 1637. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::Ordered; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::Unordered => { + // Rule at src/isa/x64/lower.isle line 1639. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::Unordered; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::UnorderedOrGreaterThan => { + // Rule at src/isa/x64/lower.isle line 1641. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::UnorderedOrGreaterThan; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::UnorderedOrGreaterThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1643. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; + let expr2_0 = FcmpImm::UnorderedOrGreaterThanOrEqual; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::UnorderedOrLessThan => { + // Rule at src/isa/x64/lower.isle line 1653. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; + let expr2_0 = FcmpImm::UnorderedOrGreaterThan; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + &FloatCC::UnorderedOrLessThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1655. + let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; + let expr2_0 = FcmpImm::UnorderedOrGreaterThanOrEqual; + let expr3_0 = + constructor_x64_cmpp(ctx, pattern6_0, expr0_0, &expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_xmm(ctx, expr3_0)?; + return Some(expr4_0); + } + _ => {} + } + } + } + } &InstructionData::IntCompare { opcode: ref pattern2_0, args: ref pattern2_1, @@ -4929,7 +5246,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { // Rule at src/isa/x64/lower.isle line 1457. @@ -5067,7 +5384,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 1959. + // Rule at src/isa/x64/lower.isle line 2068. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 0; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -5082,7 +5399,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 1967. + // Rule at src/isa/x64/lower.isle line 2076. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 4294967295; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -5213,7 +5530,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1890. + // Rule at src/isa/x64/lower.isle line 1999. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0 = constructor_do_bitrev64(ctx, expr0_0, expr1_0)?; @@ -5287,7 +5604,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2006. + // Rule at src/isa/x64/lower.isle line 2115. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5295,7 +5612,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2012. + // Rule at src/isa/x64/lower.isle line 2121. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5303,7 +5620,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2015. + // Rule at src/isa/x64/lower.isle line 2124. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5311,7 +5628,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2009. + // Rule at src/isa/x64/lower.isle line 2118. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5319,7 +5636,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2018. + // Rule at src/isa/x64/lower.isle line 2127. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5327,7 +5644,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2021. + // Rule at src/isa/x64/lower.isle line 2130. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5335,7 +5652,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2024. + // Rule at src/isa/x64/lower.isle line 2133. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5343,7 +5660,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2027. + // Rule at src/isa/x64/lower.isle line 2136. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5351,7 +5668,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2030. + // Rule at src/isa/x64/lower.isle line 2139. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5366,7 +5683,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Uload32 = pattern12_0 { - // Rule at src/isa/x64/lower.isle line 2033. + // Rule at src/isa/x64/lower.isle line 2142. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5379,7 +5696,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1893. + // Rule at src/isa/x64/lower.isle line 2002. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 1; @@ -5656,7 +5973,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1663. + // Rule at src/isa/x64/lower.isle line 1772. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -5694,7 +6011,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1705. + // Rule at src/isa/x64/lower.isle line 1814. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -5732,7 +6049,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1757. + // Rule at src/isa/x64/lower.isle line 1866. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 0; @@ -5755,7 +6072,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2108. + // Rule at src/isa/x64/lower.isle line 2217. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0: u32 = 1; @@ -5773,7 +6090,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern7_0 = C::value_type(ctx, pattern5_1); if pattern7_0 == I64 { - // Rule at src/isa/x64/lower.isle line 1980. + // Rule at src/isa/x64/lower.isle line 2089. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = I64; let expr2_0: u64 = 0; @@ -5783,7 +6100,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2131. + // Rule at src/isa/x64/lower.isle line 2240. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addss(ctx, expr0_0, &expr1_0)?; @@ -5853,7 +6170,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2142. + // Rule at src/isa/x64/lower.isle line 2251. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subss(ctx, expr0_0, &expr1_0)?; @@ -5862,7 +6179,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2153. + // Rule at src/isa/x64/lower.isle line 2262. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulss(ctx, expr0_0, &expr1_0)?; @@ -5871,7 +6188,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2164. + // Rule at src/isa/x64/lower.isle line 2273. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divss(ctx, expr0_0, &expr1_0)?; @@ -5880,7 +6197,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2175. + // Rule at src/isa/x64/lower.isle line 2284. let expr0_0: Type = F32; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -5892,7 +6209,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2344. + // Rule at src/isa/x64/lower.isle line 2453. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minss(ctx, expr0_0, expr1_0)?; @@ -5901,7 +6218,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2247. + // Rule at src/isa/x64/lower.isle line 2356. let expr0_0: Type = F32; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -5913,7 +6230,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2355. + // Rule at src/isa/x64/lower.isle line 2464. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxss(ctx, expr0_0, expr1_0)?; @@ -5934,7 +6251,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2133. + // Rule at src/isa/x64/lower.isle line 2242. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addsd(ctx, expr0_0, &expr1_0)?; @@ -5943,7 +6260,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2144. + // Rule at src/isa/x64/lower.isle line 2253. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subsd(ctx, expr0_0, &expr1_0)?; @@ -5952,7 +6269,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2155. + // Rule at src/isa/x64/lower.isle line 2264. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulsd(ctx, expr0_0, &expr1_0)?; @@ -5961,7 +6278,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2166. + // Rule at src/isa/x64/lower.isle line 2275. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divsd(ctx, expr0_0, &expr1_0)?; @@ -5970,7 +6287,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2177. + // Rule at src/isa/x64/lower.isle line 2286. let expr0_0: Type = F64; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -5982,7 +6299,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2346. + // Rule at src/isa/x64/lower.isle line 2455. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minsd(ctx, expr0_0, expr1_0)?; @@ -5991,7 +6308,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2249. + // Rule at src/isa/x64/lower.isle line 2358. let expr0_0: Type = F64; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6003,7 +6320,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2357. + // Rule at src/isa/x64/lower.isle line 2466. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxsd(ctx, expr0_0, expr1_0)?; @@ -6142,7 +6459,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1864. + // Rule at src/isa/x64/lower.isle line 1973. let expr0_0 = C::popcount_4bit_table(ctx); let expr1_0: Type = I8X16; let expr2_0 = C::popcount_low_mask(ctx); @@ -6359,7 +6676,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2121. + // Rule at src/isa/x64/lower.isle line 2230. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaddwd(ctx, expr0_0, &expr1_0)?; @@ -6499,7 +6816,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2135. + // Rule at src/isa/x64/lower.isle line 2244. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addps(ctx, expr0_0, &expr1_0)?; @@ -6508,7 +6825,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2146. + // Rule at src/isa/x64/lower.isle line 2255. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subps(ctx, expr0_0, &expr1_0)?; @@ -6517,7 +6834,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2157. + // Rule at src/isa/x64/lower.isle line 2266. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulps(ctx, expr0_0, &expr1_0)?; @@ -6526,7 +6843,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2168. + // Rule at src/isa/x64/lower.isle line 2277. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divps(ctx, expr0_0, &expr1_0)?; @@ -6535,7 +6852,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2192. + // Rule at src/isa/x64/lower.isle line 2301. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -6560,7 +6877,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2348. + // Rule at src/isa/x64/lower.isle line 2457. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -6569,7 +6886,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2255. + // Rule at src/isa/x64/lower.isle line 2364. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -6597,7 +6914,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2359. + // Rule at src/isa/x64/lower.isle line 2468. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -6639,7 +6956,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2137. + // Rule at src/isa/x64/lower.isle line 2246. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addpd(ctx, expr0_0, &expr1_0)?; @@ -6648,7 +6965,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2148. + // Rule at src/isa/x64/lower.isle line 2257. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subpd(ctx, expr0_0, &expr1_0)?; @@ -6657,7 +6974,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2159. + // Rule at src/isa/x64/lower.isle line 2268. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulpd(ctx, expr0_0, &expr1_0)?; @@ -6666,7 +6983,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2170. + // Rule at src/isa/x64/lower.isle line 2279. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divpd(ctx, expr0_0, &expr1_0)?; @@ -6675,7 +6992,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2235. + // Rule at src/isa/x64/lower.isle line 2344. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -6700,7 +7017,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2350. + // Rule at src/isa/x64/lower.isle line 2459. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -6709,7 +7026,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2298. + // Rule at src/isa/x64/lower.isle line 2407. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -6737,7 +7054,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2361. + // Rule at src/isa/x64/lower.isle line 2470. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -6816,7 +7133,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1634. + // Rule at src/isa/x64/lower.isle line 1743. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; let expr1_0 = CC::NZ; @@ -6831,7 +7148,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1586. + // Rule at src/isa/x64/lower.isle line 1695. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; let expr1_0 = CC::NBE; @@ -6844,7 +7161,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1589. + // Rule at src/isa/x64/lower.isle line 1698. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; let expr1_0 = CC::NB; @@ -6857,7 +7174,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1609. + // Rule at src/isa/x64/lower.isle line 1718. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; let expr1_0 = CC::NBE; @@ -6870,7 +7187,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1612. + // Rule at src/isa/x64/lower.isle line 1721. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; let expr1_0 = CC::NB; @@ -6883,7 +7200,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1637. + // Rule at src/isa/x64/lower.isle line 1746. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; let expr1_0 = CC::NZ; @@ -6898,7 +7215,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1580. + // Rule at src/isa/x64/lower.isle line 1689. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; let expr1_0 = CC::NP; @@ -6911,7 +7228,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1583. + // Rule at src/isa/x64/lower.isle line 1692. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; let expr1_0 = CC::P; @@ -6924,7 +7241,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1615. + // Rule at src/isa/x64/lower.isle line 1724. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; let expr1_0 = CC::B; @@ -6937,7 +7254,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1618. + // Rule at src/isa/x64/lower.isle line 1727. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; let expr1_0 = CC::BE; @@ -6950,7 +7267,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1592. + // Rule at src/isa/x64/lower.isle line 1701. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; let expr1_0 = CC::B; @@ -6963,7 +7280,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1595. + // Rule at src/isa/x64/lower.isle line 1704. let expr0_0 = constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; let expr1_0 = CC::BE; @@ -6990,14 +7307,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2094. + // Rule at src/isa/x64/lower.isle line 2203. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Bextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2075. + // Rule at src/isa/x64/lower.isle line 2184. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -7005,7 +7322,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2083. + // Rule at src/isa/x64/lower.isle line 2192. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } @@ -7013,14 +7330,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 1976. + // Rule at src/isa/x64/lower.isle line 2085. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Sextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2069. + // Rule at src/isa/x64/lower.isle line 2178. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -8216,7 +8533,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2097. + // Rule at src/isa/x64/lower.isle line 2206. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -8672,7 +8989,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2105. + // Rule at src/isa/x64/lower.isle line 2214. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0: u32 = 1; let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; @@ -8682,7 +8999,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2089. + // Rule at src/isa/x64/lower.isle line 2198. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -8796,21 +9113,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1653. + // Rule at src/isa/x64/lower.isle line 1762. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_clz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Ctz => { - // Rule at src/isa/x64/lower.isle line 1695. + // Rule at src/isa/x64/lower.isle line 1804. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_ctz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Popcnt => { - // Rule at src/isa/x64/lower.isle line 1747. + // Rule at src/isa/x64/lower.isle line 1856. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_popcnt(ctx, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; @@ -8913,7 +9230,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1658. + // Rule at src/isa/x64/lower.isle line 1767. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -8924,7 +9241,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1700. + // Rule at src/isa/x64/lower.isle line 1809. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -8935,7 +9252,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1752. + // Rule at src/isa/x64/lower.isle line 1861. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -9567,7 +9884,7 @@ pub fn constructor_do_clz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1678. + // Rule at src/isa/x64/lower.isle line 1787. let expr0_0: Type = I64; let expr1_0: i64 = -1; let expr2_0 = constructor_imm_i64(ctx, expr0_0, expr1_0)?; @@ -9594,7 +9911,7 @@ pub fn constructor_do_ctz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1719. + // Rule at src/isa/x64/lower.isle line 1828. let expr0_0: Type = I64; let expr1_0 = C::ty_bits_u64(ctx, pattern1_0); let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -9608,7 +9925,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> let pattern0_0 = arg0; if pattern0_0 == I32 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1810. + // Rule at src/isa/x64/lower.isle line 1919. let expr0_0: Type = I32; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -9673,7 +9990,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> } if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1767. + // Rule at src/isa/x64/lower.isle line 1876. let expr0_0: Type = I64; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -9747,7 +10064,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1899. + // Rule at src/isa/x64/lower.isle line 2008. let expr0_0 = C::ty_mask(ctx, pattern0_0); let expr1_0: u64 = 6148914691236517205; let expr2_0 = C::u64_and(ctx, expr0_0, expr1_0); @@ -9810,7 +10127,7 @@ pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1922. + // Rule at src/isa/x64/lower.isle line 2031. let expr0_0 = constructor_do_bitrev8(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 71777214294589695; @@ -9838,7 +10155,7 @@ pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) - pub fn constructor_do_bitrev32(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1934. + // Rule at src/isa/x64/lower.isle line 2043. let expr0_0 = constructor_do_bitrev16(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 281470681808895; @@ -9867,7 +10184,7 @@ pub fn constructor_do_bitrev64(ctx: &mut C, arg0: Type, arg1: Gpr) - let pattern0_0 = arg0; if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1946. + // Rule at src/isa/x64/lower.isle line 2055. let expr0_0 = constructor_do_bitrev32(ctx, pattern0_0, pattern2_0)?; let expr1_0: u64 = 4294967295; let expr2_0 = constructor_imm(ctx, pattern0_0, expr1_0)?; @@ -9901,7 +10218,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::fits_in_32(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2066. + // Rule at src/isa/x64/lower.isle line 2175. let expr0_0: Type = I32; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -9909,7 +10226,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_64(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2062. + // Rule at src/isa/x64/lower.isle line 2171. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -9917,7 +10234,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2056. + // Rule at src/isa/x64/lower.isle line 2165. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -9932,7 +10249,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::ty_int_bool_64(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2052. + // Rule at src/isa/x64/lower.isle line 2161. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_put_in_gpr(ctx, pattern0_0)?; let expr2_0 = constructor_spread_sign_bit(ctx, expr1_0)?; @@ -9944,7 +10261,7 @@ pub fn constructor_generic_sextend( } let pattern2_0 = arg2; if pattern2_0 == pattern1_0 { - // Rule at src/isa/x64/lower.isle line 2042. + // Rule at src/isa/x64/lower.isle line 2151. let expr0_0 = constructor_output_value(ctx, pattern0_0)?; return Some(expr0_0); } @@ -9954,7 +10271,7 @@ pub fn constructor_generic_sextend( // Generated as internal constructor for term spread_sign_bit. pub fn constructor_spread_sign_bit(ctx: &mut C, arg0: Gpr) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 2048. + // Rule at src/isa/x64/lower.isle line 2157. let expr0_0: Type = I64; let expr1_0: u8 = 63; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 3afb02f8a5..6b6e7caa05 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -253,7 +253,16 @@ macro_rules! isle_prelude_methods { } } - fn vec128(&mut self, ty: Type) -> Option { + #[inline] + fn ty_scalar_float(&mut self, ty: Type) -> Option { + match ty { + F32 | F64 => Some(ty), + _ => None, + } + } + + #[inline] + fn ty_vec128(&mut self, ty: Type) -> Option { if ty.is_vector() && ty.bits() == 128 { Some(ty) } else { diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 397c5b0b88..e440698e0b 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -269,9 +269,13 @@ (decl ty_int_bool_128 (Type) Type) (extern extractor ty_int_bool_128 ty_int_bool_128) +;; An extractor that only matches scalar floating-point types--F32 or F64. +(decl ty_scalar_float (Type) Type) +(extern extractor ty_scalar_float ty_scalar_float) + ;; An extractor that only matches 128-bit vector types. -(decl vec128 (Type) Type) -(extern extractor vec128 vec128) +(decl ty_vec128 (Type) Type) +(extern extractor ty_vec128 ty_vec128) ;; An extractor that matches everything except i64x2 (decl not_i64x2 () Type) diff --git a/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif b/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif index dbb67b4b59..bb60205667 100644 --- a/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif +++ b/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif @@ -48,8 +48,8 @@ block0(v0: f64, v1: i64): ; Inst 1: movq %rsp, %rbp ; Inst 2: movsd 0(%rdi), %xmm1 ; Inst 3: ucomisd %xmm1, %xmm0 -; Inst 4: setnp %dil -; Inst 5: setz %sil +; Inst 4: setnp %sil +; Inst 5: setz %dil ; Inst 6: andl %edi, %esi ; Inst 7: andq $1, %rsi ; Inst 8: ucomisd %xmm0, %xmm1