From 13b93969319c06caea6c2f7e852b28fa62c4d598 Mon Sep 17 00:00:00 2001 From: FreddieLiardet <79145971+FreddieLiardet@users.noreply.github.com> Date: Thu, 10 Mar 2022 00:20:06 +0000 Subject: [PATCH] Add vector compare to 0 optims (#3887) Signed-off-by: Freddie Liardet --- cranelift/codegen/src/isa/aarch64/inst.isle | 94 ++ .../codegen/src/isa/aarch64/inst/emit.rs | 44 + .../src/isa/aarch64/inst/emit_tests.rs | 103 ++- cranelift/codegen/src/isa/aarch64/inst/mod.rs | 9 + cranelift/codegen/src/isa/aarch64/lower.isle | 66 ++ .../codegen/src/isa/aarch64/lower/isle.rs | 107 ++- .../lower/isle/generated_code.manifest | 4 +- .../isa/aarch64/lower/isle/generated_code.rs | 813 ++++++++++++++---- .../filetests/isa/aarch64/compare_zero.clif | 415 +++++++++ .../filetests/runtests/simd_compare_zero.clif | 255 ++++++ 10 files changed, 1748 insertions(+), 162 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/aarch64/compare_zero.clif create mode 100644 cranelift/filetests/filetests/runtests/simd_compare_zero.clif diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 91fe89825a..734c268bb0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -1158,6 +1158,24 @@ (Cnt) ;; Compare bitwise equal to 0 (Cmeq0) + ;; Compare signed greater than or equal to 0 + (Cmge0) + ;; Compare signed greater than 0 + (Cmgt0) + ;; Compare signed less than or equal to 0 + (Cmle0) + ;; Compare signed less than 0 + (Cmlt0) + ;; Floating point compare equal to 0 + (Fcmeq0) + ;; Floating point compare greater than or equal to 0 + (Fcmge0) + ;; Floating point compare greater than 0 + (Fcmgt0) + ;; Floating point compare less than or equal to 0 + (Fcmle0) + ;; Floating point compare less than 0 + (Fcmlt0) )) ;; A vector widening operation with one argument. @@ -1997,3 +2015,79 @@ (value_regs (alu_rrr op ty x_lo y_lo) (alu_rrr op ty x_hi y_hi)))) + +;; Float vector compare helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Match 32 bit float 0 value +(decl zero_value_f32 (Ieee32) Ieee32) +(extern extractor zero_value_f32 zero_value_f32) + +;; Match 64 bit float 0 value +(decl zero_value_f64 (Ieee64) Ieee64) +(extern extractor zero_value_f64 zero_value_f64) + +;; Generate comparison to zero operator from input condition code +(decl float_cc_cmp_zero_to_vec_misc_op (FloatCC) VecMisc2) +(extern constructor float_cc_cmp_zero_to_vec_misc_op float_cc_cmp_zero_to_vec_misc_op) + +(decl float_cc_cmp_zero_to_vec_misc_op_swap (FloatCC) VecMisc2) +(extern constructor float_cc_cmp_zero_to_vec_misc_op_swap float_cc_cmp_zero_to_vec_misc_op_swap) + +;; Match valid generic compare to zero cases +(decl fcmp_zero_cond (FloatCC) FloatCC) +(extern extractor fcmp_zero_cond fcmp_zero_cond) + +;; Match not equal compare to zero separately as it requires two output instructions +(decl fcmp_zero_cond_not_eq (FloatCC) FloatCC) +(extern extractor fcmp_zero_cond_not_eq fcmp_zero_cond_not_eq) + +;; Helper for generating float compare to zero instructions where 2nd argument is zero +(decl float_cmp_zero (FloatCC Reg VectorSize) Reg) +(rule (float_cmp_zero cond rn size) + (vec_misc (float_cc_cmp_zero_to_vec_misc_op cond) rn size)) + +;; Helper for generating float compare to zero instructions in case where 1st argument is zero +(decl float_cmp_zero_swap (FloatCC Reg VectorSize) Reg) +(rule (float_cmp_zero_swap cond rn size) + (vec_misc (float_cc_cmp_zero_to_vec_misc_op_swap cond) rn size)) + +;; Helper for generating float compare equal to zero instruction +(decl fcmeq0 (Reg VectorSize) Reg) +(rule (fcmeq0 rn size) + (vec_misc (VecMisc2.Fcmeq0) rn size)) + +;; Int vector compare helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Match integer 0 value +(decl zero_value (Imm64) Imm64) +(extern extractor zero_value zero_value) + +;; Generate comparison to zero operator from input condition code +(decl int_cc_cmp_zero_to_vec_misc_op (IntCC) VecMisc2) +(extern constructor int_cc_cmp_zero_to_vec_misc_op int_cc_cmp_zero_to_vec_misc_op) + +(decl int_cc_cmp_zero_to_vec_misc_op_swap (IntCC) VecMisc2) +(extern constructor int_cc_cmp_zero_to_vec_misc_op_swap int_cc_cmp_zero_to_vec_misc_op_swap) + +;; Match valid generic compare to zero cases +(decl icmp_zero_cond (IntCC) IntCC) +(extern extractor icmp_zero_cond icmp_zero_cond) + +;; Match not equal compare to zero separately as it requires two output instructions +(decl icmp_zero_cond_not_eq (IntCC) IntCC) +(extern extractor icmp_zero_cond_not_eq icmp_zero_cond_not_eq) + +;; Helper for generating int compare to zero instructions where 2nd argument is zero +(decl int_cmp_zero (IntCC Reg VectorSize) Reg) +(rule (int_cmp_zero cond rn size) + (vec_misc (int_cc_cmp_zero_to_vec_misc_op cond) rn size)) + +;; Helper for generating int compare to zero instructions in case where 1st argument is zero +(decl int_cmp_zero_swap (IntCC Reg VectorSize) Reg) +(rule (int_cmp_zero_swap cond rn size) + (vec_misc (int_cc_cmp_zero_to_vec_misc_op_swap cond) rn size)) + +;; Helper for generating int compare equal to zero instruction +(decl cmeq0 (Reg VectorSize) Reg) +(rule (cmeq0 rn size) + (vec_misc (VecMisc2.Cmeq0) rn size)) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index d8670f5a1f..560a5b5d77 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -1765,6 +1765,50 @@ impl MachInstEmit for Inst { (0b0, 0b00101, enc_size) } VecMisc2::Cmeq0 => (0b0, 0b01001, enc_size), + VecMisc2::Cmge0 => (0b1, 0b01000, enc_size), + VecMisc2::Cmgt0 => (0b0, 0b01000, enc_size), + VecMisc2::Cmle0 => (0b1, 0b01001, enc_size), + VecMisc2::Cmlt0 => (0b0, 0b01010, enc_size), + VecMisc2::Fcmeq0 => { + debug_assert!( + size == VectorSize::Size32x2 + || size == VectorSize::Size32x4 + || size == VectorSize::Size64x2 + ); + (0b0, 0b01101, enc_size) + } + VecMisc2::Fcmge0 => { + debug_assert!( + size == VectorSize::Size32x2 + || size == VectorSize::Size32x4 + || size == VectorSize::Size64x2 + ); + (0b1, 0b01100, enc_size) + } + VecMisc2::Fcmgt0 => { + debug_assert!( + size == VectorSize::Size32x2 + || size == VectorSize::Size32x4 + || size == VectorSize::Size64x2 + ); + (0b0, 0b01100, enc_size) + } + VecMisc2::Fcmle0 => { + debug_assert!( + size == VectorSize::Size32x2 + || size == VectorSize::Size32x4 + || size == VectorSize::Size64x2 + ); + (0b1, 0b01101, enc_size) + } + VecMisc2::Fcmlt0 => { + debug_assert!( + size == VectorSize::Size32x2 + || size == VectorSize::Size32x4 + || size == VectorSize::Size64x2 + ); + (0b0, 0b01110, enc_size) + } }; sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn)); } diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 6e7c67e26d..9cc917192c 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -4518,15 +4518,114 @@ fn test_aarch64_binemit() { "cnt v23.8b, v5.8b", )); + insns.push(( + Inst::VecMisc { + op: VecMisc2::Fcmeq0, + rd: writable_vreg(5), + rn: vreg(2), + size: VectorSize::Size32x4, + }, + "45D8A04E", + "fcmeq v5.4s, v2.4s, #0.0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Fcmge0, + rd: writable_vreg(3), + rn: vreg(1), + size: VectorSize::Size64x2, + }, + "23C8E06E", + "fcmge v3.2d, v1.2d, #0.0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Fcmgt0, + rd: writable_vreg(5), + rn: vreg(7), + size: VectorSize::Size32x4, + }, + "E5C8A04E", + "fcmgt v5.4s, v7.4s, #0.0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Fcmle0, + rd: writable_vreg(10), + rn: vreg(2), + size: VectorSize::Size32x4, + }, + "4AD8A06E", + "fcmle v10.4s, v2.4s, #0.0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Fcmlt0, + rd: writable_vreg(12), + rn: vreg(12), + size: VectorSize::Size64x2, + }, + "8CE9E04E", + "fcmlt v12.2d, v12.2d, #0.0", + )); + insns.push(( Inst::VecMisc { op: VecMisc2::Cmeq0, + rd: writable_vreg(22), + rn: vreg(27), + size: VectorSize::Size16x8, + }, + "769B604E", + "cmeq v22.8h, v27.8h, #0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Cmge0, rd: writable_vreg(12), rn: vreg(27), size: VectorSize::Size16x8, }, - "6C9B604E", - "cmeq v12.8h, v27.8h, #0", + "6C8B606E", + "cmge v12.8h, v27.8h, #0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Cmgt0, + rd: writable_vreg(12), + rn: vreg(27), + size: VectorSize::Size8x16, + }, + "6C8B204E", + "cmgt v12.16b, v27.16b, #0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Cmle0, + rd: writable_vreg(1), + rn: vreg(27), + size: VectorSize::Size32x4, + }, + "619BA06E", + "cmle v1.4s, v27.4s, #0", + )); + + insns.push(( + Inst::VecMisc { + op: VecMisc2::Cmlt0, + rd: writable_vreg(0), + rn: vreg(7), + size: VectorSize::Size64x2, + }, + "E0A8E04E", + "cmlt v0.2d, v7.2d, #0", )); insns.push(( diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 4e161ea45a..a0af36d6cb 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -3175,6 +3175,15 @@ impl Inst { VecMisc2::Frintp => ("frintp", size, ""), VecMisc2::Cnt => ("cnt", size, ""), VecMisc2::Cmeq0 => ("cmeq", size, ", #0"), + VecMisc2::Cmge0 => ("cmge", size, ", #0"), + VecMisc2::Cmgt0 => ("cmgt", size, ", #0"), + VecMisc2::Cmle0 => ("cmle", size, ", #0"), + VecMisc2::Cmlt0 => ("cmlt", size, ", #0"), + VecMisc2::Fcmeq0 => ("fcmeq", size, ", #0.0"), + VecMisc2::Fcmge0 => ("fcmge", size, ", #0.0"), + VecMisc2::Fcmgt0 => ("fcmgt", size, ", #0.0"), + VecMisc2::Fcmle0 => ("fcmle", size, ", #0.0"), + VecMisc2::Fcmlt0 => ("fcmlt", size, ", #0.0"), }; let rd = show_vreg_vector(rd.to_reg(), mb_rru, size); let rn = show_vreg_vector(rn, mb_rru, size); diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 50a97d9203..e7dfb80d41 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -1124,3 +1124,69 @@ (rule (lower (has_type $I8X16 (popcnt x))) (vec_cnt x (VectorSize.Size8x16))) + +;;;; Rules for `fcmp` 32 bit ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond_not_eq cond) x (splat (f32const (zero_value_f32 y)))))) + (let ((rn Reg x) + (vec_size VectorSize (vector_size ty))) + (value_reg (not (fcmeq0 rn vec_size) vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond cond) x (splat (f32const (zero_value_f32 y)))))) + (let ((rn Reg x) + (vec_size VectorSize (vector_size ty))) + (value_reg (float_cmp_zero cond rn vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond_not_eq cond) (splat (f32const (zero_value_f32 x))) y))) + (let ((rn Reg y) + (vec_size VectorSize (vector_size ty))) + (value_reg (not (fcmeq0 rn vec_size) vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond cond) (splat (f32const (zero_value_f32 x))) y))) + (let ((rn Reg y) + (vec_size VectorSize (vector_size ty))) + (value_reg (float_cmp_zero_swap cond rn vec_size)))) + +;;;; Rules for `fcmp` 64 bit ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond_not_eq cond) x (splat (f64const (zero_value_f64 y)))))) + (let ((rn Reg x) + (vec_size VectorSize (vector_size ty))) + (value_reg (not (fcmeq0 rn vec_size) vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond cond) x (splat (f64const (zero_value_f64 y)))))) + (let ((rn Reg x) + (vec_size VectorSize (vector_size ty))) + (value_reg (float_cmp_zero cond rn vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond_not_eq cond) (splat (f64const (zero_value_f64 x))) y))) + (let ((rn Reg y) + (vec_size VectorSize (vector_size ty))) + (value_reg (not (fcmeq0 rn vec_size) vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond cond) (splat (f64const (zero_value_f64 x))) y))) + (let ((rn Reg y) + (vec_size VectorSize (vector_size ty))) + (value_reg (float_cmp_zero_swap cond rn vec_size)))) + +;;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond_not_eq cond) x (splat (iconst (zero_value y)))))) + (let ((rn Reg x) + (vec_size VectorSize (vector_size ty))) + (value_reg (not (cmeq0 rn vec_size) vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond cond) x (splat (iconst (zero_value y)))))) + (let ((rn Reg x) + (vec_size VectorSize (vector_size ty))) + (value_reg (int_cmp_zero cond rn vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond_not_eq cond) (splat (iconst (zero_value x))) y))) + (let ((rn Reg y) + (vec_size VectorSize (vector_size ty))) + (value_reg (not (cmeq0 rn vec_size) vec_size)))) + +(rule (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond cond) (splat (iconst (zero_value x))) y))) + (let ((rn Reg y) + (vec_size VectorSize (vector_size ty))) + (value_reg (int_cmp_zero_swap cond rn vec_size)))) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 56d86ccfad..73e5b6d4b7 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -6,9 +6,9 @@ pub mod generated_code; // Types that the generated ISLE code uses via `use super::*`. use super::{ writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, AtomicRmwOp, BranchTarget, - CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, Imm12, ImmLogic, ImmShift, - Inst as MInst, JTSequenceInfo, MachLabel, MoveWideConst, NarrowValueMode, Opcode, OperandSize, - PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VectorSize, NZCV, + CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FloatCC, Imm12, ImmLogic, ImmShift, + Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MoveWideConst, NarrowValueMode, Opcode, + OperandSize, PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VecMisc2, VectorSize, NZCV, }; use crate::isa::aarch64::settings::Flags as IsaFlags; use crate::machinst::isle::*; @@ -286,4 +286,105 @@ where let amount = val.value() & u8::try_from(ty.bits() - 1).unwrap(); ImmShift::maybe_from_u64(u64::from(ty.bits()) - u64::from(amount)).unwrap() } + + fn icmp_zero_cond(&mut self, cond: &IntCC) -> Option { + match cond { + &IntCC::Equal + | &IntCC::SignedGreaterThanOrEqual + | &IntCC::SignedGreaterThan + | &IntCC::SignedLessThanOrEqual + | &IntCC::SignedLessThan => Some(*cond), + _ => None, + } + } + + fn fcmp_zero_cond(&mut self, cond: &FloatCC) -> Option { + match cond { + &FloatCC::Equal + | &FloatCC::GreaterThanOrEqual + | &FloatCC::GreaterThan + | &FloatCC::LessThanOrEqual + | &FloatCC::LessThan => Some(*cond), + _ => None, + } + } + + fn fcmp_zero_cond_not_eq(&mut self, cond: &FloatCC) -> Option { + match cond { + &FloatCC::NotEqual => Some(FloatCC::NotEqual), + _ => None, + } + } + + fn icmp_zero_cond_not_eq(&mut self, cond: &IntCC) -> Option { + match cond { + &IntCC::NotEqual => Some(IntCC::NotEqual), + _ => None, + } + } + + fn float_cc_cmp_zero_to_vec_misc_op(&mut self, cond: &FloatCC) -> VecMisc2 { + match cond { + &FloatCC::Equal => VecMisc2::Fcmeq0, + &FloatCC::GreaterThanOrEqual => VecMisc2::Fcmge0, + &FloatCC::LessThanOrEqual => VecMisc2::Fcmle0, + &FloatCC::GreaterThan => VecMisc2::Fcmgt0, + &FloatCC::LessThan => VecMisc2::Fcmlt0, + _ => panic!(), + } + } + + fn int_cc_cmp_zero_to_vec_misc_op(&mut self, cond: &IntCC) -> VecMisc2 { + match cond { + &IntCC::Equal => VecMisc2::Cmeq0, + &IntCC::SignedGreaterThanOrEqual => VecMisc2::Cmge0, + &IntCC::SignedLessThanOrEqual => VecMisc2::Cmle0, + &IntCC::SignedGreaterThan => VecMisc2::Cmgt0, + &IntCC::SignedLessThan => VecMisc2::Cmlt0, + _ => panic!(), + } + } + + fn float_cc_cmp_zero_to_vec_misc_op_swap(&mut self, cond: &FloatCC) -> VecMisc2 { + match cond { + &FloatCC::Equal => VecMisc2::Fcmeq0, + &FloatCC::GreaterThanOrEqual => VecMisc2::Fcmle0, + &FloatCC::LessThanOrEqual => VecMisc2::Fcmge0, + &FloatCC::GreaterThan => VecMisc2::Fcmlt0, + &FloatCC::LessThan => VecMisc2::Fcmgt0, + _ => panic!(), + } + } + + fn int_cc_cmp_zero_to_vec_misc_op_swap(&mut self, cond: &IntCC) -> VecMisc2 { + match cond { + &IntCC::Equal => VecMisc2::Cmeq0, + &IntCC::SignedGreaterThanOrEqual => VecMisc2::Cmle0, + &IntCC::SignedLessThanOrEqual => VecMisc2::Cmge0, + &IntCC::SignedGreaterThan => VecMisc2::Cmlt0, + &IntCC::SignedLessThan => VecMisc2::Cmgt0, + _ => panic!(), + } + } + + fn zero_value(&mut self, value: Imm64) -> Option { + if value.bits() == 0 { + return Some(value); + } + None + } + + fn zero_value_f32(&mut self, value: Ieee32) -> Option { + if value.bits() == 0 { + return Some(value); + } + None + } + + fn zero_value_f64(&mut self, value: Ieee64) -> Option { + if value.bits() == 0 { + return Some(value); + } + None + } } diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index a9fae6d4d2..3cd53aaf74 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 src/prelude.isle b2bc986bcbbbb77 -src/isa/aarch64/inst.isle 3678d0a37bdb4cff -src/isa/aarch64/lower.isle 90accbfcadaea46d +src/isa/aarch64/inst.isle 19ccefb6a496d392 +src/isa/aarch64/lower.isle 90ead921762336d2 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 817d95a045..4b866a7cc3 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -111,6 +111,17 @@ pub trait Context { fn load_constant64_full(&mut self, arg0: u64) -> Reg; fn sinkable_atomic_load(&mut self, arg0: Value) -> Option; fn sink_atomic_load(&mut self, arg0: &SinkableAtomicLoad) -> Reg; + fn zero_value_f32(&mut self, arg0: Ieee32) -> Option; + fn zero_value_f64(&mut self, arg0: Ieee64) -> Option; + fn float_cc_cmp_zero_to_vec_misc_op(&mut self, arg0: &FloatCC) -> VecMisc2; + fn float_cc_cmp_zero_to_vec_misc_op_swap(&mut self, arg0: &FloatCC) -> VecMisc2; + fn fcmp_zero_cond(&mut self, arg0: &FloatCC) -> Option; + fn fcmp_zero_cond_not_eq(&mut self, arg0: &FloatCC) -> Option; + fn zero_value(&mut self, arg0: Imm64) -> Option; + fn int_cc_cmp_zero_to_vec_misc_op(&mut self, arg0: &IntCC) -> VecMisc2; + fn int_cc_cmp_zero_to_vec_misc_op_swap(&mut self, arg0: &IntCC) -> VecMisc2; + fn icmp_zero_cond(&mut self, arg0: &IntCC) -> Option; + fn icmp_zero_cond_not_eq(&mut self, arg0: &IntCC) -> Option; fn safe_divisor_from_imm64(&mut self, arg0: Imm64) -> Option; fn shift_mask(&mut self, arg0: Type) -> ImmLogic; fn negate_imm_shift(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; @@ -930,9 +941,18 @@ pub enum VecMisc2 { Frintp, Cnt, Cmeq0, + Cmge0, + Cmgt0, + Cmle0, + Cmlt0, + Fcmeq0, + Fcmge0, + Fcmgt0, + Fcmle0, + Fcmlt0, } -/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1164. +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1182. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRLongOp { Fcvtl16, @@ -942,7 +962,7 @@ pub enum VecRRLongOp { Shll32, } -/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1179. +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1197. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRNarrowOp { Xtn16, @@ -961,7 +981,7 @@ pub enum VecRRNarrowOp { Fcvtn64, } -/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1211. +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1229. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRRLongOp { Smull8, @@ -975,13 +995,13 @@ pub enum VecRRRLongOp { Umlal32, } -/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1228. +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1246. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecPairOp { Addp, } -/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1236. +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1254. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecRRPairLongOp { Saddlp8, @@ -990,14 +1010,14 @@ pub enum VecRRPairLongOp { Uaddlp16, } -/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1247. +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1265. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecLanesOp { Addv, Uminv, } -/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1256. +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1274. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum VecShiftImmOp { Shl, @@ -1005,7 +1025,7 @@ pub enum VecShiftImmOp { Sshr, } -/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1267. +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1285. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AtomicRMWOp { Add, @@ -1320,7 +1340,7 @@ pub fn constructor_movz( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1363. + // Rule at src/isa/aarch64/inst.isle line 1381. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -1341,7 +1361,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1370. + // Rule at src/isa/aarch64/inst.isle line 1388. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -1366,7 +1386,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1377. + // Rule at src/isa/aarch64/inst.isle line 1395. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_operand_size(ctx, pattern1_0)?; @@ -1394,7 +1414,7 @@ pub fn constructor_alu_rr_imm_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1384. + // Rule at src/isa/aarch64/inst.isle line 1402. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_operand_size(ctx, pattern1_0)?; @@ -1422,7 +1442,7 @@ pub fn constructor_alu_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1391. + // Rule at src/isa/aarch64/inst.isle line 1409. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_operand_size(ctx, pattern1_0)?; @@ -1450,7 +1470,7 @@ pub fn constructor_vec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1398. + // Rule at src/isa/aarch64/inst.isle line 1416. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRR { @@ -1475,7 +1495,7 @@ pub fn constructor_vec_lanes( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1405. + // Rule at src/isa/aarch64/inst.isle line 1423. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecLanes { @@ -1493,7 +1513,7 @@ pub fn constructor_vec_lanes( pub fn constructor_vec_dup(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1412. + // Rule at src/isa/aarch64/inst.isle line 1430. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecDup { @@ -1518,7 +1538,7 @@ pub fn constructor_alu_rr_imm12( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1419. + // Rule at src/isa/aarch64/inst.isle line 1437. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_operand_size(ctx, pattern1_0)?; @@ -1548,7 +1568,7 @@ pub fn constructor_alu_rrr_shift( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1426. + // Rule at src/isa/aarch64/inst.isle line 1444. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_operand_size(ctx, pattern1_0)?; @@ -1579,7 +1599,7 @@ pub fn constructor_alu_rrr_extend( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1433. + // Rule at src/isa/aarch64/inst.isle line 1451. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_operand_size(ctx, pattern1_0)?; @@ -1608,7 +1628,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1441. + // Rule at src/isa/aarch64/inst.isle line 1459. let expr0_0 = C::put_extended_in_reg(ctx, pattern3_0); let expr1_0 = C::get_extended_op(ctx, pattern3_0); let expr2_0 = @@ -1628,7 +1648,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1448. + // Rule at src/isa/aarch64/inst.isle line 1466. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1653,7 +1673,7 @@ pub fn constructor_bit_rr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1455. + // Rule at src/isa/aarch64/inst.isle line 1473. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = constructor_operand_size(ctx, pattern1_0)?; @@ -1678,7 +1698,7 @@ pub fn constructor_add_with_flags_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1462. + // Rule at src/isa/aarch64/inst.isle line 1480. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::AddS; @@ -1708,7 +1728,7 @@ pub fn constructor_adc_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1470. + // Rule at src/isa/aarch64/inst.isle line 1488. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Adc; @@ -1738,7 +1758,7 @@ pub fn constructor_sub_with_flags_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1478. + // Rule at src/isa/aarch64/inst.isle line 1496. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::SubS; @@ -1766,7 +1786,7 @@ pub fn constructor_cmp64_imm( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1485. + // Rule at src/isa/aarch64/inst.isle line 1503. let expr0_0 = ALUOp::SubS; let expr1_0 = OperandSize::Size64; let expr2_0 = C::writable_zero_reg(ctx); @@ -1791,7 +1811,7 @@ pub fn constructor_sbc_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1492. + // Rule at src/isa/aarch64/inst.isle line 1510. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Sbc; @@ -1821,7 +1841,7 @@ pub fn constructor_vec_misc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1500. + // Rule at src/isa/aarch64/inst.isle line 1518. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecMisc { @@ -1847,7 +1867,7 @@ pub fn constructor_vec_rrr_long( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1507. + // Rule at src/isa/aarch64/inst.isle line 1525. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRRLong { @@ -1876,7 +1896,7 @@ pub fn constructor_vec_rrrr_long( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1517. + // Rule at src/isa/aarch64/inst.isle line 1535. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1906,7 +1926,7 @@ pub fn constructor_vec_rr_narrow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1525. + // Rule at src/isa/aarch64/inst.isle line 1543. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRNarrow { @@ -1930,7 +1950,7 @@ pub fn constructor_vec_rr_long( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1532. + // Rule at src/isa/aarch64/inst.isle line 1550. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRLong { @@ -1952,7 +1972,7 @@ pub fn constructor_mov_to_fpu( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1539. + // Rule at src/isa/aarch64/inst.isle line 1557. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovToFpu { @@ -1977,7 +1997,7 @@ pub fn constructor_mov_to_vec( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1546. + // Rule at src/isa/aarch64/inst.isle line 1564. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -2006,7 +2026,7 @@ pub fn constructor_mov_from_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1554. + // Rule at src/isa/aarch64/inst.isle line 1572. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVec { @@ -2032,7 +2052,7 @@ pub fn constructor_mov_from_vec_signed( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1561. + // Rule at src/isa/aarch64/inst.isle line 1579. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVecSigned { @@ -2059,7 +2079,7 @@ pub fn constructor_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1568. + // Rule at src/isa/aarch64/inst.isle line 1586. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Extend { @@ -2078,7 +2098,7 @@ pub fn constructor_extend( pub fn constructor_load_acquire(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1575. + // Rule at src/isa/aarch64/inst.isle line 1593. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAcquire { @@ -2101,7 +2121,7 @@ pub fn constructor_tst_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1585. + // Rule at src/isa/aarch64/inst.isle line 1603. let expr0_0 = ALUOp::AndS; let expr1_0 = constructor_operand_size(ctx, pattern0_0)?; let expr2_0 = C::writable_zero_reg(ctx); @@ -2126,7 +2146,7 @@ pub fn constructor_csel( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1599. + // Rule at src/isa/aarch64/inst.isle line 1617. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::CSel { @@ -2148,7 +2168,7 @@ pub fn constructor_add(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1608. + // Rule at src/isa/aarch64/inst.isle line 1626. let expr0_0 = ALUOp::Add; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2164,7 +2184,7 @@ pub fn constructor_add_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1611. + // Rule at src/isa/aarch64/inst.isle line 1629. let expr0_0 = ALUOp::Add; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2180,7 +2200,7 @@ pub fn constructor_add_extend( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1614. + // Rule at src/isa/aarch64/inst.isle line 1632. let expr0_0 = ALUOp::Add; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2198,7 +2218,7 @@ pub fn constructor_add_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1617. + // Rule at src/isa/aarch64/inst.isle line 1635. let expr0_0 = ALUOp::Add; let expr1_0 = constructor_alu_rrr_shift( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2216,7 +2236,7 @@ pub fn constructor_add_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1620. + // Rule at src/isa/aarch64/inst.isle line 1638. let expr0_0 = VecALUOp::Add; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2227,7 +2247,7 @@ pub fn constructor_sub(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1625. + // Rule at src/isa/aarch64/inst.isle line 1643. let expr0_0 = ALUOp::Sub; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2243,7 +2263,7 @@ pub fn constructor_sub_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1628. + // Rule at src/isa/aarch64/inst.isle line 1646. let expr0_0 = ALUOp::Sub; let expr1_0 = constructor_alu_rr_imm12(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2259,7 +2279,7 @@ pub fn constructor_sub_extend( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1631. + // Rule at src/isa/aarch64/inst.isle line 1649. let expr0_0 = ALUOp::Sub; let expr1_0 = constructor_alu_rr_extend_reg(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2277,7 +2297,7 @@ pub fn constructor_sub_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1634. + // Rule at src/isa/aarch64/inst.isle line 1652. let expr0_0 = ALUOp::Sub; let expr1_0 = constructor_alu_rrr_shift( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2295,7 +2315,7 @@ pub fn constructor_sub_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1637. + // Rule at src/isa/aarch64/inst.isle line 1655. let expr0_0 = VecALUOp::Sub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2314,7 +2334,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1643. + // Rule at src/isa/aarch64/inst.isle line 1661. let expr0_0 = constructor_madd64(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2322,7 +2342,7 @@ pub fn constructor_madd( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1642. + // Rule at src/isa/aarch64/inst.isle line 1660. let expr0_0 = constructor_madd32(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -2334,7 +2354,7 @@ pub fn constructor_madd32(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1646. + // Rule at src/isa/aarch64/inst.isle line 1664. let expr0_0 = ALUOp3::MAdd32; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2345,7 +2365,7 @@ pub fn constructor_madd64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1649. + // Rule at src/isa/aarch64/inst.isle line 1667. let expr0_0 = ALUOp3::MAdd64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2356,7 +2376,7 @@ pub fn constructor_msub64(ctx: &mut C, arg0: Reg, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1654. + // Rule at src/isa/aarch64/inst.isle line 1672. let expr0_0 = ALUOp3::MSub64; let expr1_0 = constructor_alu_rrrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2372,7 +2392,7 @@ pub fn constructor_uqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1658. + // Rule at src/isa/aarch64/inst.isle line 1676. let expr0_0 = VecALUOp::Uqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2388,7 +2408,7 @@ pub fn constructor_sqadd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1662. + // Rule at src/isa/aarch64/inst.isle line 1680. let expr0_0 = VecALUOp::Sqadd; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2404,7 +2424,7 @@ pub fn constructor_uqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1666. + // Rule at src/isa/aarch64/inst.isle line 1684. let expr0_0 = VecALUOp::Uqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2420,7 +2440,7 @@ pub fn constructor_sqsub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1670. + // Rule at src/isa/aarch64/inst.isle line 1688. let expr0_0 = VecALUOp::Sqsub; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2431,7 +2451,7 @@ pub fn constructor_umulh(ctx: &mut C, arg0: Type, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1674. + // Rule at src/isa/aarch64/inst.isle line 1692. let expr0_0 = ALUOp::UMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2442,7 +2462,7 @@ pub fn constructor_smulh(ctx: &mut C, arg0: Type, arg1: Reg, arg2: R let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1678. + // Rule at src/isa/aarch64/inst.isle line 1696. let expr0_0 = ALUOp::SMulH; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2458,7 +2478,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1682. + // Rule at src/isa/aarch64/inst.isle line 1700. let expr0_0 = VecALUOp::Mul; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2468,7 +2488,7 @@ pub fn constructor_mul( pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1686. + // Rule at src/isa/aarch64/inst.isle line 1704. let expr0_0 = VecMisc2::Neg; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2478,7 +2498,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1690. + // Rule at src/isa/aarch64/inst.isle line 1708. let expr0_0 = VecMisc2::Rev64; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2488,7 +2508,7 @@ pub fn constructor_rev64(ctx: &mut C, arg0: Reg, arg1: &VectorSize) pub fn constructor_xtn64(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1694. + // Rule at src/isa/aarch64/inst.isle line 1712. let expr0_0 = VecRRNarrowOp::Xtn64; let expr1_0 = constructor_vec_rr_narrow(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2504,7 +2524,7 @@ pub fn constructor_addp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1698. + // Rule at src/isa/aarch64/inst.isle line 1716. let expr0_0 = VecALUOp::Addp; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2514,7 +2534,7 @@ pub fn constructor_addp( pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1702. + // Rule at src/isa/aarch64/inst.isle line 1720. let expr0_0 = VecLanesOp::Addv; let expr1_0 = constructor_vec_lanes(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2524,7 +2544,7 @@ pub fn constructor_addv(ctx: &mut C, arg0: Reg, arg1: &VectorSize) - pub fn constructor_shll32(ctx: &mut C, arg0: Reg, arg1: bool) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1706. + // Rule at src/isa/aarch64/inst.isle line 1724. let expr0_0 = VecRRLongOp::Shll32; let expr1_0 = constructor_vec_rr_long(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2542,7 +2562,7 @@ pub fn constructor_umlal32( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1710. + // Rule at src/isa/aarch64/inst.isle line 1728. let expr0_0 = VecRRRLongOp::Umlal32; let expr1_0 = constructor_vec_rrrr_long( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2560,7 +2580,7 @@ pub fn constructor_smull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1714. + // Rule at src/isa/aarch64/inst.isle line 1732. let expr0_0 = VecRRRLongOp::Smull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2576,7 +2596,7 @@ pub fn constructor_umull8( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1718. + // Rule at src/isa/aarch64/inst.isle line 1736. let expr0_0 = VecRRRLongOp::Umull8; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2592,7 +2612,7 @@ pub fn constructor_smull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1722. + // Rule at src/isa/aarch64/inst.isle line 1740. let expr0_0 = VecRRRLongOp::Smull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2608,7 +2628,7 @@ pub fn constructor_umull16( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1726. + // Rule at src/isa/aarch64/inst.isle line 1744. let expr0_0 = VecRRRLongOp::Umull16; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2624,7 +2644,7 @@ pub fn constructor_smull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1730. + // Rule at src/isa/aarch64/inst.isle line 1748. let expr0_0 = VecRRRLongOp::Smull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2640,7 +2660,7 @@ pub fn constructor_umull32( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1734. + // Rule at src/isa/aarch64/inst.isle line 1752. let expr0_0 = VecRRRLongOp::Umull32; let expr1_0 = constructor_vec_rrr_long(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2651,7 +2671,7 @@ pub fn constructor_asr(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1738. + // Rule at src/isa/aarch64/inst.isle line 1756. let expr0_0 = ALUOp::Asr; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2667,7 +2687,7 @@ pub fn constructor_asr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1741. + // Rule at src/isa/aarch64/inst.isle line 1759. let expr0_0 = ALUOp::Asr; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2678,7 +2698,7 @@ pub fn constructor_lsr(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1745. + // Rule at src/isa/aarch64/inst.isle line 1763. let expr0_0 = ALUOp::Lsr; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2694,7 +2714,7 @@ pub fn constructor_lsr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1748. + // Rule at src/isa/aarch64/inst.isle line 1766. let expr0_0 = ALUOp::Lsr; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2705,7 +2725,7 @@ pub fn constructor_lsl(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1752. + // Rule at src/isa/aarch64/inst.isle line 1770. let expr0_0 = ALUOp::Lsl; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2721,7 +2741,7 @@ pub fn constructor_lsl_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1755. + // Rule at src/isa/aarch64/inst.isle line 1773. let expr0_0 = ALUOp::Lsl; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2737,7 +2757,7 @@ pub fn constructor_a64_udiv( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1759. + // Rule at src/isa/aarch64/inst.isle line 1777. let expr0_0 = ALUOp::UDiv; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2753,7 +2773,7 @@ pub fn constructor_a64_sdiv( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1763. + // Rule at src/isa/aarch64/inst.isle line 1781. let expr0_0 = ALUOp::SDiv; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2763,7 +2783,7 @@ pub fn constructor_a64_sdiv( pub fn constructor_not(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1767. + // Rule at src/isa/aarch64/inst.isle line 1785. let expr0_0 = VecMisc2::Not; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2779,7 +2799,7 @@ pub fn constructor_orr_not( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1772. + // Rule at src/isa/aarch64/inst.isle line 1790. let expr0_0 = ALUOp::OrrNot; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2797,7 +2817,7 @@ pub fn constructor_orr_not_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1775. + // Rule at src/isa/aarch64/inst.isle line 1793. let expr0_0 = ALUOp::OrrNot; let expr1_0 = constructor_alu_rrr_shift( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2810,7 +2830,7 @@ pub fn constructor_orr(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1780. + // Rule at src/isa/aarch64/inst.isle line 1798. let expr0_0 = ALUOp::Orr; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2826,7 +2846,7 @@ pub fn constructor_orr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1783. + // Rule at src/isa/aarch64/inst.isle line 1801. let expr0_0 = ALUOp::Orr; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2842,7 +2862,7 @@ pub fn constructor_orr_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1786. + // Rule at src/isa/aarch64/inst.isle line 1804. let expr0_0 = VecALUOp::Orr; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2858,7 +2878,7 @@ pub fn constructor_and_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1791. + // Rule at src/isa/aarch64/inst.isle line 1809. let expr0_0 = ALUOp::And; let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2874,7 +2894,7 @@ pub fn constructor_and_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1794. + // Rule at src/isa/aarch64/inst.isle line 1812. let expr0_0 = VecALUOp::And; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2890,7 +2910,7 @@ pub fn constructor_eor_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1798. + // Rule at src/isa/aarch64/inst.isle line 1816. let expr0_0 = VecALUOp::Eor; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2906,7 +2926,7 @@ pub fn constructor_bic_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1802. + // Rule at src/isa/aarch64/inst.isle line 1820. let expr0_0 = VecALUOp::Bic; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2922,7 +2942,7 @@ pub fn constructor_sshl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1806. + // Rule at src/isa/aarch64/inst.isle line 1824. let expr0_0 = VecALUOp::Sshl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2938,7 +2958,7 @@ pub fn constructor_ushl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1810. + // Rule at src/isa/aarch64/inst.isle line 1828. let expr0_0 = VecALUOp::Ushl; let expr1_0 = constructor_vec_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2954,7 +2974,7 @@ pub fn constructor_a64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1815. + // Rule at src/isa/aarch64/inst.isle line 1833. let expr0_0 = ALUOp::RotR; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2970,7 +2990,7 @@ pub fn constructor_a64_rotr_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1818. + // Rule at src/isa/aarch64/inst.isle line 1836. let expr0_0 = ALUOp::RotR; let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2980,7 +3000,7 @@ pub fn constructor_a64_rotr_imm( pub fn constructor_rbit(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1823. + // Rule at src/isa/aarch64/inst.isle line 1841. let expr0_0 = BitOp::RBit; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2990,7 +3010,7 @@ pub fn constructor_rbit(ctx: &mut C, arg0: Type, arg1: Reg) -> Optio pub fn constructor_a64_clz(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1828. + // Rule at src/isa/aarch64/inst.isle line 1846. let expr0_0 = BitOp::Clz; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3000,7 +3020,7 @@ pub fn constructor_a64_clz(ctx: &mut C, arg0: Type, arg1: Reg) -> Op pub fn constructor_a64_cls(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1833. + // Rule at src/isa/aarch64/inst.isle line 1851. let expr0_0 = BitOp::Cls; let expr1_0 = constructor_bit_rr(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3011,7 +3031,7 @@ pub fn constructor_eon(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1838. + // Rule at src/isa/aarch64/inst.isle line 1856. let expr0_0 = ALUOp::EorNot; let expr1_0 = constructor_alu_rrr(ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3021,7 +3041,7 @@ pub fn constructor_eon(ctx: &mut C, arg0: Type, arg1: Reg, arg2: Reg pub fn constructor_vec_cnt(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1843. + // Rule at src/isa/aarch64/inst.isle line 1861. let expr0_0 = VecMisc2::Cnt; let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3038,7 +3058,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option }; if let Some(pattern3_0) = closure3() { if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) { - // Rule at src/isa/aarch64/inst.isle line 1858. + // Rule at src/isa/aarch64/inst.isle line 1876. let expr0_0: Type = I64; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_orr_imm(ctx, expr0_0, expr1_0, pattern4_0)?; @@ -3046,18 +3066,18 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1850. + // Rule at src/isa/aarch64/inst.isle line 1868. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1854. + // Rule at src/isa/aarch64/inst.isle line 1872. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1865. + // Rule at src/isa/aarch64/inst.isle line 1883. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } @@ -3069,17 +3089,17 @@ pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 1876. + // Rule at src/isa/aarch64/inst.isle line 1894. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1877. + // Rule at src/isa/aarch64/inst.isle line 1895. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1872. + // Rule at src/isa/aarch64/inst.isle line 1890. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3095,17 +3115,17 @@ pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I32 { - // Rule at src/isa/aarch64/inst.isle line 1885. + // Rule at src/isa/aarch64/inst.isle line 1903. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1886. + // Rule at src/isa/aarch64/inst.isle line 1904. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1881. + // Rule at src/isa/aarch64/inst.isle line 1899. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3121,12 +3141,12 @@ pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1894. + // Rule at src/isa/aarch64/inst.isle line 1912. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1890. + // Rule at src/isa/aarch64/inst.isle line 1908. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3142,12 +3162,12 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1902. + // Rule at src/isa/aarch64/inst.isle line 1920. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1898. + // Rule at src/isa/aarch64/inst.isle line 1916. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -3161,7 +3181,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op // Generated as internal constructor for term trap_if_zero_divisor. pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1907. + // Rule at src/isa/aarch64/inst.isle line 1925. let expr0_0 = C::cond_br_zero(ctx, pattern0_0); let expr1_0 = C::trap_code_division_by_zero(ctx); let expr2_0 = MInst::TrapIf { @@ -3176,12 +3196,12 @@ pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> O pub fn constructor_size_from_ty(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1913. + // Rule at src/isa/aarch64/inst.isle line 1931. let expr0_0 = OperandSize::Size64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 1912. + // Rule at src/isa/aarch64/inst.isle line 1930. let expr0_0 = OperandSize::Size32; return Some(expr0_0); } @@ -3198,7 +3218,7 @@ pub fn constructor_trap_if_div_overflow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1919. + // Rule at src/isa/aarch64/inst.isle line 1937. let expr0_0 = ALUOp::AddS; let expr1_0 = constructor_operand_size(ctx, pattern0_0)?; let expr2_0 = C::writable_zero_reg(ctx); @@ -3267,7 +3287,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0) { let pattern9_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1964. + // Rule at src/isa/aarch64/inst.isle line 1982. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = constructor_alu_rr_imm_logic( ctx, pattern0_0, pattern1_0, expr0_0, pattern8_0, @@ -3299,7 +3319,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::lshl_from_imm64(ctx, pattern10_1, pattern12_0) { let pattern14_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1970. + // Rule at src/isa/aarch64/inst.isle line 1988. let expr0_0 = C::put_in_reg(ctx, pattern14_0); let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3337,7 +3357,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 1962. + // Rule at src/isa/aarch64/inst.isle line 1980. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic( ctx, pattern0_0, pattern1_0, expr0_0, pattern9_0, @@ -3368,7 +3388,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 1968. + // Rule at src/isa/aarch64/inst.isle line 1986. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3390,7 +3410,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 1958. + // Rule at src/isa/aarch64/inst.isle line 1976. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, pattern1_0, expr0_0, expr1_0)?; @@ -3424,7 +3444,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 1978. + // Rule at src/isa/aarch64/inst.isle line 1996. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic( ctx, pattern0_0, pattern1_0, expr0_0, pattern9_0, @@ -3455,7 +3475,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 1980. + // Rule at src/isa/aarch64/inst.isle line 1998. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -3477,7 +3497,7 @@ pub fn constructor_alu_rs_imm_logic( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 1976. + // Rule at src/isa/aarch64/inst.isle line 1994. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, pattern1_0, expr0_0, expr1_0)?; @@ -3496,7 +3516,7 @@ pub fn constructor_i128_alu_bitop( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1988. + // Rule at src/isa/aarch64/inst.isle line 2006. let expr0_0 = C::put_in_regs(ctx, pattern2_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -3513,6 +3533,90 @@ pub fn constructor_i128_alu_bitop( return Some(expr12_0); } +// Generated as internal constructor for term float_cmp_zero. +pub fn constructor_float_cmp_zero( + ctx: &mut C, + arg0: &FloatCC, + arg1: Reg, + arg2: &VectorSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 2046. + let expr0_0 = C::float_cc_cmp_zero_to_vec_misc_op(ctx, pattern0_0); + let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term float_cmp_zero_swap. +pub fn constructor_float_cmp_zero_swap( + ctx: &mut C, + arg0: &FloatCC, + arg1: Reg, + arg2: &VectorSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 2051. + let expr0_0 = C::float_cc_cmp_zero_to_vec_misc_op_swap(ctx, pattern0_0); + let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term fcmeq0. +pub fn constructor_fcmeq0(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 2056. + let expr0_0 = VecMisc2::Fcmeq0; + let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term int_cmp_zero. +pub fn constructor_int_cmp_zero( + ctx: &mut C, + arg0: &IntCC, + arg1: Reg, + arg2: &VectorSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 2082. + let expr0_0 = C::int_cc_cmp_zero_to_vec_misc_op(ctx, pattern0_0); + let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term int_cmp_zero_swap. +pub fn constructor_int_cmp_zero_swap( + ctx: &mut C, + arg0: &IntCC, + arg1: Reg, + arg2: &VectorSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 2087. + let expr0_0 = C::int_cc_cmp_zero_to_vec_misc_op_swap(ctx, pattern0_0); + let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term cmeq0. +pub fn constructor_cmeq0(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 2092. + let expr0_0 = VecMisc2::Cmeq0; + let expr1_0 = constructor_vec_misc(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + // Generated as internal constructor for term lower. pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern0_0 = arg0; @@ -5094,34 +5198,433 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/aarch64/lower.isle line 75. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = constructor_vector_size(ctx, pattern2_0)?; - let expr3_0 = constructor_add_vec(ctx, expr0_0, expr1_0, &expr2_0)?; - let expr4_0 = constructor_output_reg(ctx, expr3_0)?; - return Some(expr4_0); + match &pattern4_0 { + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match pattern5_0 { + &Opcode::Iadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 75. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_vector_size(ctx, pattern2_0)?; + let expr3_0 = constructor_add_vec(ctx, expr0_0, expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_reg(ctx, expr3_0)?; + return Some(expr4_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 125. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_vector_size(ctx, pattern2_0)?; + let expr3_0 = constructor_sub_vec(ctx, expr0_0, expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_reg(ctx, expr3_0)?; + return Some(expr4_0); + } + _ => {} } - &Opcode::Isub => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/aarch64/lower.isle line 125. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = constructor_vector_size(ctx, pattern2_0)?; - let expr3_0 = constructor_sub_vec(ctx, expr0_0, expr1_0, &expr2_0)?; - let expr4_0 = constructor_output_reg(ctx, expr3_0)?; - return Some(expr4_0); - } - _ => {} } + &InstructionData::FloatCompare { + opcode: ref pattern5_0, + args: ref pattern5_1, + cond: ref pattern5_2, + } => { + if let &Opcode::Fcmp = pattern5_0 { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_0) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::Unary { + opcode: ref pattern10_0, + arg: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Splat = pattern10_0 { + if let Some(pattern12_0) = C::def_inst(ctx, pattern10_1) { + let pattern13_0 = C::inst_data(ctx, pattern12_0); + match &pattern13_0 { + &InstructionData::UnaryIeee32 { + opcode: ref pattern14_0, + imm: pattern14_1, + } => { + if let &Opcode::F32const = pattern14_0 { + if let Some(pattern16_0) = + C::zero_value_f32(ctx, pattern14_1) + { + if let Some(pattern17_0) = + C::fcmp_zero_cond(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1145. + let expr0_0 = + C::put_in_reg(ctx, pattern7_1); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = + constructor_float_cmp_zero_swap( + ctx, + &pattern17_0, + expr0_0, + &expr1_0, + )?; + let expr3_0 = + C::value_reg(ctx, expr2_0); + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + if let Some(pattern17_0) = + C::fcmp_zero_cond_not_eq( + ctx, pattern5_2, + ) + { + // Rule at src/isa/aarch64/lower.isle line 1140. + let expr0_0 = + C::put_in_reg(ctx, pattern7_1); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = constructor_fcmeq0( + ctx, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_not( + ctx, expr2_0, &expr1_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + &InstructionData::UnaryIeee64 { + opcode: ref pattern14_0, + imm: pattern14_1, + } => { + if let &Opcode::F64const = pattern14_0 { + if let Some(pattern16_0) = + C::zero_value_f64(ctx, pattern14_1) + { + if let Some(pattern17_0) = + C::fcmp_zero_cond(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1167. + let expr0_0 = + C::put_in_reg(ctx, pattern7_1); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = + constructor_float_cmp_zero_swap( + ctx, + &pattern17_0, + expr0_0, + &expr1_0, + )?; + let expr3_0 = + C::value_reg(ctx, expr2_0); + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + if let Some(pattern17_0) = + C::fcmp_zero_cond_not_eq( + ctx, pattern5_2, + ) + { + // Rule at src/isa/aarch64/lower.isle line 1162. + let expr0_0 = + C::put_in_reg(ctx, pattern7_1); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = constructor_fcmeq0( + ctx, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_not( + ctx, expr2_0, &expr1_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + _ => {} + } + } + } + } + } + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::Unary { + opcode: ref pattern10_0, + arg: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Splat = pattern10_0 { + if let Some(pattern12_0) = C::def_inst(ctx, pattern10_1) { + let pattern13_0 = C::inst_data(ctx, pattern12_0); + match &pattern13_0 { + &InstructionData::UnaryIeee32 { + opcode: ref pattern14_0, + imm: pattern14_1, + } => { + if let &Opcode::F32const = pattern14_0 { + if let Some(pattern16_0) = + C::zero_value_f32(ctx, pattern14_1) + { + if let Some(pattern17_0) = + C::fcmp_zero_cond(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1135. + let expr0_0 = + C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = + constructor_float_cmp_zero( + ctx, + &pattern17_0, + expr0_0, + &expr1_0, + )?; + let expr3_0 = + C::value_reg(ctx, expr2_0); + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + if let Some(pattern17_0) = + C::fcmp_zero_cond_not_eq( + ctx, pattern5_2, + ) + { + // Rule at src/isa/aarch64/lower.isle line 1130. + let expr0_0 = + C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = constructor_fcmeq0( + ctx, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_not( + ctx, expr2_0, &expr1_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + &InstructionData::UnaryIeee64 { + opcode: ref pattern14_0, + imm: pattern14_1, + } => { + if let &Opcode::F64const = pattern14_0 { + if let Some(pattern16_0) = + C::zero_value_f64(ctx, pattern14_1) + { + if let Some(pattern17_0) = + C::fcmp_zero_cond(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1157. + let expr0_0 = + C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = + constructor_float_cmp_zero( + ctx, + &pattern17_0, + expr0_0, + &expr1_0, + )?; + let expr3_0 = + C::value_reg(ctx, expr2_0); + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + if let Some(pattern17_0) = + C::fcmp_zero_cond_not_eq( + ctx, pattern5_2, + ) + { + // Rule at src/isa/aarch64/lower.isle line 1152. + let expr0_0 = + C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = constructor_fcmeq0( + ctx, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_not( + ctx, expr2_0, &expr1_0, + )?; + let expr4_0 = + C::value_reg(ctx, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + _ => {} + } + } + } + } + } + } + } + &InstructionData::IntCompare { + opcode: ref pattern5_0, + args: ref pattern5_1, + cond: ref pattern5_2, + } => { + if let &Opcode::Icmp = pattern5_0 { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_0) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::Unary { + opcode: ref pattern10_0, + arg: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Splat = pattern10_0 { + if let Some(pattern12_0) = C::def_inst(ctx, pattern10_1) { + let pattern13_0 = C::inst_data(ctx, pattern12_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern14_0, + imm: pattern14_1, + } = &pattern13_0 + { + if let &Opcode::Iconst = pattern14_0 { + if let Some(pattern16_0) = + C::zero_value(ctx, pattern14_1) + { + if let Some(pattern17_0) = + C::icmp_zero_cond(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1189. + let expr0_0 = + C::put_in_reg(ctx, pattern7_1); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = + constructor_int_cmp_zero_swap( + ctx, + &pattern17_0, + expr0_0, + &expr1_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + if let Some(pattern17_0) = + C::icmp_zero_cond_not_eq(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1184. + let expr0_0 = + C::put_in_reg(ctx, pattern7_1); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = constructor_cmeq0( + ctx, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_not( + ctx, expr2_0, &expr1_0, + )?; + let expr4_0 = C::value_reg(ctx, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + } + } + } + } + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::Unary { + opcode: ref pattern10_0, + arg: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Splat = pattern10_0 { + if let Some(pattern12_0) = C::def_inst(ctx, pattern10_1) { + let pattern13_0 = C::inst_data(ctx, pattern12_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern14_0, + imm: pattern14_1, + } = &pattern13_0 + { + if let &Opcode::Iconst = pattern14_0 { + if let Some(pattern16_0) = + C::zero_value(ctx, pattern14_1) + { + if let Some(pattern17_0) = + C::icmp_zero_cond(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1179. + let expr0_0 = + C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = constructor_int_cmp_zero( + ctx, + &pattern17_0, + expr0_0, + &expr1_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + if let Some(pattern17_0) = + C::icmp_zero_cond_not_eq(ctx, pattern5_2) + { + // Rule at src/isa/aarch64/lower.isle line 1174. + let expr0_0 = + C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_vector_size( + ctx, pattern2_0, + )?; + let expr2_0 = constructor_cmeq0( + ctx, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_not( + ctx, expr2_0, &expr1_0, + )?; + let expr4_0 = C::value_reg(ctx, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + } + } + } + } + } + } + _ => {} } } if let Some(pattern3_0) = C::fits_in_16(ctx, pattern2_0) { diff --git a/cranelift/filetests/filetests/isa/aarch64/compare_zero.clif b/cranelift/filetests/filetests/isa/aarch64/compare_zero.clif new file mode 100644 index 0000000000..1828d811e4 --- /dev/null +++ b/cranelift/filetests/filetests/isa/aarch64/compare_zero.clif @@ -0,0 +1,415 @@ +test compile precise-output +set unwind_info=false +target aarch64 + +function %f0(i8x16) -> b8x16 { +block0(v0: i8x16): + v1 = iconst.i8 0 + v2 = splat.i8x16 v1 + v3 = icmp eq v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmeq v0.16b, v0.16b, #0 +; Inst 1: ret +; }} + +function %f1(i16x8) -> b16x8 { +block0(v0: i16x8): + v1 = iconst.i16 0 + v2 = splat.i16x8 v1 + v3 = icmp eq v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmeq v0.8h, v0.8h, #0 +; Inst 1: ret +; }} + +function %f2(i32x4) -> b32x4 { +block0(v0: i32x4): + v1 = iconst.i32 0 + v2 = splat.i32x4 v1 + v3 = icmp ne v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 3) +; Inst 0: cmeq v0.4s, v0.4s, #0 +; Inst 1: mvn v0.16b, v0.16b +; Inst 2: ret +; }} + +function %f3(i64x2) -> b64x2 { +block0(v0: i64x2): + v1 = iconst.i64 0 + v2 = splat.i64x2 v1 + v3 = icmp ne v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 3) +; Inst 0: cmeq v0.2d, v0.2d, #0 +; Inst 1: mvn v0.16b, v0.16b +; Inst 2: ret +; }} + +function %f4(i8x16) -> b8x16 { +block0(v0: i8x16): + v1 = iconst.i8 0 + v2 = splat.i8x16 v1 + v3 = icmp sle v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmle v0.16b, v0.16b, #0 +; Inst 1: ret +; }} + +function %f5(i16x8) -> b16x8 { +block0(v0: i16x8): + v1 = iconst.i16 0 + v2 = splat.i16x8 v1 + v3 = icmp sle v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmge v0.8h, v0.8h, #0 +; Inst 1: ret +; }} + +function %f6(i32x4) -> b32x4 { +block0(v0: i32x4): + v1 = iconst.i32 0 + v2 = splat.i32x4 v1 + v3 = icmp sge v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmge v0.4s, v0.4s, #0 +; Inst 1: ret +; }} + +function %f7(i64x2) -> b64x2 { +block0(v0: i64x2): + v1 = iconst.i64 0 + v2 = splat.i64x2 v1 + v3 = icmp sge v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmle v0.2d, v0.2d, #0 +; Inst 1: ret +; }} + +function %f8(i8x16) -> b8x16 { +block0(v0: i8x16): + v1 = iconst.i8 0 + v2 = splat.i8x16 v1 + v3 = icmp slt v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmlt v0.16b, v0.16b, #0 +; Inst 1: ret +; }} + +function %f9(i16x8) -> b16x8 { +block0(v0: i16x8): + v1 = iconst.i16 0 + v2 = splat.i16x8 v1 + v3 = icmp slt v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmgt v0.8h, v0.8h, #0 +; Inst 1: ret +; }} + +function %f10(i32x4) -> b32x4 { +block0(v0: i32x4): + v1 = iconst.i32 0 + v2 = splat.i32x4 v1 + v3 = icmp sgt v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmgt v0.4s, v0.4s, #0 +; Inst 1: ret +; }} + +function %f11(i64x2) -> b64x2 { +block0(v0: i64x2): + v1 = iconst.i64 0 + v2 = splat.i64x2 v1 + v3 = icmp sgt v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: cmlt v0.2d, v0.2d, #0 +; Inst 1: ret +; }} + +function %f12(f32x4) -> b32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v2 = splat.f32x4 v1 + v3 = fcmp eq v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmeq v0.4s, v0.4s, #0.0 +; Inst 1: ret +; }} + +function %f13(f64x2) -> b64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v2 = splat.f64x2 v1 + v3 = fcmp eq v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmeq v0.2d, v0.2d, #0.0 +; Inst 1: ret +; }} + +function %f14(f64x2) -> b64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v2 = splat.f64x2 v1 + v3 = fcmp ne v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 3) +; Inst 0: fcmeq v0.2d, v0.2d, #0.0 +; Inst 1: mvn v0.16b, v0.16b +; Inst 2: ret +; }} + +function %f15(f32x4) -> b32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v2 = splat.f32x4 v1 + v3 = fcmp ne v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 3) +; Inst 0: fcmeq v0.4s, v0.4s, #0.0 +; Inst 1: mvn v0.16b, v0.16b +; Inst 2: ret +; }} + +function %f16(f32x4) -> b32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v2 = splat.f32x4 v1 + v3 = fcmp le v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmle v0.4s, v0.4s, #0.0 +; Inst 1: ret +; }} + +function %f17(f64x2) -> b64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v2 = splat.f64x2 v1 + v3 = fcmp le v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmge v0.2d, v0.2d, #0.0 +; Inst 1: ret +; }} + +function %f18(f64x2) -> b64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v2 = splat.f64x2 v1 + v3 = fcmp ge v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmge v0.2d, v0.2d, #0.0 +; Inst 1: ret +; }} + +function %f19(f32x4) -> b32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v2 = splat.f32x4 v1 + v3 = fcmp ge v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmle v0.4s, v0.4s, #0.0 +; Inst 1: ret +; }} + +function %f20(f32x4) -> b32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v2 = splat.f32x4 v1 + v3 = fcmp lt v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmlt v0.4s, v0.4s, #0.0 +; Inst 1: ret +; }} + +function %f21(f64x2) -> b64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v2 = splat.f64x2 v1 + v3 = fcmp lt v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmgt v0.2d, v0.2d, #0.0 +; Inst 1: ret +; }} + +function %f22(f64x2) -> b64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v2 = splat.f64x2 v1 + v3 = fcmp gt v0, v2 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmgt v0.2d, v0.2d, #0.0 +; Inst 1: ret +; }} + +function %f23(f32x4) -> b32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v2 = splat.f32x4 v1 + v3 = fcmp gt v2, v0 + return v3 +} + +; VCode_ShowWithRRU {{ +; Entry block: 0 +; Block 0: +; (original IR block: block0) +; (instruction range: 0 .. 2) +; Inst 0: fcmlt v0.4s, v0.4s, #0.0 +; Inst 1: ret +; }} diff --git a/cranelift/filetests/filetests/runtests/simd_compare_zero.clif b/cranelift/filetests/filetests/runtests/simd_compare_zero.clif new file mode 100644 index 0000000000..d3a8c655d1 --- /dev/null +++ b/cranelift/filetests/filetests/runtests/simd_compare_zero.clif @@ -0,0 +1,255 @@ +test run +target aarch64 + +; raw_bitcast is needed to get around issue with "bint" on aarch64 + +function %simd_icmp_eq_i8(i8x16) -> i8x16 { +block0(v0: i8x16): + v1 = iconst.i8 0 + v3 = splat.i8x16 v1 + v2 = icmp eq v0, v3 + v4 = raw_bitcast.i8x16 v2 + return v4 +} +; run: %simd_icmp_eq_i8([-1 0 1 100 -1 0 1 100 -1 0 1 100 -1 0 1 100]) == [0 0xff 0 0 0 0xff 0 0 0 0xff 0 0 0 0xff 0 0] + +function %simd_icmp_ne_i16(i16x8) -> i16x8 { +block0(v0: i16x8): + v1 = iconst.i16 0 + v3 = splat.i16x8 v1 + v2 = icmp ne v0, v3 + v4 = raw_bitcast.i16x8 v2 + return v4 +} +; run: %simd_icmp_ne_i16([-1 0 1 100 -1 0 1 100]) == [0xffff 0 0xffff 0xffff 0xffff 0 0xffff 0xffff] + +function %simd_icmp_le_i32(i32x4) -> i32x4 { +block0(v0: i32x4): + v1 = iconst.i32 0 + v3 = splat.i32x4 v1 + v2 = icmp sle v0, v3 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_icmp_le_i32([-1 0 1 100]) == [0xffffffff 0xffffffff 0 0] + +function %simd_icmp_ge_i64(i64x2) -> i64x2 { +block0(v0: i64x2): + v1 = iconst.i64 0 + v3 = splat.i64x2 v1 + v2 = icmp sge v0, v3 + v4 = raw_bitcast.i64x2 v2 + return v4 +} +; run: %simd_icmp_ge_i64([-1 0]) == [0 0xffffffffffffffff] +; run: %simd_icmp_ge_i64([1 100]) == [0xffffffffffffffff 0xffffffffffffffff] + +function %simd_icmp_lt_i8(i8x16) -> i8x16 { +block0(v0: i8x16): + v1 = iconst.i8 0 + v3 = splat.i8x16 v1 + v2 = icmp slt v0, v3 + v4 = raw_bitcast.i8x16 v2 + return v4 +} +; run: %simd_icmp_lt_i8([-1 0 1 100 -1 0 1 100 -1 0 1 100 -1 0 1 100]) == [0xff 0 0 0 0xff 0 0 0 0xff 0 0 0 0xff 0 0 0] + +function %simd_icmp_gt_i16(i16x8) -> i16x8 { +block0(v0: i16x8): + v1 = iconst.i16 0 + v3 = splat.i16x8 v1 + v2 = icmp sgt v0, v3 + v4 = raw_bitcast.i16x8 v2 + return v4 +} +; run: %simd_icmp_gt_i16([-1 0 1 100 -1 0 1 100]) == [0 0 0xffff 0xffff 0 0 0xffff 0xffff] + +function %simd_fcmp_eq_f32(f32x4) -> i32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v3 = splat.f32x4 v1 + v2 = fcmp eq v0, v3 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_fcmp_eq_f32([-0x1.0 0x0.0 0x1.0 NaN]) == [0 0xffffffff 0 0] + +function %simd_fcmp_ne_f64(f64x2) -> i64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v3 = splat.f64x2 v1 + v2 = fcmp ne v0, v3 + v4 = raw_bitcast.i64x2 v2 + return v4 +} +; run: %simd_fcmp_ne_f64([-0x1.0 0x0.0]) == [0xffffffffffffffff 0] +; run: %simd_fcmp_ne_f64([0x1.0 NaN]) == [0xffffffffffffffff 0xffffffffffffffff] + +function %simd_fcmp_le_f32(f32x4) -> i32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v3 = splat.f32x4 v1 + v2 = fcmp le v0, v3 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_fcmp_le_f32([-0x1.0 0x0.0 0x1.0 NaN]) == [0xffffffff 0xffffffff 0 0] + +function %simd_fcmp_ge_f64(f64x2) -> i64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v3 = splat.f64x2 v1 + v2 = fcmp ge v0, v3 + v4 = raw_bitcast.i64x2 v2 + return v4 +} + +; run: %simd_fcmp_ge_f64([-0x1.0 0x0.0]) == [0 0xffffffffffffffff] +; run: %simd_fcmp_ge_f64([0x1.0 NaN]) == [0xffffffffffffffff 0] + +function %simd_fcmp_lt_f32(f32x4) -> i32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v3 = splat.f32x4 v1 + v2 = fcmp lt v0, v3 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_fcmp_lt_f32([-0x1.0 0x0.0 0x1.0 NaN]) == [0xffffffff 0 0 0] + +function %simd_fcmp_gt_f64(f64x2) -> i64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v3 = splat.f64x2 v1 + v2 = fcmp gt v0, v3 + v4 = raw_bitcast.i64x2 v2 + return v4 +} + +; run: %simd_fcmp_gt_f64([-0x1.0 0x0.0]) == [0 0] +; run: %simd_fcmp_gt_f64([0x1.0 NaN]) == [0xffffffffffffffff 0] + +function %simd_icmp_eq_i32(i32x4) -> i32x4 { +block0(v0: i32x4): + v1 = iconst.i32 0 + v3 = splat.i32x4 v1 + v2 = icmp eq v3, v0 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_icmp_eq_i32([1 0 -1 100]) == [0 0xffffffff 0 0] + +function %simd_icmp_ne_i64(i64x2) -> i64x2 { +block0(v0: i64x2): + v1 = iconst.i64 0 + v3 = splat.i64x2 v1 + v2 = icmp ne v3, v0 + v4 = raw_bitcast.i64x2 v2 + return v4 +} +; run: %simd_icmp_ne_i64([-1 0]) == [0xffffffffffffffff 0] +; run: %simd_icmp_ne_i64([1 100]) == [0xffffffffffffffff 0xffffffffffffffff] + +function %simd_icmp_le_i8(i8x16) -> i8x16 { +block0(v0: i8x16): + v1 = iconst.i8 0 + v3 = splat.i8x16 v1 + v2 = icmp sle v3, v0 + v4 = raw_bitcast.i8x16 v2 + return v4 +} +; run: %simd_icmp_le_i8([-1 0 1 100 -1 0 1 100 -1 0 1 100 -1 0 1 100]) == [0 0xff 0xff 0xff 0 0xff 0xff 0xff 0 0xff 0xff 0xff 0 0xff 0xff 0xff] + +function %simd_icmp_ge_i16(i16x8) -> i16x8 { +block0(v0: i16x8): + v1 = iconst.i16 0 + v3 = splat.i16x8 v1 + v2 = icmp sge v3, v0 + v4 = raw_bitcast.i16x8 v2 + return v4 +} +; run: %simd_icmp_ge_i16([-1 0 1 100 -1 0 1 100]) == [0xffff 0xffff 0 0 0xffff 0xffff 0 0] + +function %simd_icmp_lt_i32(i32x4) -> i32x4 { +block0(v0: i32x4): + v1 = iconst.i32 0 + v3 = splat.i32x4 v1 + v2 = icmp slt v3, v0 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_icmp_lt_i32([-1 0 1 100]) == [0 0 0xffffffff 0xffffffff] + +function %simd_icmp_gt_i64(i64x2) -> i64x2 { +block0(v0: i64x2): + v1 = iconst.i64 0 + v3 = splat.i64x2 v1 + v2 = icmp sgt v3, v0 + v4 = raw_bitcast.i64x2 v2 + return v4 +} +; run: %simd_icmp_gt_i64([-1 0]) == [0xffffffffffffffff 0] +; run: %simd_icmp_gt_i64([1 100]) == [0 0] + +function %simd_fcmp_eq_f64(f64x2) -> i64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v3 = splat.f64x2 v1 + v2 = fcmp eq v3, v0 + v4 = raw_bitcast.i64x2 v2 + return v4 +} +; run: %simd_fcmp_eq_f64([-0x1.0 0x0.0]) == [0 0xffffffffffffffff] +; run: %simd_fcmp_eq_f64([0x1.0 NaN]) == [0 0] + +function %simd_fcmp_ne_f32(f32x4) -> i32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v3 = splat.f32x4 v1 + v2 = fcmp ne v3, v0 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_fcmp_ne_f32([-0x1.0 0x0.0 0x1.0 NaN]) == [0xffffffff 0 0xffffffff 0xffffffff] + +function %simd_fcmp_le_f64(f64x2) -> i64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v3 = splat.f64x2 v1 + v2 = fcmp le v3, v0 + v4 = raw_bitcast.i64x2 v2 + return v4 +} +; run: %simd_fcmp_le_f64([-0x1.0 0x0.0]) == [0 0xffffffffffffffff] +; run: %simd_fcmp_le_f64([0x1.0 NaN]) == [0xffffffffffffffff 0] + +function %simd_fcmp_ge_f32(f32x4) -> i32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v3 = splat.f32x4 v1 + v2 = fcmp ge v3, v0 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_fcmp_ge_f32([-0x1.0 0x0.0 0x1.0 NaN]) == [0xffffffff 0xffffffff 0 0] + +function %simd_fcmp_lt_f64(f64x2) -> i64x2 { +block0(v0: f64x2): + v1 = f64const 0.0 + v3 = splat.f64x2 v1 + v2 = fcmp lt v3, v0 + v4 = raw_bitcast.i64x2 v2 + return v4 +} +; run: %simd_fcmp_lt_f64([-0x1.0 0x0.0]) == [0 0] +; run: %simd_fcmp_lt_f64([0x1.0 NaN]) == [0xffffffffffffffff 0] + +function %simd_fcmp_gt_f32(f32x4) -> i32x4 { +block0(v0: f32x4): + v1 = f32const 0.0 + v3 = splat.f32x4 v1 + v2 = fcmp gt v3, v0 + v4 = raw_bitcast.i32x4 v2 + return v4 +} +; run: %simd_fcmp_gt_f32([-0x1.0 0x0.0 0x1.0 NaN]) == [0xffffffff 0 0 0]