From 1323ae417e923cd971db17cc9d3583b31c377b51 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 16 Dec 2021 11:34:24 -0800 Subject: [PATCH] Fix some 16- and 8-bit behavior in x64 backend related to rotates. Uncovered by @bjorn3 (thanks!): 8- and 16-bit rotates were not working properly in recent versions of Cranelift with part of the lowering migrated to ISLE. This PR fixes a few issues: - 8- and 16-bit rotate-left needs to mask a constant amount, if any, because we use a 32-bit rotate instruction and so don't get the appropriate shift-amount masking for free from x86 semantics. - `operand_size_from_type` was incorrect: it only handled 32- and 64-bit types and silently returned `OperandSize::Size32` for everything else. Now uses the `OperandSize::from_ty(ty)` helper as the pre-ISLE code did. Our test coverage for narrow value types is not great; this PR adds some runtests for rotl/rotr but more would always be better! --- .../lower/isle/generated_code.manifest | 2 +- .../isa/aarch64/lower/isle/generated_code.rs | 13 +- cranelift/codegen/src/isa/x64/inst.isle | 4 + cranelift/codegen/src/isa/x64/lower.isle | 17 +- cranelift/codegen/src/isa/x64/lower/isle.rs | 16 +- .../x64/lower/isle/generated_code.manifest | 6 +- .../src/isa/x64/lower/isle/generated_code.rs | 465 ++++++++++-------- cranelift/codegen/src/machinst/isle.rs | 23 + cranelift/codegen/src/prelude.isle | 13 + .../filetests/filetests/runtests/shifts.clif | 22 + 10 files changed, 352 insertions(+), 229 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 0e9aa67723..9f20d1897c 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602 -src/prelude.isle d3d2a6a42fb778231a4cdca30995324e1293a9ca8073c5a27a501535759eb51f84a6718322a93dfba4b66ee4f0c9afce7dcec0428516ef0c5bc96e8c8b76925d +src/prelude.isle 75a46b97817ad6a4c34e618b81e60876eec6fd1c83ac3ee174851e42045c951644663b2cbc31f1749ce2bc3ad9eb94fb0b877eb2c3bc4885cab7d7e87e9df1d6 src/isa/aarch64/inst.isle 8e4b8e452cf06a368c2e1d930042027a5d3bd690ab46d498d959257e9b4461d17abf244838395cd80da1fe5e2e86fc43855fb5753ca4f1643538c2ae4b3b4a1e src/isa/aarch64/lower.isle bc3db9c1e6ac186b918cc04f4d26af398f99ec36c8cdc20ec4d02d18dd57dba12e3184fea031b4ac97051c5e194a69666afb5e204807c818e6688c177f9c1b91 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index b147310b38..ed4f7e2a0d 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -34,6 +34,8 @@ pub trait Context { fn ty_bits_u16(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; + fn ty_32_or_64(&mut self, arg0: Type) -> Option; + fn ty_8_or_16(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; @@ -49,6 +51,7 @@ pub trait Context { fn first_result(&mut self, arg0: Inst) -> Option; fn inst_data(&mut self, arg0: Inst) -> InstructionData; fn value_type(&mut self, arg0: Value) -> Type; + fn ty_bits_mask(&mut self, arg0: Type) -> u64; fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>; fn def_inst(&mut self, arg0: Value) -> Option; fn trap_code_division_by_zero(&mut self) -> TrapCode; @@ -79,13 +82,13 @@ pub trait Context { fn safe_divisor_from_imm64(&mut self, arg0: Imm64) -> Option; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 246. +/// Internal type ProducesFlags: defined at src/prelude.isle line 259. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 249. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 262. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -999,7 +1002,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 259. + // Rule at src/prelude.isle line 272. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1027,7 +1030,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 267. + // Rule at src/prelude.isle line 280. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1061,7 +1064,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 277. + // Rule at src/prelude.isle line 290. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 674cd3dd19..2d8fc0999a 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -379,6 +379,10 @@ (decl imm8_from_value (Imm8Reg) Value) (extern extractor imm8_from_value imm8_from_value) +;; Mask an `Imm8Reg.Imm8`. +(decl mask_imm8_const (Imm8Reg u64) Imm8Reg) +(extern constructor mask_imm8_const mask_imm8_const) + ;; Extract a constant `RegMemImm.Imm` from a value operand. (decl simm32_from_value (RegMemImm) Value) (extern extractor simm32_from_value simm32_from_value) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index b8b22f2209..06d064f7d7 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -633,15 +633,26 @@ ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; `i64` and smaller. +;; `i16` and `i8`: we need to extend the shift amount, or mask the +;; constant. -(rule (lower (has_type (fits_in_64 ty) (rotl src amt))) +(rule (lower (has_type (ty_8_or_16 ty) (rotl src amt))) + (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) + (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (ty_8_or_16 ty) (rotl src (imm8_from_value amt)))) + (value_reg (m_rotl ty (put_in_reg src) (mask_imm8_const amt (ty_bits_mask ty))))) + +;; `i64` and `i32`: we can rely on x86's rotate-amount masking since +;; we operate on the whole register. + +(rule (lower (has_type (ty_32_or_64 ty) (rotl src amt))) ;; NB: Only the low bits of `amt` matter since we logically mask the ;; shift amount to the value's bit width. (let ((amt_ Reg (lo_reg amt))) (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) -(rule (lower (has_type (fits_in_64 ty) (rotl src (imm8_from_value amt)))) +(rule (lower (has_type (ty_32_or_64 ty) (rotl src (imm8_from_value amt)))) (value_reg (m_rotl ty (put_in_reg src) amt))) ;; `i128`. diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 2961b1487b..66b65d3468 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -57,11 +57,7 @@ where #[inline] fn operand_size_of_type(&mut self, ty: Type) -> OperandSize { - if ty.bits() == 64 { - OperandSize::Size64 - } else { - OperandSize::Size32 - } + OperandSize::from_ty(ty) } fn put_in_reg_mem(&mut self, val: Value) -> RegMem { @@ -125,6 +121,16 @@ where Some(Imm8Reg::Imm8 { imm }) } + #[inline] + fn mask_imm8_const(&mut self, imm8: &Imm8Reg, mask: u64) -> Imm8Reg { + match imm8 { + &Imm8Reg::Reg { reg } => Imm8Reg::Reg { reg }, + &Imm8Reg::Imm8 { imm } => Imm8Reg::Imm8 { + imm: imm & (mask as u8), + }, + } + } + #[inline] fn simm32_from_value(&mut self, val: Value) -> Option { let inst = self.lower_ctx.dfg().value_def(val).inst()?; diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 1288faee5e..64c2a0a0a4 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602 -src/prelude.isle d3d2a6a42fb778231a4cdca30995324e1293a9ca8073c5a27a501535759eb51f84a6718322a93dfba4b66ee4f0c9afce7dcec0428516ef0c5bc96e8c8b76925d -src/isa/x64/inst.isle b151120df3c356ac697122a8557becd8857eb725851506e844edeb85d831d461322a96d280ad84f9a23518e1e4efb607aebc0e249004148675e4cc19e89f0655 -src/isa/x64/lower.isle c9b408df0a089fb4f207838973ac775b0f9b56c86f056867c28e6bae317873d3844f74f713f9acd6fed98d3d11a2f9d19d392fe5049169dad33b1fc703b9b766 +src/prelude.isle 75a46b97817ad6a4c34e618b81e60876eec6fd1c83ac3ee174851e42045c951644663b2cbc31f1749ce2bc3ad9eb94fb0b877eb2c3bc4885cab7d7e87e9df1d6 +src/isa/x64/inst.isle 3b9c5c81e40b4de04169ac10e5b57d8de14dfefb104e565d24a303e6ccf28416acbdf585b1cae5a90c7e37310d7cdc1534054202597a7b8d2181c8eece08c29e +src/isa/x64/lower.isle c7943201b32e9eb9726466e8cc417f7e84c4c4052de31e05ab6e0ad7502a587cf1d7d9835703c4ff5a506390f7a0668741e7f3feaa1edda6396571a425949fc9 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index e43367d7fd..3f08b62a4b 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -34,6 +34,8 @@ pub trait Context { fn ty_bits_u16(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; + fn ty_32_or_64(&mut self, arg0: Type) -> Option; + fn ty_8_or_16(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; @@ -49,6 +51,7 @@ pub trait Context { fn first_result(&mut self, arg0: Inst) -> Option; fn inst_data(&mut self, arg0: Inst) -> InstructionData; fn value_type(&mut self, arg0: Value) -> Type; + fn ty_bits_mask(&mut self, arg0: Type) -> u64; fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>; fn def_inst(&mut self, arg0: Value) -> Option; fn trap_code_division_by_zero(&mut self) -> TrapCode; @@ -61,6 +64,7 @@ pub trait Context { fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; fn imm8_from_value(&mut self, arg0: Value) -> Option; + fn mask_imm8_const(&mut self, arg0: &Imm8Reg, arg1: u64) -> Imm8Reg; fn simm32_from_value(&mut self, arg0: Value) -> Option; fn simm32_from_imm64(&mut self, arg0: Imm64) -> Option; fn sinkable_load(&mut self, arg0: Value) -> Option; @@ -71,19 +75,19 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 246. +/// Internal type ProducesFlags: defined at src/prelude.isle line 259. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 249. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 262. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 408. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 412. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -127,7 +131,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 259. + // Rule at src/prelude.isle line 272. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -155,7 +159,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 267. + // Rule at src/prelude.isle line 280. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -189,7 +193,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 277. + // Rule at src/prelude.isle line 290. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -242,12 +246,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 420. + // Rule at src/isa/x64/inst.isle line 424. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 423. + // Rule at src/isa/x64/inst.isle line 427. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -271,7 +275,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 443. + // Rule at src/isa/x64/inst.isle line 447. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -279,7 +283,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 439. + // Rule at src/isa/x64/inst.isle line 443. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -292,17 +296,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 450. + // Rule at src/isa/x64/inst.isle line 454. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 451. + // Rule at src/isa/x64/inst.isle line 455. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 452. + // Rule at src/isa/x64/inst.isle line 456. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -319,7 +323,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 456. + // Rule at src/isa/x64/inst.isle line 460. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -329,40 +333,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 465. + // Rule at src/isa/x64/inst.isle line 469. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 466. + // Rule at src/isa/x64/inst.isle line 470. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 461. + // Rule at src/isa/x64/inst.isle line 465. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 462. + // Rule at src/isa/x64/inst.isle line 466. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 463. + // Rule at src/isa/x64/inst.isle line 467. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 464. + // Rule at src/isa/x64/inst.isle line 468. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -374,7 +378,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 480. + // Rule at src/isa/x64/inst.isle line 484. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -402,7 +406,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 504. + // Rule at src/isa/x64/inst.isle line 508. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -427,7 +431,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 512. + // Rule at src/isa/x64/inst.isle line 516. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -443,7 +447,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 524. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -472,7 +476,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 531. + // Rule at src/isa/x64/inst.isle line 535. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -501,7 +505,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 542. + // Rule at src/isa/x64/inst.isle line 546. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -517,7 +521,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 550. + // Rule at src/isa/x64/inst.isle line 554. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -546,7 +550,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 561. + // Rule at src/isa/x64/inst.isle line 565. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -575,7 +579,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 572. + // Rule at src/isa/x64/inst.isle line 576. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -591,7 +595,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 583. + // Rule at src/isa/x64/inst.isle line 587. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -607,7 +611,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 591. + // Rule at src/isa/x64/inst.isle line 595. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -623,7 +627,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 599. + // Rule at src/isa/x64/inst.isle line 603. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -635,7 +639,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 628. + // Rule at src/isa/x64/inst.isle line 632. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -652,7 +656,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 657. + // Rule at src/isa/x64/inst.isle line 661. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -667,7 +671,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 616. + // Rule at src/isa/x64/inst.isle line 620. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -680,7 +684,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 669. + // Rule at src/isa/x64/inst.isle line 673. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -695,7 +699,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 620. + // Rule at src/isa/x64/inst.isle line 624. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -708,7 +712,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 647. + // Rule at src/isa/x64/inst.isle line 651. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -725,7 +729,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } let pattern1_0 = arg1; if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 634. + // Rule at src/isa/x64/inst.isle line 638. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -741,7 +745,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 609. + // Rule at src/isa/x64/inst.isle line 613. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Imm { @@ -766,7 +770,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 682. + // Rule at src/isa/x64/inst.isle line 686. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -791,7 +795,7 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 691. + // Rule at src/isa/x64/inst.isle line 695. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -807,7 +811,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 696. + // Rule at src/isa/x64/inst.isle line 700. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -823,7 +827,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 701. + // Rule at src/isa/x64/inst.isle line 705. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -839,7 +843,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 706. + // Rule at src/isa/x64/inst.isle line 710. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -857,7 +861,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 711. + // Rule at src/isa/x64/inst.isle line 715. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -882,7 +886,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 720. + // Rule at src/isa/x64/inst.isle line 724. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -898,7 +902,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 725. + // Rule at src/isa/x64/inst.isle line 729. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -916,7 +920,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 730. + // Rule at src/isa/x64/inst.isle line 734. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -944,7 +948,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 738. + // Rule at src/isa/x64/inst.isle line 742. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -966,7 +970,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 745. + // Rule at src/isa/x64/inst.isle line 749. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -990,7 +994,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 752. + // Rule at src/isa/x64/inst.isle line 756. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1007,7 +1011,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 759. + // Rule at src/isa/x64/inst.isle line 763. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1018,7 +1022,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 764. + // Rule at src/isa/x64/inst.isle line 768. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1029,7 +1033,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 769. + // Rule at src/isa/x64/inst.isle line 773. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1040,7 +1044,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 774. + // Rule at src/isa/x64/inst.isle line 778. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1051,7 +1055,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 779. + // Rule at src/isa/x64/inst.isle line 783. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1062,7 +1066,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 784. + // Rule at src/isa/x64/inst.isle line 788. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1073,7 +1077,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 789. + // Rule at src/isa/x64/inst.isle line 793. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1084,7 +1088,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 794. + // Rule at src/isa/x64/inst.isle line 798. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1095,7 +1099,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 799. + // Rule at src/isa/x64/inst.isle line 803. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1106,7 +1110,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 804. + // Rule at src/isa/x64/inst.isle line 808. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1117,7 +1121,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 809. + // Rule at src/isa/x64/inst.isle line 813. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1128,7 +1132,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 814. + // Rule at src/isa/x64/inst.isle line 818. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1139,7 +1143,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 819. + // Rule at src/isa/x64/inst.isle line 823. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1150,7 +1154,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 824. + // Rule at src/isa/x64/inst.isle line 828. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1161,7 +1165,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 829. + // Rule at src/isa/x64/inst.isle line 833. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1172,7 +1176,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 834. + // Rule at src/isa/x64/inst.isle line 838. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1183,7 +1187,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 839. + // Rule at src/isa/x64/inst.isle line 843. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1194,7 +1198,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 844. + // Rule at src/isa/x64/inst.isle line 848. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1205,7 +1209,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 849. + // Rule at src/isa/x64/inst.isle line 853. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1216,7 +1220,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 854. + // Rule at src/isa/x64/inst.isle line 858. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1227,7 +1231,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 859. + // Rule at src/isa/x64/inst.isle line 863. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1238,7 +1242,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 864. + // Rule at src/isa/x64/inst.isle line 868. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1249,7 +1253,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 869. + // Rule at src/isa/x64/inst.isle line 873. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1260,7 +1264,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 878. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1271,7 +1275,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 879. + // Rule at src/isa/x64/inst.isle line 883. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1282,7 +1286,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 884. + // Rule at src/isa/x64/inst.isle line 888. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1293,7 +1297,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 893. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1304,7 +1308,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 898. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1315,7 +1319,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 903. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1326,7 +1330,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 908. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1337,7 +1341,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 913. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1348,7 +1352,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 918. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1359,7 +1363,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 923. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1370,7 +1374,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 924. + // Rule at src/isa/x64/inst.isle line 928. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1381,7 +1385,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 929. + // Rule at src/isa/x64/inst.isle line 933. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1392,7 +1396,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 938. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1403,7 +1407,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 939. + // Rule at src/isa/x64/inst.isle line 943. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1414,7 +1418,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 944. + // Rule at src/isa/x64/inst.isle line 948. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1431,7 +1435,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 949. + // Rule at src/isa/x64/inst.isle line 953. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = RegMem::Reg { reg: pattern2_0 }; @@ -1451,7 +1455,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 961. + // Rule at src/isa/x64/inst.isle line 965. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1462,7 +1466,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 966. + // Rule at src/isa/x64/inst.isle line 970. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1473,7 +1477,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 971. + // Rule at src/isa/x64/inst.isle line 975. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1484,7 +1488,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 976. + // Rule at src/isa/x64/inst.isle line 980. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1495,7 +1499,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 981. + // Rule at src/isa/x64/inst.isle line 985. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1506,7 +1510,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 986. + // Rule at src/isa/x64/inst.isle line 990. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1517,7 +1521,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 991. + // Rule at src/isa/x64/inst.isle line 995. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1528,7 +1532,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 996. + // Rule at src/isa/x64/inst.isle line 1000. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1539,7 +1543,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1001. + // Rule at src/isa/x64/inst.isle line 1005. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1550,7 +1554,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1006. + // Rule at src/isa/x64/inst.isle line 1010. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1561,7 +1565,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1011. + // Rule at src/isa/x64/inst.isle line 1015. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1572,7 +1576,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1016. + // Rule at src/isa/x64/inst.isle line 1020. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1583,7 +1587,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1021. + // Rule at src/isa/x64/inst.isle line 1025. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1594,7 +1598,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1026. + // Rule at src/isa/x64/inst.isle line 1030. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1615,7 +1619,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1031. + // Rule at src/isa/x64/inst.isle line 1035. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1643,7 +1647,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1043. + // Rule at src/isa/x64/inst.isle line 1047. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1661,7 +1665,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1052. + // Rule at src/isa/x64/inst.isle line 1056. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1686,7 +1690,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1065. + // Rule at src/isa/x64/inst.isle line 1069. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1702,7 +1706,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1072. + // Rule at src/isa/x64/inst.isle line 1076. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1711,7 +1715,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1077. + // Rule at src/isa/x64/inst.isle line 1081. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1720,7 +1724,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1082. + // Rule at src/isa/x64/inst.isle line 1086. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1729,7 +1733,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1087. + // Rule at src/isa/x64/inst.isle line 1091. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1738,7 +1742,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1092. + // Rule at src/isa/x64/inst.isle line 1096. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1752,7 +1756,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1097. + // Rule at src/isa/x64/inst.isle line 1101. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmREvex { @@ -1768,7 +1772,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1104. + // Rule at src/isa/x64/inst.isle line 1108. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1784,7 +1788,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1109. + // Rule at src/isa/x64/inst.isle line 1113. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -1802,7 +1806,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1121. + // Rule at src/isa/x64/inst.isle line 1125. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1818,7 +1822,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1128. + // Rule at src/isa/x64/inst.isle line 1132. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -1836,7 +1840,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1138. + // Rule at src/isa/x64/inst.isle line 1142. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1846,7 +1850,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1143. + // Rule at src/isa/x64/inst.isle line 1147. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1856,7 +1860,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1148. + // Rule at src/isa/x64/inst.isle line 1152. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1874,7 +1878,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1155. + // Rule at src/isa/x64/inst.isle line 1159. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -1903,7 +1907,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1171. + // Rule at src/isa/x64/inst.isle line 1175. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1919,7 +1923,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1176. + // Rule at src/isa/x64/inst.isle line 1180. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1938,7 +1942,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1189. + // Rule at src/isa/x64/inst.isle line 1193. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -1959,7 +1963,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1198. + // Rule at src/isa/x64/inst.isle line 1202. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -1982,7 +1986,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1205. + // Rule at src/isa/x64/inst.isle line 1209. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2000,7 +2004,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1210. + // Rule at src/isa/x64/inst.isle line 1214. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2020,7 +2024,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1215. + // Rule at src/isa/x64/inst.isle line 1219. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2038,7 +2042,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1220. + // Rule at src/isa/x64/inst.isle line 1224. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2050,7 +2054,7 @@ pub fn constructor_insertps( pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1225. + // Rule at src/isa/x64/inst.isle line 1229. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -2103,7 +2107,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1028. + // Rule at src/isa/x64/lower.isle line 1039. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2292,7 +2296,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 716. + // Rule at src/isa/x64/lower.isle line 727. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2383,7 +2387,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 649. + // Rule at src/isa/x64/lower.isle line 660. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -2423,7 +2427,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1025. + // Rule at src/isa/x64/lower.isle line 1036. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2468,7 +2472,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1106. + // Rule at src/isa/x64/lower.isle line 1117. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2478,7 +2482,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1128. + // Rule at src/isa/x64/lower.isle line 1139. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2488,7 +2492,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1095. + // Rule at src/isa/x64/lower.isle line 1106. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2498,7 +2502,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1117. + // Rule at src/isa/x64/lower.isle line 1128. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -2513,7 +2517,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 971. + // Rule at src/isa/x64/lower.isle line 982. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2534,7 +2538,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1109. + // Rule at src/isa/x64/lower.isle line 1120. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -2544,7 +2548,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1131. + // Rule at src/isa/x64/lower.isle line 1142. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -2554,7 +2558,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1098. + // Rule at src/isa/x64/lower.isle line 1109. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -2564,7 +2568,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1120. + // Rule at src/isa/x64/lower.isle line 1131. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -2579,7 +2583,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 974. + // Rule at src/isa/x64/lower.isle line 985. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2600,7 +2604,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1112. + // Rule at src/isa/x64/lower.isle line 1123. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -2610,7 +2614,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1134. + // Rule at src/isa/x64/lower.isle line 1145. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -2620,7 +2624,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1101. + // Rule at src/isa/x64/lower.isle line 1112. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -2630,7 +2634,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1123. + // Rule at src/isa/x64/lower.isle line 1134. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -2645,7 +2649,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 977. + // Rule at src/isa/x64/lower.isle line 988. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2663,7 +2667,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 960. + // Rule at src/isa/x64/lower.isle line 971. let expr0_0 = C::put_in_reg(ctx, pattern7_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr2_0 = constructor_andnps(ctx, expr0_0, &expr1_0)?; @@ -2723,7 +2727,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Fabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 999. + // Rule at src/isa/x64/lower.isle line 1010. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = F32X4; let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; @@ -2770,7 +2774,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 963. + // Rule at src/isa/x64/lower.isle line 974. let expr0_0 = C::put_in_reg(ctx, pattern7_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr2_0 = constructor_andnpd(ctx, expr0_0, &expr1_0)?; @@ -2785,7 +2789,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Fabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1004. + // Rule at src/isa/x64/lower.isle line 1015. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = F64X2; let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; @@ -2828,7 +2832,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 659. + // Rule at src/isa/x64/lower.isle line 670. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -2956,7 +2960,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 663. + // Rule at src/isa/x64/lower.isle line 674. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3069,7 +3073,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 966. + // Rule at src/isa/x64/lower.isle line 977. let expr0_0 = C::put_in_reg(ctx, pattern7_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr2_0 = constructor_pandn(ctx, expr0_0, &expr1_0)?; @@ -4007,7 +4011,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1033. + // Rule at src/isa/x64/lower.isle line 1044. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4134,7 +4138,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - let (pattern7_0, pattern7_1) = - C::unpack_value_array_2(ctx, &pattern5_1); - if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { - // Rule at src/isa/x64/lower.isle line 644. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = - constructor_m_rotl(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); - return Some(expr2_0); - } - // Rule at src/isa/x64/lower.isle line 638. - let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; - let expr1_0 = C::put_in_reg(ctx, pattern7_0); - let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; - let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); - } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); @@ -4432,7 +4417,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1012. + // Rule at src/isa/x64/lower.isle line 1023. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -4458,6 +4443,62 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } } + if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + if let &Opcode::Rotl = &pattern5_0 { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 655. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = constructor_m_rotl(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + // Rule at src/isa/x64/lower.isle line 649. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + } + if let Some(pattern3_0) = C::ty_8_or_16(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + if let &Opcode::Rotl = &pattern5_0 { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 643. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::ty_bits_mask(ctx, pattern3_0); + let expr2_0 = C::mask_imm8_const(ctx, &pattern8_0, expr1_0); + let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + // Rule at src/isa/x64/lower.isle line 639. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = constructor_extend_to_reg(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = C::put_in_reg(ctx, pattern7_0); + let expr4_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr5_0 = constructor_m_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); + } + } + } } return None; } @@ -4613,7 +4654,7 @@ pub fn constructor_shr_i128( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1018. + // Rule at src/isa/x64/lower.isle line 1029. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -4640,7 +4681,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1049. + // Rule at src/isa/x64/lower.isle line 1060. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -4648,7 +4689,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1052. + // Rule at src/isa/x64/lower.isle line 1063. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -4656,7 +4697,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1055. + // Rule at src/isa/x64/lower.isle line 1066. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -4665,7 +4706,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1058. + // Rule at src/isa/x64/lower.isle line 1069. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -4674,7 +4715,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1061. + // Rule at src/isa/x64/lower.isle line 1072. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -4685,7 +4726,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1082. + // Rule at src/isa/x64/lower.isle line 1093. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -4693,7 +4734,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1083. + // Rule at src/isa/x64/lower.isle line 1094. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4701,7 +4742,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1091. + // Rule at src/isa/x64/lower.isle line 1102. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 7a7ddcf6f2..b3354d807e 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -95,6 +95,11 @@ macro_rules! isle_prelude_methods { ty.bits().try_into().unwrap() } + #[inline] + fn ty_bits_mask(&mut self, ty: Type) -> u64 { + (1 << (self.ty_bits(ty) as u64)) - 1 + } + #[inline] fn ty_bits_u16(&mut self, ty: Type) -> u16 { ty.bits() @@ -118,6 +123,24 @@ macro_rules! isle_prelude_methods { } } + #[inline] + fn ty_32_or_64(&mut self, ty: Type) -> Option { + if ty.bits() == 32 || ty.bits() == 64 { + Some(ty) + } else { + None + } + } + + #[inline] + fn ty_8_or_16(&mut self, ty: Type) -> Option { + if ty.bits() == 8 || ty.bits() == 16 { + Some(ty) + } else { + None + } + } + fn vec128(&mut self, ty: Type) -> Option { if ty.is_vector() && ty.bits() == 128 { Some(ty) diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 1d5e994a3e..1d8c5ad87c 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -156,6 +156,14 @@ (decl fits_in_64 (Type) Type) (extern extractor fits_in_64 fits_in_64) +;; An extractor that maches 32- and 64-bit types only. +(decl ty_32_or_64 (Type) Type) +(extern extractor ty_32_or_64 ty_32_or_64) + +;; An extractor that maches 8- and 16-bit types only. +(decl ty_8_or_16 (Type) Type) +(extern extractor ty_8_or_16 ty_8_or_16) + ;; An extractor that only matches 128-bit vector types. (decl vec128 (Type) Type) (extern extractor vec128 vec128) @@ -230,6 +238,11 @@ (and (result_type ty) inst)) +;; Return a bitmask that will mask off a count to be within `ty`'s +;; bit-width. Used for shifts/rotates. +(decl ty_bits_mask (Type) u64) +(extern constructor ty_bits_mask ty_bits_mask) + ;; Match a multi-lane type, extracting (# bits per lane, # lanes) from the given ;; type. Will only match when there is more than one lane. (decl multi_lane (u8 u16) Type) diff --git a/cranelift/filetests/filetests/runtests/shifts.clif b/cranelift/filetests/filetests/runtests/shifts.clif index c39ff3a0d4..5f66d56191 100644 --- a/cranelift/filetests/filetests/runtests/shifts.clif +++ b/cranelift/filetests/filetests/runtests/shifts.clif @@ -318,3 +318,25 @@ block0(v0: i32, v1: i8): ; run: %sshr_i32_i8(0x40000000, 32) == 0x40000000 ; run: %sshr_i32_i8(0x40000000, 33) == 0x20000000 ; run: %sshr_i32_i8(0x40000000, 34) == 0x10000000 + +function %rotl_i8_const_37(i8) -> i8 { +block0(v0: i8): + v1 = iconst.i8 37 + v2 = rotl.i8 v0, v1 + return v2 +} + +; run: %rotl_i8_const_37(0x00) == 0x00 +; run: %rotl_i8_const_37(0x01) == 0x20 +; run: %rotl_i8_const_37(0x12) == 0x42 + +function %rotr_i8_const_37(i8) -> i8 { +block0(v0: i8): + v1 = iconst.i8 37 + v2 = rotr.i8 v0, v1 + return v2 +} + +; run: %rotr_i8_const_37(0x00) == 0x00 +; run: %rotr_i8_const_37(0x01) == 0x08 +; run: %rotr_i8_const_37(0x12) == 0x90