From 164bfeaf7e0bbe60c63e3c605266c8bc5edcfd60 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 26 Apr 2022 09:50:46 -0700 Subject: [PATCH] x64 backend: migrate stores, and remainder of loads (I128 case), to ISLE. (#4069) --- .../lower/isle/generated_code.manifest | 2 +- .../isa/aarch64/lower/isle/generated_code.rs | 73 +- .../s390x/lower/isle/generated_code.manifest | 2 +- .../isa/s390x/lower/isle/generated_code.rs | 73 +- cranelift/codegen/src/isa/x64/inst.isle | 14 + cranelift/codegen/src/isa/x64/lower.isle | 85 ++ cranelift/codegen/src/isa/x64/lower.rs | 68 +- cranelift/codegen/src/isa/x64/lower/isle.rs | 5 + .../x64/lower/isle/generated_code.manifest | 6 +- .../src/isa/x64/lower/isle/generated_code.rs | 843 +++++++++++------- cranelift/codegen/src/machinst/isle.rs | 9 + cranelift/codegen/src/prelude.isle | 18 +- 12 files changed, 792 insertions(+), 406 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 366acde7e2..fa0e304414 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 443b34b797fc8ace -src/prelude.isle afd037c4d91c875c +src/prelude.isle d8a93eb727abd7f4 src/isa/aarch64/inst.isle 77984cc33a05be7 src/isa/aarch64/lower.isle 71c7e603b0e4bdef diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 128cf2c4e5..45b02aed03 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -59,6 +59,7 @@ pub trait Context { fn ty_int_bool_128(&mut self, arg0: Type) -> Option; fn ty_scalar_float(&mut self, arg0: Type) -> Option; fn ty_vec128(&mut self, arg0: Type) -> Option; + fn ty_vec128_int(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>; @@ -131,13 +132,14 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 402. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, + Inst2 { inst1: MInst, inst2: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 418. +/// Internal type ProducesFlags: defined at src/prelude.isle line 434. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -145,7 +147,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 429. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 445. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -1064,15 +1066,54 @@ pub fn constructor_side_effect( ctx: &mut C, arg0: &SideEffectNoResult, ) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &SideEffectNoResult::Inst { + inst: ref pattern1_0, + } => { + // Rule at src/prelude.isle line 415. + let expr0_0 = C::emit(ctx, pattern1_0); + let expr1_0 = C::output_none(ctx); + return Some(expr1_0); + } + &SideEffectNoResult::Inst2 { + inst1: ref pattern1_0, + inst2: ref pattern1_1, + } => { + // Rule at src/prelude.isle line 418. + let expr0_0 = C::emit(ctx, pattern1_0); + let expr1_0 = C::emit(ctx, pattern1_1); + let expr2_0 = C::output_none(ctx); + return Some(expr2_0); + } + _ => {} + } + return None; +} + +// Generated as internal constructor for term side_effect_concat. +pub fn constructor_side_effect_concat( + ctx: &mut C, + arg0: &SideEffectNoResult, + arg1: &SideEffectNoResult, +) -> Option { let pattern0_0 = arg0; if let &SideEffectNoResult::Inst { inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 407. - let expr0_0 = C::emit(ctx, pattern1_0); - let expr1_0 = C::output_none(ctx); - return Some(expr1_0); + let pattern2_0 = arg1; + if let &SideEffectNoResult::Inst { + inst: ref pattern3_0, + } = pattern2_0 + { + // Rule at src/prelude.isle line 424. + let expr0_0 = SideEffectNoResult::Inst2 { + inst1: pattern1_0.clone(), + inst2: pattern3_0.clone(), + }; + return Some(expr0_0); + } } return None; } @@ -1088,7 +1129,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 445. + // Rule at src/prelude.isle line 461. return Some(pattern1_1); } return None; @@ -1105,7 +1146,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 450. + // Rule at src/prelude.isle line 466. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1115,7 +1156,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 452. + // Rule at src/prelude.isle line 468. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1144,7 +1185,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 459. + // Rule at src/prelude.isle line 475. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1174,7 +1215,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 484. + // Rule at src/prelude.isle line 500. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1185,7 +1226,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 490. + // Rule at src/prelude.isle line 506. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1198,7 +1239,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 502. + // Rule at src/prelude.isle line 518. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1219,7 +1260,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 478. + // Rule at src/prelude.isle line 494. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1239,7 +1280,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 519. + // Rule at src/prelude.isle line 535. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index aae4194934..f7ca087f03 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 443b34b797fc8ace -src/prelude.isle afd037c4d91c875c +src/prelude.isle d8a93eb727abd7f4 src/isa/s390x/inst.isle 8218bd9e8556446b src/isa/s390x/lower.isle 6a8de81f8dc4e568 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index 5a9d17ab09..2ea073acac 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -59,6 +59,7 @@ pub trait Context { fn ty_int_bool_128(&mut self, arg0: Type) -> Option; fn ty_scalar_float(&mut self, arg0: Type) -> Option; fn ty_vec128(&mut self, arg0: Type) -> Option; + fn ty_vec128_int(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>; @@ -154,13 +155,14 @@ pub trait Context { fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 402. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, + Inst2 { inst1: MInst, inst2: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 418. +/// Internal type ProducesFlags: defined at src/prelude.isle line 434. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -168,7 +170,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 429. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 445. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -953,15 +955,54 @@ pub fn constructor_side_effect( ctx: &mut C, arg0: &SideEffectNoResult, ) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &SideEffectNoResult::Inst { + inst: ref pattern1_0, + } => { + // Rule at src/prelude.isle line 415. + let expr0_0 = C::emit(ctx, pattern1_0); + let expr1_0 = C::output_none(ctx); + return Some(expr1_0); + } + &SideEffectNoResult::Inst2 { + inst1: ref pattern1_0, + inst2: ref pattern1_1, + } => { + // Rule at src/prelude.isle line 418. + let expr0_0 = C::emit(ctx, pattern1_0); + let expr1_0 = C::emit(ctx, pattern1_1); + let expr2_0 = C::output_none(ctx); + return Some(expr2_0); + } + _ => {} + } + return None; +} + +// Generated as internal constructor for term side_effect_concat. +pub fn constructor_side_effect_concat( + ctx: &mut C, + arg0: &SideEffectNoResult, + arg1: &SideEffectNoResult, +) -> Option { let pattern0_0 = arg0; if let &SideEffectNoResult::Inst { inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 407. - let expr0_0 = C::emit(ctx, pattern1_0); - let expr1_0 = C::output_none(ctx); - return Some(expr1_0); + let pattern2_0 = arg1; + if let &SideEffectNoResult::Inst { + inst: ref pattern3_0, + } = pattern2_0 + { + // Rule at src/prelude.isle line 424. + let expr0_0 = SideEffectNoResult::Inst2 { + inst1: pattern1_0.clone(), + inst2: pattern3_0.clone(), + }; + return Some(expr0_0); + } } return None; } @@ -977,7 +1018,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 445. + // Rule at src/prelude.isle line 461. return Some(pattern1_1); } return None; @@ -994,7 +1035,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 450. + // Rule at src/prelude.isle line 466. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1004,7 +1045,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 452. + // Rule at src/prelude.isle line 468. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1033,7 +1074,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 459. + // Rule at src/prelude.isle line 475. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1063,7 +1104,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 484. + // Rule at src/prelude.isle line 500. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1074,7 +1115,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 490. + // Rule at src/prelude.isle line 506. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1087,7 +1128,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 502. + // Rule at src/prelude.isle line 518. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1108,7 +1149,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 478. + // Rule at src/prelude.isle line 494. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1128,7 +1169,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 519. + // Rule at src/prelude.isle line 535. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 8e2dee0522..9f7a6f16ea 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -838,6 +838,11 @@ (rule (to_amode flags base offset) (amode_imm_reg_flags offset (put_in_gpr base) flags)) +;; Offsetting an Amode. Used when we need to do consecutive +;; loads/stores to adjacent addresses. +(decl amode_offset (Amode u32) Amode) +(extern constructor amode_offset amode_offset) + ;; Shift kinds. (type ShiftKind extern @@ -1404,6 +1409,15 @@ (rule (x64_pmovzxdq from) (xmm_unary_rm_r (SseOpcode.Pmovzxdq) from)) +(decl x64_movrm (Type SyntheticAmode Gpr) SideEffectNoResult) +(rule (x64_movrm ty addr data) + (let ((size OperandSize (raw_operand_size_of_type ty))) + (SideEffectNoResult.Inst (MInst.MovRM size data addr)))) + +(decl x64_xmm_movrm (SseOpcode SyntheticAmode Xmm) SideEffectNoResult) +(rule (x64_xmm_movrm op addr data) + (SideEffectNoResult.Inst (MInst.XmmMovRM op data addr))) + ;; Load a constant into an XMM register. (decl x64_xmm_load_const (Type VCodeConstant) Xmm) (rule (x64_xmm_load_const ty const) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index d76bdff004..c5170fd15c 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -2554,6 +2554,15 @@ (rule (lower (has_type (ty_vec128 ty) (load flags address offset))) (x64_movdqu (to_amode flags address offset))) +;; We can load an I128/B128 by doing two 64-bit loads. +(rule (lower (has_type (ty_int_bool_128 _) + (load flags address offset))) + (let ((addr_lo Amode (to_amode flags address offset)) + (addr_hi Amode (amode_offset addr_lo 8)) + (value_lo Reg (x64_mov addr_lo)) + (value_hi Reg (x64_mov addr_hi))) + (value_regs value_lo value_hi))) + ;; We also include widening vector loads; these sign- or zero-extend each lane ;; to the next wider width (e.g., 16x4 -> 32x4). (rule (lower (has_type $I16X8 (sload8x8 flags address offset))) @@ -2570,3 +2579,79 @@ (x64_pmovzxdq (to_amode flags address offset))) ;; TODO: Multi-register loads (I128) + +;; Rules for `store*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; 8-, 16-, 32- and 64-bit GPR stores. +(rule (lower (store flags + value @ (value_type (is_gpr_type ty)) + address + offset)) + (side_effect + (x64_movrm ty (to_amode flags address offset) value))) + +;; Explicit 8/16/32-bit opcodes. +(rule (lower (istore8 flags value address offset)) + (side_effect + (x64_movrm $I8 (to_amode flags address offset) value))) +(rule (lower (istore16 flags value address offset)) + (side_effect + (x64_movrm $I16 (to_amode flags address offset) value))) +(rule (lower (istore32 flags value address offset)) + (side_effect + (x64_movrm $I32 (to_amode flags address offset) value))) + +;; F32 stores of values in XMM registers. +(rule (lower (store flags + value @ (value_type $F32) + address + offset)) + (side_effect + (x64_xmm_movrm (SseOpcode.Movss) (to_amode flags address offset) value))) + +;; F64 stores of values in XMM registers. +(rule (lower (store flags + value @ (value_type $F64) + address + offset)) + (side_effect + (x64_xmm_movrm (SseOpcode.Movsd) (to_amode flags address offset) value))) + +;; Stores of F32X4 vectors. +(rule (lower (store flags + value @ (value_type $F32X4) + address + offset)) + (side_effect + (x64_xmm_movrm (SseOpcode.Movups) (to_amode flags address offset) value))) + +;; Stores of F64X2 vectors. +(rule (lower (store flags + value @ (value_type $F64X2) + address + offset)) + (side_effect + (x64_xmm_movrm (SseOpcode.Movupd) (to_amode flags address offset) value))) + +;; Stores of all other 128-bit vector types with integer lanes. +(rule (lower (store flags + value @ (value_type (ty_vec128_int _)) + address + offset)) + (side_effect + (x64_xmm_movrm (SseOpcode.Movdqu) (to_amode flags address offset) value))) + +;; Stores of I128/B128 values: store the two 64-bit halves separately. +(rule (lower (store flags + value @ (value_type (ty_int_bool_128 _)) + address + offset)) + (let ((value_reg ValueRegs value) + (value_lo Gpr (value_regs_get_gpr value_reg 0)) + (value_hi Gpr (value_regs_get_gpr value_reg 1)) + (addr_lo Amode (to_amode flags address offset)) + (addr_hi Amode (amode_offset addr_lo 8))) + (side_effect + (side_effect_concat + (x64_movrm $I64 addr_lo value_lo) + (x64_movrm $I64 addr_hi value_hi))))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index ac852160a7..8da6cd82e3 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2146,75 +2146,11 @@ fn lower_insn_to_regs>( | Opcode::Uload16x4 | Opcode::Sload32x2 | Opcode::Uload32x2 => { - let offset = ctx.data(insn).load_store_offset().unwrap(); - - let elem_ty = match op { - Opcode::Sload8 | Opcode::Uload8 => types::I8, - Opcode::Sload16 | Opcode::Uload16 => types::I16, - Opcode::Sload32 | Opcode::Uload32 => types::I32, - Opcode::Sload8x8 | Opcode::Uload8x8 => types::I8X8, - Opcode::Sload16x4 | Opcode::Uload16x4 => types::I16X4, - Opcode::Sload32x2 | Opcode::Uload32x2 => types::I32X2, - Opcode::Load => ctx.output_ty(insn, 0), - _ => unimplemented!(), - }; - - let amode = match op { - Opcode::Load - | Opcode::Uload8 - | Opcode::Sload8 - | Opcode::Uload16 - | Opcode::Sload16 - | Opcode::Uload32 - | Opcode::Sload32 - | Opcode::Sload8x8 - | Opcode::Uload8x8 - | Opcode::Sload16x4 - | Opcode::Uload16x4 - | Opcode::Sload32x2 - | Opcode::Uload32x2 => { - assert_eq!(inputs.len(), 1, "only one input for load operands"); - lower_to_amode(ctx, inputs[0], offset) - } - _ => unreachable!(), - }; - - if elem_ty == types::I128 { - let dsts = get_output_reg(ctx, outputs[0]); - ctx.emit(Inst::mov64_m_r(amode.clone(), dsts.regs()[0])); - ctx.emit(Inst::mov64_m_r(amode.offset(8), dsts.regs()[1])); - } else { - implemented_in_isle(ctx); - } + implemented_in_isle(ctx); } Opcode::Store | Opcode::Istore8 | Opcode::Istore16 | Opcode::Istore32 => { - let offset = ctx.data(insn).load_store_offset().unwrap(); - - let elem_ty = match op { - Opcode::Istore8 => types::I8, - Opcode::Istore16 => types::I16, - Opcode::Istore32 => types::I32, - Opcode::Store => ctx.input_ty(insn, 0), - _ => unreachable!(), - }; - - let addr = match op { - Opcode::Store | Opcode::Istore8 | Opcode::Istore16 | Opcode::Istore32 => { - assert_eq!(inputs.len(), 2, "only one input for store memory operands"); - lower_to_amode(ctx, inputs[1], offset) - } - _ => unreachable!(), - }; - - if elem_ty == types::I128 { - let srcs = put_input_in_regs(ctx, inputs[0]); - ctx.emit(Inst::store(types::I64, srcs.regs()[0], addr.clone())); - ctx.emit(Inst::store(types::I64, srcs.regs()[1], addr.offset(8))); - } else { - let src = put_input_in_reg(ctx, inputs[0]); - ctx.emit(Inst::store(elem_ty, src, addr)); - } + implemented_in_isle(ctx); } Opcode::AtomicRmw => { diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 733ef1f7a1..65635f72a7 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -544,6 +544,11 @@ where None } } + + #[inline] + fn amode_offset(&mut self, addr: &Amode, offset: u32) -> Amode { + addr.offset(offset) + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index e5dba8075f..a71192b46a 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 443b34b797fc8ace -src/prelude.isle afd037c4d91c875c -src/isa/x64/inst.isle cad03431447aca1b -src/isa/x64/lower.isle a7181571835ddd1e +src/prelude.isle d8a93eb727abd7f4 +src/isa/x64/inst.isle 2fa48b8183f9d5cb +src/isa/x64/lower.isle b7fe1c95c21edbe4 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index f810f4d284..893971f05b 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -59,6 +59,7 @@ pub trait Context { fn ty_int_bool_128(&mut self, arg0: Type) -> Option; fn ty_scalar_float(&mut self, arg0: Type) -> Option; fn ty_vec128(&mut self, arg0: Type) -> Option; + fn ty_vec128_int(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>; @@ -105,6 +106,7 @@ pub trait Context { arg1: Type, arg2: Imm64, ) -> Option; + fn amode_offset(&mut self, arg0: &Amode, arg1: u32) -> Amode; fn put_masked_in_imm8_gpr(&mut self, arg0: Value, arg1: Type) -> Imm8Gpr; fn intcc_to_cc(&mut self, arg0: &IntCC) -> CC; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; @@ -161,13 +163,14 @@ pub trait Context { fn popcount_low_mask(&mut self) -> VCodeConstant; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 402. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, + Inst2 { inst1: MInst, inst2: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 418. +/// Internal type ProducesFlags: defined at src/prelude.isle line 434. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -175,7 +178,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 429. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 445. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -516,7 +519,7 @@ pub enum MInst { }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1202. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1207. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ExtendKind { Sign, @@ -565,15 +568,54 @@ pub fn constructor_side_effect( ctx: &mut C, arg0: &SideEffectNoResult, ) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &SideEffectNoResult::Inst { + inst: ref pattern1_0, + } => { + // Rule at src/prelude.isle line 415. + let expr0_0 = C::emit(ctx, pattern1_0); + let expr1_0 = C::output_none(ctx); + return Some(expr1_0); + } + &SideEffectNoResult::Inst2 { + inst1: ref pattern1_0, + inst2: ref pattern1_1, + } => { + // Rule at src/prelude.isle line 418. + let expr0_0 = C::emit(ctx, pattern1_0); + let expr1_0 = C::emit(ctx, pattern1_1); + let expr2_0 = C::output_none(ctx); + return Some(expr2_0); + } + _ => {} + } + return None; +} + +// Generated as internal constructor for term side_effect_concat. +pub fn constructor_side_effect_concat( + ctx: &mut C, + arg0: &SideEffectNoResult, + arg1: &SideEffectNoResult, +) -> Option { let pattern0_0 = arg0; if let &SideEffectNoResult::Inst { inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 407. - let expr0_0 = C::emit(ctx, pattern1_0); - let expr1_0 = C::output_none(ctx); - return Some(expr1_0); + let pattern2_0 = arg1; + if let &SideEffectNoResult::Inst { + inst: ref pattern3_0, + } = pattern2_0 + { + // Rule at src/prelude.isle line 424. + let expr0_0 = SideEffectNoResult::Inst2 { + inst1: pattern1_0.clone(), + inst2: pattern3_0.clone(), + }; + return Some(expr0_0); + } } return None; } @@ -589,7 +631,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 445. + // Rule at src/prelude.isle line 461. return Some(pattern1_1); } return None; @@ -606,7 +648,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 450. + // Rule at src/prelude.isle line 466. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -616,7 +658,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 452. + // Rule at src/prelude.isle line 468. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -645,7 +687,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 459. + // Rule at src/prelude.isle line 475. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -675,7 +717,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 484. + // Rule at src/prelude.isle line 500. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -686,7 +728,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 490. + // Rule at src/prelude.isle line 506. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -699,7 +741,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 502. + // Rule at src/prelude.isle line 518. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -720,7 +762,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 478. + // Rule at src/prelude.isle line 494. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -740,7 +782,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 519. + // Rule at src/prelude.isle line 535. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1098,7 +1140,7 @@ pub fn constructor_to_amode( // Generated as internal constructor for term reg_to_gpr_mem_imm. pub fn constructor_reg_to_gpr_mem_imm(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1032. + // Rule at src/isa/x64/inst.isle line 1037. let expr0_0 = C::gpr_new(ctx, pattern0_0); let expr1_0 = C::gpr_to_gpr_mem_imm(ctx, expr0_0); return Some(expr1_0); @@ -1107,7 +1149,7 @@ pub fn constructor_reg_to_gpr_mem_imm(ctx: &mut C, arg0: Reg) -> Opt // Generated as internal constructor for term put_in_gpr. pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1039. + // Rule at src/isa/x64/inst.isle line 1044. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = C::gpr_new(ctx, expr0_0); return Some(expr1_0); @@ -1116,7 +1158,7 @@ pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1046. + // Rule at src/isa/x64/inst.isle line 1051. let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_gpr_mem(ctx, &expr0_0); return Some(expr1_0); @@ -1125,7 +1167,7 @@ pub fn constructor_put_in_gpr_mem(ctx: &mut C, arg0: Value) -> Optio // Generated as internal constructor for term put_in_gpr_mem_imm. pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1053. + // Rule at src/isa/x64/inst.isle line 1058. let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); return Some(expr1_0); @@ -1134,7 +1176,7 @@ pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> O // Generated as internal constructor for term put_in_xmm. pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1060. + // Rule at src/isa/x64/inst.isle line 1065. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = C::xmm_new(ctx, expr0_0); return Some(expr1_0); @@ -1143,7 +1185,7 @@ pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1067. + // Rule at src/isa/x64/inst.isle line 1072. let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); return Some(expr1_0); @@ -1152,7 +1194,7 @@ pub fn constructor_put_in_xmm_mem(ctx: &mut C, arg0: Value) -> Optio // Generated as internal constructor for term put_in_xmm_mem_imm. pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1074. + // Rule at src/isa/x64/inst.isle line 1079. let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); let expr1_0 = C::xmm_mem_imm_new(ctx, &expr0_0); return Some(expr1_0); @@ -1161,7 +1203,7 @@ pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> O // Generated as internal constructor for term output_gpr. pub fn constructor_output_gpr(ctx: &mut C, arg0: Gpr) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1079. + // Rule at src/isa/x64/inst.isle line 1084. let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); let expr1_0 = constructor_output_reg(ctx, expr0_0)?; return Some(expr1_0); @@ -1171,7 +1213,7 @@ pub fn constructor_output_gpr(ctx: &mut C, arg0: Gpr) -> Option(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1084. + // Rule at src/isa/x64/inst.isle line 1089. let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); let expr1_0 = C::gpr_to_reg(ctx, pattern1_0); let expr2_0 = C::value_regs(ctx, expr0_0, expr1_0); @@ -1181,7 +1223,7 @@ pub fn constructor_value_gprs(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> // Generated as internal constructor for term output_xmm. pub fn constructor_output_xmm(ctx: &mut C, arg0: Xmm) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1089. + // Rule at src/isa/x64/inst.isle line 1094. let expr0_0 = C::xmm_to_reg(ctx, pattern0_0); let expr1_0 = constructor_output_reg(ctx, expr0_0)?; return Some(expr1_0); @@ -1195,7 +1237,7 @@ pub fn constructor_value_regs_get_gpr( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1096. + // Rule at src/isa/x64/inst.isle line 1101. let expr0_0 = C::value_regs_get(ctx, pattern0_0, pattern1_0); let expr1_0 = C::gpr_new(ctx, expr0_0); return Some(expr1_0); @@ -1204,7 +1246,7 @@ pub fn constructor_value_regs_get_gpr( // Generated as internal constructor for term lo_gpr. pub fn constructor_lo_gpr(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1109. + // Rule at src/isa/x64/inst.isle line 1114. let expr0_0 = constructor_lo_reg(ctx, pattern0_0)?; let expr1_0 = C::gpr_new(ctx, expr0_0); return Some(expr1_0); @@ -1216,7 +1258,7 @@ pub fn constructor_sink_load_to_gpr_mem_imm( arg0: &SinkableLoad, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1192. + // Rule at src/isa/x64/inst.isle line 1197. let expr0_0 = C::sink_load(ctx, pattern0_0); let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); return Some(expr1_0); @@ -1234,12 +1276,12 @@ pub fn constructor_extend_to_gpr( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1214. + // Rule at src/isa/x64/inst.isle line 1219. let expr0_0 = constructor_put_in_gpr(ctx, pattern0_0)?; return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1217. + // Rule at src/isa/x64/inst.isle line 1222. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -1263,7 +1305,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1237. + // Rule at src/isa/x64/inst.isle line 1242. let expr0_0 = constructor_x64_movsx(ctx, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -1271,7 +1313,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1233. + // Rule at src/isa/x64/inst.isle line 1238. let expr0_0 = constructor_x64_movzx(ctx, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -1284,17 +1326,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1244. + // Rule at src/isa/x64/inst.isle line 1249. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1245. + // Rule at src/isa/x64/inst.isle line 1250. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1246. + // Rule at src/isa/x64/inst.isle line 1251. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -1311,7 +1353,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1250. + // Rule at src/isa/x64/inst.isle line 1255. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1321,40 +1363,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1259. + // Rule at src/isa/x64/inst.isle line 1264. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1260. + // Rule at src/isa/x64/inst.isle line 1265. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 1255. + // Rule at src/isa/x64/inst.isle line 1260. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 1256. + // Rule at src/isa/x64/inst.isle line 1261. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 1257. + // Rule at src/isa/x64/inst.isle line 1262. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 1258. + // Rule at src/isa/x64/inst.isle line 1263. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -1366,7 +1408,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1274. + // Rule at src/isa/x64/inst.isle line 1279. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0: Type = I32X4; let expr2_0 = constructor_sse_cmp_op(ctx, expr1_0)?; @@ -1391,7 +1433,7 @@ pub fn constructor_make_i64x2_from_lanes( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1284. + // Rule at src/isa/x64/inst.isle line 1289. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUninitializedValue { dst: expr0_0 }; @@ -1433,12 +1475,12 @@ pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 1305. + // Rule at src/isa/x64/inst.isle line 1310. let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 1306. + // Rule at src/isa/x64/inst.isle line 1311. let expr0_0 = SseOpcode::Movd; let expr1_0 = C::reg_to_gpr_mem(ctx, pattern1_0); let expr2_0 = OperandSize::Size32; @@ -1449,7 +1491,7 @@ pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 1304. + // Rule at src/isa/x64/inst.isle line 1309. let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); return Some(expr0_0); } @@ -1469,7 +1511,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1320. + // Rule at src/isa/x64/inst.isle line 1325. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::Mov64MR { src: pattern2_0.clone(), @@ -1482,7 +1524,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1325. + // Rule at src/isa/x64/inst.isle line 1330. let expr0_0 = SseOpcode::Movss; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1492,7 +1534,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1329. + // Rule at src/isa/x64/inst.isle line 1334. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1502,7 +1544,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1333. + // Rule at src/isa/x64/inst.isle line 1338. let expr0_0 = SseOpcode::Movups; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1512,7 +1554,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1337. + // Rule at src/isa/x64/inst.isle line 1342. let expr0_0 = SseOpcode::Movupd; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1522,7 +1564,7 @@ pub fn constructor_x64_load( if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1341. + // Rule at src/isa/x64/inst.isle line 1346. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1533,7 +1575,7 @@ pub fn constructor_x64_load( let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 1316. + // Rule at src/isa/x64/inst.isle line 1321. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -1549,7 +1591,7 @@ pub fn constructor_x64_load( // Generated as internal constructor for term x64_mov. pub fn constructor_x64_mov(ctx: &mut C, arg0: &Amode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1346. + // Rule at src/isa/x64/inst.isle line 1351. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::amode_to_synthetic_amode(ctx, pattern0_0); let expr2_0 = MInst::Mov64MR { @@ -1569,7 +1611,7 @@ pub fn constructor_x64_movzx( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1352. + // Rule at src/isa/x64/inst.isle line 1357. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern0_0.clone(), @@ -1589,7 +1631,7 @@ pub fn constructor_x64_movsx( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1358. + // Rule at src/isa/x64/inst.isle line 1363. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern0_0.clone(), @@ -1604,7 +1646,7 @@ pub fn constructor_x64_movsx( // Generated as internal constructor for term x64_movss_load. pub fn constructor_x64_movss_load(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1364. + // Rule at src/isa/x64/inst.isle line 1369. let expr0_0 = SseOpcode::Movss; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1613,7 +1655,7 @@ pub fn constructor_x64_movss_load(ctx: &mut C, arg0: &XmmMem) -> Opt // Generated as internal constructor for term x64_movsd_load. pub fn constructor_x64_movsd_load(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1368. + // Rule at src/isa/x64/inst.isle line 1373. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1622,7 +1664,7 @@ pub fn constructor_x64_movsd_load(ctx: &mut C, arg0: &XmmMem) -> Opt // Generated as internal constructor for term x64_movups. pub fn constructor_x64_movups(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1372. + // Rule at src/isa/x64/inst.isle line 1377. let expr0_0 = SseOpcode::Movups; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1631,7 +1673,7 @@ pub fn constructor_x64_movups(ctx: &mut C, arg0: &XmmMem) -> Option< // Generated as internal constructor for term x64_movupd. pub fn constructor_x64_movupd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1376. + // Rule at src/isa/x64/inst.isle line 1381. let expr0_0 = SseOpcode::Movupd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1640,7 +1682,7 @@ pub fn constructor_x64_movupd(ctx: &mut C, arg0: &XmmMem) -> Option< // Generated as internal constructor for term x64_movdqu. pub fn constructor_x64_movdqu(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1380. + // Rule at src/isa/x64/inst.isle line 1385. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1649,7 +1691,7 @@ pub fn constructor_x64_movdqu(ctx: &mut C, arg0: &XmmMem) -> Option< // Generated as internal constructor for term x64_pmovsxbw. pub fn constructor_x64_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1384. + // Rule at src/isa/x64/inst.isle line 1389. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1658,7 +1700,7 @@ pub fn constructor_x64_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Optio // Generated as internal constructor for term x64_pmovzxbw. pub fn constructor_x64_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1388. + // Rule at src/isa/x64/inst.isle line 1393. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1667,7 +1709,7 @@ pub fn constructor_x64_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Optio // Generated as internal constructor for term x64_pmovsxwd. pub fn constructor_x64_pmovsxwd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1392. + // Rule at src/isa/x64/inst.isle line 1397. let expr0_0 = SseOpcode::Pmovsxwd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1676,7 +1718,7 @@ pub fn constructor_x64_pmovsxwd(ctx: &mut C, arg0: &XmmMem) -> Optio // Generated as internal constructor for term x64_pmovzxwd. pub fn constructor_x64_pmovzxwd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1396. + // Rule at src/isa/x64/inst.isle line 1401. let expr0_0 = SseOpcode::Pmovzxwd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1685,7 +1727,7 @@ pub fn constructor_x64_pmovzxwd(ctx: &mut C, arg0: &XmmMem) -> Optio // Generated as internal constructor for term x64_pmovsxdq. pub fn constructor_x64_pmovsxdq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1400. + // Rule at src/isa/x64/inst.isle line 1405. let expr0_0 = SseOpcode::Pmovsxdq; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1694,12 +1736,54 @@ pub fn constructor_x64_pmovsxdq(ctx: &mut C, arg0: &XmmMem) -> Optio // Generated as internal constructor for term x64_pmovzxdq. pub fn constructor_x64_pmovzxdq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1404. + // Rule at src/isa/x64/inst.isle line 1409. let expr0_0 = SseOpcode::Pmovzxdq; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } +// Generated as internal constructor for term x64_movrm. +pub fn constructor_x64_movrm( + ctx: &mut C, + arg0: Type, + arg1: &SyntheticAmode, + arg2: Gpr, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1413. + let expr0_0 = C::raw_operand_size_of_type(ctx, pattern0_0); + let expr1_0 = MInst::MovRM { + size: expr0_0, + src: pattern2_0, + dst: pattern1_0.clone(), + }; + let expr2_0 = SideEffectNoResult::Inst { inst: expr1_0 }; + return Some(expr2_0); +} + +// Generated as internal constructor for term x64_xmm_movrm. +pub fn constructor_x64_xmm_movrm( + ctx: &mut C, + arg0: &SseOpcode, + arg1: &SyntheticAmode, + arg2: Xmm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1418. + let expr0_0 = C::xmm_to_reg(ctx, pattern2_0); + let expr1_0 = MInst::XmmMovRM { + op: pattern0_0.clone(), + src: expr0_0, + dst: pattern1_0.clone(), + }; + let expr2_0 = SideEffectNoResult::Inst { inst: expr1_0 }; + return Some(expr2_0); +} + // Generated as internal constructor for term x64_xmm_load_const. pub fn constructor_x64_xmm_load_const( ctx: &mut C, @@ -1708,7 +1792,7 @@ pub fn constructor_x64_xmm_load_const( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1409. + // Rule at src/isa/x64/inst.isle line 1423. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmLoadConst { @@ -1733,7 +1817,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1422. + // Rule at src/isa/x64/inst.isle line 1436. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -1758,7 +1842,7 @@ pub fn constructor_x64_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1430. + // Rule at src/isa/x64/inst.isle line 1444. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1774,7 +1858,7 @@ pub fn constructor_x64_add_with_flags_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1438. + // Rule at src/isa/x64/inst.isle line 1452. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -1803,7 +1887,7 @@ pub fn constructor_x64_adc_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1450. + // Rule at src/isa/x64/inst.isle line 1464. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -1832,7 +1916,7 @@ pub fn constructor_x64_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1462. + // Rule at src/isa/x64/inst.isle line 1476. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1848,7 +1932,7 @@ pub fn constructor_x64_sub_with_flags_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1470. + // Rule at src/isa/x64/inst.isle line 1484. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -1877,7 +1961,7 @@ pub fn constructor_x64_sbb_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1482. + // Rule at src/isa/x64/inst.isle line 1496. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -1906,7 +1990,7 @@ pub fn constructor_x64_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1494. + // Rule at src/isa/x64/inst.isle line 1508. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1922,7 +2006,7 @@ pub fn constructor_x64_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1502. + // Rule at src/isa/x64/inst.isle line 1516. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1938,7 +2022,7 @@ pub fn constructor_x64_and_with_flags_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1509. + // Rule at src/isa/x64/inst.isle line 1523. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::And; @@ -1963,7 +2047,7 @@ pub fn constructor_x64_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1520. + // Rule at src/isa/x64/inst.isle line 1534. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1979,7 +2063,7 @@ pub fn constructor_x64_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1528. + // Rule at src/isa/x64/inst.isle line 1542. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1991,7 +2075,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 1568. + // Rule at src/isa/x64/inst.isle line 1582. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = OperandSize::Size32; let expr2_0 = MInst::Imm { @@ -2007,7 +2091,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1597. + // Rule at src/isa/x64/inst.isle line 1611. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = SseOpcode::Xorps; @@ -2022,7 +2106,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::xmm_to_reg(ctx, expr1_0); return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 1545. + // Rule at src/isa/x64/inst.isle line 1559. let expr0_0 = SseOpcode::Movd; let expr1_0: Type = I32; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; @@ -2035,7 +2119,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1609. + // Rule at src/isa/x64/inst.isle line 1623. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = SseOpcode::Xorpd; @@ -2050,7 +2134,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::xmm_to_reg(ctx, expr1_0); return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 1551. + // Rule at src/isa/x64/inst.isle line 1565. let expr0_0 = SseOpcode::Movq; let expr1_0: Type = I64; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; @@ -2063,7 +2147,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1587. + // Rule at src/isa/x64/inst.isle line 1601. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -2082,7 +2166,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1574. + // Rule at src/isa/x64/inst.isle line 1588. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); @@ -2099,7 +2183,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr7_0 = C::gpr_to_reg(ctx, expr1_0); return Some(expr7_0); } - // Rule at src/isa/x64/inst.isle line 1538. + // Rule at src/isa/x64/inst.isle line 1552. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { @@ -2118,7 +2202,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option pub fn constructor_imm_i64(ctx: &mut C, arg0: Type, arg1: i64) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1560. + // Rule at src/isa/x64/inst.isle line 1574. let expr0_0 = C::i64_as_u64(ctx, pattern1_0); let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; return Some(expr1_0); @@ -2136,7 +2220,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1622. + // Rule at src/isa/x64/inst.isle line 1636. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -2161,7 +2245,7 @@ pub fn constructor_x64_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1632. + // Rule at src/isa/x64/inst.isle line 1646. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2177,7 +2261,7 @@ pub fn constructor_x64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1637. + // Rule at src/isa/x64/inst.isle line 1651. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2193,7 +2277,7 @@ pub fn constructor_x64_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1642. + // Rule at src/isa/x64/inst.isle line 1656. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2209,7 +2293,7 @@ pub fn constructor_x64_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1647. + // Rule at src/isa/x64/inst.isle line 1661. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2225,7 +2309,7 @@ pub fn constructor_x64_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1652. + // Rule at src/isa/x64/inst.isle line 1666. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2243,7 +2327,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1657. + // Rule at src/isa/x64/inst.isle line 1671. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -2264,7 +2348,7 @@ pub fn constructor_x64_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1666. + // Rule at src/isa/x64/inst.isle line 1680. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2280,7 +2364,7 @@ pub fn constructor_x64_cmp_imm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1671. + // Rule at src/isa/x64/inst.isle line 1685. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = RegMemImm::Imm { simm32: pattern1_0 }; let expr2_0 = C::gpr_mem_imm_new(ctx, &expr1_0); @@ -2298,7 +2382,7 @@ pub fn constructor_xmm_cmp_rm_r( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1676. + // Rule at src/isa/x64/inst.isle line 1690. let expr0_0 = MInst::XmmCmpRmR { op: pattern0_0.clone(), src: pattern1_0.clone(), @@ -2318,7 +2402,7 @@ pub fn constructor_x64_ucomis( let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == F32 { let pattern3_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1682. + // Rule at src/isa/x64/inst.isle line 1696. let expr0_0 = SseOpcode::Ucomiss; let expr1_0 = constructor_put_in_xmm(ctx, pattern0_0)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); @@ -2328,7 +2412,7 @@ pub fn constructor_x64_ucomis( } if pattern1_0 == F64 { let pattern3_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1686. + // Rule at src/isa/x64/inst.isle line 1700. let expr0_0 = SseOpcode::Ucomisd; let expr1_0 = constructor_put_in_xmm(ctx, pattern0_0)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); @@ -2349,7 +2433,7 @@ pub fn constructor_x64_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1691. + // Rule at src/isa/x64/inst.isle line 1705. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2367,7 +2451,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1698. + // Rule at src/isa/x64/inst.isle line 1712. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -2397,7 +2481,7 @@ pub fn constructor_cmove_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1706. + // Rule at src/isa/x64/inst.isle line 1720. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::XmmCmove { @@ -2428,7 +2512,7 @@ pub fn constructor_cmove_from_values( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1717. + // Rule at src/isa/x64/inst.isle line 1731. let expr0_0 = C::put_in_regs(ctx, pattern3_0); let expr1_0 = C::put_in_regs(ctx, pattern4_0); let expr2_0 = C::temp_writable_gpr(ctx); @@ -2473,7 +2557,7 @@ pub fn constructor_cmove_from_values( let pattern3_0 = arg1; let pattern4_0 = arg2; let pattern5_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1741. + // Rule at src/isa/x64/inst.isle line 1755. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern5_0)?; let expr2_0 = constructor_cmove_xmm(ctx, pattern2_0, pattern3_0, &expr0_0, expr1_0)?; @@ -2485,7 +2569,7 @@ pub fn constructor_cmove_from_values( let pattern3_0 = arg1; let pattern4_0 = arg2; let pattern5_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1738. + // Rule at src/isa/x64/inst.isle line 1752. let expr0_0 = constructor_put_in_gpr_mem(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_0)?; let expr2_0 = constructor_cmove(ctx, pattern2_0, pattern3_0, &expr0_0, expr1_0)?; @@ -2509,7 +2593,7 @@ pub fn constructor_cmove_or( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1748. + // Rule at src/isa/x64/inst.isle line 1762. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::temp_writable_gpr(ctx); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2551,7 +2635,7 @@ pub fn constructor_cmove_or_xmm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1760. + // Rule at src/isa/x64/inst.isle line 1774. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::temp_writable_xmm(ctx); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -2594,7 +2678,7 @@ pub fn constructor_cmove_or_from_values( let pattern3_0 = arg2; let pattern4_0 = arg3; let pattern5_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1775. + // Rule at src/isa/x64/inst.isle line 1789. let expr0_0 = C::put_in_regs(ctx, pattern4_0); let expr1_0 = C::put_in_regs(ctx, pattern5_0); let expr2_0 = C::temp_writable_gpr(ctx); @@ -2666,7 +2750,7 @@ pub fn constructor_cmove_or_from_values( let pattern4_0 = arg2; let pattern5_0 = arg3; let pattern6_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1797. + // Rule at src/isa/x64/inst.isle line 1811. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern6_0)?; let expr2_0 = constructor_cmove_or_xmm( @@ -2681,7 +2765,7 @@ pub fn constructor_cmove_or_from_values( let pattern4_0 = arg2; let pattern5_0 = arg3; let pattern6_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1794. + // Rule at src/isa/x64/inst.isle line 1808. let expr0_0 = constructor_put_in_gpr_mem(ctx, pattern5_0)?; let expr1_0 = constructor_put_in_gpr(ctx, pattern6_0)?; let expr2_0 = @@ -2695,7 +2779,7 @@ pub fn constructor_cmove_or_from_values( // Generated as internal constructor for term x64_setcc. pub fn constructor_x64_setcc(ctx: &mut C, arg0: &CC) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1802. + // Rule at src/isa/x64/inst.isle line 1816. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::Setcc { cc: pattern0_0.clone(), @@ -2721,7 +2805,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1810. + // Rule at src/isa/x64/inst.isle line 1824. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -2738,7 +2822,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_x64_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1817. + // Rule at src/isa/x64/inst.isle line 1831. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2749,7 +2833,7 @@ pub fn constructor_x64_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1822. + // Rule at src/isa/x64/inst.isle line 1836. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2760,7 +2844,7 @@ pub fn constructor_x64_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1827. + // Rule at src/isa/x64/inst.isle line 1841. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2771,7 +2855,7 @@ pub fn constructor_x64_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1832. + // Rule at src/isa/x64/inst.isle line 1846. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2782,7 +2866,7 @@ pub fn constructor_x64_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1837. + // Rule at src/isa/x64/inst.isle line 1851. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2793,7 +2877,7 @@ pub fn constructor_x64_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1842. + // Rule at src/isa/x64/inst.isle line 1856. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2804,7 +2888,7 @@ pub fn constructor_x64_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1847. + // Rule at src/isa/x64/inst.isle line 1861. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2815,7 +2899,7 @@ pub fn constructor_x64_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1852. + // Rule at src/isa/x64/inst.isle line 1866. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2826,7 +2910,7 @@ pub fn constructor_x64_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1857. + // Rule at src/isa/x64/inst.isle line 1871. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2837,7 +2921,7 @@ pub fn constructor_x64_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1862. + // Rule at src/isa/x64/inst.isle line 1876. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2848,7 +2932,7 @@ pub fn constructor_x64_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1867. + // Rule at src/isa/x64/inst.isle line 1881. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2859,7 +2943,7 @@ pub fn constructor_x64_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1872. + // Rule at src/isa/x64/inst.isle line 1886. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2870,7 +2954,7 @@ pub fn constructor_x64_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1877. + // Rule at src/isa/x64/inst.isle line 1891. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2881,7 +2965,7 @@ pub fn constructor_x64_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1882. + // Rule at src/isa/x64/inst.isle line 1896. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2892,7 +2976,7 @@ pub fn constructor_x64_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1887. + // Rule at src/isa/x64/inst.isle line 1901. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2903,7 +2987,7 @@ pub fn constructor_x64_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1892. + // Rule at src/isa/x64/inst.isle line 1906. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2914,7 +2998,7 @@ pub fn constructor_x64_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1897. + // Rule at src/isa/x64/inst.isle line 1911. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2925,7 +3009,7 @@ pub fn constructor_x64_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1902. + // Rule at src/isa/x64/inst.isle line 1916. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2936,7 +3020,7 @@ pub fn constructor_x64_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1907. + // Rule at src/isa/x64/inst.isle line 1921. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2947,7 +3031,7 @@ pub fn constructor_x64_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) - pub fn constructor_x64_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1912. + // Rule at src/isa/x64/inst.isle line 1926. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2958,7 +3042,7 @@ pub fn constructor_x64_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1917. + // Rule at src/isa/x64/inst.isle line 1931. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2969,7 +3053,7 @@ pub fn constructor_x64_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1922. + // Rule at src/isa/x64/inst.isle line 1936. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2980,7 +3064,7 @@ pub fn constructor_x64_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_x64_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1927. + // Rule at src/isa/x64/inst.isle line 1941. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2991,7 +3075,7 @@ pub fn constructor_x64_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) - pub fn constructor_x64_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1932. + // Rule at src/isa/x64/inst.isle line 1946. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3002,7 +3086,7 @@ pub fn constructor_x64_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) - pub fn constructor_x64_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1937. + // Rule at src/isa/x64/inst.isle line 1951. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3013,7 +3097,7 @@ pub fn constructor_x64_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) - pub fn constructor_x64_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1942. + // Rule at src/isa/x64/inst.isle line 1956. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3024,7 +3108,7 @@ pub fn constructor_x64_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1947. + // Rule at src/isa/x64/inst.isle line 1961. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3035,7 +3119,7 @@ pub fn constructor_x64_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1952. + // Rule at src/isa/x64/inst.isle line 1966. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3046,7 +3130,7 @@ pub fn constructor_x64_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1957. + // Rule at src/isa/x64/inst.isle line 1971. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3057,7 +3141,7 @@ pub fn constructor_x64_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1962. + // Rule at src/isa/x64/inst.isle line 1976. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3068,7 +3152,7 @@ pub fn constructor_x64_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1967. + // Rule at src/isa/x64/inst.isle line 1981. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3079,7 +3163,7 @@ pub fn constructor_x64_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1972. + // Rule at src/isa/x64/inst.isle line 1986. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3090,7 +3174,7 @@ pub fn constructor_x64_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1977. + // Rule at src/isa/x64/inst.isle line 1991. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3101,7 +3185,7 @@ pub fn constructor_x64_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1982. + // Rule at src/isa/x64/inst.isle line 1996. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3112,7 +3196,7 @@ pub fn constructor_x64_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmM pub fn constructor_x64_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1987. + // Rule at src/isa/x64/inst.isle line 2001. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3123,7 +3207,7 @@ pub fn constructor_x64_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmM pub fn constructor_x64_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1992. + // Rule at src/isa/x64/inst.isle line 2006. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3134,7 +3218,7 @@ pub fn constructor_x64_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1997. + // Rule at src/isa/x64/inst.isle line 2011. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3145,7 +3229,7 @@ pub fn constructor_x64_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2002. + // Rule at src/isa/x64/inst.isle line 2016. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3156,7 +3240,7 @@ pub fn constructor_x64_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_addss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2007. + // Rule at src/isa/x64/inst.isle line 2021. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Addss; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3167,7 +3251,7 @@ pub fn constructor_x64_addss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_addsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2012. + // Rule at src/isa/x64/inst.isle line 2026. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Addsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3178,7 +3262,7 @@ pub fn constructor_x64_addsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_addps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2017. + // Rule at src/isa/x64/inst.isle line 2031. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Addps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3189,7 +3273,7 @@ pub fn constructor_x64_addps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_addpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2022. + // Rule at src/isa/x64/inst.isle line 2036. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Addpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3200,7 +3284,7 @@ pub fn constructor_x64_addpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_subss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2027. + // Rule at src/isa/x64/inst.isle line 2041. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Subss; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3211,7 +3295,7 @@ pub fn constructor_x64_subss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_subsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2032. + // Rule at src/isa/x64/inst.isle line 2046. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Subsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3222,7 +3306,7 @@ pub fn constructor_x64_subsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_subps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2037. + // Rule at src/isa/x64/inst.isle line 2051. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Subps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3233,7 +3317,7 @@ pub fn constructor_x64_subps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_subpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2042. + // Rule at src/isa/x64/inst.isle line 2056. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Subpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3244,7 +3328,7 @@ pub fn constructor_x64_subpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_mulss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2047. + // Rule at src/isa/x64/inst.isle line 2061. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Mulss; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3255,7 +3339,7 @@ pub fn constructor_x64_mulss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_mulsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2052. + // Rule at src/isa/x64/inst.isle line 2066. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Mulsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3266,7 +3350,7 @@ pub fn constructor_x64_mulsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_mulps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2057. + // Rule at src/isa/x64/inst.isle line 2071. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Mulps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3277,7 +3361,7 @@ pub fn constructor_x64_mulps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_mulpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2062. + // Rule at src/isa/x64/inst.isle line 2076. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Mulpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3288,7 +3372,7 @@ pub fn constructor_x64_mulpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_divss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2067. + // Rule at src/isa/x64/inst.isle line 2081. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Divss; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3299,7 +3383,7 @@ pub fn constructor_x64_divss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_divsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2072. + // Rule at src/isa/x64/inst.isle line 2086. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Divsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3310,7 +3394,7 @@ pub fn constructor_x64_divsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_divps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2077. + // Rule at src/isa/x64/inst.isle line 2091. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Divps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3321,7 +3405,7 @@ pub fn constructor_x64_divps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_divpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2082. + // Rule at src/isa/x64/inst.isle line 2096. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Divpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3332,17 +3416,17 @@ pub fn constructor_x64_divpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 2086. + // Rule at src/isa/x64/inst.isle line 2100. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 2087. + // Rule at src/isa/x64/inst.isle line 2101. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 2088. + // Rule at src/isa/x64/inst.isle line 2102. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -3353,17 +3437,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 2091. + // Rule at src/isa/x64/inst.isle line 2105. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 2092. + // Rule at src/isa/x64/inst.isle line 2106. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 2093. + // Rule at src/isa/x64/inst.isle line 2107. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -3382,7 +3466,7 @@ pub fn constructor_x64_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2097. + // Rule at src/isa/x64/inst.isle line 2111. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -3406,7 +3490,7 @@ pub fn constructor_x64_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2111. + // Rule at src/isa/x64/inst.isle line 2125. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); @@ -3430,7 +3514,7 @@ pub fn constructor_x64_movsd_regmove( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2125. + // Rule at src/isa/x64/inst.isle line 2139. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3441,7 +3525,7 @@ pub fn constructor_x64_movsd_regmove( pub fn constructor_x64_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2130. + // Rule at src/isa/x64/inst.isle line 2144. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3459,21 +3543,21 @@ pub fn constructor_x64_pmaxs( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2135. + // Rule at src/isa/x64/inst.isle line 2149. let expr0_0 = constructor_x64_pmaxsb(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2136. + // Rule at src/isa/x64/inst.isle line 2150. let expr0_0 = constructor_x64_pmaxsw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2137. + // Rule at src/isa/x64/inst.isle line 2151. let expr0_0 = constructor_x64_pmaxsd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -3484,7 +3568,7 @@ pub fn constructor_x64_pmaxs( pub fn constructor_x64_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2140. + // Rule at src/isa/x64/inst.isle line 2154. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3495,7 +3579,7 @@ pub fn constructor_x64_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2142. + // Rule at src/isa/x64/inst.isle line 2156. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3506,7 +3590,7 @@ pub fn constructor_x64_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2144. + // Rule at src/isa/x64/inst.isle line 2158. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3524,21 +3608,21 @@ pub fn constructor_x64_pmins( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2148. + // Rule at src/isa/x64/inst.isle line 2162. let expr0_0 = constructor_x64_pminsb(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2149. + // Rule at src/isa/x64/inst.isle line 2163. let expr0_0 = constructor_x64_pminsw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2150. + // Rule at src/isa/x64/inst.isle line 2164. let expr0_0 = constructor_x64_pminsd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -3549,7 +3633,7 @@ pub fn constructor_x64_pmins( pub fn constructor_x64_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2153. + // Rule at src/isa/x64/inst.isle line 2167. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3560,7 +3644,7 @@ pub fn constructor_x64_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2155. + // Rule at src/isa/x64/inst.isle line 2169. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3571,7 +3655,7 @@ pub fn constructor_x64_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2157. + // Rule at src/isa/x64/inst.isle line 2171. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3589,21 +3673,21 @@ pub fn constructor_x64_pmaxu( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2161. + // Rule at src/isa/x64/inst.isle line 2175. let expr0_0 = constructor_x64_pmaxub(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2162. + // Rule at src/isa/x64/inst.isle line 2176. let expr0_0 = constructor_x64_pmaxuw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2163. + // Rule at src/isa/x64/inst.isle line 2177. let expr0_0 = constructor_x64_pmaxud(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -3614,7 +3698,7 @@ pub fn constructor_x64_pmaxu( pub fn constructor_x64_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2166. + // Rule at src/isa/x64/inst.isle line 2180. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3625,7 +3709,7 @@ pub fn constructor_x64_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2168. + // Rule at src/isa/x64/inst.isle line 2182. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3636,7 +3720,7 @@ pub fn constructor_x64_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2170. + // Rule at src/isa/x64/inst.isle line 2184. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3654,21 +3738,21 @@ pub fn constructor_x64_pminu( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2174. + // Rule at src/isa/x64/inst.isle line 2188. let expr0_0 = constructor_x64_pminub(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2175. + // Rule at src/isa/x64/inst.isle line 2189. let expr0_0 = constructor_x64_pminuw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2176. + // Rule at src/isa/x64/inst.isle line 2190. let expr0_0 = constructor_x64_pminud(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -3679,7 +3763,7 @@ pub fn constructor_x64_pminu( pub fn constructor_x64_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2179. + // Rule at src/isa/x64/inst.isle line 2193. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3690,7 +3774,7 @@ pub fn constructor_x64_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2181. + // Rule at src/isa/x64/inst.isle line 2195. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3701,7 +3785,7 @@ pub fn constructor_x64_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2183. + // Rule at src/isa/x64/inst.isle line 2197. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3712,7 +3796,7 @@ pub fn constructor_x64_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_x64_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2187. + // Rule at src/isa/x64/inst.isle line 2201. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3723,7 +3807,7 @@ pub fn constructor_x64_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmM pub fn constructor_x64_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2192. + // Rule at src/isa/x64/inst.isle line 2206. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3734,7 +3818,7 @@ pub fn constructor_x64_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmM pub fn constructor_x64_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2197. + // Rule at src/isa/x64/inst.isle line 2211. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -3755,7 +3839,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 2202. + // Rule at src/isa/x64/inst.isle line 2216. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -3783,7 +3867,7 @@ pub fn constructor_x64_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2214. + // Rule at src/isa/x64/inst.isle line 2228. let expr0_0 = SseOpcode::Palignr; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3805,7 +3889,7 @@ pub fn constructor_x64_cmpp( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2223. + // Rule at src/isa/x64/inst.isle line 2237. let expr0_0 = constructor_x64_cmpps(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -3813,7 +3897,7 @@ pub fn constructor_x64_cmpp( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2224. + // Rule at src/isa/x64/inst.isle line 2238. let expr0_0 = constructor_x64_cmppd(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -3830,7 +3914,7 @@ pub fn constructor_x64_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2227. + // Rule at src/isa/x64/inst.isle line 2241. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3850,7 +3934,7 @@ pub fn constructor_x64_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2238. + // Rule at src/isa/x64/inst.isle line 2252. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3870,7 +3954,7 @@ pub fn constructor_x64_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2247. + // Rule at src/isa/x64/inst.isle line 2261. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3889,7 +3973,7 @@ pub fn constructor_x64_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2256. + // Rule at src/isa/x64/inst.isle line 2270. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3910,7 +3994,7 @@ pub fn constructor_x64_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2265. + // Rule at src/isa/x64/inst.isle line 2279. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3923,7 +4007,7 @@ pub fn constructor_x64_pinsrd( pub fn constructor_x64_pmaddwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2274. + // Rule at src/isa/x64/inst.isle line 2288. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pmaddwd; let expr2_0 = MInst::XmmRmR { @@ -3947,7 +4031,7 @@ pub fn constructor_x64_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2284. + // Rule at src/isa/x64/inst.isle line 2298. let expr0_0 = SseOpcode::Insertps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3966,7 +4050,7 @@ pub fn constructor_x64_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2293. + // Rule at src/isa/x64/inst.isle line 2307. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pshufd; let expr2_0 = constructor_writable_xmm_to_r_reg(ctx, expr0_0)?; @@ -3989,7 +4073,7 @@ pub fn constructor_x64_pshufd( pub fn constructor_x64_pshufb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2305. + // Rule at src/isa/x64/inst.isle line 2319. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pshufb; let expr2_0 = MInst::XmmRmR { @@ -4011,7 +4095,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2315. + // Rule at src/isa/x64/inst.isle line 2329. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), @@ -4026,7 +4110,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term x64_pabsb. pub fn constructor_x64_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2322. + // Rule at src/isa/x64/inst.isle line 2336. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -4035,7 +4119,7 @@ pub fn constructor_x64_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2327. + // Rule at src/isa/x64/inst.isle line 2341. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -4044,7 +4128,7 @@ pub fn constructor_x64_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2332. + // Rule at src/isa/x64/inst.isle line 2346. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -4058,7 +4142,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2337. + // Rule at src/isa/x64/inst.isle line 2351. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), @@ -4073,7 +4157,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term x64_vpabsq. pub fn constructor_x64_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2344. + // Rule at src/isa/x64/inst.isle line 2358. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -4082,7 +4166,7 @@ pub fn constructor_x64_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option< // Generated as internal constructor for term x64_vpopcntb. pub fn constructor_x64_vpopcntb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2349. + // Rule at src/isa/x64/inst.isle line 2363. let expr0_0 = Avx512Opcode::Vpopcntb; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -4098,7 +4182,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2354. + // Rule at src/isa/x64/inst.isle line 2368. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), @@ -4115,7 +4199,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_x64_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2366. + // Rule at src/isa/x64/inst.isle line 2380. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4133,7 +4217,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2375. + // Rule at src/isa/x64/inst.isle line 2389. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::temp_writable_gpr(ctx); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -4162,7 +4246,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2390. + // Rule at src/isa/x64/inst.isle line 2404. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -4178,7 +4262,7 @@ pub fn constructor_xmm_rmi_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2395. + // Rule at src/isa/x64/inst.isle line 2409. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmiReg { opcode: pattern0_0.clone(), @@ -4195,7 +4279,7 @@ pub fn constructor_xmm_rmi_xmm( pub fn constructor_x64_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2405. + // Rule at src/isa/x64/inst.isle line 2419. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4205,7 +4289,7 @@ pub fn constructor_x64_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2410. + // Rule at src/isa/x64/inst.isle line 2424. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4215,7 +4299,7 @@ pub fn constructor_x64_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2415. + // Rule at src/isa/x64/inst.isle line 2429. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4225,7 +4309,7 @@ pub fn constructor_x64_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2420. + // Rule at src/isa/x64/inst.isle line 2434. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4235,7 +4319,7 @@ pub fn constructor_x64_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2425. + // Rule at src/isa/x64/inst.isle line 2439. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4245,7 +4329,7 @@ pub fn constructor_x64_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2430. + // Rule at src/isa/x64/inst.isle line 2444. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4255,7 +4339,7 @@ pub fn constructor_x64_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2435. + // Rule at src/isa/x64/inst.isle line 2449. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4265,7 +4349,7 @@ pub fn constructor_x64_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemIm pub fn constructor_x64_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2440. + // Rule at src/isa/x64/inst.isle line 2454. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -4281,7 +4365,7 @@ pub fn constructor_x64_pextrd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2445. + // Rule at src/isa/x64/inst.isle line 2459. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = SseOpcode::Pextrd; let expr2_0 = constructor_writable_gpr_to_r_reg(ctx, expr0_0)?; @@ -4314,7 +4398,7 @@ pub fn constructor_gpr_to_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2457. + // Rule at src/isa/x64/inst.isle line 2471. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::GprToXmm { op: pattern0_0.clone(), @@ -4331,7 +4415,7 @@ pub fn constructor_gpr_to_xmm( pub fn constructor_x64_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2464. + // Rule at src/isa/x64/inst.isle line 2478. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -4348,7 +4432,7 @@ pub fn constructor_x64_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Op pub fn constructor_x64_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2472. + // Rule at src/isa/x64/inst.isle line 2486. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { @@ -4364,7 +4448,7 @@ pub fn constructor_x64_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Op // Generated as internal constructor for term x64_lea. pub fn constructor_x64_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2479. + // Rule at src/isa/x64/inst.isle line 2493. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), @@ -4378,7 +4462,7 @@ pub fn constructor_x64_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Op // Generated as internal constructor for term x64_ud2. pub fn constructor_x64_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2486. + // Rule at src/isa/x64/inst.isle line 2500. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -4388,7 +4472,7 @@ pub fn constructor_x64_ud2(ctx: &mut C, arg0: &TrapCode) -> Option(ctx: &mut C) -> Option { - // Rule at src/isa/x64/inst.isle line 2491. + // Rule at src/isa/x64/inst.isle line 2505. let expr0_0 = MInst::Hlt; let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; return Some(expr1_0); @@ -4398,7 +4482,7 @@ pub fn constructor_x64_hlt(ctx: &mut C) -> Option(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2496. + // Rule at src/isa/x64/inst.isle line 2510. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Lzcnt; @@ -4418,7 +4502,7 @@ pub fn constructor_x64_lzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_x64_tzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2504. + // Rule at src/isa/x64/inst.isle line 2518. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Tzcnt; @@ -4442,7 +4526,7 @@ pub fn constructor_x64_bsr( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2512. + // Rule at src/isa/x64/inst.isle line 2526. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Bsr; @@ -4471,7 +4555,7 @@ pub fn constructor_bsr_or_else( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2521. + // Rule at src/isa/x64/inst.isle line 2535. let expr0_0 = constructor_x64_bsr(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_produces_flags_get_reg(ctx, &expr0_0)?; let expr2_0 = C::gpr_new(ctx, expr1_0); @@ -4492,7 +4576,7 @@ pub fn constructor_x64_bsf( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2532. + // Rule at src/isa/x64/inst.isle line 2546. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Bsf; @@ -4521,7 +4605,7 @@ pub fn constructor_bsf_or_else( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2541. + // Rule at src/isa/x64/inst.isle line 2555. let expr0_0 = constructor_x64_bsf(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_produces_flags_get_reg(ctx, &expr0_0)?; let expr2_0 = C::gpr_new(ctx, expr1_0); @@ -4538,7 +4622,7 @@ pub fn constructor_bsf_or_else( pub fn constructor_x64_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2552. + // Rule at src/isa/x64/inst.isle line 2566. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Popcnt; @@ -4566,7 +4650,7 @@ pub fn constructor_xmm_min_max_seq( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2560. + // Rule at src/isa/x64/inst.isle line 2574. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::XmmMinMaxSeq { @@ -4585,7 +4669,7 @@ pub fn constructor_xmm_min_max_seq( pub fn constructor_x64_minss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2568. + // Rule at src/isa/x64/inst.isle line 2582. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minss; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4604,7 +4688,7 @@ pub fn constructor_x64_minss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_minsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2575. + // Rule at src/isa/x64/inst.isle line 2589. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minsd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4623,7 +4707,7 @@ pub fn constructor_x64_minsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_minps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2583. + // Rule at src/isa/x64/inst.isle line 2597. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minps; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4642,7 +4726,7 @@ pub fn constructor_x64_minps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_minpd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2590. + // Rule at src/isa/x64/inst.isle line 2604. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Minpd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4661,7 +4745,7 @@ pub fn constructor_x64_minpd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2597. + // Rule at src/isa/x64/inst.isle line 2611. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxss; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4680,7 +4764,7 @@ pub fn constructor_x64_maxss(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2604. + // Rule at src/isa/x64/inst.isle line 2618. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxsd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4699,7 +4783,7 @@ pub fn constructor_x64_maxsd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2611. + // Rule at src/isa/x64/inst.isle line 2625. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxps; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4718,7 +4802,7 @@ pub fn constructor_x64_maxps(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O pub fn constructor_x64_maxpd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2618. + // Rule at src/isa/x64/inst.isle line 2632. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Maxpd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern1_0); @@ -4736,7 +4820,7 @@ pub fn constructor_x64_maxpd(ctx: &mut C, arg0: Xmm, arg1: Xmm) -> O // Generated as internal constructor for term x64_sqrtss. pub fn constructor_x64_sqrtss(ctx: &mut C, arg0: Xmm) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2626. + // Rule at src/isa/x64/inst.isle line 2640. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Sqrtss; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern0_0); @@ -4753,7 +4837,7 @@ pub fn constructor_x64_sqrtss(ctx: &mut C, arg0: Xmm) -> Option // Generated as internal constructor for term x64_sqrtsd. pub fn constructor_x64_sqrtsd(ctx: &mut C, arg0: Xmm) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2633. + // Rule at src/isa/x64/inst.isle line 2647. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Sqrtsd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern0_0); @@ -4770,7 +4854,7 @@ pub fn constructor_x64_sqrtsd(ctx: &mut C, arg0: Xmm) -> Option // Generated as internal constructor for term x64_sqrtps. pub fn constructor_x64_sqrtps(ctx: &mut C, arg0: Xmm) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2640. + // Rule at src/isa/x64/inst.isle line 2654. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Sqrtps; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern0_0); @@ -4787,7 +4871,7 @@ pub fn constructor_x64_sqrtps(ctx: &mut C, arg0: Xmm) -> Option // Generated as internal constructor for term x64_sqrtpd. pub fn constructor_x64_sqrtpd(ctx: &mut C, arg0: Xmm) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2647. + // Rule at src/isa/x64/inst.isle line 2661. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Sqrtpd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern0_0); @@ -4812,28 +4896,28 @@ pub fn constructor_x64_pcmpeq( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2654. + // Rule at src/isa/x64/inst.isle line 2668. let expr0_0 = constructor_x64_pcmpeqb(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2655. + // Rule at src/isa/x64/inst.isle line 2669. let expr0_0 = constructor_x64_pcmpeqw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2656. + // Rule at src/isa/x64/inst.isle line 2670. let expr0_0 = constructor_x64_pcmpeqd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2657. + // Rule at src/isa/x64/inst.isle line 2671. let expr0_0 = constructor_x64_pcmpeqq(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4844,7 +4928,7 @@ pub fn constructor_x64_pcmpeq( pub fn constructor_x64_pcmpeqb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2660. + // Rule at src/isa/x64/inst.isle line 2674. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pcmpeqb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4855,7 +4939,7 @@ pub fn constructor_x64_pcmpeqb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpeqw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2662. + // Rule at src/isa/x64/inst.isle line 2676. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pcmpeqw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4866,7 +4950,7 @@ pub fn constructor_x64_pcmpeqw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpeqd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2664. + // Rule at src/isa/x64/inst.isle line 2678. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Pcmpeqd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4877,7 +4961,7 @@ pub fn constructor_x64_pcmpeqd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpeqq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2666. + // Rule at src/isa/x64/inst.isle line 2680. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pcmpeqq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4895,28 +4979,28 @@ pub fn constructor_x64_pcmpgt( if pattern0_0 == I8X16 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2670. + // Rule at src/isa/x64/inst.isle line 2684. let expr0_0 = constructor_x64_pcmpgtb(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I16X8 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2671. + // Rule at src/isa/x64/inst.isle line 2685. let expr0_0 = constructor_x64_pcmpgtw(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2672. + // Rule at src/isa/x64/inst.isle line 2686. let expr0_0 = constructor_x64_pcmpgtd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == I64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2673. + // Rule at src/isa/x64/inst.isle line 2687. let expr0_0 = constructor_x64_pcmpgtq(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4927,7 +5011,7 @@ pub fn constructor_x64_pcmpgt( pub fn constructor_x64_pcmpgtb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2676. + // Rule at src/isa/x64/inst.isle line 2690. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pcmpgtb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4938,7 +5022,7 @@ pub fn constructor_x64_pcmpgtb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpgtw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2678. + // Rule at src/isa/x64/inst.isle line 2692. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pcmpgtw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4949,7 +5033,7 @@ pub fn constructor_x64_pcmpgtw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpgtd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2680. + // Rule at src/isa/x64/inst.isle line 2694. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Pcmpgtd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4960,7 +5044,7 @@ pub fn constructor_x64_pcmpgtd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem pub fn constructor_x64_pcmpgtq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2682. + // Rule at src/isa/x64/inst.isle line 2696. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pcmpgtq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -4970,7 +5054,7 @@ pub fn constructor_x64_pcmpgtq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem // Generated as internal constructor for term reg_to_xmm_mem. pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2738. + // Rule at src/isa/x64/inst.isle line 2752. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -4979,7 +5063,7 @@ pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option< // Generated as internal constructor for term xmm_to_reg_mem. pub fn constructor_xmm_to_reg_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2741. + // Rule at src/isa/x64/inst.isle line 2755. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_reg(ctx, expr0_0); let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4993,7 +5077,7 @@ pub fn constructor_writable_gpr_to_r_reg( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2745. + // Rule at src/isa/x64/inst.isle line 2759. let expr0_0 = C::writable_gpr_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -5005,7 +5089,7 @@ pub fn constructor_writable_gpr_to_gpr_mem( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2748. + // Rule at src/isa/x64/inst.isle line 2762. let expr0_0 = C::writable_gpr_to_gpr(ctx, pattern0_0); let expr1_0 = C::gpr_to_gpr_mem(ctx, expr0_0); return Some(expr1_0); @@ -5017,7 +5101,7 @@ pub fn constructor_writable_gpr_to_value_regs( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2751. + // Rule at src/isa/x64/inst.isle line 2765. let expr0_0 = constructor_writable_gpr_to_r_reg(ctx, pattern0_0)?; let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); @@ -5029,7 +5113,7 @@ pub fn constructor_writable_xmm_to_r_reg( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2754. + // Rule at src/isa/x64/inst.isle line 2768. let expr0_0 = C::writable_xmm_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -5041,7 +5125,7 @@ pub fn constructor_writable_xmm_to_xmm_mem( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2757. + // Rule at src/isa/x64/inst.isle line 2771. let expr0_0 = C::writable_xmm_to_xmm(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -5053,7 +5137,7 @@ pub fn constructor_writable_xmm_to_value_regs( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2760. + // Rule at src/isa/x64/inst.isle line 2774. let expr0_0 = constructor_writable_xmm_to_r_reg(ctx, pattern0_0)?; let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); @@ -5065,7 +5149,7 @@ pub fn constructor_synthetic_amode_to_gpr_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2767. + // Rule at src/isa/x64/inst.isle line 2781. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_gpr_mem(ctx, &expr0_0); return Some(expr1_0); @@ -5074,7 +5158,7 @@ pub fn constructor_synthetic_amode_to_gpr_mem( // Generated as internal constructor for term amode_to_gpr_mem. pub fn constructor_amode_to_gpr_mem(ctx: &mut C, arg0: &Amode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2765. + // Rule at src/isa/x64/inst.isle line 2779. let expr0_0 = C::amode_to_synthetic_amode(ctx, pattern0_0); let expr1_0 = constructor_synthetic_amode_to_gpr_mem(ctx, &expr0_0)?; return Some(expr1_0); @@ -5083,7 +5167,7 @@ pub fn constructor_amode_to_gpr_mem(ctx: &mut C, arg0: &Amode) -> Op // Generated as internal constructor for term amode_to_xmm_mem. pub fn constructor_amode_to_xmm_mem(ctx: &mut C, arg0: &Amode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2770. + // Rule at src/isa/x64/inst.isle line 2784. let expr0_0 = C::amode_to_synthetic_amode(ctx, pattern0_0); let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, &expr0_0)?; return Some(expr1_0); @@ -5095,7 +5179,7 @@ pub fn constructor_synthetic_amode_to_xmm_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2773. + // Rule at src/isa/x64/inst.isle line 2787. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); return Some(expr1_0); @@ -5292,6 +5376,139 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } } + &InstructionData::Store { + opcode: ref pattern2_0, + args: ref pattern2_1, + flags: pattern2_2, + offset: pattern2_3, + } => { + match pattern2_0 { + &Opcode::Store => { + let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, pattern2_1); + let pattern5_0 = C::value_type(ctx, pattern4_0); + if pattern5_0 == F32 { + // Rule at src/isa/x64/lower.isle line 2605. + let expr0_0 = SseOpcode::Movss; + let expr1_0 = + constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_xmm_movrm(ctx, &expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + if pattern5_0 == F64 { + // Rule at src/isa/x64/lower.isle line 2613. + let expr0_0 = SseOpcode::Movsd; + let expr1_0 = + constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_xmm_movrm(ctx, &expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + if pattern5_0 == F32X4 { + // Rule at src/isa/x64/lower.isle line 2621. + let expr0_0 = SseOpcode::Movups; + let expr1_0 = + constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_xmm_movrm(ctx, &expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + if pattern5_0 == F64X2 { + // Rule at src/isa/x64/lower.isle line 2629. + let expr0_0 = SseOpcode::Movupd; + let expr1_0 = + constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_xmm_movrm(ctx, &expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + if let Some(pattern6_0) = C::ty_int_bool_128(ctx, pattern5_0) { + // Rule at src/isa/x64/lower.isle line 2645. + let expr0_0 = C::put_in_regs(ctx, pattern4_0); + let expr1_0: usize = 0; + let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; + let expr3_0: usize = 1; + let expr4_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr3_0)?; + let expr5_0 = + constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr6_0: u32 = 8; + let expr7_0 = C::amode_offset(ctx, &expr5_0, expr6_0); + let expr8_0: Type = I64; + let expr9_0 = C::amode_to_synthetic_amode(ctx, &expr5_0); + let expr10_0 = constructor_x64_movrm(ctx, expr8_0, &expr9_0, expr2_0)?; + let expr11_0: Type = I64; + let expr12_0 = C::amode_to_synthetic_amode(ctx, &expr7_0); + let expr13_0 = constructor_x64_movrm(ctx, expr11_0, &expr12_0, expr4_0)?; + let expr14_0 = constructor_side_effect_concat(ctx, &expr10_0, &expr13_0)?; + let expr15_0 = constructor_side_effect(ctx, &expr14_0)?; + return Some(expr15_0); + } + if let Some(pattern6_0) = C::ty_vec128_int(ctx, pattern5_0) { + // Rule at src/isa/x64/lower.isle line 2637. + let expr0_0 = SseOpcode::Movdqu; + let expr1_0 = + constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_xmm(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_xmm_movrm(ctx, &expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + if let Some(pattern6_0) = C::is_gpr_type(ctx, pattern5_0) { + // Rule at src/isa/x64/lower.isle line 2586. + let expr0_0 = + constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr1_0 = C::amode_to_synthetic_amode(ctx, &expr0_0); + let expr2_0 = constructor_put_in_gpr(ctx, pattern4_0)?; + let expr3_0 = constructor_x64_movrm(ctx, pattern6_0, &expr1_0, expr2_0)?; + let expr4_0 = constructor_side_effect(ctx, &expr3_0)?; + return Some(expr4_0); + } + } + &Opcode::Istore8 => { + let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, pattern2_1); + // Rule at src/isa/x64/lower.isle line 2594. + let expr0_0: Type = I8; + let expr1_0 = constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_gpr(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_movrm(ctx, expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + &Opcode::Istore16 => { + let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, pattern2_1); + // Rule at src/isa/x64/lower.isle line 2597. + let expr0_0: Type = I16; + let expr1_0 = constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_gpr(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_movrm(ctx, expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + &Opcode::Istore32 => { + let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, pattern2_1); + // Rule at src/isa/x64/lower.isle line 2600. + let expr0_0: Type = I32; + let expr1_0 = constructor_to_amode(ctx, pattern2_2, pattern4_1, pattern2_3)?; + let expr2_0 = C::amode_to_synthetic_amode(ctx, &expr1_0); + let expr3_0 = constructor_put_in_gpr(ctx, pattern4_0)?; + let expr4_0 = constructor_x64_movrm(ctx, expr0_0, &expr2_0, expr3_0)?; + let expr5_0 = constructor_side_effect(ctx, &expr4_0)?; + return Some(expr5_0); + } + _ => {} + } + } &InstructionData::TernaryImm8 { opcode: ref pattern2_0, args: ref pattern2_1, @@ -7165,7 +7382,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload8x8 => { - // Rule at src/isa/x64/lower.isle line 2561. + // Rule at src/isa/x64/lower.isle line 2570. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7174,7 +7391,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2559. + // Rule at src/isa/x64/lower.isle line 2568. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7308,7 +7525,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload16x4 => { - // Rule at src/isa/x64/lower.isle line 2565. + // Rule at src/isa/x64/lower.isle line 2574. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7317,7 +7534,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2563. + // Rule at src/isa/x64/lower.isle line 2572. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7427,7 +7644,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload32x2 => { - // Rule at src/isa/x64/lower.isle line 2569. + // Rule at src/isa/x64/lower.isle line 2578. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7436,7 +7653,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2567. + // Rule at src/isa/x64/lower.isle line 2576. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -10103,6 +10320,28 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option Option { + if ty.is_vector() && ty.bits() == 128 && ty.lane_type().is_int() { + Some(ty) + } else { + None + } + } + #[inline] fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { (list, 0) diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 9922130762..d5d758b917 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -281,6 +281,11 @@ (decl ty_vec128 (Type) Type) (extern extractor ty_vec128 ty_vec128) +;; An extractor that only matches 128-bit vector types with integer +;; lanes (I8X16, I16X8, I32X4, I64X2). +(decl ty_vec128_int (Type) Type) +(extern extractor ty_vec128_int ty_vec128_int) + ;; An extractor that matches everything except i64x2 (decl not_i64x2 () Type) (extern extractor not_i64x2 not_i64x2) @@ -399,7 +404,10 @@ ;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;; -(type SideEffectNoResult (enum (Inst (inst MInst)))) +(type SideEffectNoResult (enum + (Inst (inst MInst)) + (Inst2 (inst1 MInst) + (inst2 MInst)))) ;; Create an empty `InstOutput`, but do emit the given side-effectful ;; instruction. @@ -407,6 +415,14 @@ (rule (side_effect (SideEffectNoResult.Inst inst)) (let ((_ Unit (emit inst))) (output_none))) +(rule (side_effect (SideEffectNoResult.Inst2 inst1 inst2)) + (let ((_1 Unit (emit inst1)) + (_2 Unit (emit inst2))) + (output_none))) + +(decl side_effect_concat (SideEffectNoResult SideEffectNoResult) SideEffectNoResult) +(rule (side_effect_concat (SideEffectNoResult.Inst inst1) (SideEffectNoResult.Inst inst2)) + (SideEffectNoResult.Inst2 inst1 inst2)) ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;