x64 backend: migrate stores, and remainder of loads (I128 case), to ISLE. (#4069)

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 443b34b797fc8ace
-src/prelude.isle afd037c4d91c875c
+src/prelude.isle d8a93eb727abd7f4
 src/isa/aarch64/inst.isle 77984cc33a05be7
 src/isa/aarch64/lower.isle 71c7e603b0e4bdef

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs

@@ -59,6 +59,7 @@ pub trait Context {
     fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
     fn ty_scalar_float(&mut self, arg0: Type) -> Option<Type>;
     fn ty_vec128(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_vec128_int(&mut self, arg0: Type) -> Option<Type>;
     fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
     fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
     fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>;
@@ -131,13 +132,14 @@ pub trait Context {
     fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 402.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
+    Inst2 { inst1: MInst, inst2: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 418.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 434.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -145,7 +147,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 429.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 445.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -1064,15 +1066,54 @@ pub fn constructor_side_effect<C: Context>(
     ctx: &mut C,
     arg0: &SideEffectNoResult,
 ) -> Option<InstOutput> {
+    let pattern0_0 = arg0;
+    match pattern0_0 {
+        &SideEffectNoResult::Inst {
+            inst: ref pattern1_0,
+        } => {
+            // Rule at src/prelude.isle line 415.
+            let expr0_0 = C::emit(ctx, pattern1_0);
+            let expr1_0 = C::output_none(ctx);
+            return Some(expr1_0);
+        }
+        &SideEffectNoResult::Inst2 {
+            inst1: ref pattern1_0,
+            inst2: ref pattern1_1,
+        } => {
+            // Rule at src/prelude.isle line 418.
+            let expr0_0 = C::emit(ctx, pattern1_0);
+            let expr1_0 = C::emit(ctx, pattern1_1);
+            let expr2_0 = C::output_none(ctx);
+            return Some(expr2_0);
+        }
+        _ => {}
+    }
+    return None;
+}
+
+// Generated as internal constructor for term side_effect_concat.
+pub fn constructor_side_effect_concat<C: Context>(
+    ctx: &mut C,
+    arg0: &SideEffectNoResult,
+    arg1: &SideEffectNoResult,
+) -> Option<SideEffectNoResult> {
     let pattern0_0 = arg0;
     if let &SideEffectNoResult::Inst {
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 407.
+        let pattern2_0 = arg1;
-        let expr0_0 = C::emit(ctx, pattern1_0);
+        if let &SideEffectNoResult::Inst {
-        let expr1_0 = C::output_none(ctx);
+            inst: ref pattern3_0,
-        return Some(expr1_0);
+        } = pattern2_0
+        {
+            // Rule at src/prelude.isle line 424.
+            let expr0_0 = SideEffectNoResult::Inst2 {
+                inst1: pattern1_0.clone(),
+                inst2: pattern3_0.clone(),
+            };
+            return Some(expr0_0);
+        }
     }
     return None;
 }
@@ -1088,7 +1129,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 445.
+        // Rule at src/prelude.isle line 461.
         return Some(pattern1_1);
     }
     return None;
@@ -1105,7 +1146,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 450.
+            // Rule at src/prelude.isle line 466.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1115,7 +1156,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 452.
+            // Rule at src/prelude.isle line 468.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1144,7 +1185,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 459.
+            // Rule at src/prelude.isle line 475.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -1174,7 +1215,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 484.
+                    // Rule at src/prelude.isle line 500.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1185,7 +1226,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 490.
+                    // Rule at src/prelude.isle line 506.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1198,7 +1239,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 502.
+                    // Rule at src/prelude.isle line 518.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1219,7 +1260,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 478.
+                // Rule at src/prelude.isle line 494.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1239,7 +1280,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 519.
+    // Rule at src/prelude.isle line 535.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 443b34b797fc8ace
-src/prelude.isle afd037c4d91c875c
+src/prelude.isle d8a93eb727abd7f4
 src/isa/s390x/inst.isle 8218bd9e8556446b
 src/isa/s390x/lower.isle 6a8de81f8dc4e568

cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs

@@ -59,6 +59,7 @@ pub trait Context {
     fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
     fn ty_scalar_float(&mut self, arg0: Type) -> Option<Type>;
     fn ty_vec128(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_vec128_int(&mut self, arg0: Type) -> Option<Type>;
     fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
     fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
     fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>;
@@ -154,13 +155,14 @@ pub trait Context {
     fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 402.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 407.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
+    Inst2 { inst1: MInst, inst2: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 418.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 434.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -168,7 +170,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 429.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 445.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -953,15 +955,54 @@ pub fn constructor_side_effect<C: Context>(
     ctx: &mut C,
     arg0: &SideEffectNoResult,
 ) -> Option<InstOutput> {
+    let pattern0_0 = arg0;
+    match pattern0_0 {
+        &SideEffectNoResult::Inst {
+            inst: ref pattern1_0,
+        } => {
+            // Rule at src/prelude.isle line 415.
+            let expr0_0 = C::emit(ctx, pattern1_0);
+            let expr1_0 = C::output_none(ctx);
+            return Some(expr1_0);
+        }
+        &SideEffectNoResult::Inst2 {
+            inst1: ref pattern1_0,
+            inst2: ref pattern1_1,
+        } => {
+            // Rule at src/prelude.isle line 418.
+            let expr0_0 = C::emit(ctx, pattern1_0);
+            let expr1_0 = C::emit(ctx, pattern1_1);
+            let expr2_0 = C::output_none(ctx);
+            return Some(expr2_0);
+        }
+        _ => {}
+    }
+    return None;
+}
+
+// Generated as internal constructor for term side_effect_concat.
+pub fn constructor_side_effect_concat<C: Context>(
+    ctx: &mut C,
+    arg0: &SideEffectNoResult,
+    arg1: &SideEffectNoResult,
+) -> Option<SideEffectNoResult> {
     let pattern0_0 = arg0;
     if let &SideEffectNoResult::Inst {
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 407.
+        let pattern2_0 = arg1;
-        let expr0_0 = C::emit(ctx, pattern1_0);
+        if let &SideEffectNoResult::Inst {
-        let expr1_0 = C::output_none(ctx);
+            inst: ref pattern3_0,
-        return Some(expr1_0);
+        } = pattern2_0
+        {
+            // Rule at src/prelude.isle line 424.
+            let expr0_0 = SideEffectNoResult::Inst2 {
+                inst1: pattern1_0.clone(),
+                inst2: pattern3_0.clone(),
+            };
+            return Some(expr0_0);
+        }
     }
     return None;
 }
@@ -977,7 +1018,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 445.
+        // Rule at src/prelude.isle line 461.
         return Some(pattern1_1);
     }
     return None;
@@ -994,7 +1035,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 450.
+            // Rule at src/prelude.isle line 466.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1004,7 +1045,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 452.
+            // Rule at src/prelude.isle line 468.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1033,7 +1074,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 459.
+            // Rule at src/prelude.isle line 475.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -1063,7 +1104,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 484.
+                    // Rule at src/prelude.isle line 500.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1074,7 +1115,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 490.
+                    // Rule at src/prelude.isle line 506.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1087,7 +1128,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 502.
+                    // Rule at src/prelude.isle line 518.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1108,7 +1149,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 478.
+                // Rule at src/prelude.isle line 494.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1128,7 +1169,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 519.
+    // Rule at src/prelude.isle line 535.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

cranelift/codegen/src/isa/x64/inst.isle

@@ -838,6 +838,11 @@
 (rule (to_amode flags base offset)
       (amode_imm_reg_flags offset (put_in_gpr base) flags))
 
+;; Offsetting an Amode. Used when we need to do consecutive
+;; loads/stores to adjacent addresses.
+(decl amode_offset (Amode u32) Amode)
+(extern constructor amode_offset amode_offset)
+
 ;; Shift kinds.
 
 (type ShiftKind extern
@@ -1404,6 +1409,15 @@
 (rule (x64_pmovzxdq from)
       (xmm_unary_rm_r (SseOpcode.Pmovzxdq) from))
 
+(decl x64_movrm (Type SyntheticAmode Gpr) SideEffectNoResult)
+(rule (x64_movrm ty addr data)
+      (let ((size OperandSize (raw_operand_size_of_type ty)))
+        (SideEffectNoResult.Inst (MInst.MovRM size data addr))))
+
+(decl x64_xmm_movrm (SseOpcode SyntheticAmode Xmm) SideEffectNoResult)
+(rule (x64_xmm_movrm op addr data)
+      (SideEffectNoResult.Inst (MInst.XmmMovRM op data addr)))
+
 ;; Load a constant into an XMM register.
 (decl x64_xmm_load_const (Type VCodeConstant) Xmm)
 (rule (x64_xmm_load_const ty const)

cranelift/codegen/src/isa/x64/lower.isle

@@ -2554,6 +2554,15 @@
 (rule (lower (has_type (ty_vec128 ty) (load flags address offset)))
       (x64_movdqu (to_amode flags address offset)))
 
+;; We can load an I128/B128 by doing two 64-bit loads.
+(rule (lower (has_type (ty_int_bool_128 _)
+                       (load flags address offset)))
+      (let ((addr_lo Amode (to_amode flags address offset))
+            (addr_hi Amode (amode_offset addr_lo 8))
+            (value_lo Reg (x64_mov addr_lo))
+            (value_hi Reg (x64_mov addr_hi)))
+        (value_regs value_lo value_hi)))
+
 ;; We also include widening vector loads; these sign- or zero-extend each lane
 ;; to the next wider width (e.g., 16x4 -> 32x4).
 (rule (lower (has_type $I16X8 (sload8x8 flags address offset)))
@@ -2570,3 +2579,79 @@
       (x64_pmovzxdq (to_amode flags address offset)))
 
 ;; TODO: Multi-register loads (I128)
+
+;; Rules for `store*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 8-, 16-, 32- and 64-bit GPR stores.
+(rule (lower (store flags
+                    value @ (value_type (is_gpr_type ty))
+                    address
+                    offset))
+      (side_effect
+       (x64_movrm ty (to_amode flags address offset) value)))
+
+;; Explicit 8/16/32-bit opcodes.
+(rule (lower (istore8 flags value address offset))
+      (side_effect
+       (x64_movrm $I8 (to_amode flags address offset) value)))
+(rule (lower (istore16 flags value address offset))
+      (side_effect
+       (x64_movrm $I16 (to_amode flags address offset) value)))
+(rule (lower (istore32 flags value address offset))
+      (side_effect
+       (x64_movrm $I32 (to_amode flags address offset) value)))
+
+;; F32 stores of values in XMM registers.
+(rule (lower (store flags
+                    value @ (value_type $F32)
+                    address
+                    offset))
+      (side_effect
+       (x64_xmm_movrm (SseOpcode.Movss) (to_amode flags address offset) value)))
+
+;; F64 stores of values in XMM registers.
+(rule (lower (store flags
+                    value @ (value_type $F64)
+                    address
+                    offset))
+      (side_effect
+       (x64_xmm_movrm (SseOpcode.Movsd) (to_amode flags address offset) value)))
+
+;; Stores of F32X4 vectors.
+(rule (lower (store flags
+                    value @ (value_type $F32X4)
+                    address
+                    offset))
+      (side_effect
+       (x64_xmm_movrm (SseOpcode.Movups) (to_amode flags address offset) value)))
+
+;; Stores of F64X2 vectors.
+(rule (lower (store flags
+                    value @ (value_type $F64X2)
+                    address
+                    offset))
+      (side_effect
+       (x64_xmm_movrm (SseOpcode.Movupd) (to_amode flags address offset) value)))
+
+;; Stores of all other 128-bit vector types with integer lanes.
+(rule (lower (store flags
+                    value @ (value_type (ty_vec128_int _))
+                    address
+                    offset))
+      (side_effect
+       (x64_xmm_movrm (SseOpcode.Movdqu) (to_amode flags address offset) value)))
+
+;; Stores of I128/B128 values: store the two 64-bit halves separately.
+(rule (lower (store flags
+                    value @ (value_type (ty_int_bool_128 _))
+                    address
+                    offset))
+      (let ((value_reg ValueRegs value)
+            (value_lo Gpr (value_regs_get_gpr value_reg 0))
+            (value_hi Gpr (value_regs_get_gpr value_reg 1))
+            (addr_lo Amode (to_amode flags address offset))
+            (addr_hi Amode (amode_offset addr_lo 8)))
+      (side_effect
+       (side_effect_concat
+        (x64_movrm $I64 addr_lo value_lo)
+        (x64_movrm $I64 addr_hi value_hi)))))

cranelift/codegen/src/isa/x64/lower.rs

@@ -2146,75 +2146,11 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         | Opcode::Uload16x4
         | Opcode::Sload32x2
         | Opcode::Uload32x2 => {
-            let offset = ctx.data(insn).load_store_offset().unwrap();
-
-            let elem_ty = match op {
-                Opcode::Sload8 | Opcode::Uload8 => types::I8,
-                Opcode::Sload16 | Opcode::Uload16 => types::I16,
-                Opcode::Sload32 | Opcode::Uload32 => types::I32,
-                Opcode::Sload8x8 | Opcode::Uload8x8 => types::I8X8,
-                Opcode::Sload16x4 | Opcode::Uload16x4 => types::I16X4,
-                Opcode::Sload32x2 | Opcode::Uload32x2 => types::I32X2,
-                Opcode::Load => ctx.output_ty(insn, 0),
-                _ => unimplemented!(),
-            };
-
-            let amode = match op {
-                Opcode::Load
-                | Opcode::Uload8
-                | Opcode::Sload8
-                | Opcode::Uload16
-                | Opcode::Sload16
-                | Opcode::Uload32
-                | Opcode::Sload32
-                | Opcode::Sload8x8
-                | Opcode::Uload8x8
-                | Opcode::Sload16x4
-                | Opcode::Uload16x4
-                | Opcode::Sload32x2
-                | Opcode::Uload32x2 => {
-                    assert_eq!(inputs.len(), 1, "only one input for load operands");
-                    lower_to_amode(ctx, inputs[0], offset)
-                }
-                _ => unreachable!(),
-            };
-
-            if elem_ty == types::I128 {
-                let dsts = get_output_reg(ctx, outputs[0]);
-                ctx.emit(Inst::mov64_m_r(amode.clone(), dsts.regs()[0]));
-                ctx.emit(Inst::mov64_m_r(amode.offset(8), dsts.regs()[1]));
-            } else {
             implemented_in_isle(ctx);
         }
-        }
 
         Opcode::Store | Opcode::Istore8 | Opcode::Istore16 | Opcode::Istore32 => {
-            let offset = ctx.data(insn).load_store_offset().unwrap();
+            implemented_in_isle(ctx);
-
-            let elem_ty = match op {
-                Opcode::Istore8 => types::I8,
-                Opcode::Istore16 => types::I16,
-                Opcode::Istore32 => types::I32,
-                Opcode::Store => ctx.input_ty(insn, 0),
-                _ => unreachable!(),
-            };
-
-            let addr = match op {
-                Opcode::Store | Opcode::Istore8 | Opcode::Istore16 | Opcode::Istore32 => {
-                    assert_eq!(inputs.len(), 2, "only one input for store memory operands");
-                    lower_to_amode(ctx, inputs[1], offset)
-                }
-                _ => unreachable!(),
-            };
-
-            if elem_ty == types::I128 {
-                let srcs = put_input_in_regs(ctx, inputs[0]);
-                ctx.emit(Inst::store(types::I64, srcs.regs()[0], addr.clone()));
-                ctx.emit(Inst::store(types::I64, srcs.regs()[1], addr.offset(8)));
-            } else {
-                let src = put_input_in_reg(ctx, inputs[0]);
-                ctx.emit(Inst::store(elem_ty, src, addr));
-            }
         }
 
         Opcode::AtomicRmw => {

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -544,6 +544,11 @@ where
             None
         }
     }
+
+    #[inline]
+    fn amode_offset(&mut self, addr: &Amode, offset: u32) -> Amode {
+        addr.offset(offset)
+    }
 }
 
 // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we

cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 443b34b797fc8ace
-src/prelude.isle afd037c4d91c875c
+src/prelude.isle d8a93eb727abd7f4
-src/isa/x64/inst.isle cad03431447aca1b
+src/isa/x64/inst.isle 2fa48b8183f9d5cb
-src/isa/x64/lower.isle a7181571835ddd1e
+src/isa/x64/lower.isle b7fe1c95c21edbe4

cranelift/codegen/src/machinst/isle.rs

@@ -278,6 +278,15 @@ macro_rules! isle_prelude_methods {
             }
         }
 
+        #[inline]
+        fn ty_vec128_int(&mut self, ty: Type) -> Option<Type> {
+            if ty.is_vector() && ty.bits() == 128 && ty.lane_type().is_int() {
+                Some(ty)
+            } else {
+                None
+            }
+        }
+
         #[inline]
         fn value_list_slice(&mut self, list: ValueList) -> ValueSlice {
             (list, 0)

cranelift/codegen/src/prelude.isle

@@ -281,6 +281,11 @@
 (decl ty_vec128 (Type) Type)
 (extern extractor ty_vec128 ty_vec128)
 
+;; An extractor that only matches 128-bit vector types with integer
+;; lanes (I8X16, I16X8, I32X4, I64X2).
+(decl ty_vec128_int (Type) Type)
+(extern extractor ty_vec128_int ty_vec128_int)
+
 ;; An extractor that matches everything except i64x2
 (decl not_i64x2 () Type)
 (extern extractor not_i64x2 not_i64x2)
@@ -399,7 +404,10 @@
 
 ;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;;
 
-(type SideEffectNoResult (enum (Inst (inst MInst))))
+(type SideEffectNoResult (enum
+                          (Inst (inst MInst))
+                          (Inst2 (inst1 MInst)
+                                 (inst2 MInst))))
 
 ;; Create an empty `InstOutput`, but do emit the given side-effectful
 ;; instruction.
@@ -407,6 +415,14 @@
 (rule (side_effect (SideEffectNoResult.Inst inst))
       (let ((_ Unit (emit inst)))
         (output_none)))
+(rule (side_effect (SideEffectNoResult.Inst2 inst1 inst2))
+      (let ((_1 Unit (emit inst1))
+            (_2 Unit (emit inst2)))
+        (output_none)))
+
+(decl side_effect_concat (SideEffectNoResult SideEffectNoResult) SideEffectNoResult)
+(rule (side_effect_concat (SideEffectNoResult.Inst inst1) (SideEffectNoResult.Inst inst2))
+      (SideEffectNoResult.Inst2 inst1 inst2))
 
 ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;