ISLE: port extend/reduce opcodes on x64. (#3849)

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 6b0160bfcac86902
+src/prelude.isle b2bc986bcbbbb77
 src/isa/aarch64/inst.isle 3678d0a37bdb4cff
 src/isa/aarch64/lower.isle 90accbfcadaea46d

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs

@@ -54,6 +54,8 @@ pub trait Context {
     fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
     fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
     fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
     fn vec128(&mut self, arg0: Type) -> Option<Type>;
     fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
     fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
@@ -116,13 +118,13 @@ pub trait Context {
     fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 407.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -130,7 +132,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 418.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -1063,7 +1065,7 @@ pub fn constructor_side_effect<C: Context>(
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 390.
+        // Rule at src/prelude.isle line 398.
         let expr0_0 = C::emit(ctx, pattern1_0);
         let expr1_0 = C::output_none(ctx);
         return Some(expr1_0);
@@ -1081,7 +1083,7 @@ pub fn constructor_safepoint<C: Context>(
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 396.
+        // Rule at src/prelude.isle line 404.
         let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
         let expr1_0 = C::output_none(ctx);
         return Some(expr1_0);
@@ -1100,7 +1102,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 434.
+        // Rule at src/prelude.isle line 442.
         return Some(pattern1_1);
     }
     return None;
@@ -1117,7 +1119,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 439.
+            // Rule at src/prelude.isle line 447.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1127,7 +1129,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 441.
+            // Rule at src/prelude.isle line 449.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1156,7 +1158,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 448.
+            // Rule at src/prelude.isle line 456.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -1186,7 +1188,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 473.
+                    // Rule at src/prelude.isle line 481.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1197,7 +1199,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 479.
+                    // Rule at src/prelude.isle line 487.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1210,7 +1212,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 491.
+                    // Rule at src/prelude.isle line 499.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1231,7 +1233,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 467.
+                // Rule at src/prelude.isle line 475.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1251,7 +1253,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 508.
+    // Rule at src/prelude.isle line 516.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 6b0160bfcac86902
+src/prelude.isle b2bc986bcbbbb77
 src/isa/s390x/inst.isle d91a16074ab186a8
 src/isa/s390x/lower.isle 1cc5a12adc8c75f9

cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs

@@ -54,6 +54,8 @@ pub trait Context {
     fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
     fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
     fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
     fn vec128(&mut self, arg0: Type) -> Option<Type>;
     fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
     fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
@@ -150,13 +152,13 @@ pub trait Context {
     fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 407.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -164,7 +166,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 418.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -954,7 +956,7 @@ pub fn constructor_side_effect<C: Context>(
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 390.
+        // Rule at src/prelude.isle line 398.
         let expr0_0 = C::emit(ctx, pattern1_0);
         let expr1_0 = C::output_none(ctx);
         return Some(expr1_0);
@@ -972,7 +974,7 @@ pub fn constructor_safepoint<C: Context>(
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 396.
+        // Rule at src/prelude.isle line 404.
         let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
         let expr1_0 = C::output_none(ctx);
         return Some(expr1_0);
@@ -991,7 +993,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 434.
+        // Rule at src/prelude.isle line 442.
         return Some(pattern1_1);
     }
     return None;
@@ -1008,7 +1010,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 439.
+            // Rule at src/prelude.isle line 447.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1018,7 +1020,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 441.
+            // Rule at src/prelude.isle line 449.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -1047,7 +1049,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 448.
+            // Rule at src/prelude.isle line 456.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -1077,7 +1079,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 473.
+                    // Rule at src/prelude.isle line 481.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1088,7 +1090,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 479.
+                    // Rule at src/prelude.isle line 487.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1101,7 +1103,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 491.
+                    // Rule at src/prelude.isle line 499.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1122,7 +1124,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 467.
+                // Rule at src/prelude.isle line 475.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1142,7 +1144,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 508.
+    // Rule at src/prelude.isle line 516.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

cranelift/codegen/src/isa/x64/lower.isle

@@ -1859,3 +1859,130 @@
        (cmp_imm (OperandSize.Size64) 0xffffffff src)  ;; simm32 0xffff_ffff is sign-extended to -1.
        (setcc (CC.Z))))
 
+
+;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; T -> T is a no-op.
+(rule (lower (has_type ty (uextend src @ (value_type =ty))))
+      src)
+
+;; I64 -> I128.
+(rule (lower (has_type $I128 (uextend src @ (value_type $I64))))
+      (value_regs src (imm $I64 0)))
+
+;; I{8,16,32} -> I128.
+(rule (lower (has_type $I128 (uextend src @ (value_type (fits_in_32 src_ty)))))
+      (value_regs (extend_to_gpr src $I64 (ExtendKind.Zero)) (imm $I64 0)))
+
+;; I{8,16,32} -> I64.
+(rule (lower (has_type $I64 (uextend src @ (value_type (fits_in_32 src_ty)))))
+      (extend_to_gpr src $I64 (ExtendKind.Zero)))
+
+;; I8 -> I{16,32}, I16 -> I32.
+(rule (lower (has_type (fits_in_32 dst_ty) (uextend src @ (value_type (fits_in_32 src_ty)))))
+      (extend_to_gpr src $I32 (ExtendKind.Zero)))
+
+;; I32 -> I64 with op that produces a zero-extended value in a register.
+;;
+;; As a particular x64 extra-pattern matching opportunity, all the ALU
+;; opcodes on 32-bits will zero-extend the upper 32-bits, so we can
+;; even not generate a zero-extended move in this case.
+;;
+;; (Note that we unfortunately can't factor out the
+;; insts-that-zero-upper-32 pattern into a separate extractor until we
+;; can write internal extractors with multiple rules; and we'd rather
+;; keep these here than write an external extractor containing bits of
+;; the instruction pattern.s)
+(rule (lower (has_type $I64
+                       (uextend src @ (iadd _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (iadd_ifcout _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (isub _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (imul _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (band _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (bor _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (bxor _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (ishl _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (ushr _ _))))
+      src)
+(rule (lower (has_type $I64
+                       (uextend src @ (uload32 _ _ _))))
+      src)
+
+;; Rules for `sextend` / `bextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl generic_sextend (Value Type Type) InstOutput)
+
+;; T -> T is a no-op.
+(rule (generic_sextend src ty =ty)
+      src)
+
+;; Produce upper 64 bits sign-extended from lower 64: shift right by
+;; 63 bits to spread the sign bit across the result.
+(decl spread_sign_bit (Gpr) Gpr)
+(rule (spread_sign_bit src)
+      (sar $I64 src (Imm8Reg.Imm8 63)))
+
+;; I64 -> I128.
+(rule (generic_sextend src (ty_int_bool_64 _) (ty_int_bool_128 _))
+      (value_regs src (spread_sign_bit src)))
+
+;; I{8,16,32} -> I128.
+(rule (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_128 _))
+      (let ((lo Gpr (extend_to_gpr src $I64 (ExtendKind.Sign)))
+            (hi Gpr (spread_sign_bit lo)))
+      (value_regs lo hi)))
+
+;; I{8,16,32} -> I64.
+(rule (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_64 _))
+      (extend_to_gpr src $I64 (ExtendKind.Sign)))
+
+;; I8 -> I{16,32}, I16 -> I32.
+(rule (generic_sextend src (fits_in_32 src_ty) (fits_in_32 dst_ty))
+      (extend_to_gpr src $I32 (ExtendKind.Sign)))
+
+(rule (lower
+       (has_type dst_ty
+                 (sextend src @ (value_type src_ty))))
+      (generic_sextend src src_ty dst_ty))
+
+;; Bools are stored as 0/-1 so extends must sign-extend as well.
+(rule (lower
+       (has_type dst_ty
+                 (bextend src @ (value_type src_ty))))
+      (generic_sextend src src_ty dst_ty))
+
+;; Rules for `ireduce` / `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; T -> T is always a no-op, even I128 -> I128.
+(rule (lower (has_type ty (ireduce src @ (value_type =ty))))
+      src)
+
+;; T -> I{64,32,16,8}: We can simply pass through the value: values
+;; are always stored with high bits undefined, so we can just leave
+;; them be.
+(rule (lower (has_type (fits_in_64 ty) (ireduce src)))
+      (value_regs_get_gpr src 0))
+
+;; Likewise for breduce.
+
+(rule (lower (has_type ty (breduce src @ (value_type =ty))))
+      src)
+
+(rule (lower (has_type (fits_in_64 ty) (breduce src)))
+      (value_regs_get_gpr src 0))

cranelift/codegen/src/isa/x64/lower.rs

@@ -71,25 +71,6 @@ fn matches_input<C: LowerCtx<I = Inst>>(
     })
 }
 
-/// Returns whether the given specified `input` is a result produced by an instruction with any of
-/// the opcodes specified in `ops`.
-fn matches_input_any<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-    ops: &[Opcode],
-) -> Option<IRInst> {
-    let inputs = ctx.get_input_as_source_or_const(input.insn, input.input);
-    inputs.inst.and_then(|(src_inst, _)| {
-        let data = ctx.data(src_inst);
-        for &op in ops {
-            if data.opcode() == op {
-                return Some(src_inst);
-            }
-        }
-        None
-    })
-}
-
 /// Emits instruction(s) to generate the given 64-bit constant value into a newly-allocated
 /// temporary register, returning that register.
 fn generate_constant<C: LowerCtx<I = Inst>>(ctx: &mut C, ty: Type, c: u64) -> ValueRegs<Reg> {
@@ -917,122 +898,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         | Opcode::Popcnt
         | Opcode::Bitrev
         | Opcode::IsNull
-        | Opcode::IsInvalid => implemented_in_isle(ctx),
+        | Opcode::IsInvalid
-
+        | Opcode::Uextend
-        Opcode::Uextend | Opcode::Sextend | Opcode::Breduce | Opcode::Bextend | Opcode::Ireduce => {
+        | Opcode::Sextend
-            let src_ty = ctx.input_ty(insn, 0);
+        | Opcode::Breduce
-            let dst_ty = ctx.output_ty(insn, 0);
+        | Opcode::Bextend
-
+        | Opcode::Ireduce => implemented_in_isle(ctx),
-            if src_ty == types::I128 {
-                assert!(dst_ty.bits() <= 64);
-                assert!(op == Opcode::Ireduce);
-                let src = put_input_in_regs(ctx, inputs[0]);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                ctx.emit(Inst::gen_move(dst, src.regs()[0], types::I64));
-            } else if dst_ty == types::I128 {
-                assert!(src_ty.bits() <= 64);
-                let src = put_input_in_reg(ctx, inputs[0]);
-                let dst = get_output_reg(ctx, outputs[0]);
-                assert!(op == Opcode::Uextend || op == Opcode::Sextend);
-                // Extend to 64 bits first.
-
-                let ext_mode = ExtMode::new(src_ty.bits(), /* dst bits = */ 64);
-                if let Some(ext_mode) = ext_mode {
-                    if op == Opcode::Sextend {
-                        ctx.emit(Inst::movsx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0]));
-                    } else {
-                        ctx.emit(Inst::movzx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0]));
-                    }
-                } else {
-                    ctx.emit(Inst::mov64_rm_r(RegMem::reg(src), dst.regs()[0]));
-                }
-
-                // Now generate the top 64 bits.
-                if op == Opcode::Sextend {
-                    // Sign-extend: move dst[0] into dst[1] and arithmetic-shift right by 63 bits
-                    // to spread the sign bit across all bits.
-                    ctx.emit(Inst::gen_move(
-                        dst.regs()[1],
-                        dst.regs()[0].to_reg(),
-                        types::I64,
-                    ));
-                    ctx.emit(Inst::shift_r(
-                        OperandSize::Size64,
-                        ShiftKind::ShiftRightArithmetic,
-                        Some(63),
-                        dst.regs()[1],
-                    ));
-                } else {
-                    // Zero-extend: just zero the top word.
-                    ctx.emit(Inst::alu_rmi_r(
-                        OperandSize::Size64,
-                        AluRmiROpcode::Xor,
-                        RegMemImm::reg(dst.regs()[1].to_reg()),
-                        dst.regs()[1],
-                    ));
-                }
-            } else {
-                // Sextend requires a sign-extended move, but all the other opcodes are simply a move
-                // from a zero-extended source. Here is why this works, in each case:
-                //
-                // - Breduce, Bextend: changing width of a boolean. We
-                //   represent a bool as a 0 or -1, so Breduce can mask, while
-                //   Bextend must sign-extend.
-                //
-                // - Ireduce: changing width of an integer. Smaller ints are stored with undefined
-                // high-order bits, so we can simply do a copy.
-                let is_sextend = match op {
-                    Opcode::Sextend | Opcode::Bextend => true,
-                    _ => false,
-                };
-                if src_ty == types::I32 && dst_ty == types::I64 && !is_sextend {
-                    // As a particular x64 extra-pattern matching opportunity, all the ALU opcodes on
-                    // 32-bits will zero-extend the upper 32-bits, so we can even not generate a
-                    // zero-extended move in this case.
-                    // TODO add loads and shifts here.
-                    if let Some(_) = matches_input_any(
-                        ctx,
-                        inputs[0],
-                        &[
-                            Opcode::Iadd,
-                            Opcode::IaddIfcout,
-                            Opcode::Isub,
-                            Opcode::Imul,
-                            Opcode::Band,
-                            Opcode::Bor,
-                            Opcode::Bxor,
-                        ],
-                    ) {
-                        let src = put_input_in_reg(ctx, inputs[0]);
-                        let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                        ctx.emit(Inst::gen_move(dst, src, types::I64));
-                        return Ok(());
-                    }
-                }
-
-                let src = input_to_reg_mem(ctx, inputs[0]);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                let ext_mode = ExtMode::new(src_ty.bits(), dst_ty.bits());
-                assert_eq!(
-                    src_ty.bits() < dst_ty.bits(),
-                    ext_mode.is_some(),
-                    "unexpected extension: {} -> {}",
-                    src_ty,
-                    dst_ty
-                );
-
-                if let Some(ext_mode) = ext_mode {
-                    if is_sextend {
-                        ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst));
-                    } else {
-                        ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst));
-                    }
-                } else {
-                    ctx.emit(Inst::mov64_rm_r(src, dst));
-                }
-            }
-        }
 
         Opcode::Bint => {
             // Booleans are stored as all-zeroes (0) or all-ones (-1). We AND

cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 6b0160bfcac86902
+src/prelude.isle b2bc986bcbbbb77
 src/isa/x64/inst.isle 67eb719e568c2a81
-src/isa/x64/lower.isle 142626fe062fd7d7
+src/isa/x64/lower.isle 2d06b233fb3a1e1c

cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs

@@ -54,6 +54,8 @@ pub trait Context {
     fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
     fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
     fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
     fn vec128(&mut self, arg0: Type) -> Option<Type>;
     fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
     fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
@@ -147,13 +149,13 @@ pub trait Context {
     fn popcount_low_mask(&mut self) -> VCodeConstant;
 }
 
-/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385.
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
 #[derive(Clone, Debug)]
 pub enum SideEffectNoResult {
     Inst { inst: MInst },
 }
 
-/// Internal type ProducesFlags: defined at src/prelude.isle line 407.
+/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
 #[derive(Clone, Debug)]
 pub enum ProducesFlags {
     ProducesFlagsSideEffect { inst: MInst },
@@ -161,7 +163,7 @@ pub enum ProducesFlags {
     ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
 }
 
-/// Internal type ConsumesFlags: defined at src/prelude.isle line 418.
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
 #[derive(Clone, Debug)]
 pub enum ConsumesFlags {
     ConsumesFlagsReturnsResultWithProducer {
@@ -552,7 +554,7 @@ pub fn constructor_side_effect<C: Context>(
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 390.
+        // Rule at src/prelude.isle line 398.
         let expr0_0 = C::emit(ctx, pattern1_0);
         let expr1_0 = C::output_none(ctx);
         return Some(expr1_0);
@@ -570,7 +572,7 @@ pub fn constructor_safepoint<C: Context>(
         inst: ref pattern1_0,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 396.
+        // Rule at src/prelude.isle line 404.
         let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
         let expr1_0 = C::output_none(ctx);
         return Some(expr1_0);
@@ -589,7 +591,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
         result: pattern1_1,
     } = pattern0_0
     {
-        // Rule at src/prelude.isle line 434.
+        // Rule at src/prelude.isle line 442.
         return Some(pattern1_1);
     }
     return None;
@@ -606,7 +608,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 439.
+            // Rule at src/prelude.isle line 447.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -616,7 +618,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
             inst: ref pattern1_0,
             result: pattern1_1,
         } => {
-            // Rule at src/prelude.isle line 441.
+            // Rule at src/prelude.isle line 449.
             let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
                 inst: pattern1_0.clone(),
             };
@@ -645,7 +647,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
             result: pattern3_1,
         } = pattern2_0
         {
-            // Rule at src/prelude.isle line 448.
+            // Rule at src/prelude.isle line 456.
             let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
             let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
                 inst1: pattern1_0.clone(),
@@ -675,7 +677,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst: ref pattern3_0,
                     result: pattern3_1,
                 } => {
-                    // Rule at src/prelude.isle line 473.
+                    // Rule at src/prelude.isle line 481.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -686,7 +688,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst2: ref pattern3_1,
                     result: pattern3_2,
                 } => {
-                    // Rule at src/prelude.isle line 479.
+                    // Rule at src/prelude.isle line 487.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -699,7 +701,7 @@ pub fn constructor_with_flags<C: Context>(
                     inst4: ref pattern3_3,
                     result: pattern3_4,
                 } => {
-                    // Rule at src/prelude.isle line 491.
+                    // Rule at src/prelude.isle line 499.
                     let expr0_0 = C::emit(ctx, pattern1_0);
                     let expr1_0 = C::emit(ctx, pattern3_0);
                     let expr2_0 = C::emit(ctx, pattern3_1);
@@ -720,7 +722,7 @@ pub fn constructor_with_flags<C: Context>(
                 result: pattern3_1,
             } = pattern2_0
             {
-                // Rule at src/prelude.isle line 467.
+                // Rule at src/prelude.isle line 475.
                 let expr0_0 = C::emit(ctx, pattern1_0);
                 let expr1_0 = C::emit(ctx, pattern3_0);
                 let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -740,7 +742,7 @@ pub fn constructor_with_flags_reg<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/prelude.isle line 508.
+    // Rule at src/prelude.isle line 516.
     let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
     let expr1_0: usize = 0;
     let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -4260,7 +4262,8 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
                 arg: pattern5_1,
             } = &pattern4_0
             {
-                if let &Opcode::Bitrev = pattern5_0 {
+                match pattern5_0 {
+                    &Opcode::Bitrev => {
                         // Rule at src/isa/x64/lower.isle line 1780.
                         let expr0_0: Type = I64;
                         let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?;
@@ -4268,6 +4271,118 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
                         let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
                         return Some(expr3_0);
                     }
+                    &Opcode::Uextend => {
+                        if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) {
+                            let pattern8_0 = C::inst_data(ctx, pattern7_0);
+                            match &pattern8_0 {
+                                &InstructionData::Binary {
+                                    opcode: ref pattern9_0,
+                                    args: ref pattern9_1,
+                                } => {
+                                    match pattern9_0 {
+                                        &Opcode::Iadd => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1896.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::Isub => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1902.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::Imul => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1905.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::IaddIfcout => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1899.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::Band => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1908.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::Bor => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1911.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::Bxor => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1914.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::Ishl => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1917.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        &Opcode::Ushr => {
+                                            let (pattern11_0, pattern11_1) =
+                                                C::unpack_value_array_2(ctx, pattern9_1);
+                                            // Rule at src/isa/x64/lower.isle line 1920.
+                                            let expr0_0 =
+                                                constructor_output_value(ctx, pattern5_1)?;
+                                            return Some(expr0_0);
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                                &InstructionData::Load {
+                                    opcode: ref pattern9_0,
+                                    arg: pattern9_1,
+                                    flags: pattern9_2,
+                                    offset: pattern9_3,
+                                } => {
+                                    if let &Opcode::Uload32 = pattern9_0 {
+                                        // Rule at src/isa/x64/lower.isle line 1923.
+                                        let expr0_0 = constructor_output_value(ctx, pattern5_1)?;
+                                        return Some(expr0_0);
+                                    }
+                                }
+                                _ => {}
+                            }
+                        }
+                        let pattern7_0 = C::value_type(ctx, pattern5_1);
+                        if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) {
+                            // Rule at src/isa/x64/lower.isle line 1878.
+                            let expr0_0: Type = I64;
+                            let expr1_0 = ExtendKind::Zero;
+                            let expr2_0 =
+                                constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?;
+                            let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
+                            return Some(expr3_0);
+                        }
+                    }
+                    _ => {}
+                }
             }
         }
         if pattern2_0 == I128 {
@@ -4630,6 +4745,33 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
                             let expr18_0 = C::output(ctx, expr17_0);
                             return Some(expr18_0);
                         }
+                        &Opcode::Uextend => {
+                            let pattern7_0 = C::value_type(ctx, pattern5_1);
+                            if pattern7_0 == I64 {
+                                // Rule at src/isa/x64/lower.isle line 1870.
+                                let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                                let expr1_0: Type = I64;
+                                let expr2_0: u64 = 0;
+                                let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?;
+                                let expr4_0 = C::value_regs(ctx, expr0_0, expr3_0);
+                                let expr5_0 = C::output(ctx, expr4_0);
+                                return Some(expr5_0);
+                            }
+                            if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 1874.
+                                let expr0_0: Type = I64;
+                                let expr1_0 = ExtendKind::Zero;
+                                let expr2_0 =
+                                    constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::gpr_to_reg(ctx, expr2_0);
+                                let expr4_0: Type = I64;
+                                let expr5_0: u64 = 0;
+                                let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?;
+                                let expr7_0 = C::value_regs(ctx, expr3_0, expr6_0);
+                                let expr8_0 = C::output(ctx, expr7_0);
+                                return Some(expr8_0);
+                            }
+                        }
                         _ => {}
                     }
                 }
@@ -5390,6 +5532,52 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
                     }
                 }
             }
+            &InstructionData::Unary {
+                opcode: ref pattern4_0,
+                arg: pattern4_1,
+            } => {
+                match pattern4_0 {
+                    &Opcode::Breduce => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        if pattern6_0 == pattern2_0 {
+                            // Rule at src/isa/x64/lower.isle line 1984.
+                            let expr0_0 = constructor_output_value(ctx, pattern4_1)?;
+                            return Some(expr0_0);
+                        }
+                    }
+                    &Opcode::Bextend => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        // Rule at src/isa/x64/lower.isle line 1965.
+                        let expr0_0 =
+                            constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?;
+                        return Some(expr0_0);
+                    }
+                    &Opcode::Ireduce => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        if pattern6_0 == pattern2_0 {
+                            // Rule at src/isa/x64/lower.isle line 1973.
+                            let expr0_0 = constructor_output_value(ctx, pattern4_1)?;
+                            return Some(expr0_0);
+                        }
+                    }
+                    &Opcode::Uextend => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        if pattern6_0 == pattern2_0 {
+                            // Rule at src/isa/x64/lower.isle line 1866.
+                            let expr0_0 = constructor_output_value(ctx, pattern4_1)?;
+                            return Some(expr0_0);
+                        }
+                    }
+                    &Opcode::Sextend => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        // Rule at src/isa/x64/lower.isle line 1959.
+                        let expr0_0 =
+                            constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?;
+                        return Some(expr0_0);
+                    }
+                    _ => {}
+                }
+            }
             _ => {}
         }
         if let Some(()) = C::avx512vl_enabled(ctx, pattern2_0) {
@@ -6568,6 +6756,27 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
                 _ => {}
             }
         }
+        if let Some(pattern3_0) = C::fits_in_32(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Unary {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+            } = &pattern4_0
+            {
+                if let &Opcode::Uextend = pattern5_0 {
+                    let pattern7_0 = C::value_type(ctx, pattern5_1);
+                    if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) {
+                        // Rule at src/isa/x64/lower.isle line 1882.
+                        let expr0_0: Type = I32;
+                        let expr1_0 = ExtendKind::Zero;
+                        let expr2_0 =
+                            constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?;
+                        let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
+                        return Some(expr3_0);
+                    }
+                }
+            }
+        }
         if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) {
             let pattern4_0 = C::inst_data(ctx, pattern0_0);
             match &pattern4_0 {
@@ -6993,6 +7202,22 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
                             let expr2_0 = constructor_output_gpr(ctx, expr1_0)?;
                             return Some(expr2_0);
                         }
+                        &Opcode::Breduce => {
+                            // Rule at src/isa/x64/lower.isle line 1987.
+                            let expr0_0 = C::put_in_regs(ctx, pattern5_1);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?;
+                            let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Ireduce => {
+                            // Rule at src/isa/x64/lower.isle line 1979.
+                            let expr0_0 = C::put_in_regs(ctx, pattern5_1);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?;
+                            let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
+                            return Some(expr3_0);
+                        }
                         _ => {}
                     }
                 }
@@ -8192,3 +8417,77 @@ pub fn constructor_do_bitrev64<C: Context>(ctx: &mut C, arg0: Type, arg1: Gpr) -
     }
     return None;
 }
+
+// Generated as internal constructor for term generic_sextend.
+pub fn constructor_generic_sextend<C: Context>(
+    ctx: &mut C,
+    arg0: Value,
+    arg1: Type,
+    arg2: Type,
+) -> Option<InstOutput> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
+        let pattern3_0 = arg2;
+        if let Some(pattern4_0) = C::fits_in_32(ctx, pattern3_0) {
+            // Rule at src/isa/x64/lower.isle line 1956.
+            let expr0_0: Type = I32;
+            let expr1_0 = ExtendKind::Sign;
+            let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?;
+            let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
+            return Some(expr3_0);
+        }
+        if let Some(pattern4_0) = C::ty_int_bool_64(ctx, pattern3_0) {
+            // Rule at src/isa/x64/lower.isle line 1952.
+            let expr0_0: Type = I64;
+            let expr1_0 = ExtendKind::Sign;
+            let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?;
+            let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
+            return Some(expr3_0);
+        }
+        if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) {
+            // Rule at src/isa/x64/lower.isle line 1946.
+            let expr0_0: Type = I64;
+            let expr1_0 = ExtendKind::Sign;
+            let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?;
+            let expr3_0 = constructor_spread_sign_bit(ctx, expr2_0)?;
+            let expr4_0 = C::gpr_to_reg(ctx, expr2_0);
+            let expr5_0 = C::gpr_to_reg(ctx, expr3_0);
+            let expr6_0 = C::value_regs(ctx, expr4_0, expr5_0);
+            let expr7_0 = C::output(ctx, expr6_0);
+            return Some(expr7_0);
+        }
+    }
+    if let Some(pattern2_0) = C::ty_int_bool_64(ctx, pattern1_0) {
+        let pattern3_0 = arg2;
+        if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) {
+            // Rule at src/isa/x64/lower.isle line 1942.
+            let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+            let expr1_0 = constructor_put_in_gpr(ctx, pattern0_0)?;
+            let expr2_0 = constructor_spread_sign_bit(ctx, expr1_0)?;
+            let expr3_0 = C::gpr_to_reg(ctx, expr2_0);
+            let expr4_0 = C::value_regs(ctx, expr0_0, expr3_0);
+            let expr5_0 = C::output(ctx, expr4_0);
+            return Some(expr5_0);
+        }
+    }
+    let pattern2_0 = arg2;
+    if pattern2_0 == pattern1_0 {
+        // Rule at src/isa/x64/lower.isle line 1932.
+        let expr0_0 = constructor_output_value(ctx, pattern0_0)?;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term spread_sign_bit.
+pub fn constructor_spread_sign_bit<C: Context>(ctx: &mut C, arg0: Gpr) -> Option<Gpr> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/x64/lower.isle line 1938.
+    let expr0_0: Type = I64;
+    let expr1_0: u8 = 63;
+    let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 };
+    let expr3_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr2_0);
+    let expr4_0 = constructor_sar(ctx, expr0_0, pattern0_0, &expr3_0)?;
+    return Some(expr4_0);
+}

cranelift/codegen/src/machinst/isle.rs

@@ -237,6 +237,22 @@ macro_rules! isle_prelude_methods {
             }
         }
 
+        #[inline]
+        fn ty_int_bool_64(&mut self, ty: Type) -> Option<Type> {
+            match ty {
+                I64 | B64 => Some(ty),
+                _ => None,
+            }
+        }
+
+        #[inline]
+        fn ty_int_bool_128(&mut self, ty: Type) -> Option<Type> {
+            match ty {
+                I128 | B128 => Some(ty),
+                _ => None,
+            }
+        }
+
         fn vec128(&mut self, ty: Type) -> Option<Type> {
             if ty.is_vector() && ty.bits() == 128 {
                 Some(ty)

cranelift/codegen/src/prelude.isle

@@ -253,14 +253,22 @@
 (decl fits_in_64 (Type) Type)
 (extern extractor fits_in_64 fits_in_64)
 
-;; An extractor that maches 32- and 64-bit types only.
+;; An extractor that matches 32- and 64-bit types only.
 (decl ty_32_or_64 (Type) Type)
 (extern extractor ty_32_or_64 ty_32_or_64)
 
-;; An extractor that maches 8- and 16-bit types only.
+;; An extractor that matches 8- and 16-bit types only.
 (decl ty_8_or_16 (Type) Type)
 (extern extractor ty_8_or_16 ty_8_or_16)
 
+;; An extractor that matches I64 or B64.
+(decl ty_int_bool_64 (Type) Type)
+(extern extractor ty_int_bool_64 ty_int_bool_64)
+
+;; An extractor that matches I128 or B128.
+(decl ty_int_bool_128 (Type) Type)
+(extern extractor ty_int_bool_128 ty_int_bool_128)
+
 ;; An extractor that only matches 128-bit vector types.
 (decl vec128 (Type) Type)
 (extern extractor vec128 vec128)