[s390x, abi_impl] Add i128 support (#4598)

This adds full i128 support to the s390x target, including new filetests
and enabling the existing i128 runtest on s390x.

The ABI requires that i128 is passed and returned via implicit pointer,
but the front end still generates direct i128 types in call.  This means
we have to implement ABI support to implicitly convert i128 types to
pointers when passing arguments.

To do so, we add a new variant ABIArg::ImplicitArg.  This acts like
StructArg, except that the value type is the actual target type,
not a pointer type.  The required conversions have to be inserted
in the prologue and at function call sites.

Note that when dereferencing the implicit pointer in the prologue,
we may require a temp register: the pointer may be passed on the
stack so it needs to be loaded first, but the value register may
be in the wrong class for pointer values.  In this case, we use
the "stack limit" register, which should be available at this
point in the prologue.

For return values, we use a mechanism similar to the one used for
supporting multiple return values in the Wasmtime ABI.  The only
difference is that the hidden pointer to the return buffer must
be the *first*, not last, argument in this case.

(This implements the second half of issue #4565.)

cranelift/codegen/src/isa/s390x/abi.rs

@@ -239,8 +239,15 @@ impl ABIMachineSpec for S390xMachineDeps {
             next_stack = REG_SAVE_AREA_SIZE as u64;
         }
 
+        // In the SystemV ABI, the return area pointer is the first argument,
+        // so we need to leave room for it if required.  (In the Wasmtime ABI,
+        // the return area pointer is the last argument and is handled below.)
+        if add_ret_area_ptr && !call_conv.extends_wasmtime() {
+            next_gpr += 1;
+        }
+
         for i in 0..params.len() {
-            let param = &params[i];
+            let mut param = params[i];
 
             // Validate "purpose".
             match &param.purpose {
@@ -259,26 +266,41 @@ impl ABIMachineSpec for S390xMachineDeps {
             let intreg = in_int_reg(param.value_type);
             let fltreg = in_flt_reg(param.value_type);
             let vecreg = in_vec_reg(param.value_type);
-            debug_assert!(intreg as i32 + fltreg as i32 + vecreg as i32 == 1);
+            debug_assert!(intreg as i32 + fltreg as i32 + vecreg as i32 <= 1);
 
-            let (next_reg, candidate) = if intreg {
+            let (next_reg, candidate, implicit_ref) = if intreg {
                 let candidate = match args_or_rets {
                     ArgsOrRets::Args => get_intreg_for_arg(next_gpr),
                     ArgsOrRets::Rets => get_intreg_for_ret(next_gpr),
                 };
-                (&mut next_gpr, candidate)
+                (&mut next_gpr, candidate, None)
             } else if fltreg {
                 let candidate = match args_or_rets {
                     ArgsOrRets::Args => get_fltreg_for_arg(next_fpr),
                     ArgsOrRets::Rets => get_fltreg_for_ret(next_fpr),
                 };
-                (&mut next_fpr, candidate)
+                (&mut next_fpr, candidate, None)
-            } else {
+            } else if vecreg {
                 let candidate = match args_or_rets {
                     ArgsOrRets::Args => get_vecreg_for_arg(next_vr),
                     ArgsOrRets::Rets => get_vecreg_for_ret(next_vr),
                 };
-                (&mut next_vr, candidate)
+                (&mut next_vr, candidate, None)
+            } else if call_conv.extends_wasmtime() {
+                panic!("i128 args/return values not supported in the Wasmtime ABI");
+            } else {
+                assert!(param.extension == ir::ArgumentExtension::None);
+                // We must pass this by implicit reference.
+                if args_or_rets == ArgsOrRets::Rets {
+                    // For return values, just force them to memory.
+                    (&mut next_gpr, None, None)
+                } else {
+                    // For arguments, implicitly convert to pointer type.
+                    let implicit_ref = Some(param.value_type);
+                    param = ir::AbiParam::new(types::I64);
+                    let candidate = get_intreg_for_arg(next_gpr);
+                    (&mut next_gpr, candidate, implicit_ref)
+                }
             };
 
             // In the Wasmtime ABI only the first return value can be in a register.
@@ -336,6 +358,17 @@ impl ABIMachineSpec for S390xMachineDeps {
                     size: size as u64,
                     purpose: param.purpose,
                 });
+            } else if let Some(ty) = implicit_ref {
+                assert!(
+                    (ty_bits(ty) / 8) % 8 == 0,
+                    "implicit argument size is not properly aligned"
+                );
+                ret.push(ABIArg::ImplicitPtrArg {
+                    pointer: slot,
+                    offset: 0,
+                    ty,
+                    purpose: param.purpose,
+                });
             } else {
                 ret.push(ABIArg::Slots {
                     slots: smallvec![slot],
@@ -348,6 +381,13 @@ impl ABIMachineSpec for S390xMachineDeps {
 
         let extra_arg = if add_ret_area_ptr {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
+            // The return pointer is passed either as first argument
+            // (in the SystemV ABI) or as last argument (Wasmtime ABI).
+            let next_gpr = if call_conv.extends_wasmtime() {
+                next_gpr
+            } else {
+                0
+            };
             if let Some(reg) = get_intreg_for_arg(next_gpr) {
                 ret.push(ABIArg::reg(
                     reg.to_real_reg().unwrap(),
@@ -370,7 +410,7 @@ impl ABIMachineSpec for S390xMachineDeps {
         };
 
         // After all arguments are in their well-defined location,
-        // allocate buffers for all StructArg arguments.
+        // allocate buffers for all StructArg or ImplicitPtrArg arguments.
         for i in 0..ret.len() {
             match &mut ret[i] {
                 &mut ABIArg::StructArg {
@@ -381,6 +421,12 @@ impl ABIMachineSpec for S390xMachineDeps {
                     *offset = next_stack as i64;
                     next_stack += size;
                 }
+                &mut ABIArg::ImplicitPtrArg {
+                    ref mut offset, ty, ..
+                } => {
+                    *offset = next_stack as i64;
+                    next_stack += (ty_bits(ty) / 8) as u64;
+                }
                 _ => {}
             }
         }

cranelift/codegen/src/isa/s390x/inst.isle

@@ -600,6 +600,20 @@
       (rn Reg)
       (rm Reg))
 
+    ;; Synthetic instruction to compare signed 128-bit values.
+    ;; Sets CC 1 if rn > rm, sets a different CC otherwise.
+    (VecInt128SCmpHi
+      (tmp WritableReg)
+      (rn Reg)
+      (rm Reg))
+
+    ;; Synthetic instruction to compare unsigned 128-bit values.
+    ;; Sets CC 1 if rn > rm, sets a different CC otherwise.
+    (VecInt128UCmpHi
+      (tmp WritableReg)
+      (rn Reg)
+      (rm Reg))
+
     ;; 128-bit vector load instruction.
     (VecLoad
       (rd WritableReg)
@@ -996,10 +1010,12 @@
     (Add16x8)
     (Add32x4)
     (Add64x2)
+    (Add128)
     (Sub8x16)
     (Sub16x8)
     (Sub32x4)
     (Sub64x2)
+    (Sub128)
     ;; Multiplication (64-bit not supported)
     (Mul8x16)
     (Mul16x8)
@@ -1105,6 +1121,15 @@
     (Popcnt16x8)
     (Popcnt32x4)
     (Popcnt64x2)
+    ;; Count leading/trailing zeros
+    (Clz8x16)
+    (Clz16x8)
+    (Clz32x4)
+    (Clz64x2)
+    (Ctz8x16)
+    (Ctz16x8)
+    (Ctz32x4)
+    (Ctz64x2)
     ;; Unpack
     (UnpackULow8x16)
     (UnpackULow16x8)
@@ -1316,6 +1341,10 @@
 (decl gpr64_ty (Type) Type)
 (extern extractor gpr64_ty gpr64_ty)
 
+;; Types that can be operated on using 128-bit vector instructions
+(decl vr128_ty (Type) Type)
+(extern extractor vr128_ty vr128_ty)
+
 
 ;; Helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1498,6 +1527,22 @@
         (and_uimm16shifted ty reg (uimm16shifted (u8_as_u16 mask) 0))))
 (rule (mask_amt_reg (gpr64_ty ty) reg) reg)
 
+;; Load a shift amount into a GPR.
+(decl amt_reg (Value) Reg)
+(rule (amt_reg amt @ (value_type (fits_in_64 _))) amt)
+(rule (amt_reg amt @ (value_type (vr128_ty _)))
+      (vec_extract_lane $I64X2 amt 1 (zero_reg)))
+
+;; Load a shift amount into a VR, replicated across all 16 bytes.
+(decl amt_vr (Value) Reg)
+(rule (amt_vr amt @ (value_type (fits_in_64 _)))
+      (vec_replicate_lane $I8X16
+        (vec_insert_lane_undef $I8X16 amt 0 (zero_reg)) 0))
+(rule (amt_vr amt @ (value_type (vr128_ty _)))
+      (vec_replicate_lane $I8X16 amt 15))
+(rule (amt_vr (u64_from_value amt))
+      (vec_imm_splat $I8X16 amt))
+
 
 ;; Helpers for condition codes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -2181,6 +2226,20 @@
       (let ((tmp WritableReg (temp_writable_reg ty)))
         (ProducesFlags.ProducesFlagsSideEffect (MInst.VecFloatCmpS op tmp src1 src2))))
 
+;; Helper for emitting `MInst.VecInt128SCmpHi` instructions.
+(decl vec_int128_scmphi (Reg Reg) ProducesBool)
+(rule (vec_int128_scmphi src1 src2)
+      (let ((tmp WritableReg (temp_writable_reg $I128)))
+        (bool (ProducesFlags.ProducesFlagsSideEffect (MInst.VecInt128SCmpHi tmp src1 src2))
+              (mask_as_cond 4))))
+
+;; Helper for emitting `MInst.VecInt128UCmpHi` instructions.
+(decl vec_int128_ucmphi (Reg Reg) ProducesBool)
+(rule (vec_int128_ucmphi src1 src2)
+      (let ((tmp WritableReg (temp_writable_reg $I128)))
+        (bool (ProducesFlags.ProducesFlagsSideEffect (MInst.VecInt128UCmpHi tmp src1 src2))
+              (mask_as_cond 4))))
+
 ;; Helper for emitting `MInst.VecLoad` instructions.
 (decl vec_load (Type MemArg) Reg)
 (rule (vec_load ty addr)
@@ -2228,7 +2287,7 @@
 
 ;; Helper for emitting `MInst.VecLoadConst` instructions.
 (decl vec_load_const (Type u128) Reg)
-(rule (vec_load_const (ty_vec128 ty) n)
+(rule (vec_load_const (vr128_ty ty) n)
       (let ((dst WritableReg (temp_writable_reg ty))
             (_ Unit (emit (MInst.VecLoadConst dst n))))
         dst))
@@ -2242,7 +2301,7 @@
 
 ;; Helper for emitting `MInst.VecImmByteMask` instructions.
 (decl vec_imm_byte_mask (Type u16) Reg)
-(rule (vec_imm_byte_mask (ty_vec128 ty) n)
+(rule (vec_imm_byte_mask (vr128_ty ty) n)
       (let ((dst WritableReg (temp_writable_reg ty))
             (_ Unit (emit (MInst.VecImmByteMask dst n))))
         dst))
@@ -2494,7 +2553,7 @@
 (rule (emit_mov $F64 dst src)
       (emit (MInst.FpuMove64 dst src)))
 
-(rule (emit_mov (ty_vec128 ty) dst src)
+(rule (emit_mov (vr128_ty ty) dst src)
       (emit (MInst.VecMov dst src)))
 
 ;; Allocate a temporary (writable) register, initialized as a copy of the input.
@@ -2545,7 +2604,7 @@
       (emit_side_effect (vec_store_lane $F32X4 reg mem 0)))
 (rule (emit_arg_store $F64 reg mem)
       (emit_side_effect (vec_store_lane $F64X2 reg mem 0)))
-(rule (emit_arg_store (ty_vec128 ty) reg mem)
+(rule (emit_arg_store (vr128_ty ty) reg mem)
       (emit_side_effect (vec_store reg mem)))
 
 (decl emit_arg_load (Type MemArg) Reg)
@@ -2556,7 +2615,7 @@
 (rule (emit_arg_load $R64 mem) (load64 mem))
 (rule (emit_arg_load $F32 mem) (vec_load_lane_undef $F32X4 mem 0))
 (rule (emit_arg_load $F64 mem) (vec_load_lane_undef $F64X2 mem 0))
-(rule (emit_arg_load (ty_vec128 ty) mem) (vec_load ty mem))
+(rule (emit_arg_load (vr128_ty ty) mem) (vec_load ty mem))
 
 ;; Helpers to emit a memory copy (MVC or memcpy libcall).
 (decl emit_memcpy (MemArg MemArg u64) Unit)
@@ -2578,6 +2637,11 @@
             (src MemArg (memarg_reg_plus_off val 0 0 (memflags_trusted)))
             (_ Unit (emit_memcpy dst src size)))
         (output_none)))
+(rule (copy_to_buffer base (abi_arg_implicit_pointer _ offset ty)
+                      val @ (value_type ty))
+      (let ((mem MemArg (memarg_stack_off base offset))
+            (_ Unit (emit_arg_store ty val mem)))
+        (output_none)))
 
 ;; Copy a single argument/return value to its slots.
 ;; For oversized arguments, set the slot to the buffer address.
@@ -2587,6 +2651,9 @@
 (rule (copy_to_arg base (abi_arg_struct_pointer slot offset _) _)
       (let ((ptr Reg (load_addr (memarg_stack_off base offset))))
         (copy_reg_to_arg_slot base slot ptr)))
+(rule (copy_to_arg base (abi_arg_implicit_pointer slot offset _) _)
+      (let ((ptr Reg (load_addr (memarg_stack_off base offset))))
+        (copy_reg_to_arg_slot base slot ptr)))
 
 ;; Copy a single argument/return value from its slots.
 (decl copy_from_arg (i64 ABIArg) ValueRegs)
@@ -2711,11 +2778,11 @@
 
 ;; Allocate a temporary register, initialized with a vector immediate.
 (decl vec_imm (Type u128) Reg)
-(rule (vec_imm (ty_vec128 ty) 0)
+(rule (vec_imm (vr128_ty ty) 0)
       (vec_imm_byte_mask ty 0))
-(rule (vec_imm (ty_vec128 ty) (u64_pair n n))
+(rule (vec_imm (vr128_ty ty) (u64_pair n n))
       (vec_imm_splat $I64X2 n))
-(rule (vec_imm (ty_vec128 ty) n)
+(rule (vec_imm (vr128_ty ty) n)
       (vec_load_const ty n))
 
 ;; Variant with replicated immediate.
@@ -2772,6 +2839,13 @@
 (rule (ty_ext64 $I32) $I64)
 (rule (ty_ext64 $I64) $I64)
 
+;; 128-bit vector type with lane type `Type`.
+(decl ty_vec128_from_lane_ty (Type) Type)
+(rule (ty_vec128_from_lane_ty $I8) $I8X16)
+(rule (ty_vec128_from_lane_ty $I16) $I16X8)
+(rule (ty_vec128_from_lane_ty $I32) $I32X4)
+(rule (ty_vec128_from_lane_ty $I64) $I64X2)
+
 ;; Zero-extend a register from a smaller `Type` into a 32-bit destination.  (Non-SSA form.)
 ;; This handles both integer and boolean input types.
 (decl emit_zext32_reg (WritableReg Type Reg) Unit)
@@ -3050,7 +3124,7 @@
 (rule (emit_cmov_reg $F64 dst cond src)
       (ConsumesFlags.ConsumesFlagsReturnsReg (MInst.FpuCMov64 dst cond src)
                                              dst))
-(rule (emit_cmov_reg (ty_vec128 ty) dst cond src)
+(rule (emit_cmov_reg (vr128_ty ty) dst cond src)
       (ConsumesFlags.ConsumesFlagsReturnsReg (MInst.VecCMov dst cond src)
                                              dst))
 
@@ -3427,7 +3501,7 @@
 (rule (vec_merge_high ty x y) (vec_rrr ty (vecop_merge_high ty) x y))
 
 
-;; Helpers for generating `clz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Helpers for generating `clz` and `ctz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Count leading zeroes.  For a zero input, return the specified value.
 (decl clz_reg (i16 Reg) RegPair)
@@ -3449,6 +3523,26 @@
                               (intcc_as_cond (IntCC.Equal)) zeroval))))
         dst))
 
+;; Vector count leading zeros.
+(decl vecop_clz (Type) VecUnaryOp)
+(rule (vecop_clz $I8X16) (VecUnaryOp.Clz8x16))
+(rule (vecop_clz $I16X8) (VecUnaryOp.Clz16x8))
+(rule (vecop_clz $I32X4) (VecUnaryOp.Clz32x4))
+(rule (vecop_clz $I64X2) (VecUnaryOp.Clz64x2))
+
+(decl vec_clz (Type Reg) Reg)
+(rule (vec_clz ty x) (vec_rr ty (vecop_clz ty) x))
+
+;; Vector count trailing zeros.
+(decl vecop_ctz (Type) VecUnaryOp)
+(rule (vecop_ctz $I8X16) (VecUnaryOp.Ctz8x16))
+(rule (vecop_ctz $I16X8) (VecUnaryOp.Ctz16x8))
+(rule (vecop_ctz $I32X4) (VecUnaryOp.Ctz32x4))
+(rule (vecop_ctz $I64X2) (VecUnaryOp.Ctz64x2))
+
+(decl vec_ctz (Type Reg) Reg)
+(rule (vec_ctz ty x) (vec_rr ty (vecop_ctz ty) x))
+
 
 ;; Helpers for generating saturating integer instructions ;;;;;;;;;;;;;;;;;;;;;;
 
@@ -3536,6 +3630,7 @@
 (rule (vecop_add $I16X8) (VecBinaryOp.Add16x8))
 (rule (vecop_add $I32X4) (VecBinaryOp.Add32x4))
 (rule (vecop_add $I64X2) (VecBinaryOp.Add64x2))
+(rule (vecop_add $I128) (VecBinaryOp.Add128))
 
 (decl vec_add (Type Reg Reg) Reg)
 (rule (vec_add ty x y) (vec_rrr ty (vecop_add ty) x y))
@@ -3602,6 +3697,7 @@
 (rule (vecop_sub $I16X8) (VecBinaryOp.Sub16x8))
 (rule (vecop_sub $I32X4) (VecBinaryOp.Sub32x4))
 (rule (vecop_sub $I64X2) (VecBinaryOp.Sub64x2))
+(rule (vecop_sub $I128) (VecBinaryOp.Sub128))
 
 (decl vec_sub (Type Reg Reg) Reg)
 (rule (vec_sub ty x y) (vec_rrr ty (vecop_sub ty) x y))
@@ -3826,7 +3922,7 @@
 (rule (and_mem ty x y) (alu_rx ty (aluop_and ty) x y))
 
 (decl vec_and (Type Reg Reg) Reg)
-(rule (vec_and (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.And128) x y))
+(rule (vec_and (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.And128) x y))
 
 
 ;; Helpers for generating `or` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3848,7 +3944,7 @@
 (rule (or_mem ty x y) (alu_rx ty (aluop_or ty) x y))
 
 (decl vec_or (Type Reg Reg) Reg)
-(rule (vec_or (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.Orr128) x y))
+(rule (vec_or (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.Orr128) x y))
 
 
 ;; Helpers for generating `xor` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3871,7 +3967,7 @@
       (push_alu_uimm32shifted ib (aluop_xor ty) dst src imm))
 
 (decl vec_xor (Type Reg Reg) Reg)
-(rule (vec_xor (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.Xor128) x y))
+(rule (vec_xor (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.Xor128) x y))
 
 
 ;; Helpers for generating `not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3905,7 +4001,7 @@
 (rule (not_and_reg ty x y) (alu_rrr ty (aluop_not_and ty) x y))
 
 (decl vec_not_and (Type Reg Reg) Reg)
-(rule (vec_not_and (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.NotAnd128) x y))
+(rule (vec_not_and (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.NotAnd128) x y))
 
 
 ;; Helpers for generating `not_or` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3918,7 +4014,7 @@
 (rule (not_or_reg ty x y) (alu_rrr ty (aluop_not_or ty) x y))
 
 (decl vec_not_or (Type Reg Reg) Reg)
-(rule (vec_not_or (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.NotOrr128) x y))
+(rule (vec_not_or (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.NotOrr128) x y))
 
 
 ;; Helpers for generating `not_xor` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3931,7 +4027,7 @@
 (rule (not_xor_reg ty x y) (alu_rrr ty (aluop_not_xor ty) x y))
 
 (decl vec_not_xor (Type Reg Reg) Reg)
-(rule (vec_not_xor (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.NotXor128) x y))
+(rule (vec_not_xor (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.NotXor128) x y))
 
 
 ;; Helpers for generating `and_not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3944,7 +4040,7 @@
 (rule (and_not_reg ty x y) (alu_rrr ty (aluop_and_not ty) x y))
 
 (decl vec_and_not (Type Reg Reg) Reg)
-(rule (vec_and_not (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.AndNot128) x y))
+(rule (vec_and_not (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.AndNot128) x y))
 
 
 ;; Helpers for generating `or_not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3957,7 +4053,7 @@
 (rule (or_not_reg ty x y) (alu_rrr ty (aluop_or_not ty) x y))
 
 (decl vec_or_not (Type Reg Reg) Reg)
-(rule (vec_or_not (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.OrrNot128) x y))
+(rule (vec_or_not (vr128_ty ty) x y) (vec_rrr ty (VecBinaryOp.OrrNot128) x y))
 
 
 ;; Helpers for generating `bitpermute` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -4101,6 +4197,9 @@
 (decl vec_lshl_by_byte (Reg Reg) Reg)
 (rule (vec_lshl_by_byte x y) (vec_rrr $I8X16 (VecBinaryOp.LShLByByte128) x y))
 
+(decl vec_lshl_by_bit (Reg Reg) Reg)
+(rule (vec_lshl_by_bit x y) (vec_rrr $I8X16 (VecBinaryOp.LShLByBit128) x y))
+
 
 ;; Helpers for generating `lshr` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -4133,6 +4232,9 @@
 (decl vec_lshr_by_byte (Reg Reg) Reg)
 (rule (vec_lshr_by_byte x y) (vec_rrr $I8X16 (VecBinaryOp.LShRByByte128) x y))
 
+(decl vec_lshr_by_bit (Reg Reg) Reg)
+(rule (vec_lshr_by_bit x y) (vec_rrr $I8X16 (VecBinaryOp.LShRByBit128) x y))
+
 
 ;; Helpers for generating `ashr` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -4165,6 +4267,9 @@
 (decl vec_ashr_by_byte (Reg Reg) Reg)
 (rule (vec_ashr_by_byte x y) (vec_rrr $I8X16 (VecBinaryOp.AShRByByte128) x y))
 
+(decl vec_ashr_by_bit (Reg Reg) Reg)
+(rule (vec_ashr_by_bit x y) (vec_rrr $I8X16 (VecBinaryOp.AShRByBit128) x y))
+
 
 ;; Helpers for generating `popcnt` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/codegen/src/isa/s390x/inst/emit.rs

@@ -2548,10 +2548,12 @@ impl MachInstEmit for Inst {
                     VecBinaryOp::Add16x8 => (0xe7f3, 1),       // VAH
                     VecBinaryOp::Add32x4 => (0xe7f3, 2),       // VAF
                     VecBinaryOp::Add64x2 => (0xe7f3, 3),       // VAG
+                    VecBinaryOp::Add128 => (0xe7f3, 4),        // VAQ
                     VecBinaryOp::Sub8x16 => (0xe7f7, 0),       // VSB
                     VecBinaryOp::Sub16x8 => (0xe7f7, 1),       // VSH
                     VecBinaryOp::Sub32x4 => (0xe7f7, 2),       // VSF
                     VecBinaryOp::Sub64x2 => (0xe7f7, 3),       // VSG
+                    VecBinaryOp::Sub128 => (0xe7f7, 4),        // VSQ
                     VecBinaryOp::Mul8x16 => (0xe7a2, 0),       // VMLB
                     VecBinaryOp::Mul16x8 => (0xe7a2, 1),       // VMLHW
                     VecBinaryOp::Mul32x4 => (0xe7a2, 2),       // VMLF
@@ -2650,6 +2652,14 @@ impl MachInstEmit for Inst {
                     VecUnaryOp::Popcnt16x8 => (0xe750, 1),      // VPOPCTH
                     VecUnaryOp::Popcnt32x4 => (0xe750, 2),      // VPOPCTF
                     VecUnaryOp::Popcnt64x2 => (0xe750, 3),      // VPOPCTG
+                    VecUnaryOp::Clz8x16 => (0xe753, 0),         // VCLZB
+                    VecUnaryOp::Clz16x8 => (0xe753, 1),         // VCLZH
+                    VecUnaryOp::Clz32x4 => (0xe753, 2),         // VCLZF
+                    VecUnaryOp::Clz64x2 => (0xe753, 3),         // VCLZG
+                    VecUnaryOp::Ctz8x16 => (0xe752, 0),         // VCTZB
+                    VecUnaryOp::Ctz16x8 => (0xe752, 1),         // VCTZH
+                    VecUnaryOp::Ctz32x4 => (0xe752, 2),         // VCTZF
+                    VecUnaryOp::Ctz64x2 => (0xe752, 3),         // VCTZG
                     VecUnaryOp::UnpackULow8x16 => (0xe7d4, 0),  // VUPLLB
                     VecUnaryOp::UnpackULow16x8 => (0xe7d4, 1),  // VUPLLH
                     VecUnaryOp::UnpackULow32x4 => (0xe7d4, 2),  // VUPLLF
@@ -2781,6 +2791,45 @@ impl MachInstEmit for Inst {
 
                 put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 0, m6));
             }
+            &Inst::VecInt128SCmpHi { tmp, rn, rm } | &Inst::VecInt128UCmpHi { tmp, rn, rm } => {
+                // Synthetic instruction to compare 128-bit values.
+                // Sets CC 1 if rn > rm, sets a different CC otherwise.
+                let tmp = allocs.next_writable(tmp);
+                let rn = allocs.next(rn);
+                let rm = allocs.next(rm);
+
+                // Use VECTOR ELEMENT COMPARE to compare the high parts.
+                // Swap the inputs to get:
+                //    CC 1 if high(rn) > high(rm)
+                //    CC 2 if high(rn) < high(rm)
+                //    CC 0 if high(rn) == high(rm)
+                let (opcode, m3) = match self {
+                    &Inst::VecInt128SCmpHi { .. } => (0xe7db, 3), // VECG
+                    &Inst::VecInt128UCmpHi { .. } => (0xe7d9, 3), // VECLG
+                    _ => unreachable!(),
+                };
+                put(sink, &enc_vrr_a(opcode, rm, rn, m3, 0, 0));
+
+                // If CC != 0, we'd done, so jump over the next instruction.
+                let opcode = 0xa74; // BCR
+                put(sink, &enc_ri_c(opcode, 7, 4 + 6));
+
+                // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
+                // Since we already know the high parts are equal, the CC
+                // result will only depend on the low parts:
+                //     CC 1 if low(rn) > low(rm)
+                //     CC 3 if low(rn) <= low(rm)
+                let inst = Inst::VecIntCmpS {
+                    op: VecIntCmpOp::UCmpHi64x2,
+                    // N.B.: This is the first write to tmp, and it happens
+                    // after all uses of rn and rm.  If this were to ever
+                    // change, tmp would have to become an early-def.
+                    rd: tmp,
+                    rn,
+                    rm,
+                };
+                inst.emit(&[], sink, emit_info, state);
+            }
 
             &Inst::VecLoad { rd, ref mem } | &Inst::VecLoadRev { rd, ref mem } => {
                 let rd = allocs.next_writable(rd);

cranelift/codegen/src/isa/s390x/inst/emit_tests.rs

@@ -8170,6 +8170,16 @@ fn test_s390x_binemit() {
         "E748C00038F3",
         "vag %v20, %v8, %v12",
     ));
+    insns.push((
+        Inst::VecRRR {
+            op: VecBinaryOp::Add128,
+            rd: writable_vr(20),
+            rn: vr(8),
+            rm: vr(12),
+        },
+        "E748C00048F3",
+        "vaq %v20, %v8, %v12",
+    ));
     insns.push((
         Inst::VecRRR {
             op: VecBinaryOp::Sub8x16,
@@ -8210,6 +8220,16 @@ fn test_s390x_binemit() {
         "E748C00038F7",
         "vsg %v20, %v8, %v12",
     ));
+    insns.push((
+        Inst::VecRRR {
+            op: VecBinaryOp::Sub128,
+            rd: writable_vr(20),
+            rn: vr(8),
+            rm: vr(12),
+        },
+        "E748C00048F7",
+        "vsq %v20, %v8, %v12",
+    ));
     insns.push((
         Inst::VecRRR {
             op: VecBinaryOp::Mul8x16,
@@ -9089,6 +9109,78 @@ fn test_s390x_binemit() {
         "E74800003850",
         "vpopctg %v20, %v8",
     ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Clz8x16,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800000853",
+        "vclzb %v20, %v8",
+    ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Clz16x8,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800001853",
+        "vclzh %v20, %v8",
+    ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Clz32x4,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800002853",
+        "vclzf %v20, %v8",
+    ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Clz64x2,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800003853",
+        "vclzg %v20, %v8",
+    ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Ctz8x16,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800000852",
+        "vctzb %v20, %v8",
+    ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Ctz16x8,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800001852",
+        "vctzh %v20, %v8",
+    ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Ctz32x4,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800002852",
+        "vctzf %v20, %v8",
+    ));
+    insns.push((
+        Inst::VecRR {
+            op: VecUnaryOp::Ctz64x2,
+            rd: writable_vr(20),
+            rn: vr(8),
+        },
+        "E74800003852",
+        "vctzg %v20, %v8",
+    ));
     insns.push((
         Inst::VecRR {
             op: VecUnaryOp::UnpackULow8x16,
@@ -9780,6 +9872,24 @@ fn test_s390x_binemit() {
         "E748C01038F9",
         "vchlgs %v20, %v8, %v12",
     ));
+    insns.push((
+        Inst::VecInt128SCmpHi {
+            tmp: writable_vr(20),
+            rn: vr(8),
+            rm: vr(12),
+        },
+        "E7C8000030DBA7740005E748C01038F9",
+        "vecg %v12, %v8 ; jne 10 ; vchlgs %v20, %v8, %v12",
+    ));
+    insns.push((
+        Inst::VecInt128UCmpHi {
+            tmp: writable_vr(20),
+            rn: vr(8),
+            rm: vr(12),
+        },
+        "E7C8000030D9A7740005E748C01038F9",
+        "veclg %v12, %v8 ; jne 10 ; vchlgs %v20, %v8, %v12",
+    ));
 
     insns.push((
         Inst::VecFloatCmp {

cranelift/codegen/src/isa/s390x/inst/mod.rs

@@ -180,6 +180,8 @@ impl Inst {
             | Inst::VecIntCmpS { .. }
             | Inst::VecFloatCmp { .. }
             | Inst::VecFloatCmpS { .. }
+            | Inst::VecInt128SCmpHi { .. }
+            | Inst::VecInt128UCmpHi { .. }
             | Inst::VecLoad { .. }
             | Inst::VecStore { .. }
             | Inst::VecLoadReplicate { .. }
@@ -394,6 +396,7 @@ impl Inst {
                 lane_imm: 0,
             },
             _ if ty.is_vector() && ty.bits() == 128 => Inst::VecLoad { rd: into_reg, mem },
+            types::B128 | types::I128 => Inst::VecLoad { rd: into_reg, mem },
             _ => unimplemented!("gen_load({})", ty),
         }
     }
@@ -418,6 +421,7 @@ impl Inst {
                 lane_imm: 0,
             },
             _ if ty.is_vector() && ty.bits() == 128 => Inst::VecStore { rd: from_reg, mem },
+            types::B128 | types::I128 => Inst::VecStore { rd: from_reg, mem },
             _ => unimplemented!("gen_store({})", ty),
         }
     }
@@ -736,6 +740,11 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
             collector.reg_use(rn);
             collector.reg_use(rm);
         }
+        &Inst::VecInt128SCmpHi { tmp, rn, rm, .. } | &Inst::VecInt128UCmpHi { tmp, rn, rm, .. } => {
+            collector.reg_def(tmp);
+            collector.reg_use(rn);
+            collector.reg_use(rm);
+        }
         &Inst::VecLoad { rd, ref mem, .. } => {
             collector.reg_def(rd);
             memarg_operands(mem, collector);
@@ -979,6 +988,11 @@ impl MachInst for Inst {
             .only_reg()
             .expect("multi-reg values not supported yet");
         match ty {
+            types::I128 | types::B128 => {
+                let mut ret = SmallVec::new();
+                ret.push(Inst::load_vec_constant(to_reg, value));
+                ret
+            }
             _ if ty.is_vector() && ty.bits() == 128 => {
                 let mut ret = SmallVec::new();
                 ret.push(Inst::load_vec_constant(to_reg, value));
@@ -1037,8 +1051,8 @@ impl MachInst for Inst {
             types::R64 => Ok((&[RegClass::Int], &[types::R64])),
             types::F32 => Ok((&[RegClass::Float], &[types::F32])),
             types::F64 => Ok((&[RegClass::Float], &[types::F64])),
-            types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
+            types::I128 => Ok((&[RegClass::Float], &[types::I128])),
-            types::B128 => Ok((&[RegClass::Int, RegClass::Int], &[types::B64, types::B64])),
+            types::B128 => Ok((&[RegClass::Float], &[types::B128])),
             _ if ty.is_vector() && ty.bits() == 128 => Ok((&[RegClass::Float], &[types::I8X16])),
             // FIXME: We don't really have IFLAGS, but need to allow it here
             // for now to support the SelectifSpectreGuard instruction.
@@ -2202,10 +2216,12 @@ impl Inst {
                     VecBinaryOp::Add16x8 => "vah",
                     VecBinaryOp::Add32x4 => "vaf",
                     VecBinaryOp::Add64x2 => "vag",
+                    VecBinaryOp::Add128 => "vaq",
                     VecBinaryOp::Sub8x16 => "vsb",
                     VecBinaryOp::Sub16x8 => "vsh",
                     VecBinaryOp::Sub32x4 => "vsf",
                     VecBinaryOp::Sub64x2 => "vsg",
+                    VecBinaryOp::Sub128 => "vsq",
                     VecBinaryOp::Mul8x16 => "vmlb",
                     VecBinaryOp::Mul16x8 => "vmlhw",
                     VecBinaryOp::Mul32x4 => "vmlf",
@@ -2303,6 +2319,14 @@ impl Inst {
                     VecUnaryOp::Popcnt16x8 => "vpopcth",
                     VecUnaryOp::Popcnt32x4 => "vpopctf",
                     VecUnaryOp::Popcnt64x2 => "vpopctg",
+                    VecUnaryOp::Clz8x16 => "vclzb",
+                    VecUnaryOp::Clz16x8 => "vclzh",
+                    VecUnaryOp::Clz32x4 => "vclzf",
+                    VecUnaryOp::Clz64x2 => "vclzg",
+                    VecUnaryOp::Ctz8x16 => "vctzb",
+                    VecUnaryOp::Ctz16x8 => "vctzh",
+                    VecUnaryOp::Ctz32x4 => "vctzf",
+                    VecUnaryOp::Ctz64x2 => "vctzg",
                     VecUnaryOp::UnpackULow8x16 => "vupllb",
                     VecUnaryOp::UnpackULow16x8 => "vupllh",
                     VecUnaryOp::UnpackULow32x4 => "vupllf",
@@ -2425,6 +2449,20 @@ impl Inst {
                 let rm = pretty_print_reg(rm, allocs);
                 format!("{}{} {}, {}, {}", op, s, rd, rn, rm)
             }
+            &Inst::VecInt128SCmpHi { tmp, rn, rm } | &Inst::VecInt128UCmpHi { tmp, rn, rm } => {
+                let op = match self {
+                    &Inst::VecInt128SCmpHi { .. } => "vecg",
+                    &Inst::VecInt128UCmpHi { .. } => "veclg",
+                    _ => unreachable!(),
+                };
+                let tmp = pretty_print_reg(tmp.to_reg(), allocs);
+                let rn = pretty_print_reg(rn, allocs);
+                let rm = pretty_print_reg(rm, allocs);
+                format!(
+                    "{} {}, {} ; jne 10 ; vchlgs {}, {}, {}",
+                    op, rm, rn, tmp, rn, rm
+                )
+            }
             &Inst::VecLoad { rd, ref mem } | &Inst::VecLoadRev { rd, ref mem } => {
                 let opcode = match self {
                     &Inst::VecLoad { .. } => "vl",

cranelift/codegen/src/isa/s390x/lower.isle

@@ -60,6 +60,21 @@
       x)
 
 
+;;;; Rules for `iconcat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (vr128_ty ty) (iconcat x y)))
+      (mov_to_vec128 ty y x))
+
+
+;;;; Rules for `isplit` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (gpr64_ty ty) (isplit x)))
+      (let ((x_reg Reg x)
+            (x_hi Reg (vec_extract_lane $I64X2 x_reg 0 (zero_reg)))
+            (x_lo Reg (vec_extract_lane $I64X2 x_reg 1 (zero_reg))))
+        (output_pair x_lo x_hi)))
+
+
 ;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Add two registers.
@@ -105,7 +120,7 @@
       (add_mem_sext32 ty y (sink_sload32 x)))
 
 ;; Add two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (iadd x y)))
+(rule (lower (has_type (vr128_ty ty) (iadd x y)))
       (vec_add ty x y))
 
 
@@ -169,7 +184,7 @@
       (sub_mem_sext32 ty x (sink_sload32 y)))
 
 ;; Sub two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (isub x y)))
+(rule (lower (has_type (vr128_ty ty) (isub x y)))
       (vec_sub ty x y))
 
 
@@ -191,22 +206,6 @@
                                                   (vec_unpacks_low ty y))))
 
 
-;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; Absolute value of a register.
-;; For types smaller than 32-bit, the input value must be sign-extended.
-(rule (lower (has_type (fits_in_64 ty) (iabs x)))
-      (abs_reg (ty_ext32 ty) (put_in_reg_sext32 x)))
-
-;; Absolute value of a sign-extended register.
-(rule (lower (has_type (fits_in_64 ty) (iabs (sext32_value x))))
-      (abs_reg_sext32 ty x))
-
-;; Absolute value of a vector register.
-(rule (lower (has_type (ty_vec128 ty) (iabs x)))
-      (vec_abs ty x))
-
-
 ;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; N.B.: the second output of `iadd_ifcout` is meant to be the `iflags` value
@@ -248,6 +247,31 @@
       (output_ifcout (add_logical_mem_zext32 ty y (sink_uload32 x))))
 
 
+;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Absolute value of a register.
+;; For types smaller than 32-bit, the input value must be sign-extended.
+(rule (lower (has_type (fits_in_64 ty) (iabs x)))
+      (abs_reg (ty_ext32 ty) (put_in_reg_sext32 x)))
+
+;; Absolute value of a sign-extended register.
+(rule (lower (has_type (fits_in_64 ty) (iabs (sext32_value x))))
+      (abs_reg_sext32 ty x))
+
+;; Absolute value of a vector register.
+(rule (lower (has_type (ty_vec128 ty) (iabs x)))
+      (vec_abs ty x))
+
+;; Absolute value of a 128-bit integer.
+(rule (lower (has_type $I128 (iabs x)))
+      (let ((zero Reg (vec_imm $I128 0))
+            (pos Reg x)
+            (neg Reg (vec_sub $I128 zero pos))
+            (rep Reg (vec_replicate_lane $I64X2 pos 0))
+            (mask Reg (vec_cmph $I64X2 zero rep)))
+        (vec_select $I128 neg pos mask)))
+
+
 ;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Negate a register.
@@ -262,6 +286,10 @@
 (rule (lower (has_type (ty_vec128 ty) (ineg x)))
       (vec_neg ty x))
 
+;; Negate a 128-bit integer.
+(rule (lower (has_type $I128 (ineg x)))
+      (vec_sub $I128 (vec_imm $I128 0) x))
+
 
 ;;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -344,7 +372,7 @@
 
 ;; Multiply two vector registers, using a helper.
 (decl vec_mul_impl (Type Reg Reg) Reg)
-(rule (lower (has_type (ty_vec128 ty) (imul x y)))
+(rule (lower (has_type (vr128_ty ty) (imul x y)))
       (vec_mul_impl ty x y))
 
 ;; Multiply two vector registers - byte, halfword, and word.
@@ -360,6 +388,20 @@
                      (mul_reg $I64 (vec_extract_lane $I64X2 x 1 (zero_reg))
                                    (vec_extract_lane $I64X2 y 1 (zero_reg)))))
 
+;; Multiply two vector registers - quadword.
+(rule (vec_mul_impl $I128 x y)
+      (let ((x_hi Reg (vec_extract_lane $I64X2 x 0 (zero_reg)))
+            (x_lo Reg (vec_extract_lane $I64X2 x 1 (zero_reg)))
+            (y_hi Reg (vec_extract_lane $I64X2 y 0 (zero_reg)))
+            (y_lo Reg (vec_extract_lane $I64X2 y 1 (zero_reg)))
+            (lo_pair RegPair (umul_wide x_lo y_lo))
+            (res_lo Reg (copy_reg $I64 (regpair_lo lo_pair)))
+            (res_hi_1 Reg (copy_reg $I64 (regpair_hi lo_pair)))
+            (res_hi_2 Reg (mul_reg $I64 x_lo y_hi))
+            (res_hi_3 Reg (mul_reg $I64 x_hi y_lo))
+            (res_hi Reg (add_reg $I64 res_hi_3 (add_reg $I64 res_hi_2 res_hi_1))))
+        (mov_to_vec128 $I64X2 res_hi res_lo)))
+
 
 ;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -666,7 +708,7 @@
 
 ;; Shift left, shift amount in register.
 (rule (lower (has_type (fits_in_64 ty) (ishl x y)))
-      (let ((masked_amt Reg (mask_amt_reg ty y)))
+      (let ((masked_amt Reg (mask_amt_reg ty (amt_reg y))))
         (lshl_reg ty x masked_amt)))
 
 ;; Shift left, immediate shift amount.
@@ -676,13 +718,18 @@
 
 ;; Vector shift left, shift amount in register.
 (rule (lower (has_type (ty_vec128 ty) (ishl x y)))
-      (vec_lshl_reg ty x y))
+      (vec_lshl_reg ty x (amt_reg y)))
 
 ;; Vector shift left, immediate shift amount.
 (rule (lower (has_type (ty_vec128 ty) (ishl x (i64_from_value y))))
       (let ((masked_amt u8 (mask_amt_imm ty y)))
         (vec_lshl_imm ty x masked_amt)))
 
+;; 128-bit vector shift left.
+(rule (lower (has_type $I128 (ishl x y)))
+      (let ((amt Reg (amt_vr y)))
+        (vec_lshl_by_bit (vec_lshl_by_byte x amt) amt)))
+
 
 ;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -690,7 +737,7 @@
 ;; For types smaller than 32-bit, the input value must be zero-extended.
 (rule (lower (has_type (fits_in_64 ty) (ushr x y)))
       (let ((ext_reg Reg (put_in_reg_zext32 x))
-            (masked_amt Reg (mask_amt_reg ty y)))
+            (masked_amt Reg (mask_amt_reg ty (amt_reg y))))
         (lshr_reg (ty_ext32 ty) ext_reg masked_amt)))
 
 ;; Shift right logical, immediate shift amount.
@@ -702,13 +749,18 @@
 
 ;; Vector shift right logical, shift amount in register.
 (rule (lower (has_type (ty_vec128 ty) (ushr x y)))
-      (vec_lshr_reg ty x y))
+      (vec_lshr_reg ty x (amt_reg y)))
 
 ;; Vector shift right logical, immediate shift amount.
 (rule (lower (has_type (ty_vec128 ty) (ushr x (i64_from_value y))))
       (let ((masked_amt u8 (mask_amt_imm ty y)))
         (vec_lshr_imm ty x masked_amt)))
 
+;; 128-bit vector shift right logical.
+(rule (lower (has_type $I128 (ushr x y)))
+      (let ((amt Reg (amt_vr y)))
+        (vec_lshr_by_bit (vec_lshr_by_byte x amt) amt)))
+
 
 ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -716,7 +768,7 @@
 ;; For types smaller than 32-bit, the input value must be sign-extended.
 (rule (lower (has_type (fits_in_64 ty) (sshr x y)))
       (let ((ext_reg Reg (put_in_reg_sext32 x))
-            (masked_amt Reg (mask_amt_reg ty y)))
+            (masked_amt Reg (mask_amt_reg ty (amt_reg y))))
         (ashr_reg (ty_ext32 ty) ext_reg masked_amt)))
 
 ;; Shift right arithmetic, immediate shift amount.
@@ -728,19 +780,24 @@
 
 ;; Vector shift right arithmetic, shift amount in register.
 (rule (lower (has_type (ty_vec128 ty) (sshr x y)))
-      (vec_ashr_reg ty x y))
+      (vec_ashr_reg ty x (amt_reg y)))
 
 ;; Vector shift right arithmetic, immediate shift amount.
 (rule (lower (has_type (ty_vec128 ty) (sshr x (i64_from_value y))))
       (let ((masked_amt u8 (mask_amt_imm ty y)))
         (vec_ashr_imm ty x masked_amt)))
 
+;; 128-bit vector shift right arithmetic.
+(rule (lower (has_type $I128 (sshr x y)))
+      (let ((amt Reg (amt_vr y)))
+        (vec_ashr_by_bit (vec_ashr_by_byte x amt) amt)))
+
 
 ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Rotate left, shift amount in register.  32-bit or 64-bit types.
 (rule (lower (has_type (ty_32_or_64 ty) (rotl x y)))
-      (rot_reg ty x y))
+      (rot_reg ty x (amt_reg y)))
 
 ;; Rotate left arithmetic, immediate shift amount.  32-bit or 64-bit types.
 (rule (lower (has_type (ty_32_or_64 ty) (rotl x (i64_from_value y))))
@@ -752,8 +809,8 @@
 (rule (lower (has_type (ty_8_or_16 ty) (rotl x y)))
       (let ((ext_reg Reg (put_in_reg_zext32 x))
             (ext_ty Type (ty_ext32 ty))
-            (pos_amt Reg y)
+            (pos_amt Reg (amt_reg y))
-            (neg_amt Reg (neg_reg ty pos_amt))
+            (neg_amt Reg (neg_reg $I32 pos_amt))
             (masked_pos_amt Reg (mask_amt_reg ty pos_amt))
             (masked_neg_amt Reg (mask_amt_reg ty neg_amt)))
         (or_reg ty (lshl_reg ext_ty ext_reg masked_pos_amt)
@@ -772,20 +829,30 @@
 
 ;; Vector rotate left, shift amount in register.
 (rule (lower (has_type (ty_vec128 ty) (rotl x y)))
-      (vec_rot_reg ty x y))
+      (vec_rot_reg ty x (amt_reg y)))
 
 ;; Vector rotate left, immediate shift amount.
 (rule (lower (has_type (ty_vec128 ty) (rotl x (i64_from_value y))))
       (let ((masked_amt u8 (mask_amt_imm ty y)))
         (vec_rot_imm ty x masked_amt)))
 
+;; 128-bit full vector rotate left.
+;; Implemented via a pair of 128-bit full vector shifts.
+(rule (lower (has_type $I128 (rotl x y)))
+      (let ((x_reg Reg x)
+            (pos_amt Reg (amt_vr y))
+            (neg_amt Reg (vec_neg $I8X16 pos_amt)))
+        (vec_or $I128
+          (vec_lshl_by_bit (vec_lshl_by_byte x_reg pos_amt) pos_amt)
+          (vec_lshr_by_bit (vec_lshr_by_byte x_reg neg_amt) neg_amt))))
+
 
 ;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Rotate right, shift amount in register.  32-bit or 64-bit types.
 ;; Implemented as rotate left with negated rotate amount.
 (rule (lower (has_type (ty_32_or_64 ty) (rotr x y)))
-      (let ((negated_amt Reg (neg_reg ty y)))
+      (let ((negated_amt Reg (neg_reg $I32 (amt_reg y))))
         (rot_reg ty x negated_amt)))
 
 ;; Rotate right arithmetic, immediate shift amount.  32-bit or 64-bit types.
@@ -799,8 +866,8 @@
 (rule (lower (has_type (ty_8_or_16 ty) (rotr x y)))
       (let ((ext_reg Reg (put_in_reg_zext32 x))
             (ext_ty Type (ty_ext32 ty))
-            (pos_amt Reg y)
+            (pos_amt Reg (amt_reg y))
-            (neg_amt Reg (neg_reg ty pos_amt))
+            (neg_amt Reg (neg_reg $I32 pos_amt))
             (masked_pos_amt Reg (mask_amt_reg ty pos_amt))
             (masked_neg_amt Reg (mask_amt_reg ty neg_amt)))
         (or_reg ty (lshl_reg ext_ty ext_reg masked_neg_amt)
@@ -820,7 +887,7 @@
 ;; Vector rotate right, shift amount in register.
 ;; Implemented as rotate left with negated rotate amount.
 (rule (lower (has_type (ty_vec128 ty) (rotr x y)))
-      (let ((negated_amt Reg (neg_reg $I32 y)))
+      (let ((negated_amt Reg (neg_reg $I32 (amt_reg y))))
         (vec_rot_reg ty x negated_amt)))
 
 ;; Vector rotate right, immediate shift amount.
@@ -829,13 +896,27 @@
       (let ((negated_amt u8 (mask_amt_imm ty y)))
         (vec_rot_imm ty x negated_amt)))
 
+;; 128-bit full vector rotate right.
+;; Implemented via a pair of 128-bit full vector shifts.
+(rule (lower (has_type $I128 (rotr x y)))
+      (let ((x_reg Reg x)
+            (pos_amt Reg (amt_vr y))
+            (neg_amt Reg (vec_neg $I8X16 pos_amt)))
+        (vec_or $I128
+          (vec_lshl_by_bit (vec_lshl_by_byte x_reg neg_amt) neg_amt)
+          (vec_lshr_by_bit (vec_lshr_by_byte x_reg pos_amt) pos_amt))))
+
 
 ;;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; Always a no-op.
+;; Up to 64-bit source type: Always a no-op.
-(rule (lower (ireduce x))
+(rule (lower (ireduce x @ (value_type (fits_in_64 _ty))))
       x)
 
+;; 128-bit source type: Extract the low half.
+(rule (lower (ireduce x @ (value_type (vr128_ty _ty))))
+      (vec_extract_lane $I64X2 x 1 (zero_reg)))
+
 
 ;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -847,6 +928,11 @@
 (rule (lower (has_type (gpr64_ty _ty) (uextend x)))
       (put_in_reg_zext64 x))
 
+;; 128-bit target types.
+(rule (lower (has_type (vr128_ty _ty) (uextend x @ (value_type src_ty))))
+      (let ((ty Type (ty_vec128_from_lane_ty src_ty)))
+        (vec_insert_lane ty (vec_imm ty 0) x (be_lane_idx ty 0) (zero_reg))))
+
 
 ;;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -858,6 +944,11 @@
 (rule (lower (has_type (gpr64_ty _ty) (sextend x)))
       (put_in_reg_sext64 x))
 
+;; 128-bit target types.
+(rule (lower (has_type (vr128_ty ty) (sextend x)))
+      (let ((x_ext Reg (put_in_reg_sext64 x)))
+        (mov_to_vec128 ty (ashr_imm $I64 x_ext 63) x_ext)))
+
 
 ;;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -914,7 +1005,7 @@
       (not_reg ty x))
 
 ;; Vector version using vector NOR.
-(rule (lower (has_type (ty_vec128 ty) (bnot x)))
+(rule (lower (has_type (vr128_ty ty) (bnot x)))
       (vec_not ty x))
 
 
@@ -941,7 +1032,7 @@
       (and_mem ty y (sink_load x)))
 
 ;; And two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (band x y)))
+(rule (lower (has_type (vr128_ty ty) (band x y)))
       (vec_and ty x y))
 
 ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -967,7 +1058,7 @@
       (or_mem ty y (sink_load x)))
 
 ;; Or two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (bor x y)))
+(rule (lower (has_type (vr128_ty ty) (bor x y)))
       (vec_or ty x y))
 
 
@@ -990,7 +1081,7 @@
       (xor_mem ty y (sink_load x)))
 
 ;; Xor two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (bxor x y)))
+(rule (lower (has_type (vr128_ty ty) (bxor x y)))
       (vec_xor ty x y))
 
 
@@ -1005,7 +1096,7 @@
       (and_reg ty x (not_reg ty y)))
 
 ;; And-not two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (band_not x y)))
+(rule (lower (has_type (vr128_ty ty) (band_not x y)))
       (vec_and_not ty x y))
 
 
@@ -1020,7 +1111,7 @@
       (or_reg ty x (not_reg ty y)))
 
 ;; Or-not two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (bor_not x y)))
+(rule (lower (has_type (vr128_ty ty) (bor_not x y)))
       (vec_or_not ty x y))
 
 
@@ -1035,7 +1126,7 @@
       (not_reg ty (xor_reg ty x y)))
 
 ;; Xor-not two vector registers.
-(rule (lower (has_type (ty_vec128 ty) (bxor_not x y)))
+(rule (lower (has_type (vr128_ty ty) (bxor_not x y)))
       (vec_not_xor ty x y))
 
 
@@ -1056,7 +1147,7 @@
         (or_reg ty if_false if_true)))
 
 ;; Bitselect vector registers.
-(rule (lower (has_type (ty_vec128 ty) (bitselect x y z)))
+(rule (lower (has_type (vr128_ty ty) (bitselect x y z)))
       (vec_select ty y z x))
 
 
@@ -1069,10 +1160,14 @@
 
 ;;;; Rules for `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; Always a no-op.
+;; Up to 64-bit source type: Always a no-op.
-(rule (lower (breduce x))
+(rule (lower (breduce x @ (value_type (fits_in_64 _ty))))
       x)
 
+;; 128-bit source type: Extract the low half.
+(rule (lower (breduce x @ (value_type (vr128_ty _ty))))
+      (vec_extract_lane $I64X2 x 1 (zero_reg)))
+
 
 ;;;; Rules for `bextend` and `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1091,12 +1186,18 @@
 (rule (cast_bool (fits_in_16 _ty) x @ (value_type $B16)) x)
 (rule (cast_bool (fits_in_32 _ty) x @ (value_type $B32)) x)
 (rule (cast_bool (fits_in_64 _ty) x @ (value_type $B64)) x)
+(rule (cast_bool (vr128_ty _ty) x @ (value_type $B128)) x)
+(rule (cast_bool (fits_in_64 _ty) x @ (value_type $B128))
+      (vec_extract_lane $I64X2 x 1 (zero_reg)))
 
 ;; Single-bit values are sign-extended via a pair of shifts.
 (rule (cast_bool (gpr32_ty ty) x @ (value_type $B1))
       (ashr_imm $I32 (lshl_imm $I32 x 31) 31))
 (rule (cast_bool (gpr64_ty ty) x @ (value_type $B1))
       (ashr_imm $I64 (lshl_imm $I64 x 63) 63))
+(rule (cast_bool (vr128_ty ty) x @ (value_type $B1))
+      (let ((gpr Reg (ashr_imm $I64 (lshl_imm $I64 x 63) 63)))
+        (mov_to_vec128 ty gpr gpr)))
 
 ;; Other values are just sign-extended normally.
 (rule (cast_bool (gpr32_ty _ty) x @ (value_type $B8))
@@ -1109,22 +1210,41 @@
       (sext64_reg $I16 x))
 (rule (cast_bool (gpr64_ty _ty) x @ (value_type $B32))
       (sext64_reg $I32 x))
+(rule (cast_bool (vr128_ty ty) x @ (value_type (gpr32_ty src_ty)))
+      (let ((x_ext Reg (sext64_reg src_ty x)))
+        (mov_to_vec128 ty x_ext x_ext)))
+(rule (cast_bool (vr128_ty ty) x @ (value_type (gpr64_ty src_ty)))
+      (mov_to_vec128 ty x x))
 
 
 ;;;; Rules for `bint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; Mask with 1 to get a 0/1 result (8- or 16-bit types).
+;; Mask with 1 to get a 0/1 result (8- or 16-bit result types).
-(rule (lower (has_type (fits_in_16 ty) (bint x)))
+(rule (lower (has_type (fits_in_16 ty) (bint x @ (value_type (fits_in_64 _)))))
       (and_uimm16shifted ty x (uimm16shifted 1 0)))
 
-;; Mask with 1 to get a 0/1 result (32-bit types).
+;; Mask with 1 to get a 0/1 result (32-bit result types).
-(rule (lower (has_type (fits_in_32 ty) (bint x)))
+(rule (lower (has_type (fits_in_32 ty) (bint x @ (value_type (fits_in_64 _)))))
       (and_uimm32shifted ty x (uimm32shifted 1 0)))
 
-;; Mask with 1 to get a 0/1 result (64-bit types).
+;; Mask with 1 to get a 0/1 result (64-bit result types).
-(rule (lower (has_type (fits_in_64 ty) (bint x)))
+(rule (lower (has_type (fits_in_64 ty) (bint x @ (value_type (fits_in_64 _)))))
       (and_reg ty x (imm ty 1)))
 
+;; Mask with 1 to get a 0/1 result (128-bit result types).
+(rule (lower (has_type (vr128_ty ty) (bint x @ (value_type (fits_in_64 _)))))
+      (let ((x_ext Reg (and_uimm16shifted $I8 x (uimm16shifted 1 0))))
+        (vec_insert_lane $I8X16 (vec_imm ty 0) x_ext 15 (zero_reg))))
+
+;; Mask with 1 to get a 0/1 result (128-bit source types).
+(rule (lower (has_type (fits_in_64 ty) (bint x @ (value_type (vr128_ty _)))))
+      (let ((x_gpr Reg (vec_extract_lane $I8X16 x 15 (zero_reg))))
+        (and_uimm16shifted ty x_gpr (uimm16shifted 1 0))))
+
+;; Mask with 1 to get a 0/1 result (128-bit source and result types).
+(rule (lower (has_type (vr128_ty ty) (bint x @ (value_type (vr128_ty _)))))
+      (vec_and ty x (vec_imm ty 1)))
+
 
 ;;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1146,6 +1266,16 @@
             (clz RegPair (clz_reg 64 ext_reg)))
         (clz_offset ty (regpair_hi clz))))
 
+;; Count leading zeros, 128-bit full vector.
+(rule (lower (has_type $I128 (clz x)))
+      (let ((clz_vec Reg (vec_clz $I64X2 x))
+            (zero Reg (vec_imm $I64X2 0))
+            (clz_hi Reg (vec_permute_dw_imm $I64X2 zero 0 clz_vec 0))
+            (clz_lo Reg (vec_permute_dw_imm $I64X2 zero 0 clz_vec 1))
+            (clz_sum Reg (vec_add $I64X2 clz_hi clz_lo))
+            (mask Reg (vec_cmpeq $I64X2 clz_hi (vec_imm_splat $I64X2 64))))
+        (vec_select $I128 clz_sum clz_hi mask)))
+
 
 ;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1170,6 +1300,20 @@
             (clz RegPair (clz_reg 64 inv_reg)))
         (cls_offset ty (regpair_hi clz))))
 
+;; Count leading sign-bit copies, 128-bit full vector.
+(rule (lower (has_type $I128 (cls x)))
+      (let ((x_reg Reg x)
+            (ones Reg (vec_imm_splat $I8X16 255))
+            (signbit_copies Reg (vec_ashr_by_bit (vec_ashr_by_byte x_reg ones) ones))
+            (inv_reg Reg (vec_xor $I128 x_reg signbit_copies))
+            (clz_vec Reg (vec_clz $I64X2 inv_reg))
+            (zero Reg (vec_imm $I64X2 0))
+            (clz_hi Reg (vec_permute_dw_imm $I64X2 zero 0 clz_vec 0))
+            (clz_lo Reg (vec_permute_dw_imm $I64X2 zero 0 clz_vec 1))
+            (clz_sum Reg (vec_add $I64X2 clz_hi clz_lo))
+            (mask Reg (vec_cmpeq $I64X2 clz_hi (vec_imm_splat $I64X2 64))))
+        (vec_add $I128 (vec_select $I128 clz_sum clz_hi mask) ones)))
+
 
 ;;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1207,6 +1351,16 @@
             (clz RegPair (clz_reg -1 lastbit)))
         (sub_reg $I64 (imm $I64 63) (regpair_hi clz))))
 
+;; Count trailing zeros, 128-bit full vector.
+(rule (lower (has_type $I128 (ctz x)))
+      (let ((ctz_vec Reg (vec_ctz $I64X2 x))
+            (zero Reg (vec_imm $I64X2 0))
+            (ctz_hi Reg (vec_permute_dw_imm $I64X2 zero 0 ctz_vec 0))
+            (ctz_lo Reg (vec_permute_dw_imm $I64X2 zero 0 ctz_vec 1))
+            (ctz_sum Reg (vec_add $I64X2 ctz_hi ctz_lo))
+            (mask Reg (vec_cmpeq $I64X2 ctz_lo (vec_imm_splat $I64X2 64))))
+        (vec_select $I128 ctz_sum ctz_lo mask)))
+
 
 ;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1249,6 +1403,14 @@
 (rule (lower (has_type (ty_vec128 ty) (popcnt x)))
       (vec_popcnt ty x))
 
+;; Population count, 128-bit full vector.
+(rule (lower (has_type $I128 (popcnt x)))
+      (let ((popcnt_vec Reg (vec_popcnt $I64X2 x))
+            (zero Reg (vec_imm $I64X2 0))
+            (popcnt_hi Reg (vec_permute_dw_imm $I64X2 zero 0 popcnt_vec 0))
+            (popcnt_lo Reg (vec_permute_dw_imm $I64X2 zero 0 popcnt_vec 1)))
+        (vec_add $I64X2 popcnt_hi popcnt_lo)))
+
 
 ;;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -2262,16 +2424,16 @@
       (vec_load_lane_little_undef $F64X2 (lower_address flags addr offset) 0))
 
 ;; Load 128-bit big-endian vector values.
-(rule (lower (has_type (ty_vec128 ty) (load flags @ (bigendian) addr offset)))
+(rule (lower (has_type (vr128_ty ty) (load flags @ (bigendian) addr offset)))
       (vec_load ty (lower_address flags addr offset)))
 
 ;; Load 128-bit little-endian vector values (z15 instruction).
-(rule (lower (has_type (and (vxrs_ext2_enabled) (ty_vec128 ty))
+(rule (lower (has_type (and (vxrs_ext2_enabled) (vr128_ty ty))
                        (load flags @ (littleendian) addr offset)))
       (vec_loadrev ty (lower_address flags addr offset)))
 
 ;; Load 128-bit little-endian vector values (via GPRs on z14).
-(rule (lower (has_type (and (vxrs_ext2_disabled) (ty_vec128 ty))
+(rule (lower (has_type (and (vxrs_ext2_disabled) (vr128_ty ty))
                        (load flags @ (littleendian) addr offset)))
       (let ((lo_addr MemArg (lower_address_bias flags addr offset 0))
             (hi_addr MemArg (lower_address_bias flags addr offset 8))
@@ -2494,17 +2656,17 @@
 
 ;; Store 128-bit big-endian vector type.
 (rule (lower (store flags @ (bigendian)
-                    val @ (value_type (ty_vec128 ty)) addr offset))
+                    val @ (value_type (vr128_ty ty)) addr offset))
       (side_effect (vec_store val (lower_address flags addr offset))))
 
 ;; Store 128-bit little-endian vector type (z15 instruction).
 (rule (lower (store flags @ (littleendian)
-                    val @ (value_type (and (ty_vec128 ty) (vxrs_ext2_enabled))) addr offset))
+                    val @ (value_type (and (vr128_ty ty) (vxrs_ext2_enabled))) addr offset))
       (side_effect (vec_storerev val (lower_address flags addr offset))))
 
 ;; Store 128-bit little-endian vector type (via GPRs on z14).
 (rule (lower (store flags @ (littleendian)
-                    val @ (value_type (and (ty_vec128 ty) (vxrs_ext2_disabled))) addr offset))
+                    val @ (value_type (and (vr128_ty ty) (vxrs_ext2_disabled))) addr offset))
       (let ((lo_addr MemArg (lower_address_bias flags addr offset 0))
             (hi_addr MemArg (lower_address_bias flags addr offset 8))
             (lo_val Reg (vec_extract_lane $I64X2 val 1 (zero_reg)))
@@ -3028,10 +3190,10 @@
 (decl icmp_val (bool IntCC Value Value) ProducesBool)
 
 ;; Dispatch for signed comparisons.
-(rule (icmp_val allow_mem int_cc @ (signed) x y)
+(rule (icmp_val allow_mem int_cc @ (signed) x @ (value_type (fits_in_64 _)) y)
       (bool (icmps_val allow_mem x y) (intcc_as_cond int_cc)))
 ;; Dispatch for unsigned comparisons.
-(rule (icmp_val allow_mem int_cc @ (unsigned) x y)
+(rule (icmp_val allow_mem int_cc @ (unsigned) x @ (value_type (fits_in_64 _)) y)
       (bool (icmpu_val allow_mem x y) (intcc_as_cond int_cc)))
 
 
@@ -3104,6 +3266,39 @@
 (rule (icmpu_val $true x @ (value_type (fits_in_64 ty)) (sinkable_uload32 y))
       (icmpu_mem_zext32 ty x (sink_uload32 y)))
 
+
+;; Compare 128-bit integers for equality.
+;; Implemented via element-wise comparison using the all-element true CC flag.
+(rule (icmp_val _ (IntCC.Equal) x @ (value_type (vr128_ty _)) y)
+      (bool (vec_cmpeqs $I64X2 x y)
+            (floatcc_as_cond (FloatCC.Equal))))
+(rule (icmp_val _ (IntCC.NotEqual) x @ (value_type (vr128_ty _)) y)
+      (bool (vec_cmpeqs $I64X2 x y)
+            (floatcc_as_cond (FloatCC.NotEqual))))
+
+;; Compare (signed) 128-bit integers for relational inequality.
+;; Implemented via synthetic instruction using VECG and VCHLGS.
+(rule (icmp_val _ (IntCC.SignedGreaterThan) x @ (value_type (vr128_ty ty)) y)
+      (vec_int128_scmphi x y))
+(rule (icmp_val _ (IntCC.SignedLessThan) x @ (value_type (vr128_ty ty)) y)
+      (vec_int128_scmphi y x))
+(rule (icmp_val _ (IntCC.SignedGreaterThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+      (invert_bool (vec_int128_scmphi y x)))
+(rule (icmp_val _ (IntCC.SignedLessThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+      (invert_bool (vec_int128_scmphi x y)))
+
+;; Compare (unsigned) 128-bit integers for relational inequality.
+;; Implemented via synthetic instruction using VECLG and VCHLGS.
+(rule (icmp_val _ (IntCC.UnsignedGreaterThan) x @ (value_type (vr128_ty ty)) y)
+      (vec_int128_ucmphi x y))
+(rule (icmp_val _ (IntCC.UnsignedLessThan) x @ (value_type (vr128_ty ty)) y)
+      (vec_int128_ucmphi y x))
+(rule (icmp_val _ (IntCC.UnsignedGreaterThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+      (invert_bool (vec_int128_ucmphi y x)))
+(rule (icmp_val _ (IntCC.UnsignedLessThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+      (invert_bool (vec_int128_ucmphi x y)))
+
+
 ;; Vector `icmp` produces a boolean vector.
 ;; We need to handle the various IntCC flags separately here.
 
@@ -3384,6 +3579,9 @@
 (rule (value_nonzero val @ (value_type (gpr64_ty ty)))
       (bool (icmps_simm16 $I64 (put_in_reg val) 0)
                           (intcc_as_cond (IntCC.NotEqual))))
+(rule (value_nonzero val @ (value_type (vr128_ty ty)))
+      (bool (vec_cmpeqs $I64X2 val (vec_imm $I64X2 0))
+            (floatcc_as_cond (FloatCC.NotEqual))))
 
 ;; Main `select` entry point.  Lower the `value_nonzero` result.
 (rule (lower (has_type ty (select val_cond val_true val_false)))

cranelift/codegen/src/isa/s390x/lower.rs

@@ -50,6 +50,8 @@ impl LowerBackend for S390xBackend {
             | Opcode::F64const
             | Opcode::Vconst
             | Opcode::Null
+            | Opcode::Isplit
+            | Opcode::Iconcat
             | Opcode::Iadd
             | Opcode::IaddIfcout
             | Opcode::Isub
@@ -201,8 +203,6 @@ impl LowerBackend for S390xBackend {
             | Opcode::TlsValue
             | Opcode::GetPinnedReg
             | Opcode::SetPinnedReg
-            | Opcode::Isplit
-            | Opcode::Iconcat
             | Opcode::Vsplit
             | Opcode::Vconcat
             | Opcode::DynamicStackLoad

cranelift/codegen/src/isa/s390x/lower/isle.rs

@@ -247,6 +247,15 @@ where
         }
     }
 
+    #[inline]
+    fn vr128_ty(&mut self, ty: Type) -> Option<Type> {
+        match ty {
+            I128 | B128 => Some(ty),
+            _ if ty.is_vector() && ty.bits() == 128 => Some(ty),
+            _ => None,
+        }
+    }
+
     #[inline]
     fn uimm32shifted(&mut self, n: u32, shift: u8) -> UImm32Shifted {
         UImm32Shifted::maybe_with_shift(n, shift).unwrap()

cranelift/codegen/src/machinst/abi.rs

@@ -17,14 +17,14 @@ pub trait ABICallee {
     /// The instruction type for the ISA associated with this ABI.
     type I: VCodeInst;
 
-    /// Does the ABI-body code need a temp reg (and if so, of what type)? One
+    /// Does the ABI-body code need temp registers (and if so, of what type)?
-    /// will be provided to `init()` as the `maybe_tmp` arg if so.
+    /// They will be provided to `init()` as the `temps` arg if so.
-    fn temp_needed(&self) -> Option<Type>;
+    fn temps_needed(&self) -> Vec<Type>;
 
     /// Initialize. This is called after the ABICallee is constructed because it
-    /// may be provided with a temp vreg, which can only be allocated once the
+    /// may be provided with a vector of temp vregs, which can only be allocated
-    /// lowering context exists.
+    /// once the lowering context exists.
-    fn init(&mut self, maybe_tmp: Option<Writable<Reg>>);
+    fn init(&mut self, temps: Vec<Writable<Reg>>);
 
     /// Access the (possibly legalized) signature.
     fn signature(&self) -> &Signature;

cranelift/codegen/src/machinst/abi_impl.rs

@@ -196,6 +196,19 @@ pub enum ABIArg {
         /// Purpose of this arg.
         purpose: ir::ArgumentPurpose,
     },
+    /// Implicit argument. Similar to a StructArg, except that we have the
+    /// target type, not a pointer type, at the CLIF-level. This argument is
+    /// still being passed via reference implicitly.
+    ImplicitPtrArg {
+        /// Register or stack slot holding a pointer to the buffer.
+        pointer: ABIArgSlot,
+        /// Offset of the argument buffer.
+        offset: i64,
+        /// Type of the implicit argument.
+        ty: Type,
+        /// Purpose of this arg.
+        purpose: ir::ArgumentPurpose,
+    },
 }
 
 impl ABIArg {
@@ -604,6 +617,12 @@ impl ABISig {
                         }
                     }
                 }
+                &ABIArg::ImplicitPtrArg { ref pointer, .. } => match pointer {
+                    &ABIArgSlot::Reg { reg, .. } => {
+                        uses.push(Reg::from(reg));
+                    }
+                    _ => {}
+                },
             }
         }
 
@@ -707,6 +726,8 @@ pub struct ABICalleeImpl<M: ABIMachineSpec> {
     total_frame_size: Option<u32>,
     /// The register holding the return-area pointer, if needed.
     ret_area_ptr: Option<Writable<Reg>>,
+    /// Temp registers required for argument setup, if needed.
+    arg_temp_reg: Vec<Option<Writable<Reg>>>,
     /// Calling convention this function expects.
     call_conv: isa::CallConv,
     /// The settings controlling this function's compilation.
@@ -845,6 +866,7 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
             fixed_frame_storage_size: 0,
             total_frame_size: None,
             ret_area_ptr: None,
+            arg_temp_reg: vec![],
             call_conv,
             flags,
             isa_flags: isa_flags.clone(),
@@ -1033,18 +1055,39 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         &self.ir_sig
     }
 
-    fn temp_needed(&self) -> Option<Type> {
+    fn temps_needed(&self) -> Vec<Type> {
-        if self.sig.stack_ret_arg.is_some() {
+        let mut temp_tys = vec![];
-            Some(M::word_type())
+        for arg in &self.sig.args {
-        } else {
+            match arg {
-            None
+                &ABIArg::ImplicitPtrArg { pointer, .. } => match &pointer {
+                    &ABIArgSlot::Reg { .. } => {}
+                    &ABIArgSlot::Stack { ty, .. } => {
+                        temp_tys.push(ty);
+                    }
+                },
+                _ => {}
+            }
         }
+        if self.sig.stack_ret_arg.is_some() {
+            temp_tys.push(M::word_type());
+        }
+        temp_tys
     }
 
-    fn init(&mut self, maybe_tmp: Option<Writable<Reg>>) {
+    fn init(&mut self, temps: Vec<Writable<Reg>>) {
+        let mut temps_iter = temps.into_iter();
+        for arg in &self.sig.args {
+            let temp = match arg {
+                &ABIArg::ImplicitPtrArg { pointer, .. } => match &pointer {
+                    &ABIArgSlot::Reg { .. } => None,
+                    &ABIArgSlot::Stack { .. } => Some(temps_iter.next().unwrap()),
+                },
+                _ => None,
+            };
+            self.arg_temp_reg.push(temp);
+        }
         if self.sig.stack_ret_arg.is_some() {
-            assert!(maybe_tmp.is_some());
+            self.ret_area_ptr = Some(temps_iter.next().unwrap());
-            self.ret_area_ptr = maybe_tmp;
         }
     }
 
@@ -1150,6 +1193,28 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     ));
                 }
             }
+            &ABIArg::ImplicitPtrArg { pointer, ty, .. } => {
+                let into_reg = into_regs.only_reg().unwrap();
+                // We need to dereference the pointer.
+                let base = match &pointer {
+                    &ABIArgSlot::Reg { reg, .. } => Reg::from(reg),
+                    &ABIArgSlot::Stack { offset, ty, .. } => {
+                        // In this case we need a temp register to hold the address.
+                        // This was allocated in the `init` routine.
+                        let addr_reg = self.arg_temp_reg[idx].unwrap();
+                        insts.push(M::gen_load_stack(
+                            StackAMode::FPOffset(
+                                M::fp_to_arg_offset(self.call_conv, &self.flags) + offset,
+                                ty,
+                            ),
+                            addr_reg,
+                            ty,
+                        ));
+                        addr_reg.to_reg()
+                    }
+                };
+                insts.push(M::gen_load_base_offset(into_reg, base, 0, ty));
+            }
         }
         insts
     }
@@ -1241,6 +1306,9 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
             &ABIArg::StructArg { .. } => {
                 panic!("StructArg in return position is unsupported");
             }
+            &ABIArg::ImplicitPtrArg { .. } => {
+                panic!("ImplicitPtrArg in return position is unsupported");
+            }
         }
         ret
     }
@@ -1690,6 +1758,7 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     ctx.emit(insn);
                 }
             }
+            &ABIArg::ImplicitPtrArg { .. } => unimplemented!(), // Only supported via ISLE.
         }
     }
 
@@ -1772,6 +1841,7 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
             &ABIArg::StructArg { pointer, .. } => {
                 assert!(pointer.is_none()); // Only supported via ISLE.
             }
+            &ABIArg::ImplicitPtrArg { .. } => unimplemented!(), // Only supported via ISLE.
         }
     }
 
@@ -1805,6 +1875,9 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
             &ABIArg::StructArg { .. } => {
                 panic!("StructArg not supported in return position");
             }
+            &ABIArg::ImplicitPtrArg { .. } => {
+                panic!("ImplicitPtrArg not supported in return position");
+            }
         }
     }
 

cranelift/codegen/src/machinst/isle.rs

@@ -847,6 +847,18 @@ macro_rules! isle_prelude_methods {
             }
         }
 
+        fn abi_arg_implicit_pointer(&mut self, arg: &ABIArg) -> Option<(ABIArgSlot, i64, Type)> {
+            match arg {
+                &ABIArg::ImplicitPtrArg {
+                    pointer,
+                    offset,
+                    ty,
+                    ..
+                } => Some((pointer, offset, ty)),
+                _ => None,
+            }
+        }
+
         fn abi_stackslot_addr(
             &mut self,
             dst: WritableReg,

cranelift/codegen/src/machinst/lower.rs

@@ -1051,13 +1051,15 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
     pub fn lower<B: LowerBackend<MInst = I>>(mut self, backend: &B) -> CodegenResult<VCode<I>> {
         trace!("about to lower function: {:?}", self.f);
 
-        // Initialize the ABI object, giving it a temp if requested.
+        // Initialize the ABI object, giving it temps if requested.
-        let maybe_tmp = if let Some(temp_ty) = self.vcode.abi().temp_needed() {
+        let temps = self
-            Some(self.alloc_tmp(temp_ty).only_reg().unwrap())
+            .vcode
-        } else {
+            .abi()
-            None
+            .temps_needed()
-        };
+            .into_iter()
-        self.vcode.abi().init(maybe_tmp);
+            .map(|temp_ty| self.alloc_tmp(temp_ty).only_reg().unwrap())
+            .collect::<Vec<_>>();
+        self.vcode.abi().init(temps);
 
         // Get the pinned reg here (we only parameterize this function on `B`,
         // not the whole `Lower` impl).

cranelift/codegen/src/prelude.isle

@@ -906,6 +906,11 @@
 (decl abi_arg_struct_pointer (ABIArgSlot i64 u64) ABIArg)
 (extern extractor abi_arg_struct_pointer abi_arg_struct_pointer)
 
+;; Extractor to detect the special case where a non-struct argument
+;; is implicitly passed by reference using a hidden pointer.
+(decl abi_arg_implicit_pointer (ABIArgSlot i64 Type) ABIArg)
+(extern extractor abi_arg_implicit_pointer abi_arg_implicit_pointer)
+
 ;; Convert a real register number into a virtual register.
 (decl real_reg_to_reg (RealReg) Reg)
 (extern constructor real_reg_to_reg real_reg_to_reg)

cranelift/filetests/filetests/isa/s390x/arithmetic.clif

@@ -1,9 +1,18 @@
 test compile precise-output
 target s390x
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+function %iadd_i128(i128, i128) -> i128 {
-;; IADD
+block0(v0: i128, v1: i128):
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+  v2 = iadd.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vaq %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
 
 function %iadd_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
@@ -323,6 +332,19 @@ block0(v0: i32, v1: i64):
 ;   alr %r2, %r4
 ;   br %r14
 
+function %isub_i128(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = isub.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vsq %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %isub_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = isub.i64 v0, v1
@@ -540,6 +562,22 @@ block0(v0: i8, v1: i64):
 ;   sr %r2, %r4
 ;   br %r14
 
+function %iabs_i128(i128) -> i128 {
+block0(v0: i128):
+  v1 = iabs.i128 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vgbm %v5, 0
+;   vsq %v7, %v5, %v0
+;   vrepg %v17, %v0, 0
+;   vchg %v19, %v5, %v17
+;   vsel %v21, %v7, %v0, %v19
+;   vst %v21, 0(%r2)
+;   br %r14
+
 function %iabs_i64(i64) -> i64 {
 block0(v0: i64):
   v1 = iabs.i64 v0
@@ -593,6 +631,19 @@ block0(v0: i8):
 ;   lpr %r2, %r5
 ;   br %r14
 
+function %ineg_i128(i128) -> i128 {
+block0(v0: i128):
+  v1 = ineg.i128 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vgbm %v5, 0
+;   vsq %v7, %v5, %v0
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %ineg_i64(i64) -> i64 {
 block0(v0: i64):
   v1 = ineg.i64 v0
@@ -644,6 +695,32 @@ block0(v0: i8):
 ;   lcr %r2, %r2
 ;   br %r14
 
+function %imul_i128(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = imul.i128 v0, v1
+  return v2
+}
+
+;   stmg %r13, %r15, 104(%r15)
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   lgdr %r5, %f0
+;   vlgvg %r3, %v0, 1
+;   lgdr %r4, %f1
+;   vlgvg %r1, %v1, 1
+;   lgr %r13, %r1
+;   mlgr %r0, %r3
+;   msgr %r3, %r4
+;   lgr %r4, %r13
+;   msgr %r5, %r4
+;   agr %r3, %r0
+;   agr %r5, %r3
+;   vlvgp %v5, %r5, %r1
+;   vst %v5, 0(%r2)
+;   lmg %r13, %r15, 104(%r15)
+;   br %r14
+
 function %imul_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = imul.i64 v0, v1

cranelift/filetests/filetests/isa/s390x/bitops.clif

@@ -36,6 +36,25 @@ target s390x
 ;; CLZ
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+function %clz_i128(i128) -> i128 {
+block0(v0: i128):
+    v1 = clz v0
+    return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vclzg %v5, %v0
+;   vgbm %v7, 0
+;   vpdi %v17, %v7, %v5, 0
+;   vpdi %v19, %v7, %v5, 1
+;   vag %v21, %v17, %v19
+;   vrepig %v23, 64
+;   vceqg %v25, %v17, %v23
+;   vsel %v27, %v21, %v17, %v25
+;   vst %v27, 0(%r2)
+;   br %r14
+
 function %clz_i64(i64) -> i64 {
 block0(v0: i64):
     v1 = clz v0
@@ -83,6 +102,30 @@ block0(v0: i8):
 ;   ahik %r2, %r0, -56
 ;   br %r14
 
+function %cls_i128(i128) -> i128 {
+block0(v0: i128):
+    v1 = cls v0
+    return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vrepib %v5, 255
+;   vsrab %v7, %v0, %v5
+;   vsra %v17, %v7, %v5
+;   vx %v19, %v0, %v17
+;   vclzg %v21, %v19
+;   vgbm %v23, 0
+;   vpdi %v25, %v23, %v21, 0
+;   vpdi %v27, %v23, %v21, 1
+;   vag %v29, %v25, %v27
+;   vrepig %v31, 64
+;   vceqg %v1, %v25, %v31
+;   vsel %v3, %v29, %v25, %v1
+;   vaq %v5, %v3, %v5
+;   vst %v5, 0(%r2)
+;   br %r14
+
 function %cls_i64(i64) -> i64 {
 block0(v0: i64):
     v1 = cls v0
@@ -138,6 +181,25 @@ block0(v0: i8):
 ;   ahik %r2, %r0, -57
 ;   br %r14
 
+function %ctz_i128(i128) -> i128 {
+block0(v0: i128):
+    v1 = ctz v0
+    return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vctzg %v5, %v0
+;   vgbm %v7, 0
+;   vpdi %v17, %v7, %v5, 0
+;   vpdi %v19, %v7, %v5, 1
+;   vag %v21, %v17, %v19
+;   vrepig %v23, 64
+;   vceqg %v25, %v19, %v23
+;   vsel %v27, %v21, %v19, %v25
+;   vst %v27, 0(%r2)
+;   br %r14
+
 function %ctz_i64(i64) -> i64 {
 block0(v0: i64):
     v1 = ctz v0
@@ -198,6 +260,22 @@ block0(v0: i8):
 ;   srk %r2, %r5, %r0
 ;   br %r14
 
+function %popcnt_i128(i128) -> i128 {
+block0(v0: i128):
+    v1 = popcnt v0
+    return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vpopctg %v5, %v0
+;   vgbm %v7, 0
+;   vpdi %v17, %v7, %v5, 0
+;   vpdi %v19, %v7, %v5, 1
+;   vag %v21, %v17, %v19
+;   vst %v21, 0(%r2)
+;   br %r14
+
 function %popcnt_i64(i64) -> i64 {
 block0(v0: i64):
     v1 = popcnt v0

cranelift/filetests/filetests/isa/s390x/bitwise.clif

@@ -4,9 +4,18 @@ target s390x
 
 ; FIXME: add immediate operand versions
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+function %band_i128(i128, i128) -> i128 {
-;; BAND
+block0(v0: i128, v1: i128):
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+  v2 = band.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vn %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
 
 function %band_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
@@ -105,6 +114,19 @@ block0(v0: i8, v1: i64):
 ;   nr %r2, %r4
 ;   br %r14
 
+function %bor_i128(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = bor.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vo %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bor_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = bor.i64 v0, v1
@@ -202,6 +224,19 @@ block0(v0: i8, v1: i64):
 ;   or %r2, %r4
 ;   br %r14
 
+function %bxor_i128(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = bxor.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vx %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bxor_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = bxor.i64 v0, v1
@@ -299,6 +334,19 @@ block0(v0: i8, v1: i64):
 ;   xr %r2, %r4
 ;   br %r14
 
+function %band_not_i128(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = band_not.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vnc %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %band_not_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = band_not.i64 v0, v1
@@ -344,6 +392,19 @@ block0(v0: i8, v1: i8):
 ;   nr %r2, %r3
 ;   br %r14
 
+function %bor_not_i128(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = bor_not.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   voc %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bor_not_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = bor_not.i64 v0, v1
@@ -389,6 +450,19 @@ block0(v0: i8, v1: i8):
 ;   or %r2, %r3
 ;   br %r14
 
+function %bxor_not_i128(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = bxor_not.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vnx %v7, %v0, %v1
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bxor_not_i64(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = bxor_not.i64 v0, v1
@@ -434,6 +508,18 @@ block0(v0: i8, v1: i8):
 ;   xilf %r2, 4294967295
 ;   br %r14
 
+function %bnot_i128(i128) -> i128 {
+block0(v0: i128):
+  v1 = bnot.i128 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vno %v5, %v0, %v0
+;   vst %v5, 0(%r2)
+;   br %r14
+
 function %bnot_i64(i64) -> i64 {
 block0(v0: i64):
   v1 = bnot.i64 v0
@@ -475,6 +561,20 @@ block0(v0: i8):
 ;   xilf %r2, 4294967295
 ;   br %r14
 
+function %bitselect_i128(i128, i128, i128) -> i128 {
+block0(v0: i128, v1: i128, v2: i128):
+  v3 = bitselect.i128 v0, v1, v2
+  return v3
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vl %v2, 0(%r5)
+;   vsel %v17, %v1, %v2, %v0
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %bitselect_i64(i64, i64, i64) -> i64 {
 block0(v0: i64, v1: i64, v2: i64):
   v3 = bitselect.i64 v0, v1, v2

cranelift/filetests/filetests/isa/s390x/call.clif

@@ -183,3 +183,38 @@ block0(v0: i64, v1: i32, v2: i32, v3: i32, v4: i16, v5: i16, v6: i16, v7: i8, v8
 ;   lmg %r7, %r15, 72(%r15)
 ;   br %r14
 
+function %incoming_args_i128(i128, i128, i128, i128, i128, i128, i128, i128) -> i128 {
+block0(v0: i128, v1: i128, v2: i128, v3: i128, v4: i128, v5: i128, v6: i128, v7: i128):
+    v8 = iadd v0, v1
+    v9 = iadd v2, v3
+    v10 = iadd v4, v5
+    v11 = iadd v6, v7
+    v12 = iadd v8, v9
+    v13 = iadd v10, v11
+    v14 = iadd v12, v13
+    return v14
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vl %v2, 0(%r5)
+;   vl %v3, 0(%r6)
+;   lg %r3, 160(%r15)
+;   vl %v4, 0(%r3)
+;   lg %r3, 168(%r15)
+;   vl %v5, 0(%r3)
+;   lg %r5, 176(%r15)
+;   vl %v6, 0(%r5)
+;   lg %r4, 184(%r15)
+;   vl %v7, 0(%r4)
+;   vaq %v17, %v0, %v1
+;   vaq %v18, %v2, %v3
+;   vaq %v19, %v4, %v5
+;   vaq %v20, %v6, %v7
+;   vaq %v17, %v17, %v18
+;   vaq %v18, %v19, %v20
+;   vaq %v17, %v17, %v18
+;   vst %v17, 0(%r2)
+;   br %r14
+

cranelift/filetests/filetests/isa/s390x/concat-split.clif

@@ -0,0 +1,26 @@
+test compile precise-output
+target s390x
+
+function %iconcat_i64(i64, i64) -> i128 {
+block0(v0: i64, v1: i64):
+  v2 = iconcat.i64 v0, v1
+  return v2
+}
+
+; block0:
+;   vlvgp %v7, %r4, %r3
+;   vst %v7, 0(%r2)
+;   br %r14
+
+function %isplit_i128(i128) -> i64, i64 {
+block0(v0: i128):
+  v1, v2 = isplit.i128 v0
+  return v1, v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   lgdr %r3, %f0
+;   vlgvg %r2, %v0, 1
+;   br %r14
+

cranelift/filetests/filetests/isa/s390x/conversions.clif

@@ -1,9 +1,29 @@
 test compile precise-output
 target s390x
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+function %uextend_i64_i128(i64) -> i128 {
-;; UEXTEND
+block0(v0: i64):
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+  v1 = uextend.i128 v0
+  return v1
+}
+
+; block0:
+;   vgbm %v5, 0
+;   vlvgg %v5, %r3, 1
+;   vst %v5, 0(%r2)
+;   br %r14
+
+function %uextend_i32_i128(i32) -> i128 {
+block0(v0: i32):
+  v1 = uextend.i128 v0
+  return v1
+}
+
+; block0:
+;   vgbm %v5, 0
+;   vlvgf %v5, %r3, 3
+;   vst %v5, 0(%r2)
+;   br %r14
 
 function %uextend_i32_i64(i32) -> i64 {
 block0(v0: i32):
@@ -15,6 +35,18 @@ block0(v0: i32):
 ;   llgfr %r2, %r2
 ;   br %r14
 
+function %uextend_i16_i128(i16) -> i128 {
+block0(v0: i16):
+  v1 = uextend.i128 v0
+  return v1
+}
+
+; block0:
+;   vgbm %v5, 0
+;   vlvgh %v5, %r3, 7
+;   vst %v5, 0(%r2)
+;   br %r14
+
 function %uextend_i16_i64(i16) -> i64 {
 block0(v0: i16):
   v1 = uextend.i64 v0
@@ -35,6 +67,18 @@ block0(v0: i16):
 ;   llhr %r2, %r2
 ;   br %r14
 
+function %uextend_i8_i128(i8) -> i128 {
+block0(v0: i8):
+  v1 = uextend.i128 v0
+  return v1
+}
+
+; block0:
+;   vgbm %v5, 0
+;   vlvgb %v5, %r3, 15
+;   vst %v5, 0(%r2)
+;   br %r14
+
 function %uextend_i8_i64(i8) -> i64 {
 block0(v0: i8):
   v1 = uextend.i64 v0
@@ -65,6 +109,31 @@ block0(v0: i8):
 ;   llcr %r2, %r2
 ;   br %r14
 
+function %sextend_i64_i128(i64) -> i128 {
+block0(v0: i64):
+  v1 = sextend.i128 v0
+  return v1
+}
+
+; block0:
+;   srag %r4, %r3, 63
+;   vlvgp %v7, %r4, %r3
+;   vst %v7, 0(%r2)
+;   br %r14
+
+function %sextend_i32_i128(i32) -> i128 {
+block0(v0: i32):
+  v1 = sextend.i128 v0
+  return v1
+}
+
+; block0:
+;   lgfr %r3, %r3
+;   srag %r5, %r3, 63
+;   vlvgp %v17, %r5, %r3
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %sextend_i32_i64(i32) -> i64 {
 block0(v0: i32):
   v1 = sextend.i64 v0
@@ -75,6 +144,19 @@ block0(v0: i32):
 ;   lgfr %r2, %r2
 ;   br %r14
 
+function %sextend_i16_i128(i16) -> i128 {
+block0(v0: i16):
+  v1 = sextend.i128 v0
+  return v1
+}
+
+; block0:
+;   lghr %r3, %r3
+;   srag %r5, %r3, 63
+;   vlvgp %v17, %r5, %r3
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %sextend_i16_i64(i16) -> i64 {
 block0(v0: i16):
   v1 = sextend.i64 v0
@@ -95,6 +177,19 @@ block0(v0: i16):
 ;   lhr %r2, %r2
 ;   br %r14
 
+function %sextend_i8_i128(i8) -> i128 {
+block0(v0: i8):
+  v1 = sextend.i128 v0
+  return v1
+}
+
+; block0:
+;   lgbr %r3, %r3
+;   srag %r5, %r3, 63
+;   vlvgp %v17, %r5, %r3
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %sextend_i8_i64(i8) -> i64 {
 block0(v0: i8):
   v1 = sextend.i64 v0
@@ -125,6 +220,50 @@ block0(v0: i8):
 ;   lbr %r2, %r2
 ;   br %r14
 
+function %ireduce_i128_i64(i128) -> i64 {
+block0(v0: i128):
+  v1 = ireduce.i64 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %ireduce_i128_i32(i128) -> i32 {
+block0(v0: i128):
+  v1 = ireduce.i32 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %ireduce_i128_i16(i128) -> i16 {
+block0(v0: i128):
+  v1 = ireduce.i16 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %ireduce_i128_i8(i128) -> i8 {
+block0(v0: i128):
+  v1 = ireduce.i8 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
 function %ireduce_i64_i32(i64, i64) -> i32 {
 block0(v0: i64, v1: i64):
   v2 = ireduce.i32 v1
@@ -185,6 +324,29 @@ block0(v0: i16, v1: i16):
 ;   lgr %r2, %r3
 ;   br %r14
 
+function %bextend_b64_b128(b64) -> b128 {
+block0(v0: b64):
+  v1 = bextend.b128 v0
+  return v1
+}
+
+; block0:
+;   vlvgp %v5, %r3, %r3
+;   vst %v5, 0(%r2)
+;   br %r14
+
+function %bextend_b32_b128(b32) -> b128 {
+block0(v0: b32):
+  v1 = bextend.b128 v0
+  return v1
+}
+
+; block0:
+;   lgfr %r3, %r3
+;   vlvgp %v7, %r3, %r3
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bextend_b32_b64(b32) -> b64 {
 block0(v0: b32):
   v1 = bextend.b64 v0
@@ -195,6 +357,18 @@ block0(v0: b32):
 ;   lgfr %r2, %r2
 ;   br %r14
 
+function %bextend_b16_b128(b16) -> b128 {
+block0(v0: b16):
+  v1 = bextend.b128 v0
+  return v1
+}
+
+; block0:
+;   lghr %r3, %r3
+;   vlvgp %v7, %r3, %r3
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bextend_b16_b64(b16) -> b64 {
 block0(v0: b16):
   v1 = bextend.b64 v0
@@ -215,6 +389,18 @@ block0(v0: b16):
 ;   lhr %r2, %r2
 ;   br %r14
 
+function %bextend_b8_b128(b8) -> b128 {
+block0(v0: b8):
+  v1 = bextend.b128 v0
+  return v1
+}
+
+; block0:
+;   lgbr %r3, %r3
+;   vlvgp %v7, %r3, %r3
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bextend_b8_b64(b8) -> b64 {
 block0(v0: b8):
   v1 = bextend.b64 v0
@@ -245,6 +431,19 @@ block0(v0: b8):
 ;   lbr %r2, %r2
 ;   br %r14
 
+function %bextend_b1_b128(b1) -> b128 {
+block0(v0: b1):
+  v1 = bextend.b128 v0
+  return v1
+}
+
+; block0:
+;   sllg %r3, %r3, 63
+;   srag %r5, %r3, 63
+;   vlvgp %v17, %r5, %r5
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %bextend_b1_b64(b1) -> b64 {
 block0(v0: b1):
   v1 = bextend.b64 v0
@@ -289,6 +488,61 @@ block0(v0: b1):
 ;   srak %r2, %r5, 31
 ;   br %r14
 
+function %breduce_b128_b64(b128) -> b64 {
+block0(v0: b128):
+  v1 = breduce.b64 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %breduce_b128_b32(b128) -> b32 {
+block0(v0: b128):
+  v1 = breduce.b32 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %breduce_b128_b16(b128) -> b16 {
+block0(v0: b128):
+  v1 = breduce.b16 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %breduce_b128_b8(b128) -> b8 {
+block0(v0: b128):
+  v1 = breduce.b8 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %breduce_b128_b1(b128) -> b1 {
+block0(v0: b128):
+  v1 = breduce.b1 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
 function %breduce_b64_b32(b64, b64) -> b32 {
 block0(v0: b64, v1: b64):
   v2 = breduce.b32 v1
@@ -389,6 +643,72 @@ block0(v0: b8, v1: b8):
 ;   lgr %r2, %r3
 ;   br %r14
 
+function %bmask_b128_i128(b128) -> i128 {
+block0(v0: b128):
+  v1 = bmask.i128 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vst %v0, 0(%r2)
+;   br %r14
+
+function %bmask_b128_i64(b128) -> i64 {
+block0(v0: b128):
+  v1 = bmask.i64 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %bmask_b128_i32(b128) -> i32 {
+block0(v0: b128):
+  v1 = bmask.i32 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %bmask_b128_i16(b128) -> i16 {
+block0(v0: b128):
+  v1 = bmask.i16 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %bmask_b128_i8(b128) -> i8 {
+block0(v0: b128):
+  v1 = bmask.i8 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   br %r14
+
+function %bmask_b64_i128(b64, b64) -> i128 {
+block0(v0: b64, v1: b64):
+  v2 = bmask.i128 v1
+  return v2
+}
+
+; block0:
+;   vlvgp %v7, %r4, %r4
+;   vst %v7, 0(%r2)
+;   br %r14
+
 function %bmask_b64_i64(b64, b64) -> i64 {
 block0(v0: b64, v1: b64):
   v2 = bmask.i64 v1
@@ -429,6 +749,18 @@ block0(v0: b64, v1: b64):
 ;   lgr %r2, %r3
 ;   br %r14
 
+function %bmask_b32_i128(b32, b32) -> i128 {
+block0(v0: b32, v1: b32):
+  v2 = bmask.i128 v1
+  return v2
+}
+
+; block0:
+;   lgfr %r5, %r4
+;   vlvgp %v17, %r5, %r5
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %bmask_b32_i64(b32, b32) -> i64 {
 block0(v0: b32, v1: b32):
   v2 = bmask.i64 v1
@@ -469,6 +801,18 @@ block0(v0: b32, v1: b32):
 ;   lgr %r2, %r3
 ;   br %r14
 
+function %bmask_b16_i128(b16, b16) -> i128 {
+block0(v0: b16, v1: b16):
+  v2 = bmask.i128 v1
+  return v2
+}
+
+; block0:
+;   lghr %r5, %r4
+;   vlvgp %v17, %r5, %r5
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %bmask_b16_i64(b16, b16) -> i64 {
 block0(v0: b16, v1: b16):
   v2 = bmask.i64 v1
@@ -509,6 +853,18 @@ block0(v0: b16, v1: b16):
 ;   lgr %r2, %r3
 ;   br %r14
 
+function %bmask_b8_i128(b8, b8) -> i128 {
+block0(v0: b8, v1: b8):
+  v2 = bmask.i128 v1
+  return v2
+}
+
+; block0:
+;   lgbr %r5, %r4
+;   vlvgp %v17, %r5, %r5
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %bmask_b8_i64(b8, b8) -> i64 {
 block0(v0: b8, v1: b8):
   v2 = bmask.i64 v1
@@ -549,6 +905,19 @@ block0(v0: b8, v1: b8):
 ;   lgr %r2, %r3
 ;   br %r14
 
+function %bmask_b1_i128(b1, b1) -> i128 {
+block0(v0: b1, v1: b1):
+  v2 = bmask.i128 v1
+  return v2
+}
+
+; block0:
+;   sllg %r5, %r4, 63
+;   srag %r3, %r5, 63
+;   vlvgp %v19, %r3, %r3
+;   vst %v19, 0(%r2)
+;   br %r14
+
 function %bmask_b1_i64(b1, b1) -> i64 {
 block0(v0: b1, v1: b1):
   v2 = bmask.i64 v1
@@ -593,6 +962,80 @@ block0(v0: b1, v1: b1):
 ;   srak %r2, %r3, 31
 ;   br %r14
 
+function %bint_b128_i128(b128) -> i128 {
+block0(v0: b128):
+  v1 = bint.i128 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   bras %r1, 20 ; data.u128 0x00000000000000000000000000000001 ; vl %v5, 0(%r1)
+;   vn %v7, %v0, %v5
+;   vst %v7, 0(%r2)
+;   br %r14
+
+function %bint_b128_i64(b128) -> i64 {
+block0(v0: b128):
+  v1 = bint.i64 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvb %r2, %v0, 15
+;   nill %r2, 1
+;   br %r14
+
+function %bint_b128_i32(b128) -> i32 {
+block0(v0: b128):
+  v1 = bint.i32 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvb %r2, %v0, 15
+;   nill %r2, 1
+;   br %r14
+
+function %bint_b128_i16(b128) -> i16 {
+block0(v0: b128):
+  v1 = bint.i16 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvb %r2, %v0, 15
+;   nill %r2, 1
+;   br %r14
+
+function %bint_b128_i8(b128) -> i8 {
+block0(v0: b128):
+  v1 = bint.i8 v0
+  return v1
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvb %r2, %v0, 15
+;   nill %r2, 1
+;   br %r14
+
+function %bint_b64_i128(b64) -> i128 {
+block0(v0: b64):
+  v1 = bint.i128 v0
+  return v1
+}
+
+; block0:
+;   nill %r3, 1
+;   vgbm %v16, 0
+;   vlvgb %v16, %r3, 15
+;   vst %v16, 0(%r2)
+;   br %r14
+
 function %bint_b64_i64(b64) -> i64 {
 block0(v0: b64):
   v1 = bint.i64 v0
@@ -634,6 +1077,19 @@ block0(v0: b64):
 ;   nill %r2, 1
 ;   br %r14
 
+function %bint_b32_i128(b32) -> i128 {
+block0(v0: b32):
+  v1 = bint.i128 v0
+  return v1
+}
+
+; block0:
+;   nill %r3, 1
+;   vgbm %v16, 0
+;   vlvgb %v16, %r3, 15
+;   vst %v16, 0(%r2)
+;   br %r14
+
 function %bint_b32_i64(b32) -> i64 {
 block0(v0: b32):
   v1 = bint.i64 v0
@@ -675,6 +1131,19 @@ block0(v0: b32):
 ;   nill %r2, 1
 ;   br %r14
 
+function %bint_b16_i128(b16) -> i128 {
+block0(v0: b16):
+  v1 = bint.i128 v0
+  return v1
+}
+
+; block0:
+;   nill %r3, 1
+;   vgbm %v16, 0
+;   vlvgb %v16, %r3, 15
+;   vst %v16, 0(%r2)
+;   br %r14
+
 function %bint_b16_i64(b16) -> i64 {
 block0(v0: b16):
   v1 = bint.i64 v0
@@ -716,6 +1185,19 @@ block0(v0: b16):
 ;   nill %r2, 1
 ;   br %r14
 
+function %bint_b8_i128(b8) -> i128 {
+block0(v0: b8):
+  v1 = bint.i128 v0
+  return v1
+}
+
+; block0:
+;   nill %r3, 1
+;   vgbm %v16, 0
+;   vlvgb %v16, %r3, 15
+;   vst %v16, 0(%r2)
+;   br %r14
+
 function %bint_b8_i64(b8) -> i64 {
 block0(v0: b8):
   v1 = bint.i64 v0
@@ -757,6 +1239,19 @@ block0(v0: b8):
 ;   nill %r2, 1
 ;   br %r14
 
+function %bint_b1_i128(b1) -> i128 {
+block0(v0: b1):
+  v1 = bint.i128 v0
+  return v1
+}
+
+; block0:
+;   nill %r3, 1
+;   vgbm %v16, 0
+;   vlvgb %v16, %r3, 15
+;   vst %v16, 0(%r2)
+;   br %r14
+
 function %bint_b1_i64(b1) -> i64 {
 block0(v0: b1):
   v1 = bint.i64 v0

cranelift/filetests/filetests/isa/s390x/icmp-i128.clif

@@ -0,0 +1,143 @@
+test compile precise-output
+target s390x
+
+function %icmp_eq_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 eq v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   vceqgs %v5, %v0, %v1
+;   lhi %r2, 0
+;   lochie %r2, 1
+;   br %r14
+
+function %icmp_ne_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 ne v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   vceqgs %v5, %v0, %v1
+;   lhi %r2, 0
+;   lochine %r2, 1
+;   br %r14
+
+function %icmp_slt_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 slt v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   vecg %v0, %v1 ; jne 10 ; vchlgs %v5, %v1, %v0
+;   lhi %r2, 0
+;   lochil %r2, 1
+;   br %r14
+
+function %icmp_sgt_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 sgt v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   vecg %v1, %v0 ; jne 10 ; vchlgs %v5, %v0, %v1
+;   lhi %r2, 0
+;   lochil %r2, 1
+;   br %r14
+
+function %icmp_sle_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 sle v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   vecg %v1, %v0 ; jne 10 ; vchlgs %v5, %v0, %v1
+;   lhi %r2, 0
+;   lochinl %r2, 1
+;   br %r14
+
+function %icmp_sge_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 sge v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   vecg %v0, %v1 ; jne 10 ; vchlgs %v5, %v1, %v0
+;   lhi %r2, 0
+;   lochinl %r2, 1
+;   br %r14
+
+function %icmp_ult_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 ult v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   veclg %v0, %v1 ; jne 10 ; vchlgs %v5, %v1, %v0
+;   lhi %r2, 0
+;   lochil %r2, 1
+;   br %r14
+
+function %icmp_ugt_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 ugt v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   veclg %v1, %v0 ; jne 10 ; vchlgs %v5, %v0, %v1
+;   lhi %r2, 0
+;   lochil %r2, 1
+;   br %r14
+
+function %icmp_ule_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 ule v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   veclg %v1, %v0 ; jne 10 ; vchlgs %v5, %v0, %v1
+;   lhi %r2, 0
+;   lochinl %r2, 1
+;   br %r14
+
+function %icmp_uge_i128(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = icmp.i128 uge v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vl %v1, 0(%r3)
+;   veclg %v0, %v1 ; jne 10 ; vchlgs %v5, %v1, %v0
+;   lhi %r2, 0
+;   lochinl %r2, 1
+;   br %r14
+

cranelift/filetests/filetests/isa/s390x/shift-rotate.clif

@@ -1,9 +1,75 @@
 test compile precise-output
 target s390x
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+function %rotr_i128_vr(i128, i128) -> i128 {
-;; ROTR
+block0(v0: i128, v1: i128):
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+  v2 = rotr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vrepb %v7, %v1, 15
+;   vlcb %v17, %v7
+;   vslb %v19, %v0, %v17
+;   vsl %v21, %v19, %v17
+;   vsrlb %v23, %v0, %v7
+;   vsrl %v25, %v23, %v7
+;   vo %v27, %v21, %v25
+;   vst %v27, 0(%r2)
+;   br %r14
+
+function %rotr_i128_reg(i128, i64) -> i128 {
+block0(v0: i128, v1: i64):
+  v2 = rotr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vlvgb %v7, %r4, 0
+;   vrepb %v17, %v7, 0
+;   vlcb %v19, %v17
+;   vslb %v21, %v0, %v19
+;   vsl %v23, %v21, %v19
+;   vsrlb %v25, %v0, %v17
+;   vsrl %v27, %v25, %v17
+;   vo %v29, %v23, %v27
+;   vst %v29, 0(%r2)
+;   br %r14
+
+function %rotr_i128_imm(i128) -> i128 {
+block0(v0: i128):
+  v1 = iconst.i32 17
+  v2 = rotr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vrepib %v5, 17
+;   vlcb %v7, %v5
+;   vslb %v17, %v0, %v7
+;   vsl %v19, %v17, %v7
+;   vsrlb %v21, %v0, %v5
+;   vsrl %v23, %v21, %v5
+;   vo %v25, %v19, %v23
+;   vst %v25, 0(%r2)
+;   br %r14
+
+function %rotr_i64_vr(i64, i128) -> i64 {
+block0(v0: i64, v1: i128):
+  v2 = rotr.i64 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   lcr %r5, %r3
+;   rllg %r2, %r2, 0(%r5)
+;   br %r14
 
 function %rotr_i64_reg(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
@@ -12,7 +78,7 @@ block0(v0: i64, v1: i64):
 }
 
 ; block0:
-;   lcgr %r3, %r3
+;   lcr %r3, %r3
 ;   rllg %r2, %r2, 0(%r3)
 ;   br %r14
 
@@ -27,6 +93,19 @@ block0(v0: i64):
 ;   rllg %r2, %r2, 47
 ;   br %r14
 
+function %rotr_i32_vr(i32, i128) -> i32 {
+block0(v0: i32, v1: i128):
+  v2 = rotr.i32 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   lcr %r5, %r3
+;   rll %r2, %r2, 0(%r5)
+;   br %r14
+
 function %rotr_i32_reg(i32, i32) -> i32 {
 block0(v0: i32, v1: i32):
   v2 = rotr.i32 v0, v1
@@ -49,6 +128,24 @@ block0(v0: i32):
 ;   rll %r2, %r2, 15
 ;   br %r14
 
+function %rotr_i16_vr(i16, i128) -> i16 {
+block0(v0: i16, v1: i128):
+  v2 = rotr.i16 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   llhr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   lcr %r4, %r5
+;   nill %r5, 15
+;   nill %r4, 15
+;   sllk %r4, %r3, 0(%r4)
+;   srlk %r5, %r3, 0(%r5)
+;   ork %r2, %r4, %r5
+;   br %r14
+
 function %rotr_i16_reg(i16, i16) -> i16 {
 block0(v0: i16, v1: i16):
   v2 = rotr.i16 v0, v1
@@ -79,6 +176,24 @@ block0(v0: i16):
 ;   ork %r2, %r3, %r5
 ;   br %r14
 
+function %rotr_i8_vr(i8, i128) -> i8 {
+block0(v0: i8, v1: i128):
+  v2 = rotr.i8 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   llcr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   lcr %r4, %r5
+;   nill %r5, 7
+;   nill %r4, 7
+;   sllk %r4, %r3, 0(%r4)
+;   srlk %r5, %r3, 0(%r5)
+;   ork %r2, %r4, %r5
+;   br %r14
+
 function %rotr_i8_reg(i8, i8) -> i8 {
 block0(v0: i8, v1: i8):
   v2 = rotr.i8 v0, v1
@@ -109,6 +224,75 @@ block0(v0: i8):
 ;   ork %r2, %r3, %r5
 ;   br %r14
 
+function %rotl_i128_vr(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = rotl.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vrepb %v7, %v1, 15
+;   vlcb %v17, %v7
+;   vslb %v19, %v0, %v7
+;   vsl %v21, %v19, %v7
+;   vsrlb %v23, %v0, %v17
+;   vsrl %v25, %v23, %v17
+;   vo %v27, %v21, %v25
+;   vst %v27, 0(%r2)
+;   br %r14
+
+function %rotl_i128_reg(i128, i64) -> i128 {
+block0(v0: i128, v1: i64):
+  v2 = rotl.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vlvgb %v7, %r4, 0
+;   vrepb %v17, %v7, 0
+;   vlcb %v19, %v17
+;   vslb %v21, %v0, %v17
+;   vsl %v23, %v21, %v17
+;   vsrlb %v25, %v0, %v19
+;   vsrl %v27, %v25, %v19
+;   vo %v29, %v23, %v27
+;   vst %v29, 0(%r2)
+;   br %r14
+
+function %rotl_i128_imm(i128) -> i128 {
+block0(v0: i128):
+  v1 = iconst.i32 17
+  v2 = rotl.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vrepib %v5, 17
+;   vlcb %v7, %v5
+;   vslb %v17, %v0, %v5
+;   vsl %v19, %v17, %v5
+;   vsrlb %v21, %v0, %v7
+;   vsrl %v23, %v21, %v7
+;   vo %v25, %v19, %v23
+;   vst %v25, 0(%r2)
+;   br %r14
+
+function %rotl_i64_vr(i64, i128) -> i64 {
+block0(v0: i64, v1: i128):
+  v2 = rotl.i64 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   rllg %r2, %r2, 0(%r3)
+;   br %r14
+
 function %rotl_i64_reg(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = rotl.i64 v0, v1
@@ -130,6 +314,18 @@ block0(v0: i64):
 ;   rllg %r2, %r2, 17
 ;   br %r14
 
+function %rotl_i32_vr(i32, i128) -> i32 {
+block0(v0: i32, v1: i128):
+  v2 = rotl.i32 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   rll %r2, %r2, 0(%r3)
+;   br %r14
+
 function %rotl_i32_reg(i32, i32) -> i32 {
 block0(v0: i32, v1: i32):
   v2 = rotl.i32 v0, v1
@@ -151,6 +347,24 @@ block0(v0: i32):
 ;   rll %r2, %r2, 17
 ;   br %r14
 
+function %rotl_i16_vr(i16, i128) -> i16 {
+block0(v0: i16, v1: i128):
+  v2 = rotl.i16 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   llhr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   lcr %r4, %r5
+;   nill %r5, 15
+;   nill %r4, 15
+;   sllk %r5, %r3, 0(%r5)
+;   srlk %r2, %r3, 0(%r4)
+;   ork %r2, %r5, %r2
+;   br %r14
+
 function %rotl_i16_reg(i16, i16) -> i16 {
 block0(v0: i16, v1: i16):
   v2 = rotl.i16 v0, v1
@@ -181,6 +395,24 @@ block0(v0: i16):
 ;   ork %r2, %r3, %r5
 ;   br %r14
 
+function %rotl_i8_vr(i8, i128) -> i8 {
+block0(v0: i8, v1: i128):
+  v2 = rotl.i8 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   llcr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   lcr %r4, %r5
+;   nill %r5, 7
+;   nill %r4, 7
+;   sllk %r5, %r3, 0(%r5)
+;   srlk %r2, %r3, 0(%r4)
+;   ork %r2, %r5, %r2
+;   br %r14
+
 function %rotl_i8_reg(i8, i8) -> i8 {
 block0(v0: i8, v1: i8):
   v2 = rotl.i8 v0, v1
@@ -211,6 +443,63 @@ block0(v0: i8):
 ;   ork %r2, %r3, %r5
 ;   br %r14
 
+function %ushr_i128_vr(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = ushr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vrepb %v7, %v1, 15
+;   vsrlb %v17, %v0, %v7
+;   vsrl %v19, %v17, %v7
+;   vst %v19, 0(%r2)
+;   br %r14
+
+function %ushr_i128_reg(i128, i64) -> i128 {
+block0(v0: i128, v1: i64):
+  v2 = ushr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vlvgb %v7, %r4, 0
+;   vrepb %v17, %v7, 0
+;   vsrlb %v19, %v0, %v17
+;   vsrl %v21, %v19, %v17
+;   vst %v21, 0(%r2)
+;   br %r14
+
+function %ushr_i128_imm(i128) -> i128 {
+block0(v0: i128):
+  v1 = iconst.i32 17
+  v2 = ushr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vrepib %v5, 17
+;   vsrlb %v7, %v0, %v5
+;   vsrl %v17, %v7, %v5
+;   vst %v17, 0(%r2)
+;   br %r14
+
+function %ushr_i64_vr(i64, i128) -> i64 {
+block0(v0: i64, v1: i128):
+  v2 = ushr.i64 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   srlg %r2, %r2, 0(%r3)
+;   br %r14
+
 function %ushr_i64_reg(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = ushr.i64 v0, v1
@@ -232,6 +521,19 @@ block0(v0: i64):
 ;   srlg %r2, %r2, 17
 ;   br %r14
 
+function %ushr_i32_vr(i32, i128) -> i32 {
+block0(v0: i32, v1: i128):
+  v2 = ushr.i32 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   nill %r3, 31
+;   srlk %r2, %r2, 0(%r3)
+;   br %r14
+
 function %ushr_i32_reg(i32, i32) -> i32 {
 block0(v0: i32, v1: i32):
   v2 = ushr.i32 v0, v1
@@ -254,6 +556,20 @@ block0(v0: i32):
 ;   srlk %r2, %r2, 17
 ;   br %r14
 
+function %ushr_i16_vr(i16, i128) -> i16 {
+block0(v0: i16, v1: i128):
+  v2 = ushr.i16 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   llhr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   nill %r5, 15
+;   srlk %r2, %r3, 0(%r5)
+;   br %r14
+
 function %ushr_i16_reg(i16, i16) -> i16 {
 block0(v0: i16, v1: i16):
   v2 = ushr.i16 v0, v1
@@ -278,6 +594,20 @@ block0(v0: i16):
 ;   srlk %r2, %r5, 10
 ;   br %r14
 
+function %ushr_i8_vr(i8, i128) -> i8 {
+block0(v0: i8, v1: i128):
+  v2 = ushr.i8 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   llcr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   nill %r5, 7
+;   srlk %r2, %r3, 0(%r5)
+;   br %r14
+
 function %ushr_i8_reg(i8, i8) -> i8 {
 block0(v0: i8, v1: i8):
   v2 = ushr.i8 v0, v1
@@ -302,6 +632,63 @@ block0(v0: i8):
 ;   srlk %r2, %r5, 3
 ;   br %r14
 
+function %ishl_i128_vr(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = ishl.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vrepb %v7, %v1, 15
+;   vslb %v17, %v0, %v7
+;   vsl %v19, %v17, %v7
+;   vst %v19, 0(%r2)
+;   br %r14
+
+function %ishl_i128_reg(i128, i64) -> i128 {
+block0(v0: i128, v1: i64):
+  v2 = ishl.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vlvgb %v7, %r4, 0
+;   vrepb %v17, %v7, 0
+;   vslb %v19, %v0, %v17
+;   vsl %v21, %v19, %v17
+;   vst %v21, 0(%r2)
+;   br %r14
+
+function %ishl_i128_imm(i128) -> i128 {
+block0(v0: i128):
+  v1 = iconst.i32 17
+  v2 = ishl.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vrepib %v5, 17
+;   vslb %v7, %v0, %v5
+;   vsl %v17, %v7, %v5
+;   vst %v17, 0(%r2)
+;   br %r14
+
+function %ishl_i64_vr(i64, i128) -> i64 {
+block0(v0: i64, v1: i128):
+  v2 = ishl.i64 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   sllg %r2, %r2, 0(%r3)
+;   br %r14
+
 function %ishl_i64_reg(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = ishl.i64 v0, v1
@@ -323,6 +710,19 @@ block0(v0: i64):
 ;   sllg %r2, %r2, 17
 ;   br %r14
 
+function %ishl_i32_vr(i32, i128) -> i32 {
+block0(v0: i32, v1: i128):
+  v2 = ishl.i32 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   nill %r3, 31
+;   sllk %r2, %r2, 0(%r3)
+;   br %r14
+
 function %ishl_i32_reg(i32, i32) -> i32 {
 block0(v0: i32, v1: i32):
   v2 = ishl.i32 v0, v1
@@ -345,6 +745,19 @@ block0(v0: i32):
 ;   sllk %r2, %r2, 17
 ;   br %r14
 
+function %ishl_i16_vr(i16, i128) -> i16 {
+block0(v0: i16, v1: i128):
+  v2 = ishl.i16 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   nill %r3, 15
+;   sllk %r2, %r2, 0(%r3)
+;   br %r14
+
 function %ishl_i16_reg(i16, i16) -> i16 {
 block0(v0: i16, v1: i16):
   v2 = ishl.i16 v0, v1
@@ -367,6 +780,19 @@ block0(v0: i16):
 ;   sllk %r2, %r2, 10
 ;   br %r14
 
+function %ishl_i8_vr(i8, i128) -> i8 {
+block0(v0: i8, v1: i128):
+  v2 = ishl.i8 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   nill %r3, 7
+;   sllk %r2, %r2, 0(%r3)
+;   br %r14
+
 function %ishl_i8_reg(i8, i8) -> i8 {
 block0(v0: i8, v1: i8):
   v2 = ishl.i8 v0, v1
@@ -389,6 +815,63 @@ block0(v0: i8):
 ;   sllk %r2, %r2, 3
 ;   br %r14
 
+function %sshr_i128_vr(i128, i128) -> i128 {
+block0(v0: i128, v1: i128):
+  v2 = sshr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vl %v1, 0(%r4)
+;   vrepb %v7, %v1, 15
+;   vsrab %v17, %v0, %v7
+;   vsra %v19, %v17, %v7
+;   vst %v19, 0(%r2)
+;   br %r14
+
+function %sshr_i128_reg(i128, i64) -> i128 {
+block0(v0: i128, v1: i64):
+  v2 = sshr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vlvgb %v7, %r4, 0
+;   vrepb %v17, %v7, 0
+;   vsrab %v19, %v0, %v17
+;   vsra %v21, %v19, %v17
+;   vst %v21, 0(%r2)
+;   br %r14
+
+function %sshr_i128_imm(i128) -> i128 {
+block0(v0: i128):
+  v1 = iconst.i32 17
+  v2 = sshr.i128 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v0, 0(%r3)
+;   vrepib %v5, 17
+;   vsrab %v7, %v0, %v5
+;   vsra %v17, %v7, %v5
+;   vst %v17, 0(%r2)
+;   br %r14
+
+function %sshr_i64_vr(i64, i128) -> i64 {
+block0(v0: i64, v1: i128):
+  v2 = sshr.i64 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   srag %r2, %r2, 0(%r3)
+;   br %r14
+
 function %sshr_i64_reg(i64, i64) -> i64 {
 block0(v0: i64, v1: i64):
   v2 = sshr.i64 v0, v1
@@ -410,6 +893,19 @@ block0(v0: i64):
 ;   srag %r2, %r2, 17
 ;   br %r14
 
+function %sshr_i32_vr(i32, i128) -> i32 {
+block0(v0: i32, v1: i128):
+  v2 = sshr.i32 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   vlgvg %r3, %v1, 1
+;   nill %r3, 31
+;   srak %r2, %r2, 0(%r3)
+;   br %r14
+
 function %sshr_i32_reg(i32, i32) -> i32 {
 block0(v0: i32, v1: i32):
   v2 = sshr.i32 v0, v1
@@ -432,6 +928,20 @@ block0(v0: i32):
 ;   srak %r2, %r2, 17
 ;   br %r14
 
+function %sshr_i16_vr(i16, i128) -> i16 {
+block0(v0: i16, v1: i128):
+  v2 = sshr.i16 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   lhr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   nill %r5, 15
+;   srak %r2, %r3, 0(%r5)
+;   br %r14
+
 function %sshr_i16_reg(i16, i16) -> i16 {
 block0(v0: i16, v1: i16):
   v2 = sshr.i16 v0, v1
@@ -456,6 +966,20 @@ block0(v0: i16):
 ;   srak %r2, %r5, 10
 ;   br %r14
 
+function %sshr_i8_vr(i8, i128) -> i8 {
+block0(v0: i8, v1: i128):
+  v2 = sshr.i8 v0, v1
+  return v2
+}
+
+; block0:
+;   vl %v1, 0(%r3)
+;   lbr %r3, %r2
+;   vlgvg %r5, %v1, 1
+;   nill %r5, 7
+;   srak %r2, %r3, 0(%r5)
+;   br %r14
+
 function %sshr_i8_reg(i8, i8) -> i8 {
 block0(v0: i8, v1: i8):
   v2 = sshr.i8 v0, v1

cranelift/filetests/filetests/isa/s390x/vecmem-arch13.clif

@@ -107,6 +107,17 @@ block0(v0: i64):
 ;   vl %v24, 0(%r2)
 ;   br %r14
 
+function %load_i128_big(i64) -> i128 {
+block0(v0: i64):
+  v1 = load.i128 big v0
+  return v1
+}
+
+; block0:
+;   vl %v5, 0(%r3)
+;   vst %v5, 0(%r2)
+;   br %r14
+
 function %load_f32x4_big(i64) -> f32x4 {
 block0(v0: i64):
   v1 = load.f32x4 big v0
@@ -167,6 +178,17 @@ block0(v0: i64x2, v1: i64):
 ;   vst %v24, 0(%r2)
 ;   br %r14
 
+function %store_i128_big(i128, i64) {
+block0(v0: i128, v1: i64):
+  store.i128 big v0, v1
+  return
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vst %v0, 0(%r3)
+;   br %r14
+
 function %store_f32x4_big(f32x4, i64) {
 block0(v0: f32x4, v1: i64):
   store.f32x4 big v0, v1
@@ -293,6 +315,17 @@ block0(v0: i64):
 ;   vlbrq %v24, 0(%r2)
 ;   br %r14
 
+function %load_i128_little(i64) -> i128 {
+block0(v0: i64):
+  v1 = load.i128 little v0
+  return v1
+}
+
+; block0:
+;   vlbrq %v5, 0(%r3)
+;   vst %v5, 0(%r2)
+;   br %r14
+
 function %load_f32x4_little(i64) -> f32x4 {
 block0(v0: i64):
   v1 = load.f32x4 little v0
@@ -353,6 +386,17 @@ block0(v0: i64x2, v1: i64):
 ;   vstbrq %v24, 0(%r2)
 ;   br %r14
 
+function %store_i128_little(i128, i64) {
+block0(v0: i128, v1: i64):
+  store.i128 little v0, v1
+  return
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vstbrq %v0, 0(%r3)
+;   br %r14
+
 function %store_f32x4_little(f32x4, i64) {
 block0(v0: f32x4, v1: i64):
   store.f32x4 little v0, v1

cranelift/filetests/filetests/isa/s390x/vecmem.clif

@@ -107,6 +107,17 @@ block0(v0: i64):
 ;   vl %v24, 0(%r2)
 ;   br %r14
 
+function %load_i128_big(i64) -> i128 {
+block0(v0: i64):
+  v1 = load.i128 big v0
+  return v1
+}
+
+; block0:
+;   vl %v5, 0(%r3)
+;   vst %v5, 0(%r2)
+;   br %r14
+
 function %load_f32x4_big(i64) -> f32x4 {
 block0(v0: i64):
   v1 = load.f32x4 big v0
@@ -167,6 +178,17 @@ block0(v0: i64x2, v1: i64):
 ;   vst %v24, 0(%r2)
 ;   br %r14
 
+function %store_i128_big(i128, i64) {
+block0(v0: i128, v1: i64):
+  store.i128 big v0, v1
+  return
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vst %v0, 0(%r3)
+;   br %r14
+
 function %store_f32x4_big(f32x4, i64) {
 block0(v0: f32x4, v1: i64):
   store.f32x4 big v0, v1
@@ -307,6 +329,19 @@ block0(v0: i64):
 ;   vlvgp %v24, %r3, %r5
 ;   br %r14
 
+function %load_i128_little(i64) -> i128 {
+block0(v0: i64):
+  v1 = load.i128 little v0
+  return v1
+}
+
+; block0:
+;   lrvg %r4, 0(%r3)
+;   lrvg %r5, 8(%r3)
+;   vlvgp %v17, %r5, %r4
+;   vst %v17, 0(%r2)
+;   br %r14
+
 function %load_f32x4_little(i64) -> f32x4 {
 block0(v0: i64):
   v1 = load.f32x4 little v0
@@ -408,6 +443,20 @@ block0(v0: i64x2, v1: i64):
 ;   strvg %r4, 8(%r2)
 ;   br %r14
 
+function %store_i128_little(i128, i64) {
+block0(v0: i128, v1: i64):
+  store.i128 little v0, v1
+  return
+}
+
+; block0:
+;   vl %v0, 0(%r2)
+;   vlgvg %r2, %v0, 1
+;   lgdr %r4, %f0
+;   strvg %r2, 0(%r3)
+;   strvg %r4, 8(%r3)
+;   br %r14
+
 function %store_f32x4_little(f32x4, i64) {
 block0(v0: f32x4, v1: i64):
   store.f32x4 little v0, v1

cranelift/filetests/filetests/runtests/i128-arithmetic.clif

@@ -2,6 +2,7 @@ test interpret
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %add_i128(i128, i128) -> i128 {

cranelift/filetests/filetests/runtests/i128-bandnot.clif

@@ -1,5 +1,6 @@
 test run
 target aarch64
+target s390x
 
 function %band_not_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):

cranelift/filetests/filetests/runtests/i128-bextend.clif

@@ -1,6 +1,7 @@
 test interpret
 test run
 target aarch64
+target s390x
 
 function %bextend_b1_b128(b1) -> b128 {
 block0(v0: b1):

cranelift/filetests/filetests/runtests/i128-bint.clif

@@ -2,6 +2,7 @@ test interpret
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %bint_b1_i128_true() -> i128 {

cranelift/filetests/filetests/runtests/i128-bitops-count.clif

@@ -1,6 +1,7 @@
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %ctz_i128(i128) -> i128 {

cranelift/filetests/filetests/runtests/i128-bitops.clif

@@ -1,6 +1,7 @@
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %bnot_i128(i128) -> i128 {

cranelift/filetests/filetests/runtests/i128-bmask.clif

@@ -1,6 +1,7 @@
 test interpret
 test run
 target aarch64
+target s390x
 
 function %bmask_b128_i128(b128) -> i128 {
 block0(v0: b128):

cranelift/filetests/filetests/runtests/i128-bornot.clif

@@ -1,5 +1,6 @@
 test run
 target aarch64
+target s390x
 
 function %bor_not_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):

cranelift/filetests/filetests/runtests/i128-br.clif

@@ -1,6 +1,7 @@
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 

cranelift/filetests/filetests/runtests/i128-bricmp-overflow.clif

@@ -0,0 +1,61 @@
+test run
+target aarch64
+
+; TODO: Merge this with the main i128-bricmp file when s390x supports overflows.
+; See: https://github.com/bytecodealliance/wasmtime/issues/3060
+
+function %i128_bricmp_of(i128, i128) -> b1 {
+block0(v0: i128,v1: i128):
+    br_icmp.i128 of v0, v1, block2
+    jump block1
+
+block1:
+    v2 = bconst.b1 false
+    return v2
+
+block2:
+    v3 = bconst.b1 true
+    return v3
+}
+; run: %i128_bricmp_of(0, 0) == false
+; run: %i128_bricmp_of(0, 1) == false
+; run: %i128_bricmp_of(0, -1) == false
+; run: %i128_bricmp_of(-1, -1) == false
+; run: %i128_bricmp_of(0x80000000_00000000_00000000_00000000, 0) == false
+; run: %i128_bricmp_of(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0) == false
+; run: %i128_bricmp_of(1, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == false
+; run: %i128_bricmp_of(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 1) == false
+; run: %i128_bricmp_of(0x80000000_00000000_00000000_00000000, 1) == true
+; run: %i128_bricmp_of(1, 0x80000000_00000000_00000000_00000000) == true
+; run: %i128_bricmp_of(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x80000000_00000000_00000000_00000000) == true
+; run: %i128_bricmp_of(0x80000000_00000000_00000000_00000000, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == true
+; run: %i128_bricmp_of(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000000) == false
+; run: %i128_bricmp_of(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000001) == false
+
+function %i128_bricmp_nof(i128, i128) -> b1 {
+block0(v0: i128,v1: i128):
+    br_icmp.i128 nof v0, v1, block2
+    jump block1
+
+block1:
+    v2 = bconst.b1 false
+    return v2
+
+block2:
+    v3 = bconst.b1 true
+    return v3
+}
+; run: %i128_bricmp_nof(0, 0) == true
+; run: %i128_bricmp_nof(0, 1) == true
+; run: %i128_bricmp_nof(0, -1) == true
+; run: %i128_bricmp_nof(-1, -1) == true
+; run: %i128_bricmp_nof(0x80000000_00000000_00000000_00000000, 0) == true
+; run: %i128_bricmp_nof(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0) == true
+; run: %i128_bricmp_nof(1, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == true
+; run: %i128_bricmp_nof(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 1) == true
+; run: %i128_bricmp_nof(0x80000000_00000000_00000000_00000000, 1) == false
+; run: %i128_bricmp_nof(1, 0x80000000_00000000_00000000_00000000) == false
+; run: %i128_bricmp_nof(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x80000000_00000000_00000000_00000000) == false
+; run: %i128_bricmp_nof(0x80000000_00000000_00000000_00000000, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == false
+; run: %i128_bricmp_nof(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000000) == true
+; run: %i128_bricmp_nof(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000001) == true

cranelift/filetests/filetests/runtests/i128-bricmp.clif

@@ -1,5 +1,6 @@
 test run
 target aarch64
+target s390x
 
 function %i128_bricmp_eq(i128, i128) -> b1 {
 block0(v0: i128, v1: i128):
@@ -245,45 +246,3 @@ block2:
     v3 = bconst.b1 true
     return v3
 }
-; run: %i128_bricmp_of(0, 0) == false
-; run: %i128_bricmp_of(0, 1) == false
-; run: %i128_bricmp_of(0, -1) == false
-; run: %i128_bricmp_of(-1, -1) == false
-; run: %i128_bricmp_of(0x80000000_00000000_00000000_00000000, 0) == false
-; run: %i128_bricmp_of(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0) == false
-; run: %i128_bricmp_of(1, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == false
-; run: %i128_bricmp_of(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 1) == false
-; run: %i128_bricmp_of(0x80000000_00000000_00000000_00000000, 1) == true
-; run: %i128_bricmp_of(1, 0x80000000_00000000_00000000_00000000) == true
-; run: %i128_bricmp_of(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x80000000_00000000_00000000_00000000) == true
-; run: %i128_bricmp_of(0x80000000_00000000_00000000_00000000, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == true
-; run: %i128_bricmp_of(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000000) == false
-; run: %i128_bricmp_of(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000001) == false
-
-function %i128_bricmp_nof(i128, i128) -> b1 {
-block0(v0: i128,v1: i128):
-    br_icmp.i128 nof v0, v1, block2
-    jump block1
-
-block1:
-    v2 = bconst.b1 false
-    return v2
-
-block2:
-    v3 = bconst.b1 true
-    return v3
-}
-; run: %i128_bricmp_nof(0, 0) == true
-; run: %i128_bricmp_nof(0, 1) == true
-; run: %i128_bricmp_nof(0, -1) == true
-; run: %i128_bricmp_nof(-1, -1) == true
-; run: %i128_bricmp_nof(0x80000000_00000000_00000000_00000000, 0) == true
-; run: %i128_bricmp_nof(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0) == true
-; run: %i128_bricmp_nof(1, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == true
-; run: %i128_bricmp_nof(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 1) == true
-; run: %i128_bricmp_nof(0x80000000_00000000_00000000_00000000, 1) == false
-; run: %i128_bricmp_nof(1, 0x80000000_00000000_00000000_00000000) == false
-; run: %i128_bricmp_nof(0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x80000000_00000000_00000000_00000000) == false
-; run: %i128_bricmp_nof(0x80000000_00000000_00000000_00000000, 0x7FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF) == false
-; run: %i128_bricmp_nof(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000000) == true
-; run: %i128_bricmp_nof(0x4FFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF, 0x30000000_00000000_00000000_00000001) == true

cranelift/filetests/filetests/runtests/i128-bxornot.clif

@@ -1,5 +1,6 @@
 test run
 target aarch64
+target s390x
 
 function %bxor_not_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):

cranelift/filetests/filetests/runtests/i128-cls.clif

@@ -1,5 +1,6 @@
 test run
 target aarch64
+target s390x
 
 function %cls_i128(i128) -> i128 {
 block0(v0: i128):

cranelift/filetests/filetests/runtests/i128-concat-split.clif

@@ -1,6 +1,7 @@
 test interpret
 test run
 target aarch64
+target s390x
 target x86_64
 
 function %iconcat_isplit(i64, i64) -> i64, i64 {

cranelift/filetests/filetests/runtests/i128-const.clif

@@ -2,7 +2,7 @@ test interpret
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
-; target s390x TODO: Not yet implemented on s390x
+target s390x
 target x86_64
 
 function %i128_const_0() -> i128 {

cranelift/filetests/filetests/runtests/i128-extend.clif

@@ -2,6 +2,7 @@ test interpret
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %i128_uextend_i64(i64) -> i128 {

cranelift/filetests/filetests/runtests/i128-icmp.clif

@@ -2,6 +2,7 @@ test interpret
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %icmp_eq_i128(i128, i128) -> b1 {

cranelift/filetests/filetests/runtests/i128-ireduce.clif

@@ -2,6 +2,7 @@ test interpret
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %ireduce_128_64(i128) -> i64 {

cranelift/filetests/filetests/runtests/i128-load-store.clif

@@ -2,6 +2,7 @@ test run
 set enable_llvm_abi_extensions=true
 target x86_64
 target aarch64
+target s390x
 
 function %i128_stack_store_load(i128) -> b1 {
     ss0 = explicit_slot 16

cranelift/filetests/filetests/runtests/i128-rotate.clif

@@ -1,6 +1,7 @@
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %rotl(i128, i8) -> i128 {

cranelift/filetests/filetests/runtests/i128-select.clif

@@ -1,6 +1,7 @@
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64
 
 function %i128_select(b1, i128, i128) -> i128 {

cranelift/filetests/filetests/runtests/i128-shifts-small-types.clif

@@ -1,5 +1,6 @@
 test run
 target aarch64
+target s390x
 
 ; TODO: Merge this with the main i128-shifts file when x86_64 passes these.
 

cranelift/filetests/filetests/runtests/i128-shifts.clif

@@ -1,6 +1,7 @@
 test run
 set enable_llvm_abi_extensions=true
 target aarch64
+target s390x
 target x86_64