Fix some 16- and 8-bit behavior in x64 backend related to rotates.

Uncovered by @bjorn3 (thanks!): 8- and 16-bit rotates were not working
properly in recent versions of Cranelift with part of the lowering
migrated to ISLE.

This PR fixes a few issues:

- 8- and 16-bit rotate-left needs to mask a constant amount, if any,
  because we use a 32-bit rotate instruction and so don't get the
  appropriate shift-amount masking for free from x86 semantics.

- `operand_size_from_type` was incorrect: it only handled 32- and 64-bit
  types and silently returned `OperandSize::Size32` for everything else.
  Now uses the `OperandSize::from_ty(ty)` helper as the pre-ISLE code
  did.

Our test coverage for narrow value types is not great; this PR adds some
runtests for rotl/rotr but more would always be better!

cranelift/codegen/src/isa/x64/inst.isle

@@ -379,6 +379,10 @@
 (decl imm8_from_value (Imm8Reg) Value)
 (extern extractor imm8_from_value imm8_from_value)
 
+;; Mask an `Imm8Reg.Imm8`.
+(decl mask_imm8_const (Imm8Reg u64) Imm8Reg)
+(extern constructor mask_imm8_const mask_imm8_const)
+
 ;; Extract a constant `RegMemImm.Imm` from a value operand.
 (decl simm32_from_value (RegMemImm) Value)
 (extern extractor simm32_from_value simm32_from_value)

cranelift/codegen/src/isa/x64/lower.isle

@@ -633,15 +633,26 @@
 
 ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; `i64` and smaller.
+;; `i16` and `i8`: we need to extend the shift amount, or mask the
+;; constant.
 
-(rule (lower (has_type (fits_in_64 ty) (rotl src amt)))
+(rule (lower (has_type (ty_8_or_16 ty) (rotl src amt)))
+      (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero))))
+        (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
+
+(rule (lower (has_type (ty_8_or_16 ty) (rotl src (imm8_from_value amt))))
+      (value_reg (m_rotl ty (put_in_reg src) (mask_imm8_const amt (ty_bits_mask ty)))))
+
+;; `i64` and `i32`: we can rely on x86's rotate-amount masking since
+;;  we operate on the whole register.
+
+(rule (lower (has_type (ty_32_or_64 ty) (rotl src amt)))
       ;; NB: Only the low bits of `amt` matter since we logically mask the
       ;; shift amount to the value's bit width.
       (let ((amt_ Reg (lo_reg amt)))
         (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
 
-(rule (lower (has_type (fits_in_64 ty) (rotl src (imm8_from_value amt))))
+(rule (lower (has_type (ty_32_or_64 ty) (rotl src (imm8_from_value amt))))
       (value_reg (m_rotl ty (put_in_reg src) amt)))
 
 ;; `i128`.

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -57,11 +57,7 @@ where
 
     #[inline]
     fn operand_size_of_type(&mut self, ty: Type) -> OperandSize {
-        if ty.bits() == 64 {
-            OperandSize::Size64
-        } else {
-            OperandSize::Size32
-        }
+        OperandSize::from_ty(ty)
     }
 
     fn put_in_reg_mem(&mut self, val: Value) -> RegMem {
@@ -125,6 +121,16 @@ where
         Some(Imm8Reg::Imm8 { imm })
     }
 
+    #[inline]
+    fn mask_imm8_const(&mut self, imm8: &Imm8Reg, mask: u64) -> Imm8Reg {
+        match imm8 {
+            &Imm8Reg::Reg { reg } => Imm8Reg::Reg { reg },
+            &Imm8Reg::Imm8 { imm } => Imm8Reg::Imm8 {
+                imm: imm & (mask as u8),
+            },
+        }
+    }
+
     #[inline]
     fn simm32_from_value(&mut self, val: Value) -> Option<RegMemImm> {
         let inst = self.lower_ctx.dfg().value_def(val).inst()?;

cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602
-src/prelude.isle d3d2a6a42fb778231a4cdca30995324e1293a9ca8073c5a27a501535759eb51f84a6718322a93dfba4b66ee4f0c9afce7dcec0428516ef0c5bc96e8c8b76925d
-src/isa/x64/inst.isle b151120df3c356ac697122a8557becd8857eb725851506e844edeb85d831d461322a96d280ad84f9a23518e1e4efb607aebc0e249004148675e4cc19e89f0655
-src/isa/x64/lower.isle c9b408df0a089fb4f207838973ac775b0f9b56c86f056867c28e6bae317873d3844f74f713f9acd6fed98d3d11a2f9d19d392fe5049169dad33b1fc703b9b766
+src/prelude.isle 75a46b97817ad6a4c34e618b81e60876eec6fd1c83ac3ee174851e42045c951644663b2cbc31f1749ce2bc3ad9eb94fb0b877eb2c3bc4885cab7d7e87e9df1d6
+src/isa/x64/inst.isle 3b9c5c81e40b4de04169ac10e5b57d8de14dfefb104e565d24a303e6ccf28416acbdf585b1cae5a90c7e37310d7cdc1534054202597a7b8d2181c8eece08c29e
+src/isa/x64/lower.isle c7943201b32e9eb9726466e8cc417f7e84c4c4052de31e05ab6e0ad7502a587cf1d7d9835703c4ff5a506390f7a0668741e7f3feaa1edda6396571a425949fc9