cranelift: Use GPR newtypes extensively in x64 lowering (#3798)

We already defined the `Gpr` newtype and used it in a few places, and we already
defined the `Xmm` newtype and used it extensively. This finishes the transition
to using the newtypes extensively in lowering by making use of `Gpr` in more
places.

Fixes #3685

cranelift/codegen/src/isa/x64/inst.isle

@@ -16,9 +16,9 @@
        ;; Integer arithmetic/bit-twiddling.
        (AluRmiR (size OperandSize) ;; 4 or 8
                 (op AluRmiROpcode)
-                (src1 Reg)
+                (src1 Gpr)
-                (src2 RegMemImm)
+                (src2 GprMemImm)
-                (dst WritableReg))
+                (dst WritableGpr))
 
        ;; Instructions on general-purpose registers that only read src and
        ;; defines dst (dst is not modified). `bsr`, etc.
@@ -40,19 +40,19 @@
        ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
        (Div (size OperandSize) ;; 1, 2, 4, or 8
             (signed bool)
-            (divisor RegMem)
+            (divisor GprMem)
-            (dividend Reg)
+            (dividend Gpr)
-            (dst_quotient WritableReg)
+            (dst_quotient WritableGpr)
-            (dst_remainder WritableReg))
+            (dst_remainder WritableGpr))
 
        ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX *
        ;; rhs`.
        (MulHi (size OperandSize)
               (signed bool)
-              (src1 Reg)
+              (src1 Gpr)
-              (src2 RegMem)
+              (src2 GprMem)
-              (dst_lo WritableReg)
+              (dst_lo WritableGpr)
-              (dst_hi WritableReg))
+              (dst_hi WritableGpr))
 
        ;; A synthetic sequence to implement the right inline checks for
        ;; remainder and division, assuming the dividend is in %rax.
@@ -69,32 +69,32 @@
        ;; regalloc failures where %rdx is live before its first def!
        (CheckedDivOrRemSeq (kind DivOrRemKind)
                            (size OperandSize)
-                           (dividend Reg)
+                           (dividend Gpr)
                            ;; The divisor operand. Note it's marked as modified
                            ;; so that it gets assigned a register different from
                            ;; the temporary.
-                           (divisor WritableReg)
+                           (divisor WritableGpr)
-                           (dst_quotient WritableReg)
+                           (dst_quotient WritableGpr)
-                           (dst_remainder WritableReg)
+                           (dst_remainder WritableGpr)
-                           (tmp OptionWritableReg))
+                           (tmp OptionWritableGpr))
 
        ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd
        ;; cdq cqo) or al into ah: (cbw)
        (SignExtendData (size OperandSize) ;; 1, 2, 4, or 8
-                       (src Reg)
+                       (src Gpr)
-                       (dst WritableReg))
+                       (dst WritableGpr))
 
        ;; Constant materialization: (imm32 imm64) reg.
        ;;
        ;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32.
        (Imm (dst_size OperandSize) ;; 4 or 8
             (simm64 u64)
-            (dst WritableReg))
+            (dst WritableGpr))
 
        ;; GPR to GPR move: mov (64 32) reg reg.
        (MovRR (size OperandSize) ;; 4 or 8
-              (src Reg)
+              (src Gpr)
-              (dst WritableReg))
+              (dst WritableGpr))
 
        ;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr
        ;; reg.
@@ -103,12 +103,12 @@
        ;; zero-extend rule makes it unnecessary. For that case we emit the
        ;; equivalent "movl AM, reg32".
        (MovzxRmR (ext_mode ExtMode)
-                 (src RegMem)
+                 (src GprMem)
-                 (dst WritableReg))
+                 (dst WritableGpr))
 
        ;; A plain 64-bit integer load, since MovZX_RM_R can't represent that.
        (Mov64MR (src SyntheticAmode)
-                (dst WritableReg))
+                (dst WritableGpr))
 
        ;; Loads the memory address of addr into dst.
        (LoadEffectiveAddress (addr SyntheticAmode)
@@ -116,22 +116,22 @@
 
        ;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg.
        (MovsxRmR (ext_mode ExtMode)
-                 (src RegMem)
+                 (src GprMem)
-                 (dst WritableReg))
+                 (dst WritableGpr))
 
        ;; Integer stores: mov (b w l q) reg addr.
        (MovRM (size OperandSize) ;; 1, 2, 4, or 8
-              (src Reg)
+              (src Gpr)
               (dst SyntheticAmode))
 
        ;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
        (ShiftR (size OperandSize) ;; 1, 2, 4, or 8
                (kind ShiftKind)
-               (src Reg)
+               (src Gpr)
-               ;; shift count: `Imm8Reg::Imm8(0 .. #bits-in-type - 1)` or
+               ;; shift count: `Imm8Gpr::Imm8(0 .. #bits-in-type - 1)` or
-               ;; `Imm8Reg::Reg(r)` where `r` get's move mitosis'd into `%cl`.
+               ;; `Imm8Reg::Gpr(r)` where `r` get's move mitosis'd into `%cl`.
-               (num_bits Imm8Reg)
+               (num_bits Imm8Gpr)
-               (dst WritableReg))
+               (dst WritableGpr))
 
        ;; Arithmetic SIMD shifts.
        (XmmRmiReg (opcode SseOpcode)
@@ -142,30 +142,30 @@
        ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
        (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8
                 (opcode CmpOpcode)
-                (src RegMemImm)
+                (src GprMemImm)
-                (dst Reg))
+                (dst Gpr))
 
        ;; Materializes the requested condition code in the destinaton reg.
        (Setcc (cc CC)
-              (dst WritableReg))
+              (dst WritableGpr))
 
        ;; Integer conditional move.
        ;;
        ;; Overwrites the destination register.
        (Cmove (size OperandSize)
               (cc CC)
-              (consequent RegMem)
+              (consequent GprMem)
-              (alternative Reg)
+              (alternative Gpr)
-              (dst WritableReg))
+              (dst WritableGpr))
 
        ;; =========================================
        ;; Stack manipulation.
 
        ;; pushq (reg addr imm)
-       (Push64 (src RegMemImm))
+       (Push64 (src GprMemImm))
 
        ;; popq reg
-       (Pop64 (dst WritableReg))
+       (Pop64 (dst WritableGpr))
 
        ;; =========================================
        ;; Floating-point operations.
@@ -221,7 +221,7 @@
        ;; XMM (scalar) unary op (from integer to float reg): movd, movq,
        ;; cvtsi2s{s,d}
        (GprToXmm (op SseOpcode)
-                 (src RegMem)
+                 (src GprMem)
                  (dst WritableXmm)
                  (src_size OperandSize))
 
@@ -272,21 +272,21 @@
        ;; registers.
        (XmmMinMaxSeq (size OperandSize)
                      (is_min bool)
-                     (lhs Reg)
+                     (lhs Xmm)
-                     (rhs_dst WritableReg))
+                     (rhs_dst WritableXmm))
 
        ;; XMM (scalar) conditional move.
        ;;
        ;; Overwrites the destination register if cc is set.
        (XmmCmove (size OperandSize)
                  (cc CC)
-                 (src RegMem)
+                 (src XmmMem)
-                 (dst WritableReg))
+                 (dst WritableXmm))
 
        ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
        (XmmCmpRmR (op SseOpcode)
-                  (src RegMem)
+                  (src XmmMem)
-                  (dst Reg))
+                  (dst Xmm))
 
        ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm
        ;; (reg addr) reg
@@ -780,8 +780,8 @@
 ;; As a side effect, this marks the value as used.
 ;;
 ;; This is used when lowering various shifts and rotates.
-(decl put_masked_in_imm8_reg (Value Type) Imm8Reg)
+(decl put_masked_in_imm8_gpr (Value Type) Imm8Gpr)
-(extern constructor put_masked_in_imm8_reg put_masked_in_imm8_reg)
+(extern constructor put_masked_in_imm8_gpr put_masked_in_imm8_gpr)
 
 (type CC extern
       (enum O
@@ -825,14 +825,21 @@
 
 (type Gpr (primitive Gpr))
 (type WritableGpr (primitive WritableGpr))
+(type OptionWritableGpr (primitive OptionWritableGpr))
 (type GprMem extern (enum))
 (type GprMemImm extern (enum))
+(type Imm8Gpr extern (enum))
 
 (type Xmm (primitive Xmm))
 (type WritableXmm (primitive WritableXmm))
+(type OptionWritableXmm (primitive OptionWritableXmm))
 (type XmmMem extern (enum))
 (type XmmMemImm extern (enum))
 
+;; Convert an `Imm8Reg` into an `Imm8Gpr`.
+(decl imm8_reg_to_imm8_gpr (Imm8Reg) Imm8Gpr)
+(extern constructor imm8_reg_to_imm8_gpr imm8_reg_to_imm8_gpr)
+
 ;; Convert a `WritableGpr` to a `WritableReg`.
 (decl writable_gpr_to_reg (WritableGpr) WritableReg)
 (extern constructor writable_gpr_to_reg writable_gpr_to_reg)
@@ -857,6 +864,14 @@
 (decl gpr_to_reg (Gpr) Reg)
 (extern constructor gpr_to_reg gpr_to_reg)
 
+;; Convert an `Gpr` to a `GprMem`.
+(decl gpr_to_gpr_mem (Gpr) GprMem)
+(extern constructor gpr_to_gpr_mem gpr_to_gpr_mem)
+
+;; Convert an `Gpr` to a `GprMemImm`.
+(decl gpr_to_gpr_mem_imm (Gpr) GprMemImm)
+(extern constructor gpr_to_gpr_mem_imm gpr_to_gpr_mem_imm)
+
 ;; Convert an `Xmm` to a `Reg`.
 (decl xmm_to_reg (Xmm) Reg)
 (extern constructor xmm_to_reg xmm_to_reg)
@@ -984,6 +999,25 @@
 (rule (value_xmm x)
       (value_reg (xmm_to_reg x)))
 
+;; Get the `n`th reg in a `ValueRegs` and construct a GPR from it.
+;;
+;; Asserts that the register is a GPR.
+(decl value_regs_get_gpr (ValueRegs usize) Gpr)
+(rule (value_regs_get_gpr regs n)
+      (gpr_new (value_regs_get regs n)))
+
+;; Convert a `Gpr` to an `Imm8Gpr`.
+(decl gpr_to_imm8_gpr (Gpr) Imm8Gpr)
+(extern constructor gpr_to_imm8_gpr gpr_to_imm8_gpr)
+
+;; Convert an 8-bit immediate into an `Imm8Gpr`.
+(decl imm8_to_imm8_gpr (u8) Imm8Gpr)
+(extern constructor imm8_to_imm8_gpr imm8_to_imm8_gpr)
+
+;; Get the low half of the given `Value` as a GPR.
+(decl lo_gpr (Value) Gpr)
+(rule (lo_gpr regs) (gpr_new (lo_reg regs)))
+
 ;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; These should only be used for legalization purposes, when we can't otherwise
@@ -1014,15 +1048,15 @@
 ;; `Imm8Reg.Imm8`. This is used for shifts and rotates, so that we don't try and
 ;; shift/rotate more bits than the type has available, per Cranelift's
 ;; semantics.
-(decl const_to_type_masked_imm8 (u64 Type) Imm8Reg)
+(decl const_to_type_masked_imm8 (u64 Type) Imm8Gpr)
 (extern constructor const_to_type_masked_imm8 const_to_type_masked_imm8)
 
-;; Extract a constant `RegMemImm.Imm` from a value operand.
+;; Extract a constant `GprMemImm.Imm` from a value operand.
-(decl simm32_from_value (RegMemImm) Value)
+(decl simm32_from_value (GprMemImm) Value)
 (extern extractor simm32_from_value simm32_from_value)
 
 ;; Extract a constant `RegMemImm.Imm` from an `Imm64` immediate.
-(decl simm32_from_imm64 (RegMemImm) Imm64)
+(decl simm32_from_imm64 (GprMemImm) Imm64)
 (extern extractor simm32_from_imm64 simm32_from_imm64)
 
 ;; A load that can be sunk into another operation.
@@ -1041,6 +1075,10 @@
 (decl sink_load (SinkableLoad) RegMemImm)
 (extern constructor sink_load sink_load)
 
+(decl sink_load_to_gpr_mem_imm (SinkableLoad) GprMemImm)
+(rule (sink_load_to_gpr_mem_imm load)
+      (gpr_mem_imm_new (sink_load load)))
+
 ;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (type ExtKind extern
@@ -1057,13 +1095,13 @@
 (extern constructor ext_mode ext_mode)
 
 ;; Put the given value into a register, but extended as the given type.
-(decl extend_to_reg (Value Type ExtendKind) Reg)
+(decl extend_to_gpr (Value Type ExtendKind) Gpr)
 
 ;; If the value is already of the requested type, no extending is necessary.
-(rule (extend_to_reg (and val (value_type ty)) =ty _kind)
+(rule (extend_to_gpr (and val (value_type ty)) =ty _kind)
-      (put_in_reg val))
+      (put_in_gpr val))
 
-(rule (extend_to_reg (and val (value_type from_ty))
+(rule (extend_to_gpr (and val (value_type from_ty))
                      to_ty
                      kind)
       (let ((from_bits u16 (ty_bits_u16 from_ty))
@@ -1073,10 +1111,10 @@
         (extend kind
                 to_ty
                 (ext_mode from_bits to_bits)
-                (put_in_reg_mem val))))
+                (put_in_gpr_mem val))))
 
-;; Do a sign or zero extension of the given `RegMem`.
+;; Do a sign or zero extension of the given `GprMem`.
-(decl extend (ExtendKind Type ExtMode RegMem) Reg)
+(decl extend (ExtendKind Type ExtMode GprMem) Gpr)
 
 ;; Zero extending uses `movzx`.
 (rule (extend (ExtendKind.Zero) ty mode src)
@@ -1166,14 +1204,14 @@
 (decl x64_load (Type SyntheticAmode ExtKind) Reg)
 
 (rule (x64_load (fits_in_32 ty) addr (ExtKind.SignExtend))
-      (movsx ty
+      (gpr_to_reg (movsx ty
-             (ext_mode (ty_bytes ty) 8)
+                         (ext_mode (ty_bytes ty) 8)
-             (synthetic_amode_to_reg_mem addr)))
+                         (reg_mem_to_gpr_mem (synthetic_amode_to_reg_mem addr)))))
 
 (rule (x64_load $I64 addr _ext_kind)
-      (let ((dst WritableReg (temp_writable_reg $I64))
+      (let ((dst WritableGpr (temp_writable_gpr))
             (_ Unit (emit (MInst.Mov64MR addr dst))))
-        (writable_reg_to_reg dst)))
+        (gpr_to_reg (writable_gpr_to_gpr dst))))
 
 (rule (x64_load $F32 addr _ext_kind)
       (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss)
@@ -1202,15 +1240,15 @@
 ;; only gets defined the once.
 
 ;; Helper for emitting `MInst.AluRmiR` instructions.
-(decl alu_rmi_r (Type AluRmiROpcode Reg RegMemImm) Reg)
+(decl alu_rmi_r (Type AluRmiROpcode Gpr GprMemImm) Gpr)
 (rule (alu_rmi_r ty opcode src1 src2)
-      (let ((dst WritableReg (temp_writable_reg ty))
+      (let ((dst WritableGpr (temp_writable_gpr))
             (size OperandSize (operand_size_of_type_32_64 ty))
             (_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst))))
-        (writable_reg_to_reg dst)))
+        (writable_gpr_to_gpr dst)))
 
 ;; Helper for emitting `add` instructions.
-(decl add (Type Reg RegMemImm) Reg)
+(decl add (Type Gpr GprMemImm) Gpr)
 (rule (add ty src1 src2)
       (alu_rmi_r ty
                  (AluRmiROpcode.Add)
@@ -1218,29 +1256,29 @@
                  src2))
 
 ;; Helper for creating `add` instructions whose flags are also used.
-(decl add_with_flags (Type Reg RegMemImm) ProducesFlags)
+(decl add_with_flags (Type Gpr GprMemImm) ProducesFlags)
 (rule (add_with_flags ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg ty)))
+      (let ((dst WritableGpr (temp_writable_gpr)))
         (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
                                                     (AluRmiROpcode.Add)
                                                     src1
                                                     src2
                                                     dst)
-                                     (writable_reg_to_reg dst))))
+                                     (gpr_to_reg (writable_gpr_to_gpr dst)))))
 
 ;; Helper for creating `adc` instructions.
-(decl adc (Type Reg RegMemImm) ConsumesFlags)
+(decl adc (Type Gpr GprMemImm) ConsumesFlags)
 (rule (adc ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg ty)))
+      (let ((dst WritableGpr (temp_writable_gpr)))
         (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
                                                     (AluRmiROpcode.Adc)
                                                     src1
                                                     src2
                                                     dst)
-                                     (writable_reg_to_reg dst))))
+                                     (gpr_to_reg (writable_gpr_to_gpr dst)))))
 
 ;; Helper for emitting `sub` instructions.
-(decl sub (Type Reg RegMemImm) Reg)
+(decl sub (Type Gpr GprMemImm) Gpr)
 (rule (sub ty src1 src2)
       (alu_rmi_r ty
                  (AluRmiROpcode.Sub)
@@ -1248,29 +1286,29 @@
                  src2))
 
 ;; Helper for creating `sub` instructions whose flags are also used.
-(decl sub_with_flags (Type Reg RegMemImm) ProducesFlags)
+(decl sub_with_flags (Type Gpr GprMemImm) ProducesFlags)
 (rule (sub_with_flags ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg ty)))
+      (let ((dst WritableGpr (temp_writable_gpr)))
         (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
                                                     (AluRmiROpcode.Sub)
                                                     src1
                                                     src2
                                                     dst)
-                                     (writable_reg_to_reg dst))))
+                                     (gpr_to_reg (writable_gpr_to_gpr dst)))))
 
 ;; Helper for creating `sbb` instructions.
-(decl sbb (Type Reg RegMemImm) ConsumesFlags)
+(decl sbb (Type Gpr GprMemImm) ConsumesFlags)
 (rule (sbb ty src1 src2)
-      (let ((dst WritableReg (temp_writable_reg ty)))
+      (let ((dst WritableGpr (temp_writable_gpr)))
         (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
                                                     (AluRmiROpcode.Sbb)
                                                     src1
                                                     src2
                                                     dst)
-                                     (writable_reg_to_reg dst))))
+                                     (gpr_to_reg (writable_gpr_to_gpr dst)))))
 
 ;; Helper for creating `mul` instructions.
-(decl mul (Type Reg RegMemImm) Reg)
+(decl mul (Type Gpr GprMemImm) Gpr)
 (rule (mul ty src1 src2)
       (alu_rmi_r ty
                  (AluRmiROpcode.Mul)
@@ -1278,10 +1316,7 @@
                  src2))
 
 ;; Helper for emitting `and` instructions.
-;;
+(decl x64_and (Type Gpr GprMemImm) Gpr)
-;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin
-;; `and` operator.
-(decl x64_and (Type Reg RegMemImm) Reg)
 (rule (x64_and ty src1 src2)
       (alu_rmi_r ty
                  (AluRmiROpcode.And)
@@ -1289,7 +1324,7 @@
                  src2))
 
 ;; Helper for emitting `or` instructions.
-(decl or (Type Reg RegMemImm) Reg)
+(decl or (Type Gpr GprMemImm) Gpr)
 (rule (or ty src1 src2)
       (alu_rmi_r ty
                  (AluRmiROpcode.Or)
@@ -1297,7 +1332,7 @@
                  src2))
 
 ;; Helper for emitting `xor` instructions.
-(decl xor (Type Reg RegMemImm) Reg)
+(decl xor (Type Gpr GprMemImm) Gpr)
 (rule (xor ty src1 src2)
       (alu_rmi_r ty
                  (AluRmiROpcode.Xor)
@@ -1309,10 +1344,10 @@
 
 ;; Integer immediates.
 (rule (imm (fits_in_64 ty) simm64)
-      (let ((dst WritableReg (temp_writable_reg ty))
+      (let ((dst WritableGpr (temp_writable_gpr))
             (size OperandSize (operand_size_of_type_32_64 ty))
             (_ Unit (emit (MInst.Imm size simm64 dst))))
-        (writable_reg_to_reg dst)))
+        (gpr_to_reg (writable_gpr_to_gpr dst))))
 
 ;; `f32` immediates.
 (rule (imm $F32 bits)
@@ -1332,21 +1367,21 @@
 ;; Special case for when a 64-bit immediate fits into 32-bits. We can use a
 ;; 32-bit move that zero-extends the value, which has a smaller encoding.
 (rule (imm $I64 (nonzero_u64_fits_in_u32 x))
-      (let ((dst WritableReg (temp_writable_reg $I64))
+      (let ((dst WritableGpr (temp_writable_gpr))
             (_ Unit (emit (MInst.Imm (OperandSize.Size32) x dst))))
-        (writable_reg_to_reg dst)))
+        (gpr_to_reg (writable_gpr_to_gpr dst))))
 
 ;; Special case for integer zero immediates: turn them into an `xor r, r`.
 (rule (imm (fits_in_64 ty) 0)
-      (let ((wr WritableReg (temp_writable_reg ty))
+      (let ((wgpr WritableGpr (temp_writable_gpr))
-            (r Reg (writable_reg_to_reg wr))
+            (g Gpr (writable_gpr_to_gpr wgpr))
             (size OperandSize (operand_size_of_type_32_64 ty))
             (_ Unit (emit (MInst.AluRmiR size
                                          (AluRmiROpcode.Xor)
-                                         r
+                                         g
-                                         (RegMemImm.Reg r)
+                                         (gpr_to_gpr_mem_imm g)
-                                         wr))))
+                                         wgpr))))
-        r))
+        (gpr_to_reg g)))
 
 ;; Special case for zero immediates with vector types, they turn into an xor
 ;; specific to the vector type.
@@ -1384,44 +1419,42 @@
 ;; TODO: use cmpeqpd for all 1s
 
 ;; Helper for creating `MInst.ShifR` instructions.
-(decl shift_r (Type ShiftKind Reg Imm8Reg) Reg)
+(decl shift_r (Type ShiftKind Gpr Imm8Gpr) Gpr)
 (rule (shift_r ty kind src1 src2)
-      (let ((dst WritableReg (temp_writable_reg ty))
+      (let ((dst WritableGpr (temp_writable_gpr))
             ;; Use actual 8/16-bit instructions when appropriate: we
             ;; rely on their shift-amount-masking semantics.
             (size OperandSize (raw_operand_size_of_type ty))
             (_ Unit (emit (MInst.ShiftR size kind src1 src2 dst))))
-        (writable_reg_to_reg dst)))
+        (writable_gpr_to_gpr dst)))
 
-;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach
+;; Helper for creating `rotl` instructions.
-;; inst", to disambiguate this from clif's `rotl`).
+(decl x64_rotl (Type Gpr Imm8Gpr) Gpr)
-(decl x64_rotl (Type Reg Imm8Reg) Reg)
 (rule (x64_rotl ty src1 src2)
       (shift_r ty (ShiftKind.RotateLeft) src1 src2))
 
-;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach
+;; Helper for creating `rotr` instructions.
-;; inst", to disambiguate this from clif's `rotr`).
+(decl x64_rotr (Type Gpr Imm8Gpr) Gpr)
-(decl x64_rotr (Type Reg Imm8Reg) Reg)
 (rule (x64_rotr ty src1 src2)
       (shift_r ty (ShiftKind.RotateRight) src1 src2))
 
 ;; Helper for creating `shl` instructions.
-(decl shl (Type Reg Imm8Reg) Reg)
+(decl shl (Type Gpr Imm8Gpr) Gpr)
 (rule (shl ty src1 src2)
       (shift_r ty (ShiftKind.ShiftLeft) src1 src2))
 
 ;; Helper for creating logical shift-right instructions.
-(decl shr (Type Reg Imm8Reg) Reg)
+(decl shr (Type Gpr Imm8Gpr) Gpr)
 (rule (shr ty src1 src2)
       (shift_r ty (ShiftKind.ShiftRightLogical) src1 src2))
 
 ;; Helper for creating arithmetic shift-right instructions.
-(decl sar (Type Reg Imm8Reg) Reg)
+(decl sar (Type Gpr Imm8Gpr) Gpr)
 (rule (sar ty src1 src2)
       (shift_r ty (ShiftKind.ShiftRightArithmetic) src1 src2))
 
 ;; Helper for creating `MInst.CmpRmiR` instructions.
-(decl cmp_rmi_r (OperandSize CmpOpcode RegMemImm Reg) ProducesFlags)
+(decl cmp_rmi_r (OperandSize CmpOpcode GprMemImm Gpr) ProducesFlags)
 (rule (cmp_rmi_r size opcode src1 src2)
       (ProducesFlags.ProducesFlags (MInst.CmpRmiR size
                                                   opcode
@@ -1430,36 +1463,36 @@
                                    (invalid_reg)))
 
 ;; Helper for creating `cmp` instructions.
-(decl cmp (OperandSize RegMemImm Reg) ProducesFlags)
+(decl cmp (OperandSize GprMemImm Gpr) ProducesFlags)
 (rule (cmp size src1 src2)
       (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2))
 
 ;; Helper for creating `test` instructions.
-(decl test (OperandSize RegMemImm Reg) ProducesFlags)
+(decl test (OperandSize GprMemImm Gpr) ProducesFlags)
 (rule (test size src1 src2)
       (cmp_rmi_r size (CmpOpcode.Test) src1 src2))
 
 ;; Helper for creating `MInst.Cmove` instructions.
-(decl cmove (Type CC RegMem Reg) ConsumesFlags)
+(decl cmove (Type CC GprMem Gpr) ConsumesFlags)
 (rule (cmove ty cc consequent alternative)
-      (let ((dst WritableReg (temp_writable_reg ty))
+      (let ((dst WritableGpr (temp_writable_gpr))
             (size OperandSize (operand_size_of_type_32_64 ty)))
         (ConsumesFlags.ConsumesFlags (MInst.Cmove size cc consequent alternative dst)
-                                     (writable_reg_to_reg dst))))
+                                     (gpr_to_reg (writable_gpr_to_gpr dst)))))
 
 ;; Helper for creating `MInst.MovzxRmR` instructions.
-(decl movzx (Type ExtMode RegMem) Reg)
+(decl movzx (Type ExtMode GprMem) Gpr)
 (rule (movzx ty mode src)
-      (let ((dst WritableReg (temp_writable_reg ty))
+      (let ((dst WritableGpr (temp_writable_gpr))
             (_ Unit (emit (MInst.MovzxRmR mode src dst))))
-        (writable_reg_to_reg dst)))
+        (writable_gpr_to_gpr dst)))
 
 ;; Helper for creating `MInst.MovsxRmR` instructions.
-(decl movsx (Type ExtMode RegMem) Reg)
+(decl movsx (Type ExtMode GprMem) Gpr)
 (rule (movsx ty mode src)
-      (let ((dst WritableReg (temp_writable_reg ty))
+      (let ((dst WritableGpr (temp_writable_gpr))
             (_ Unit (emit (MInst.MovsxRmR mode src dst))))
-        (writable_reg_to_reg dst)))
+        (writable_gpr_to_gpr dst)))
 
 ;; Helper for creating `MInst.XmmRmR` instructions.
 (decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm)
@@ -1926,10 +1959,10 @@
 ;; Helper for creating `MInst.MulHi` instructions.
 ;;
 ;; Returns the (lo, hi) register halves of the multiplication.
-(decl mul_hi (Type bool Reg RegMem) ValueRegs)
+(decl mul_hi (Type bool Gpr GprMem) ValueRegs)
 (rule (mul_hi ty signed src1 src2)
-      (let ((dst_lo WritableReg (temp_writable_reg ty))
+      (let ((dst_lo WritableGpr (temp_writable_gpr))
-            (dst_hi WritableReg (temp_writable_reg ty))
+            (dst_hi WritableGpr (temp_writable_gpr))
             (size OperandSize (operand_size_of_type_32_64 ty))
             (_ Unit (emit (MInst.MulHi size
                                        signed
@@ -1937,12 +1970,12 @@
                                        src2
                                        dst_lo
                                        dst_hi))))
-        (value_regs (writable_reg_to_reg dst_lo)
+        (value_gprs (writable_gpr_to_gpr dst_lo)
-                    (writable_reg_to_reg dst_hi))))
+                    (writable_gpr_to_gpr dst_hi))))
 
 ;; Helper for creating `mul` instructions that return both the lower and
 ;; (unsigned) higher halves of the result.
-(decl mulhi_u (Type Reg RegMem) ValueRegs)
+(decl mulhi_u (Type Gpr GprMem) ValueRegs)
 (rule (mulhi_u ty src1 src2)
       (mul_hi ty $false src1 src2))
 
@@ -2026,7 +2059,7 @@
 (decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm)
 (rule (gpr_to_xmm op src size)
       (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.GprToXmm op (gpr_mem_to_reg_mem src) dst size))))
+            (_ Unit (emit (MInst.GprToXmm op src dst size))))
         (writable_xmm_to_xmm dst)))
 
 ;; Helper for creating `not` instructions.

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -48,8 +48,10 @@ macro_rules! newtype_of_reg {
     (
         $newtype_reg:ident,
         $newtype_writable_reg:ident,
+        $newtype_option_writable_reg:ident,
         $newtype_reg_mem:ident,
         $newtype_reg_mem_imm:ident,
+        $newtype_imm8_reg:ident,
         |$check_reg:ident| $check:expr
     ) => {
         /// A newtype wrapper around `Reg`.
@@ -122,6 +124,9 @@ macro_rules! newtype_of_reg {
 
         pub type $newtype_writable_reg = Writable<$newtype_reg>;
 
+        #[allow(dead_code)] // Used by some newtypes and not others.
+        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
+
         impl ToWritableReg for $newtype_writable_reg {
             fn to_writable_reg(&self) -> Writable<Reg> {
                 Writable::from_reg(self.to_reg().to_reg())
@@ -218,6 +223,11 @@ macro_rules! newtype_of_reg {
                     _ => true,
                 });
             }
+
+            #[allow(dead_code)] // Used by some newtypes and not others.
+            pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
+                self.0.get_regs_as_uses(collector);
+            }
         }
 
         impl PrettyPrint for $newtype_reg_mem {
@@ -290,6 +300,11 @@ macro_rules! newtype_of_reg {
                     _ => true,
                 });
             }
+
+            #[allow(dead_code)] // Used by some newtypes and not others.
+            pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
+                self.0.get_regs_as_uses(collector);
+            }
         }
 
         impl PrettyPrint for $newtype_reg_mem_imm {
@@ -303,18 +318,60 @@ macro_rules! newtype_of_reg {
                 self.0.show_rru_sized(mb_rru, size)
             }
         }
+
+        /// A newtype wrapper around `Imm8Reg`.
+        #[derive(Clone, Debug)]
+        #[allow(dead_code)] // Used by some newtypes and not others.
+        pub struct $newtype_imm8_reg(Imm8Reg);
+
+        impl From<$newtype_reg> for $newtype_imm8_reg {
+            fn from(r: $newtype_reg) -> Self {
+                Self(Imm8Reg::Reg { reg: r.to_reg() })
+            }
+        }
+
+        impl $newtype_imm8_reg {
+            /// Construct this newtype from the given `Imm8Reg`, or return
+            /// `None` if the `Imm8Reg` is not a valid instance of this newtype.
+            #[allow(dead_code)] // Used by some newtypes and not others.
+            pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
+                match imm8_reg {
+                    Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
+                    Imm8Reg::Reg { reg: $check_reg } if $check => Some(Self(imm8_reg)),
+                    Imm8Reg::Reg { reg: _ } => None,
+                }
+            }
+
+            /// Convert this newtype into its underlying `Imm8Reg`.
+            #[allow(dead_code)] // Used by some newtypes and not others.
+            pub fn to_imm8_reg(self) -> Imm8Reg {
+                self.0
+            }
+        }
     };
 }
 
 // Define a newtype of `Reg` for general-purpose registers.
-newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| {
+newtype_of_reg!(
-    reg.get_class() == RegClass::I64
+    Gpr,
-});
+    WritableGpr,
+    OptionWritableGpr,
+    GprMem,
+    GprMemImm,
+    Imm8Gpr,
+    |reg| reg.get_class() == RegClass::I64
+);
 
 // Define a newtype of `Reg` for XMM registers.
-newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| {
+newtype_of_reg!(
-    reg.get_class() == RegClass::V128
+    Xmm,
-});
+    WritableXmm,
+    OptionWritableXmm,
+    XmmMem,
+    XmmMemImm,
+    Imm8Xmm,
+    |reg| reg.get_class() == RegClass::V128
+);
 
 /// A possible addressing mode (amode) that can be used in instructions.
 /// These denote a 64-bit value only.

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -156,15 +156,15 @@ pub(crate) fn emit(
             if *op == AluRmiROpcode::Mul {
                 // We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so
                 // we have to special-case it.
-                match src2 {
+                match src2.clone().to_reg_mem_imm() {
                     RegMemImm::Reg { reg: reg_e } => {
                         emit_std_reg_reg(
                             sink,
                             LegacyPrefixes::None,
                             0x0FAF,
                             2,
-                            reg_g.to_reg(),
+                            reg_g.to_reg().to_reg(),
-                            *reg_e,
+                            reg_e,
                             rex,
                         );
                     }
@@ -178,14 +178,14 @@ pub(crate) fn emit(
                             LegacyPrefixes::None,
                             0x0FAF,
                             2,
-                            reg_g.to_reg(),
+                            reg_g.to_reg().to_reg(),
                             &amode,
                             rex,
                         );
                     }
 
                     RegMemImm::Imm { simm32 } => {
-                        let use_imm8 = low8_will_sign_extend_to_32(*simm32);
+                        let use_imm8 = low8_will_sign_extend_to_32(simm32);
                         let opcode = if use_imm8 { 0x6B } else { 0x69 };
                         // Yes, really, reg_g twice.
                         emit_std_reg_reg(
@@ -193,11 +193,11 @@ pub(crate) fn emit(
                             LegacyPrefixes::None,
                             opcode,
                             1,
-                            reg_g.to_reg(),
+                            reg_g.to_reg().to_reg(),
-                            reg_g.to_reg(),
+                            reg_g.to_reg().to_reg(),
                             rex,
                         );
-                        emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32);
+                        emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32);
                     }
                 }
             } else {
@@ -215,11 +215,11 @@ pub(crate) fn emit(
                 };
                 assert!(!(is_8bit && *size == OperandSize::Size64));
 
-                match src2 {
+                match src2.clone().to_reg_mem_imm() {
                     RegMemImm::Reg { reg: reg_e } => {
                         if is_8bit {
-                            rex.always_emit_if_8bit_needed(*reg_e);
+                            rex.always_emit_if_8bit_needed(reg_e);
-                            rex.always_emit_if_8bit_needed(reg_g.to_reg());
+                            rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg());
                         }
                         // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R
                         // duality). Do this too, so as to be able to compare generated machine
@@ -229,15 +229,15 @@ pub(crate) fn emit(
                             LegacyPrefixes::None,
                             opcode_r,
                             1,
-                            *reg_e,
+                            reg_e,
-                            reg_g.to_reg(),
+                            reg_g.to_reg().to_reg(),
                             rex,
                         );
                     }
 
                     RegMemImm::Mem { addr } => {
                         if is_8bit {
-                            rex.always_emit_if_8bit_needed(reg_g.to_reg());
+                            rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg());
                         }
                         // Here we revert to the "normal" G-E ordering.
                         let amode = addr.finalize(state, sink);
@@ -248,7 +248,7 @@ pub(crate) fn emit(
                             LegacyPrefixes::None,
                             opcode_m,
                             1,
-                            reg_g.to_reg(),
+                            reg_g.to_reg().to_reg(),
                             &amode,
                             rex,
                         );
@@ -256,10 +256,10 @@ pub(crate) fn emit(
 
                     RegMemImm::Imm { simm32 } => {
                         assert!(!is_8bit);
-                        let use_imm8 = low8_will_sign_extend_to_32(*simm32);
+                        let use_imm8 = low8_will_sign_extend_to_32(simm32);
                         let opcode = if use_imm8 { 0x83 } else { 0x81 };
                         // And also here we use the "normal" G-E ordering.
-                        let enc_g = int_reg_enc(reg_g.to_reg());
+                        let enc_g = int_reg_enc(reg_g.to_reg().to_reg());
                         emit_std_enc_enc(
                             sink,
                             LegacyPrefixes::None,
@@ -269,7 +269,7 @@ pub(crate) fn emit(
                             enc_g,
                             rex,
                         );
-                        emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32);
+                        emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32);
                     }
                 }
             }
@@ -377,9 +377,9 @@ pub(crate) fn emit(
             sink.add_trap(loc, TrapCode::IntegerDivisionByZero);
 
             let subopcode = if *signed { 7 } else { 6 };
-            match divisor {
+            match divisor.clone().to_reg_mem() {
                 RegMem::Reg { reg } => {
-                    let src = int_reg_enc(*reg);
+                    let src = int_reg_enc(reg);
                     emit_std_enc_enc(
                         sink,
                         prefix,
@@ -387,7 +387,7 @@ pub(crate) fn emit(
                         1,
                         subopcode,
                         src,
-                        RexFlags::from((*size, *reg)),
+                        RexFlags::from((*size, reg)),
                     )
                 }
                 RegMem::Mem { addr: src } => {
@@ -428,9 +428,9 @@ pub(crate) fn emit(
             };
 
             let subopcode = if *signed { 5 } else { 4 };
-            match src2 {
+            match src2.clone().to_reg_mem() {
                 RegMem::Reg { reg } => {
-                    let src = int_reg_enc(*reg);
+                    let src = int_reg_enc(reg);
                     emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags)
                 }
                 RegMem::Mem { addr: src } => {
@@ -504,7 +504,7 @@ pub(crate) fn emit(
             // $done:
 
             // Check if the divisor is zero, first.
-            let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg());
+            let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg().to_reg());
             inst.emit(sink, info, state);
 
             let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero);
@@ -512,7 +512,8 @@ pub(crate) fn emit(
 
             let (do_op, done_label) = if kind.is_signed() {
                 // Now check if the divisor is -1.
-                let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg());
+                let inst =
+                    Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg().to_reg());
                 inst.emit(sink, info, state);
 
                 let do_op = sink.get_label();
@@ -537,12 +538,16 @@ pub(crate) fn emit(
                     if *size == OperandSize::Size64 {
                         let tmp = tmp.expect("temporary for i64 sdiv");
 
-                        let inst = Inst::imm(OperandSize::Size64, 0x8000000000000000, tmp);
+                        let inst = Inst::imm(
+                            OperandSize::Size64,
+                            0x8000000000000000,
+                            tmp.to_writable_reg(),
+                        );
                         inst.emit(sink, info, state);
 
                         let inst = Inst::cmp_rmi_r(
                             OperandSize::Size64,
-                            RegMemImm::reg(tmp.to_reg()),
+                            RegMemImm::reg(tmp.to_reg().to_reg()),
                             regs::rax(),
                         );
                         inst.emit(sink, info, state);
@@ -576,7 +581,11 @@ pub(crate) fn emit(
                 inst.emit(sink, info, state);
             }
 
-            let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg()));
+            let inst = Inst::div(
+                *size,
+                kind.is_signed(),
+                RegMem::reg(divisor.to_reg().to_reg()),
+            );
             inst.emit(sink, info, state);
 
             // Lowering takes care of moving the result back into the right register, see comment
@@ -626,8 +635,8 @@ pub(crate) fn emit(
                 LegacyPrefixes::None,
                 0x89,
                 1,
-                *src,
+                src.to_reg(),
-                dst.to_reg(),
+                dst.to_reg().to_reg(),
                 RexFlags::from(*size),
             );
         }
@@ -664,12 +673,12 @@ pub(crate) fn emit(
                 }
             };
 
-            match src {
+            match src.clone().to_reg_mem() {
                 RegMem::Reg { reg: src } => {
                     match ext_mode {
                         ExtMode::BL | ExtMode::BQ => {
                             // A redundant REX prefix must be emitted for certain register inputs.
-                            rex_flags.always_emit_if_8bit_needed(*src);
+                            rex_flags.always_emit_if_8bit_needed(src);
                         }
                         _ => {}
                     }
@@ -678,8 +687,8 @@ pub(crate) fn emit(
                         LegacyPrefixes::None,
                         opcodes,
                         num_opcodes,
-                        dst.to_reg(),
+                        dst.to_reg().to_reg(),
-                        *src,
+                        src,
                         rex_flags,
                     )
                 }
@@ -694,7 +703,7 @@ pub(crate) fn emit(
                         LegacyPrefixes::None,
                         opcodes,
                         num_opcodes,
-                        dst.to_reg(),
+                        dst.to_reg().to_reg(),
                         src,
                         rex_flags,
                     )
@@ -712,7 +721,7 @@ pub(crate) fn emit(
                 LegacyPrefixes::None,
                 0x8B,
                 1,
-                dst.to_reg(),
+                dst.to_reg().to_reg(),
                 src,
                 RexFlags::set_w(),
             )
@@ -758,12 +767,12 @@ pub(crate) fn emit(
                 }
             };
 
-            match src {
+            match src.clone().to_reg_mem() {
                 RegMem::Reg { reg: src } => {
                     match ext_mode {
                         ExtMode::BL | ExtMode::BQ => {
                             // A redundant REX prefix must be emitted for certain register inputs.
-                            rex_flags.always_emit_if_8bit_needed(*src);
+                            rex_flags.always_emit_if_8bit_needed(src);
                         }
                         _ => {}
                     }
@@ -772,8 +781,8 @@ pub(crate) fn emit(
                         LegacyPrefixes::None,
                         opcodes,
                         num_opcodes,
-                        dst.to_reg(),
+                        dst.to_reg().to_reg(),
-                        *src,
+                        src,
                         rex_flags,
                     )
                 }
@@ -788,7 +797,7 @@ pub(crate) fn emit(
                         LegacyPrefixes::None,
                         opcodes,
                         num_opcodes,
-                        dst.to_reg(),
+                        dst.to_reg().to_reg(),
                         src,
                         rex_flags,
                     )
@@ -812,13 +821,13 @@ pub(crate) fn emit(
             // This is one of the few places where the presence of a
             // redundant REX prefix changes the meaning of the
             // instruction.
-            let rex = RexFlags::from((*size, *src));
+            let rex = RexFlags::from((*size, src.to_reg()));
 
             //  8-bit: MOV r8, r/m8 is (REX.W==0) 88 /r
             // 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
             // 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
             // 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
-            emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *src, dst, rex);
+            emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src.to_reg(), dst, rex);
         }
 
         Inst::ShiftR {
@@ -837,10 +846,10 @@ pub(crate) fn emit(
                 ShiftKind::ShiftRightArithmetic => 7,
             };
             let enc_dst = int_reg_enc(dst.to_reg());
-            let rex_flags = RexFlags::from((*size, dst.to_reg()));
+            let rex_flags = RexFlags::from((*size, dst.to_reg().to_reg()));
-            match num_bits {
+            match num_bits.clone().to_imm8_reg() {
                 Imm8Reg::Reg { reg } => {
-                    debug_assert_eq!(*reg, regs::rcx());
+                    debug_assert_eq!(reg, regs::rcx());
                     let (opcode, prefix) = match size {
                         OperandSize::Size8 => (0xD2, LegacyPrefixes::None),
                         OperandSize::Size16 => (0xD3, LegacyPrefixes::_66),
@@ -870,7 +879,7 @@ pub(crate) fn emit(
                     // When the shift amount is 1, there's an even shorter encoding, but we don't
                     // bother with that nicety here.
                     emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags);
-                    sink.put1(*num_bits);
+                    sink.put1(num_bits);
                 }
             }
         }
@@ -963,13 +972,13 @@ pub(crate) fn emit(
                 prefix = LegacyPrefixes::_66;
             }
             // A redundant REX prefix can change the meaning of this instruction.
-            let mut rex = RexFlags::from((*size, *reg_g));
+            let mut rex = RexFlags::from((*size, reg_g.to_reg()));
 
-            match src_e {
+            match src_e.clone().to_reg_mem_imm() {
                 RegMemImm::Reg { reg: reg_e } => {
                     if *size == OperandSize::Size8 {
                         // Check whether the E register forces the use of a redundant REX.
-                        rex.always_emit_if_8bit_needed(*reg_e);
+                        rex.always_emit_if_8bit_needed(reg_e);
                     }
 
                     // Use the swapped operands encoding for CMP, to stay consistent with the output of
@@ -980,7 +989,7 @@ pub(crate) fn emit(
                         (OperandSize::Size8, false) => 0x84,
                         (_, false) => 0x85,
                     };
-                    emit_std_reg_reg(sink, prefix, opcode, 1, *reg_e, *reg_g, rex);
+                    emit_std_reg_reg(sink, prefix, opcode, 1, reg_e, reg_g.to_reg(), rex);
                 }
 
                 RegMemImm::Mem { addr } => {
@@ -992,13 +1001,23 @@ pub(crate) fn emit(
                         (OperandSize::Size8, false) => 0x84,
                         (_, false) => 0x85,
                     };
-                    emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *reg_g, addr, rex);
+                    emit_std_reg_mem(
+                        sink,
+                        state,
+                        info,
+                        prefix,
+                        opcode,
+                        1,
+                        reg_g.to_reg(),
+                        addr,
+                        rex,
+                    );
                 }
 
                 RegMemImm::Imm { simm32 } => {
                     // FIXME JRS 2020Feb11: there are shorter encodings for
                     // cmp $imm, rax/eax/ax/al.
-                    let use_imm8 = is_cmp && low8_will_sign_extend_to_32(*simm32);
+                    let use_imm8 = is_cmp && low8_will_sign_extend_to_32(simm32);
 
                     // And also here we use the "normal" G-E ordering.
                     let opcode = if is_cmp {
@@ -1018,9 +1037,9 @@ pub(crate) fn emit(
                     };
                     let subopcode = if is_cmp { 7 } else { 0 };
 
-                    let enc_g = int_reg_enc(*reg_g);
+                    let enc_g = int_reg_enc(reg_g.to_reg());
                     emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_g, rex);
-                    emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, *simm32);
+                    emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, simm32);
                 }
             }
         }
@@ -1056,9 +1075,17 @@ pub(crate) fn emit(
                 _ => unreachable!("invalid size spec for cmove"),
             };
             let opcode = 0x0F40 + cc.get_enc() as u32;
-            match consequent {
+            match consequent.clone().to_reg_mem() {
                 RegMem::Reg { reg: reg_e } => {
-                    emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex_flags);
+                    emit_std_reg_reg(
+                        sink,
+                        prefix,
+                        opcode,
+                        2,
+                        reg_g.to_reg().to_reg(),
+                        reg_e,
+                        rex_flags,
+                    );
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
@@ -1069,7 +1096,7 @@ pub(crate) fn emit(
                         prefix,
                         opcode,
                         2,
-                        reg_g.to_reg(),
+                        reg_g.to_reg().to_reg(),
                         addr,
                         rex_flags,
                     );
@@ -1090,7 +1117,7 @@ pub(crate) fn emit(
             } else {
                 SseOpcode::Movss
             };
-            let inst = Inst::xmm_unary_rm_r(op, src.clone(), *dst);
+            let inst = Inst::xmm_unary_rm_r(op, src.clone().to_reg_mem(), dst.to_writable_reg());
             inst.emit(sink, info, state);
 
             sink.bind_label(next);
@@ -1101,9 +1128,9 @@ pub(crate) fn emit(
                 sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow);
             }
 
-            match src {
+            match src.clone().to_reg_mem_imm() {
                 RegMemImm::Reg { reg } => {
-                    let enc_reg = int_reg_enc(*reg);
+                    let enc_reg = int_reg_enc(reg);
                     let rex = 0x40 | ((enc_reg >> 3) & 1);
                     if rex != 0x40 {
                         sink.put1(rex);
@@ -1127,12 +1154,12 @@ pub(crate) fn emit(
                 }
 
                 RegMemImm::Imm { simm32 } => {
-                    if low8_will_sign_extend_to_64(*simm32) {
+                    if low8_will_sign_extend_to_64(simm32) {
                         sink.put1(0x6A);
-                        sink.put1(*simm32 as u8);
+                        sink.put1(simm32 as u8);
                     } else {
                         sink.put1(0x68);
-                        sink.put4(*simm32);
+                        sink.put4(simm32);
                     }
                 }
             }
@@ -1711,7 +1738,8 @@ pub(crate) fn emit(
                 _ => unreachable!(),
             };
 
-            let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(*lhs), rhs_dst.to_reg());
+            let inst =
+                Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_reg().to_reg());
             inst.emit(sink, info, state);
 
             one_way_jmp(sink, CC::NZ, do_min_max);
@@ -1721,7 +1749,7 @@ pub(crate) fn emit(
             // and negative zero. These instructions merge the sign bits in that
             // case, and are no-ops otherwise.
             let op = if *is_min { or_op } else { and_op };
-            let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst);
+            let inst = Inst::xmm_rm_r(op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg());
             inst.emit(sink, info, state);
 
             let inst = Inst::jmp_known(done);
@@ -1731,13 +1759,17 @@ pub(crate) fn emit(
             // read-only operand: perform an addition between the two operands, which has the
             // desired NaN propagation effects.
             sink.bind_label(propagate_nan);
-            let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst);
+            let inst = Inst::xmm_rm_r(add_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg());
             inst.emit(sink, info, state);
 
             one_way_jmp(sink, CC::P, done);
 
             sink.bind_label(do_min_max);
-            let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst);
+            let inst = Inst::xmm_rm_r(
+                min_max_op,
+                RegMem::reg(lhs.to_reg()),
+                rhs_dst.to_writable_reg(),
+            );
             inst.emit(sink, info, state);
 
             sink.bind_label(done);
@@ -1890,17 +1922,9 @@ pub(crate) fn emit(
                 _ => panic!("unexpected opcode {:?}", op),
             };
             let rex = RexFlags::from(*src_size);
-            match src_e {
+            match src_e.clone().to_reg_mem() {
                 RegMem::Reg { reg: reg_e } => {
-                    emit_std_reg_reg(
+                    emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg().to_reg(), reg_e, rex);
-                        sink,
-                        prefix,
-                        opcode,
-                        2,
-                        reg_g.to_reg().to_reg(),
-                        *reg_e,
-                        rex,
-                    );
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
@@ -1928,13 +1952,23 @@ pub(crate) fn emit(
                 _ => unimplemented!("Emit xmm cmp rm r"),
             };
 
-            match src {
+            match src.clone().to_reg_mem() {
                 RegMem::Reg { reg } => {
-                    emit_std_reg_reg(sink, prefix, opcode, len, *dst, *reg, rex);
+                    emit_std_reg_reg(sink, prefix, opcode, len, dst.to_reg(), reg, rex);
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
-                    emit_std_reg_mem(sink, state, info, prefix, opcode, len, *dst, addr, rex);
+                    emit_std_reg_mem(
+                        sink,
+                        state,
+                        info,
+                        prefix,
+                        opcode,
+                        len,
+                        dst.to_reg(),
+                        addr,
+                        rex,
+                    );
                 }
             }
         }

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -133,9 +133,9 @@ impl Inst {
         Self::AluRmiR {
             size,
             op,
-            src1: dst.to_reg(),
+            src1: Gpr::new(dst.to_reg()).unwrap(),
-            src2: src,
+            src2: GprMemImm::new(src).unwrap(),
-            dst,
+            dst: WritableGpr::from_writable_reg(dst).unwrap(),
         }
     }
 
@@ -174,10 +174,10 @@ impl Inst {
         Inst::Div {
             size,
             signed,
-            divisor,
+            divisor: GprMem::new(divisor).unwrap(),
-            dividend: regs::rax(),
+            dividend: Gpr::new(regs::rax()).unwrap(),
-            dst_quotient: Writable::from_reg(regs::rax()),
+            dst_quotient: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
-            dst_remainder: Writable::from_reg(regs::rdx()),
+            dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
         }
     }
 
@@ -191,10 +191,10 @@ impl Inst {
         Inst::MulHi {
             size,
             signed,
-            src1: regs::rax(),
+            src1: Gpr::new(regs::rax()).unwrap(),
-            src2: rhs,
+            src2: GprMem::new(rhs).unwrap(),
-            dst_lo: Writable::from_reg(regs::rax()),
+            dst_lo: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
-            dst_hi: Writable::from_reg(regs::rdx()),
+            dst_hi: WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
         }
     }
 
@@ -211,19 +211,19 @@ impl Inst {
         Inst::CheckedDivOrRemSeq {
             kind,
             size,
-            divisor,
+            divisor: WritableGpr::from_writable_reg(divisor).unwrap(),
-            dividend: regs::rax(),
+            dividend: Gpr::new(regs::rax()).unwrap(),
-            dst_quotient: Writable::from_reg(regs::rax()),
+            dst_quotient: Writable::from_reg(Gpr::new(regs::rax()).unwrap()),
-            dst_remainder: Writable::from_reg(regs::rdx()),
+            dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
-            tmp,
+            tmp: tmp.map(|tmp| WritableGpr::from_writable_reg(tmp).unwrap()),
         }
     }
 
     pub(crate) fn sign_extend_data(size: OperandSize) -> Inst {
         Inst::SignExtendData {
             size,
-            src: regs::rax(),
+            src: Gpr::new(regs::rax()).unwrap(),
-            dst: Writable::from_reg(regs::rdx()),
+            dst: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
         }
     }
 
@@ -239,7 +239,7 @@ impl Inst {
         Inst::Imm {
             dst_size,
             simm64,
-            dst,
+            dst: WritableGpr::from_writable_reg(dst).unwrap(),
         }
     }
 
@@ -247,6 +247,8 @@ impl Inst {
         debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
         debug_assert!(src.get_class() == RegClass::I64);
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
+        let src = Gpr::new(src).unwrap();
+        let dst = WritableGpr::from_writable_reg(dst).unwrap();
         Inst::MovRR { size, src, dst }
     }
 
@@ -360,7 +362,7 @@ impl Inst {
         debug_assert!(dst.to_reg().get_class() == RegClass::V128);
         Inst::GprToXmm {
             op,
-            src,
+            src: GprMem::new(src).unwrap(),
             dst: WritableXmm::from_writable_reg(dst).unwrap(),
             src_size,
         }
@@ -369,6 +371,8 @@ impl Inst {
     pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst {
         src.assert_regclass_is(RegClass::V128);
         debug_assert!(dst.get_class() == RegClass::V128);
+        let src = XmmMem::new(src).unwrap();
+        let dst = Xmm::new(dst).unwrap();
         Inst::XmmCmpRmR { op, src, dst }
     }
 
@@ -457,8 +461,8 @@ impl Inst {
         Inst::XmmMinMaxSeq {
             size,
             is_min,
-            lhs,
+            lhs: Xmm::new(lhs).unwrap(),
-            rhs_dst,
+            rhs_dst: WritableXmm::from_writable_reg(rhs_dst).unwrap(),
         }
     }
 
@@ -483,6 +487,8 @@ impl Inst {
     pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
         src.assert_regclass_is(RegClass::I64);
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
+        let src = GprMem::new(src).unwrap();
+        let dst = WritableGpr::from_writable_reg(dst).unwrap();
         Inst::MovzxRmR { ext_mode, src, dst }
     }
 
@@ -500,6 +506,8 @@ impl Inst {
     pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
         src.assert_regclass_is(RegClass::I64);
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
+        let src = GprMem::new(src).unwrap();
+        let dst = WritableGpr::from_writable_reg(dst).unwrap();
         Inst::MovsxRmR { ext_mode, src, dst }
     }
 
@@ -507,7 +515,7 @@ impl Inst {
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
         Inst::Mov64MR {
             src: src.into(),
-            dst,
+            dst: WritableGpr::from_writable_reg(dst).unwrap(),
         }
     }
 
@@ -524,7 +532,7 @@ impl Inst {
         debug_assert!(src.get_class() == RegClass::I64);
         Inst::MovRM {
             size,
-            src,
+            src: Gpr::new(src).unwrap(),
             dst: dst.into(),
         }
     }
@@ -552,12 +560,13 @@ impl Inst {
         Inst::ShiftR {
             size,
             kind,
-            src: dst.to_reg(),
+            src: Gpr::new(dst.to_reg()).unwrap(),
-            num_bits: match num_bits {
+            num_bits: Imm8Gpr::new(match num_bits {
                 Some(imm) => Imm8Reg::Imm8 { imm },
                 None => Imm8Reg::Reg { reg: regs::rcx() },
-            },
+            })
-            dst,
+            .unwrap(),
+            dst: WritableGpr::from_writable_reg(dst).unwrap(),
         }
     }
 
@@ -568,8 +577,8 @@ impl Inst {
         debug_assert_eq!(dst.get_class(), RegClass::I64);
         Inst::CmpRmiR {
             size,
-            src,
+            src: GprMemImm::new(src).unwrap(),
-            dst,
+            dst: Gpr::new(dst).unwrap(),
             opcode: CmpOpcode::Cmp,
         }
     }
@@ -580,8 +589,8 @@ impl Inst {
         debug_assert_eq!(dst.get_class(), RegClass::I64);
         Inst::CmpRmiR {
             size,
-            src,
+            src: GprMemImm::new(src).unwrap(),
-            dst,
+            dst: Gpr::new(dst).unwrap(),
             opcode: CmpOpcode::Test,
         }
     }
@@ -594,6 +603,7 @@ impl Inst {
 
     pub(crate) fn setcc(cc: CC, dst: Writable<Reg>) -> Inst {
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
+        let dst = WritableGpr::from_writable_reg(dst).unwrap();
         Inst::Setcc { cc, dst }
     }
 
@@ -607,9 +617,9 @@ impl Inst {
         Inst::Cmove {
             size,
             cc,
-            consequent: src,
+            consequent: GprMem::new(src).unwrap(),
-            alternative: dst.to_reg(),
+            alternative: Gpr::new(dst.to_reg()).unwrap(),
-            dst,
+            dst: WritableGpr::from_writable_reg(dst).unwrap(),
         }
     }
 
@@ -617,16 +627,20 @@ impl Inst {
         debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
         src.assert_regclass_is(RegClass::V128);
         debug_assert!(dst.to_reg().get_class() == RegClass::V128);
+        let src = XmmMem::new(src).unwrap();
+        let dst = WritableXmm::from_writable_reg(dst).unwrap();
         Inst::XmmCmove { size, cc, src, dst }
     }
 
     pub(crate) fn push64(src: RegMemImm) -> Inst {
         src.assert_regclass_is(RegClass::I64);
+        let src = GprMemImm::new(src).unwrap();
         Inst::Push64 { src }
     }
 
     pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
+        let dst = WritableGpr::from_writable_reg(dst).unwrap();
         Inst::Pop64 { dst }
     }
 
@@ -780,7 +794,7 @@ impl Inst {
     fn produces_const(&self) -> bool {
         match self {
             Self::AluRmiR { op, src2, dst, .. } => {
-                src2.to_reg() == Some(dst.to_reg())
+                src2.clone().to_reg_mem_imm().to_reg() == Some(dst.to_reg().to_reg())
                     && (*op == AluRmiROpcode::Xor || *op == AluRmiROpcode::Sub)
             }
 
@@ -838,7 +852,11 @@ impl Inst {
             Inst::AluRmiR { src1, dst, .. } => {
                 if *src1 != dst.to_reg() {
                     debug_assert!(src1.is_virtual());
-                    insts.push(Self::gen_move(*dst, *src1, types::I64));
+                    insts.push(Self::gen_move(
+                        dst.to_writable_reg(),
+                        src1.to_reg(),
+                        types::I64,
+                    ));
                     *src1 = dst.to_reg();
                 }
                 insts.push(self);
@@ -883,7 +901,11 @@ impl Inst {
             } => {
                 if *alternative != dst.to_reg() {
                     debug_assert!(alternative.is_virtual());
-                    insts.push(Self::mov_r_r(*size, *alternative, *dst));
+                    insts.push(Self::mov_r_r(
+                        *size,
+                        alternative.to_reg(),
+                        dst.to_writable_reg(),
+                    ));
                     *alternative = dst.to_reg();
                 }
                 insts.push(self);
@@ -916,22 +938,30 @@ impl Inst {
                     debug_assert!(dividend.is_virtual());
                     insts.push(Self::gen_move(
                         Writable::from_reg(regs::rax()),
-                        *dividend,
+                        dividend.to_reg(),
                         types::I64,
                     ));
-                    *dividend = regs::rax();
+                    *dividend = Gpr::new(regs::rax()).unwrap();
                 }
                 let mut quotient_mov = None;
                 if dst_quotient.to_reg() != regs::rax() {
                     debug_assert!(dst_quotient.to_reg().is_virtual());
-                    quotient_mov = Some(Self::gen_move(*dst_quotient, regs::rax(), types::I64));
+                    quotient_mov = Some(Self::gen_move(
-                    *dst_quotient = Writable::from_reg(regs::rax());
+                        dst_quotient.to_writable_reg(),
+                        regs::rax(),
+                        types::I64,
+                    ));
+                    *dst_quotient = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
                 }
                 let mut remainder_mov = None;
                 if dst_remainder.to_reg() != regs::rdx() {
                     debug_assert!(dst_remainder.to_reg().is_virtual());
-                    remainder_mov = Some(Self::gen_move(*dst_remainder, regs::rdx(), types::I64));
+                    remainder_mov = Some(Self::gen_move(
-                    *dst_remainder = Writable::from_reg(regs::rdx());
+                        dst_remainder.to_writable_reg(),
+                        regs::rdx(),
+                        types::I64,
+                    ));
+                    *dst_remainder = Writable::from_reg(Gpr::new(regs::rdx()).unwrap());
                 }
                 insts.push(self);
                 insts.extend(quotient_mov);
@@ -947,22 +977,30 @@ impl Inst {
                     debug_assert!(src1.is_virtual());
                     insts.push(Self::gen_move(
                         Writable::from_reg(regs::rax()),
-                        *src1,
+                        src1.to_reg(),
                         types::I64,
                     ));
-                    *src1 = regs::rax();
+                    *src1 = Gpr::new(regs::rax()).unwrap();
                 }
                 let mut dst_lo_mov = None;
                 if dst_lo.to_reg() != regs::rax() {
                     debug_assert!(dst_lo.to_reg().is_virtual());
-                    dst_lo_mov = Some(Self::gen_move(*dst_lo, regs::rax(), types::I64));
+                    dst_lo_mov = Some(Self::gen_move(
-                    *dst_lo = Writable::from_reg(regs::rax());
+                        dst_lo.to_writable_reg(),
+                        regs::rax(),
+                        types::I64,
+                    ));
+                    *dst_lo = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
                 }
                 let mut dst_hi_mov = None;
                 if dst_hi.to_reg() != regs::rdx() {
                     debug_assert!(dst_hi.to_reg().is_virtual());
-                    dst_hi_mov = Some(Self::gen_move(*dst_hi, regs::rdx(), types::I64));
+                    dst_hi_mov = Some(Self::gen_move(
-                    *dst_hi = Writable::from_reg(regs::rdx());
+                        dst_hi.to_writable_reg(),
+                        regs::rdx(),
+                        types::I64,
+                    ));
+                    *dst_hi = Writable::from_reg(Gpr::new(regs::rdx()).unwrap());
                 }
                 insts.push(self);
                 insts.extend(dst_lo_mov);
@@ -973,16 +1011,20 @@ impl Inst {
                     debug_assert!(src.is_virtual());
                     insts.push(Self::gen_move(
                         Writable::from_reg(regs::rax()),
-                        *src,
+                        src.to_reg(),
                         types::I64,
                     ));
-                    *src = regs::rax();
+                    *src = Gpr::new(regs::rax()).unwrap();
                 }
                 let mut dst_mov = None;
                 if dst.to_reg() != regs::rax() {
                     debug_assert!(dst.to_reg().is_virtual());
-                    dst_mov = Some(Self::gen_move(*dst, dst.to_reg(), types::I64));
+                    dst_mov = Some(Self::gen_move(
-                    *dst = Writable::from_reg(regs::rax());
+                        dst.to_writable_reg(),
+                        dst.to_reg().to_reg(),
+                        types::I64,
+                    ));
+                    *dst = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
                 }
                 insts.push(self);
                 insts.extend(dst_mov);
@@ -992,18 +1034,22 @@ impl Inst {
             } => {
                 if *src != dst.to_reg() {
                     debug_assert!(src.is_virtual());
-                    insts.push(Self::gen_move(*dst, *src, types::I64));
+                    insts.push(Self::gen_move(
+                        dst.to_writable_reg(),
+                        src.to_reg(),
+                        types::I64,
+                    ));
                     *src = dst.to_reg();
                 }
-                if let Imm8Reg::Reg { reg } = num_bits {
+                if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() {
-                    if *reg != regs::rcx() {
+                    if reg != regs::rcx() {
                         debug_assert!(reg.is_virtual());
                         insts.push(Self::gen_move(
                             Writable::from_reg(regs::rcx()),
-                            *reg,
+                            reg,
                             types::I64,
                         ));
-                        *reg = regs::rcx();
+                        *num_bits = Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap();
                     }
                 }
                 insts.push(self);
@@ -1146,7 +1192,7 @@ impl PrettyPrint for Inst {
                 "{} {}, {}",
                 ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())),
                 src2.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())),
-                show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
+                show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
             ),
 
             Inst::UnaryRmR { src, dst, op, size } => format!(
@@ -1208,7 +1254,7 @@ impl PrettyPrint for Inst {
                     DivOrRemKind::SignedRem => "srem",
                     DivOrRemKind::UnsignedRem => "urem",
                 },
-                show_ireg_sized(divisor.to_reg(), mb_rru, size.to_bytes()),
+                show_ireg_sized(divisor.to_reg().to_reg(), mb_rru, size.to_bytes()),
             ),
 
             Inst::SignExtendData { size, .. } => match size {
@@ -1276,8 +1322,8 @@ impl PrettyPrint for Inst {
                     },
                     format!("f{}", size.to_bits())
                 ),
-                show_ireg_sized(*lhs, mb_rru, 8),
+                show_ireg_sized(lhs.to_reg(), mb_rru, 8),
-                show_ireg_sized(rhs_dst.to_reg(), mb_rru, 8),
+                show_ireg_sized(rhs_dst.to_reg().to_reg(), mb_rru, 8),
             ),
 
             Inst::XmmRmRImm {
@@ -1342,7 +1388,7 @@ impl PrettyPrint for Inst {
                 "{} {}, {}",
                 ljustify(op.to_string()),
                 src.show_rru_sized(mb_rru, 8),
-                show_ireg_sized(*dst, mb_rru, 8),
+                show_ireg_sized(dst.to_reg(), mb_rru, 8),
             ),
 
             Inst::CvtUint64ToFloatSeq {
@@ -1405,14 +1451,14 @@ impl PrettyPrint for Inst {
                         "{} ${}, {}",
                         ljustify("movabsq".to_string()),
                         *simm64 as i64,
-                        show_ireg_sized(dst.to_reg(), mb_rru, 8)
+                        show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8)
                     )
                 } else {
                     format!(
                         "{} ${}, {}",
                         ljustify("movl".to_string()),
                         (*simm64 as u32) as i32,
-                        show_ireg_sized(dst.to_reg(), mb_rru, 4)
+                        show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4)
                     )
                 }
             }
@@ -1420,8 +1466,8 @@ impl PrettyPrint for Inst {
             Inst::MovRR { size, src, dst } => format!(
                 "{} {}, {}",
                 ljustify2("mov".to_string(), suffix_lq(*size)),
-                show_ireg_sized(*src, mb_rru, size.to_bytes()),
+                show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()),
-                show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
+                show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
             ),
 
             Inst::MovzxRmR {
@@ -1432,14 +1478,14 @@ impl PrettyPrint for Inst {
                         "{} {}, {}",
                         ljustify("movl".to_string()),
                         src.show_rru_sized(mb_rru, ext_mode.src_size()),
-                        show_ireg_sized(dst.to_reg(), mb_rru, 4)
+                        show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4)
                     )
                 } else {
                     format!(
                         "{} {}, {}",
                         ljustify2("movz".to_string(), ext_mode.to_string()),
                         src.show_rru_sized(mb_rru, ext_mode.src_size()),
-                        show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
+                        show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size())
                     )
                 }
             }
@@ -1464,13 +1510,13 @@ impl PrettyPrint for Inst {
                 "{} {}, {}",
                 ljustify2("movs".to_string(), ext_mode.to_string()),
                 src.show_rru_sized(mb_rru, ext_mode.src_size()),
-                show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
+                show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size())
             ),
 
             Inst::MovRM { size, src, dst, .. } => format!(
                 "{} {}, {}",
                 ljustify2("mov".to_string(), suffix_bwlq(*size)),
-                show_ireg_sized(*src, mb_rru, size.to_bytes()),
+                show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()),
                 dst.show_rru(mb_rru)
             ),
 
@@ -1480,19 +1526,19 @@ impl PrettyPrint for Inst {
                 num_bits,
                 dst,
                 ..
-            } => match num_bits {
+            } => match num_bits.clone().to_imm8_reg() {
                 Imm8Reg::Reg { reg } => format!(
                     "{} {}, {}",
                     ljustify2(kind.to_string(), suffix_bwlq(*size)),
-                    show_ireg_sized(*reg, mb_rru, 1),
+                    show_ireg_sized(reg, mb_rru, 1),
-                    show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
+                    show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
                 ),
 
                 Imm8Reg::Imm8 { imm: num_bits } => format!(
                     "{} ${}, {}",
                     ljustify2(kind.to_string(), suffix_bwlq(*size)),
                     num_bits,
-                    show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
+                    show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
                 ),
             },
 
@@ -1519,14 +1565,14 @@ impl PrettyPrint for Inst {
                     "{} {}, {}",
                     ljustify2(op.to_string(), suffix_bwlq(*size)),
                     src.show_rru_sized(mb_rru, size.to_bytes()),
-                    show_ireg_sized(*dst, mb_rru, size.to_bytes())
+                    show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
                 )
             }
 
             Inst::Setcc { cc, dst } => format!(
                 "{} {}",
                 ljustify2("set".to_string(), cc.to_string()),
-                show_ireg_sized(dst.to_reg(), mb_rru, 1)
+                show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 1)
             ),
 
             Inst::Cmove {
@@ -1539,7 +1585,7 @@ impl PrettyPrint for Inst {
                 "{} {}, {}",
                 ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))),
                 src.show_rru_sized(mb_rru, size.to_bytes()),
-                show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
+                show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
             ),
 
             Inst::XmmCmove { size, cc, src, dst } => {
@@ -1552,7 +1598,7 @@ impl PrettyPrint for Inst {
                         "ss"
                     },
                     src.show_rru_sized(mb_rru, size.to_bytes()),
-                    show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
+                    show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
                 )
             }
 
@@ -1693,10 +1739,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
             debug_assert_eq!(*src1, dst.to_reg());
             if inst.produces_const() {
                 // No need to account for src2, since src2 == dst.
-                collector.add_def(*dst);
+                collector.add_def(dst.to_writable_reg());
             } else {
                 src2.get_regs_as_uses(collector);
-                collector.add_mod(*dst);
+                collector.add_mod(dst.to_writable_reg());
             }
         }
         Inst::Not { src, dst, .. } => {
@@ -1760,9 +1806,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
             // the rdx register *before* the instruction, which is not too bad.
             collector.add_mod(Writable::from_reg(regs::rax()));
             collector.add_mod(Writable::from_reg(regs::rdx()));
-            collector.add_mod(*divisor);
+            collector.add_mod(divisor.to_writable_reg());
             if let Some(tmp) = tmp {
-                collector.add_def(*tmp);
+                collector.add_def(tmp.to_writable_reg());
             }
         }
         Inst::SignExtendData { size, src, dst } => {
@@ -1852,8 +1898,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()),
         Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst),
         Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => {
-            collector.add_use(*lhs);
+            collector.add_use(lhs.to_reg());
-            collector.add_mod(*rhs_dst);
+            collector.add_mod(rhs_dst.to_writable_reg());
         }
         Inst::XmmRmiReg {
             src1, src2, dst, ..
@@ -1868,14 +1914,14 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         }
         Inst::XmmCmpRmR { src, dst, .. } => {
             src.get_regs_as_uses(collector);
-            collector.add_use(*dst);
+            collector.add_use(dst.to_reg());
         }
         Inst::Imm { dst, .. } => {
-            collector.add_def(*dst);
+            collector.add_def(dst.to_writable_reg());
         }
         Inst::MovRR { src, dst, .. } => {
-            collector.add_use(*src);
+            collector.add_use(src.to_reg());
-            collector.add_def(*dst);
+            collector.add_def(dst.to_writable_reg());
         }
         Inst::XmmToGpr { src, dst, .. } => {
             collector.add_use(src.to_reg());
@@ -1918,11 +1964,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         }
         Inst::MovzxRmR { src, dst, .. } => {
             src.get_regs_as_uses(collector);
-            collector.add_def(*dst);
+            collector.add_def(dst.to_writable_reg());
         }
         Inst::Mov64MR { src, dst, .. } => {
             src.get_regs_as_uses(collector);
-            collector.add_def(*dst)
+            collector.add_def(dst.to_writable_reg())
         }
         Inst::LoadEffectiveAddress { addr: src, dst } => {
             src.get_regs_as_uses(collector);
@@ -1930,41 +1976,44 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         }
         Inst::MovsxRmR { src, dst, .. } => {
             src.get_regs_as_uses(collector);
-            collector.add_def(*dst);
+            collector.add_def(dst.to_writable_reg());
         }
         Inst::MovRM { src, dst, .. } => {
-            collector.add_use(*src);
+            collector.add_use(src.to_reg());
             dst.get_regs_as_uses(collector);
         }
         Inst::ShiftR { num_bits, dst, .. } => {
-            if let Imm8Reg::Reg { reg } = num_bits {
+            if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() {
-                debug_assert_eq!(*reg, regs::rcx());
+                debug_assert_eq!(reg, regs::rcx());
                 collector.add_use(regs::rcx());
             }
-            collector.add_mod(*dst);
+            collector.add_mod(dst.to_writable_reg());
         }
         Inst::CmpRmiR { src, dst, .. } => {
             src.get_regs_as_uses(collector);
-            collector.add_use(*dst); // yes, really `add_use`
+            collector.add_use(dst.to_reg()); // yes, really `add_use`
         }
         Inst::Setcc { dst, .. } => {
-            collector.add_def(*dst);
+            collector.add_def(dst.to_writable_reg());
         }
         Inst::Cmove {
             consequent: src,
             dst,
             ..
-        }
+        } => {
-        | Inst::XmmCmove { src, dst, .. } => {
             src.get_regs_as_uses(collector);
-            collector.add_mod(*dst);
+            collector.add_mod(dst.to_writable_reg());
+        }
+        Inst::XmmCmove { src, dst, .. } => {
+            src.get_regs_as_uses(collector);
+            collector.add_mod(dst.to_writable_reg());
         }
         Inst::Push64 { src } => {
             src.get_regs_as_uses(collector);
             collector.add_mod(Writable::from_reg(regs::rsp()));
         }
         Inst::Pop64 { dst } => {
-            collector.add_def(*dst);
+            collector.add_def(dst.to_writable_reg());
         }
 
         Inst::CallKnown {
@@ -2149,11 +2198,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             debug_assert_eq!(*src1, dst.to_reg());
             if produces_const {
                 src2.map_as_def(mapper);
-                mapper.map_def(dst);
+                dst.map_def(mapper);
                 *src1 = dst.to_reg();
             } else {
                 src2.map_uses(mapper);
-                mapper.map_mod(dst);
+                dst.map_mod(mapper);
                 *src1 = dst.to_reg();
             }
         }
@@ -2165,9 +2214,9 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
         Inst::Div { divisor, .. } => divisor.map_uses(mapper),
         Inst::MulHi { src2, .. } => src2.map_uses(mapper),
         Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => {
-            mapper.map_mod(divisor);
+            divisor.map_mod(mapper);
             if let Some(tmp) = tmp {
-                mapper.map_def(tmp)
+                tmp.map_def(mapper)
             }
         }
         Inst::SignExtendData { .. } => {}
@@ -2275,8 +2324,8 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ref mut rhs_dst,
             ..
         } => {
-            mapper.map_use(lhs);
+            lhs.map_use(mapper);
-            mapper.map_mod(rhs_dst);
+            rhs_dst.map_mod(mapper);
         }
         Inst::XmmMovRM {
             ref mut src,
@@ -2292,16 +2341,16 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ..
         } => {
             src.map_uses(mapper);
-            mapper.map_use(dst);
+            dst.map_use(mapper);
         }
-        Inst::Imm { ref mut dst, .. } => mapper.map_def(dst),
+        Inst::Imm { ref mut dst, .. } => dst.map_def(mapper),
         Inst::MovRR {
             ref mut src,
             ref mut dst,
             ..
         } => {
-            mapper.map_use(src);
+            src.map_use(mapper);
-            mapper.map_def(dst);
+            dst.map_def(mapper);
         }
         Inst::XmmToGpr {
             ref mut src,
@@ -2356,11 +2405,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ..
         } => {
             src.map_uses(mapper);
-            mapper.map_def(dst);
+            dst.map_def(mapper);
         }
         Inst::Mov64MR { src, dst, .. } => {
             src.map_uses(mapper);
-            mapper.map_def(dst);
+            dst.map_def(mapper);
         }
         Inst::LoadEffectiveAddress { addr: src, dst } => {
             src.map_uses(mapper);
@@ -2372,14 +2421,14 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ..
         } => {
             src.map_uses(mapper);
-            mapper.map_def(dst);
+            dst.map_def(mapper);
         }
         Inst::MovRM {
             ref mut src,
             ref mut dst,
             ..
         } => {
-            mapper.map_use(src);
+            src.map_use(mapper);
             dst.map_uses(mapper);
         }
         Inst::ShiftR {
@@ -2388,7 +2437,7 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ..
         } => {
             debug_assert_eq!(*src, dst.to_reg());
-            mapper.map_mod(dst);
+            dst.map_mod(mapper);
             *src = dst.to_reg();
         }
         Inst::CmpRmiR {
@@ -2397,9 +2446,9 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ..
         } => {
             src.map_uses(mapper);
-            mapper.map_use(dst);
+            dst.map_use(mapper);
         }
-        Inst::Setcc { ref mut dst, .. } => mapper.map_def(dst),
+        Inst::Setcc { ref mut dst, .. } => dst.map_def(mapper),
         Inst::Cmove {
             consequent: ref mut src,
             ref mut dst,
@@ -2407,7 +2456,7 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ..
         } => {
             src.map_uses(mapper);
-            mapper.map_mod(dst);
+            dst.map_mod(mapper);
             *alternative = dst.to_reg();
         }
         Inst::XmmCmove {
@@ -2416,11 +2465,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
             ..
         } => {
             src.map_uses(mapper);
-            mapper.map_mod(dst);
+            dst.map_mod(mapper);
         }
         Inst::Push64 { ref mut src } => src.map_uses(mapper),
         Inst::Pop64 { ref mut dst } => {
-            mapper.map_def(dst);
+            dst.map_def(mapper);
         }
 
         Inst::CallKnown {
@@ -2520,7 +2569,7 @@ impl MachInst for Inst {
             // conceivably use `movl %reg, %reg` to zero out the top 32 bits of
             // %reg.
             Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => {
-                Some((*dst, *src))
+                Some((dst.to_writable_reg(), src.to_reg()))
             }
             // Note as well that MOVS[S|D] when used in the `XmmUnaryRmR` context are pure moves of
             // scalar floating-point values (and annotate `dst` as `def`s to the register allocator)
@@ -2578,12 +2627,15 @@ impl MachInst for Inst {
                 size: OperandSize::Size8,
                 src,
                 dst: SyntheticAmode::NominalSPOffset { simm32 },
-            } => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)),
+            } => Some(MachInstStackOpInfo::StoreNomSPOff(
+                src.to_reg(),
+                *simm32 as i64,
+            )),
             Self::Mov64MR {
                 src: SyntheticAmode::NominalSPOffset { simm32 },
                 dst,
             } => Some(MachInstStackOpInfo::LoadNomSPOff(
-                dst.to_reg(),
+                dst.to_reg().to_reg(),
                 *simm32 as i64,
             )),
             _ => None,

cranelift/codegen/src/isa/x64/lower.isle

@@ -63,33 +63,33 @@
 ;; Add two registers.
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd x y)))
-      (value_reg (add ty
+      (value_gpr (add ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (RegMemImm.Reg (put_in_reg y)))))
+                      (gpr_to_gpr_mem_imm (put_in_gpr y)))))
 
 ;; Add a register and an immediate.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd x (simm32_from_value y))))
-      (value_reg (add ty (put_in_reg x) y)))
+      (value_gpr (add ty (put_in_gpr x) y)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd (simm32_from_value x) y)))
-      (value_reg (add ty (put_in_reg y) x)))
+      (value_gpr (add ty (put_in_gpr y) x)))
 
 ;; Add a register and memory.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd x (sinkable_load y))))
-      (value_reg (add ty
+      (value_gpr (add ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (sink_load y))))
+                      (sink_load_to_gpr_mem_imm y))))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd (sinkable_load x) y)))
-      (value_reg (add ty
+      (value_gpr (add ty
-                      (put_in_reg y)
+                      (put_in_gpr y)
-                      (sink_load x))))
+                      (sink_load_to_gpr_mem_imm x))))
 
 ;; SSE.
 
@@ -117,15 +117,15 @@
 (rule (lower (has_type $I128 (iadd x y)))
       ;; Get the high/low registers for `x`.
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1)))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1)))
         ;; Get the high/low registers for `y`.
         (let ((y_regs ValueRegs (put_in_regs y))
-              (y_lo Reg (value_regs_get y_regs 0))
+              (y_lo Gpr (value_regs_get_gpr y_regs 0))
-              (y_hi Reg (value_regs_get y_regs 1)))
+              (y_hi Gpr (value_regs_get_gpr y_regs 1)))
           ;; Do an add followed by an add-with-carry.
-          (with_flags (add_with_flags $I64 x_lo (RegMemImm.Reg y_lo))
+          (with_flags (add_with_flags $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
-                      (adc $I64 x_hi (RegMemImm.Reg y_hi))))))
+                      (adc $I64 x_hi (gpr_to_gpr_mem_imm y_hi))))))
 
 ;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -166,42 +166,42 @@
 ;; Add two registers.
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout x y)))
-      (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
+      (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
-        (value_regs (add ty
+        (value_gprs (add ty
-                         (put_in_reg x)
+                        (put_in_gpr x)
-                         (RegMemImm.Reg (put_in_reg y)))
+                        (put_in_gpr_mem_imm y))
-                    unused_iflags)))
+                   unused_iflags)))
 
 ;; Add a register and an immediate.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout x (simm32_from_value y))))
-      (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
+      (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
-        (value_regs (add ty (put_in_reg x) y)
+        (value_gprs (add ty (put_in_gpr x) y)
                     unused_iflags)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout (simm32_from_value x) y)))
-      (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
+      (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
-        (value_regs (add ty (put_in_reg y) x)
+        (value_gprs (add ty (put_in_gpr y) x)
                     unused_iflags)))
 
 ;; Add a register and memory.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout x (sinkable_load y))))
-      (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
+      (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
-        (value_regs (add ty
+        (value_gprs (add ty
-                         (put_in_reg x)
+                         (put_in_gpr x)
-                         (sink_load y))
+                         (sink_load_to_gpr_mem_imm y))
                     unused_iflags)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout (sinkable_load x) y)))
-      (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
+      (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
-        (value_regs (add ty
+        (value_gprs (add ty
-                         (put_in_reg y)
+                         (put_in_gpr y)
-                         (sink_load x))
+                         (sink_load_to_gpr_mem_imm x))
                     unused_iflags)))
 
 ;; (No `iadd_ifcout` for `i128`.)
@@ -212,30 +212,30 @@
 
 ;; When the immediate fits in a `RegMemImm.Imm`, use that.
 (rule (lower (has_type (fits_in_64 ty) (iadd_imm y (simm32_from_imm64 x))))
-      (value_reg (add ty (put_in_reg y) x)))
+      (value_gpr (add ty (put_in_gpr y) x)))
 
 ;; Otherwise, put the immediate into a register.
 (rule (lower (has_type (fits_in_64 ty) (iadd_imm y (u64_from_imm64 x))))
-      (value_reg (add ty (put_in_reg y) (RegMemImm.Reg (imm ty x)))))
+      (value_gpr (add ty (put_in_gpr y) (gpr_to_gpr_mem_imm (gpr_new (imm ty x))))))
 
 ;; `i128`
 
 ;; When the immediate fits in a `RegMemImm.Imm`, use that.
 (rule (lower (has_type $I128 (iadd_imm y (simm32_from_imm64 x))))
       (let ((y_regs ValueRegs (put_in_regs y))
-            (y_lo Reg (value_regs_get y_regs 0))
+            (y_lo Gpr (value_regs_get_gpr y_regs 0))
-            (y_hi Reg (value_regs_get y_regs 1)))
+            (y_hi Gpr (value_regs_get_gpr y_regs 1)))
         (with_flags (add_with_flags $I64 y_lo x)
-                    (adc $I64 y_hi (RegMemImm.Imm 0)))))
+                    (adc $I64 y_hi (gpr_mem_imm_new (RegMemImm.Imm 0))))))
 
 ;; Otherwise, put the immediate into a register.
 (rule (lower (has_type $I128 (iadd_imm y (u64_from_imm64 x))))
       (let ((y_regs ValueRegs (put_in_regs y))
-            (y_lo Reg (value_regs_get y_regs 0))
+            (y_lo Gpr (value_regs_get_gpr y_regs 0))
-            (y_hi Reg (value_regs_get y_regs 1))
+            (y_hi Gpr (value_regs_get_gpr y_regs 1))
-            (x_lo Reg (imm $I64 x)))
+            (x_lo Gpr (gpr_new (imm $I64 x))))
-        (with_flags (add_with_flags $I64 y_lo (RegMemImm.Reg x_lo))
+        (with_flags (add_with_flags $I64 y_lo (gpr_to_gpr_mem_imm x_lo))
-                    (adc $I64 y_hi (RegMemImm.Imm 0)))))
+                    (adc $I64 y_hi (gpr_mem_imm_new (RegMemImm.Imm 0))))))
 
 ;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -244,21 +244,21 @@
 ;; Sub two registers.
 (rule (lower (has_type (fits_in_64 ty)
                        (isub x y)))
-      (value_reg (sub ty
+      (value_gpr (sub ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (RegMemImm.Reg (put_in_reg y)))))
+                      (put_in_gpr_mem_imm y))))
 
 ;; Sub a register and an immediate.
 (rule (lower (has_type (fits_in_64 ty)
                        (isub x (simm32_from_value y))))
-      (value_reg (sub ty (put_in_reg x) y)))
+      (value_gpr (sub ty (put_in_gpr x) y)))
 
 ;; Sub a register and memory.
 (rule (lower (has_type (fits_in_64 ty)
                        (isub x (sinkable_load y))))
-      (value_reg (sub ty
+      (value_gpr (sub ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (sink_load y))))
+                      (sink_load_to_gpr_mem_imm y))))
 
 ;; SSE.
 
@@ -286,15 +286,15 @@
 (rule (lower (has_type $I128 (isub x y)))
       ;; Get the high/low registers for `x`.
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1)))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1)))
         ;; Get the high/low registers for `y`.
         (let ((y_regs ValueRegs (put_in_regs y))
-              (y_lo Reg (value_regs_get y_regs 0))
+              (y_lo Gpr (value_regs_get_gpr y_regs 0))
-              (y_hi Reg (value_regs_get y_regs 1)))
+              (y_hi Gpr (value_regs_get_gpr y_regs 1)))
           ;; Do a sub followed by an sub-with-borrow.
-          (with_flags (sub_with_flags $I64 x_lo (RegMemImm.Reg y_lo))
+          (with_flags (sub_with_flags $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
-                      (sbb $I64 x_hi (RegMemImm.Reg y_hi))))))
+                      (sbb $I64 x_hi (gpr_to_gpr_mem_imm y_hi))))))
 
 ;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -326,36 +326,36 @@
 
 ;; And two registers.
 (rule (lower (has_type (fits_in_64 ty) (band x y)))
-      (value_reg (x64_and ty
+      (value_gpr (x64_and ty
-                          (put_in_reg x)
+                          (put_in_gpr x)
-                          (RegMemImm.Reg (put_in_reg y)))))
+                          (put_in_gpr_mem_imm y))))
 
 ;; And with a memory operand.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (band x (sinkable_load y))))
-      (value_reg (x64_and ty
+      (value_gpr (x64_and ty
-                          (put_in_reg x)
+                          (put_in_gpr x)
-                          (sink_load y))))
+                          (sink_load_to_gpr_mem_imm y))))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (band (sinkable_load x) y)))
-      (value_reg (x64_and ty
+      (value_gpr (x64_and ty
-                          (put_in_reg y)
+                          (put_in_gpr y)
-                          (sink_load x))))
+                          (sink_load_to_gpr_mem_imm x))))
 
 ;; And with an immediate.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (band x (simm32_from_value y))))
-      (value_reg (x64_and ty
+      (value_gpr (x64_and ty
-                          (put_in_reg x)
+                          (put_in_gpr x)
                           y)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (band (simm32_from_value x) y)))
-      (value_reg (x64_and ty
+      (value_gpr (x64_and ty
-                          (put_in_reg y)
+                          (put_in_gpr y)
                           x)))
 
 ;; SSE.
@@ -375,23 +375,23 @@
 
 (rule (lower (has_type $I128 (band x y)))
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1))
             (y_regs ValueRegs (put_in_regs y))
-            (y_lo Reg (value_regs_get y_regs 0))
+            (y_lo Gpr (value_regs_get_gpr y_regs 0))
-            (y_hi Reg (value_regs_get y_regs 1)))
+            (y_hi Gpr (value_regs_get_gpr y_regs 1)))
-        (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo))
+        (value_gprs (x64_and $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
-                    (x64_and $I64 x_hi (RegMemImm.Reg y_hi)))))
+                    (x64_and $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))
 
 (rule (lower (has_type $B128 (band x y)))
       ;; Booleans are always `0` or `1`, so we only need to do the `and` on the
       ;; low half. The high half is always zero but, rather than generate a new
       ;; zero, we just reuse `x`'s high half which is already zero.
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1))
-            (y_lo Reg (lo_reg y)))
+            (y_lo Gpr (lo_gpr y)))
-        (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo))
+        (value_gprs (x64_and $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
                     x_hi)))
 
 ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -400,36 +400,36 @@
 
 ;; Or two registers.
 (rule (lower (has_type (fits_in_64 ty) (bor x y)))
-      (value_reg (or ty
+      (value_gpr (or ty
-                     (put_in_reg x)
+                     (put_in_gpr x)
-                     (RegMemImm.Reg (put_in_reg y)))))
+                     (put_in_gpr_mem_imm y))))
 
 ;; Or with a memory operand.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bor x (sinkable_load y))))
-      (value_reg (or ty
+      (value_gpr (or ty
-                     (put_in_reg x)
+                     (put_in_gpr x)
-                     (sink_load y))))
+                     (sink_load_to_gpr_mem_imm y))))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bor (sinkable_load x) y)))
-      (value_reg (or ty
+      (value_gpr (or ty
-                     (put_in_reg y)
+                     (put_in_gpr y)
-                     (sink_load x))))
+                     (sink_load_to_gpr_mem_imm x))))
 
 ;; Or with an immediate.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bor x (simm32_from_value y))))
-      (value_reg (or ty
+      (value_gpr (or ty
-                     (put_in_reg x)
+                     (put_in_gpr x)
                      y)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bor (simm32_from_value x) y)))
-      (value_reg (or ty
+      (value_gpr (or ty
-                     (put_in_reg y)
+                     (put_in_gpr y)
                      x)))
 
 ;; SSE.
@@ -449,12 +449,12 @@
 
 (decl or_i128 (ValueRegs ValueRegs) ValueRegs)
 (rule (or_i128 x y)
-      (let ((x_lo Reg (value_regs_get x 0))
+      (let ((x_lo Gpr (value_regs_get_gpr x 0))
-            (x_hi Reg (value_regs_get x 1))
+            (x_hi Gpr (value_regs_get_gpr x 1))
-            (y_lo Reg (value_regs_get y 0))
+            (y_lo Gpr (value_regs_get_gpr y 0))
-            (y_hi Reg (value_regs_get y 1)))
+            (y_hi Gpr (value_regs_get_gpr y 1)))
-        (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo))
+        (value_gprs (or $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
-                    (or $I64 x_hi (RegMemImm.Reg y_hi)))))
+                    (or $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))
 
 (rule (lower (has_type $I128 (bor x y)))
       (or_i128 (put_in_regs x) (put_in_regs y)))
@@ -464,10 +464,10 @@
       ;; low half. The high half is always zero but, rather than generate a new
       ;; zero, we just reuse `x`'s high half which is already zero.
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1))
-            (y_lo Reg (lo_reg y)))
+            (y_lo Gpr (lo_gpr y)))
-        (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo))
+        (value_gprs (or $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
                     x_hi)))
 
 ;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -476,36 +476,36 @@
 
 ;; Xor two registers.
 (rule (lower (has_type (fits_in_64 ty) (bxor x y)))
-      (value_reg (xor ty
+      (value_gpr (xor ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (RegMemImm.Reg (put_in_reg y)))))
+                      (put_in_gpr_mem_imm y))))
 
 ;; Xor with a memory operand.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bxor x (sinkable_load y))))
-      (value_reg (xor ty
+      (value_gpr (xor ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (sink_load y))))
+                      (sink_load_to_gpr_mem_imm y))))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bxor (sinkable_load x) y)))
-      (value_reg (xor ty
+      (value_gpr (xor ty
-                      (put_in_reg y)
+                      (put_in_gpr y)
-                      (sink_load x))))
+                      (sink_load_to_gpr_mem_imm x))))
 
 ;; Xor with an immediate.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bxor x (simm32_from_value y))))
-      (value_reg (xor ty
+      (value_gpr (xor ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
                       y)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (bxor (simm32_from_value x) y)))
-      (value_reg (xor ty
+      (value_gpr (xor ty
-                      (put_in_reg y)
+                      (put_in_gpr y)
                       x)))
 
 ;; SSE.
@@ -517,23 +517,23 @@
 
 (rule (lower (has_type $I128 (bxor x y)))
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1))
             (y_regs ValueRegs (put_in_regs y))
-            (y_lo Reg (value_regs_get y_regs 0))
+            (y_lo Gpr (value_regs_get_gpr y_regs 0))
-            (y_hi Reg (value_regs_get y_regs 1)))
+            (y_hi Gpr (value_regs_get_gpr y_regs 1)))
-        (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo))
+        (value_gprs (xor $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
-                    (xor $I64 x_hi (RegMemImm.Reg y_hi)))))
+                    (xor $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))
 
 (rule (lower (has_type $B128 (bxor x y)))
       ;; Booleans are always `0` or `1`, so we only need to do the `xor` on the
       ;; low half. The high half is always zero but, rather than generate a new
       ;; zero, we just reuse `x`'s high half which is already zero.
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1))
-            (y_lo Reg (lo_reg y)))
+            (y_lo Gpr (lo_gpr y)))
-        (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo))
+        (value_gprs (xor $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
                     x_hi)))
 
 ;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -541,38 +541,49 @@
 ;; `i64` and smaller.
 
 (rule (lower (has_type (fits_in_64 ty) (ishl src amt)))
-      (value_reg (shl ty (put_in_reg src) (put_masked_in_imm8_reg amt ty))))
+      (value_gpr (shl ty (put_in_gpr src) (put_masked_in_imm8_gpr amt ty))))
 
 ;; `i128`.
 
-(decl shl_i128 (ValueRegs Reg) ValueRegs)
+(decl shl_i128 (ValueRegs Gpr) ValueRegs)
 (rule (shl_i128 src amt)
       ;; Unpack the registers that make up the 128-bit value being shifted.
-      (let ((src_lo Reg (value_regs_get src 0))
+      (let ((src_lo Gpr (value_regs_get_gpr src 0))
-            (src_hi Reg (value_regs_get src 1))
+            (src_hi Gpr (value_regs_get_gpr src 1))
             ;; Do two 64-bit shifts.
-            (lo_shifted Reg (shl $I64 src_lo (Imm8Reg.Reg amt)))
+            (lo_shifted Gpr (shl $I64 src_lo (gpr_to_imm8_gpr amt)))
-            (hi_shifted Reg (shl $I64 src_hi (Imm8Reg.Reg amt)))
+            (hi_shifted Gpr (shl $I64 src_hi (gpr_to_imm8_gpr amt)))
             ;; `src_lo >> (64 - amt)` are the bits to carry over from the lo
             ;; into the hi.
-            (carry Reg (shr $I64 src_lo (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
+            (carry Gpr (shr $I64
-            (zero Reg (imm $I64 0))
+                            src_lo
+                            (gpr_to_imm8_gpr (sub $I64
+                                                  (gpr_new (imm $I64 64))
+                                                  (gpr_to_gpr_mem_imm amt)))))
+            (zero Gpr (gpr_new (imm $I64 0)))
             ;; Nullify the carry if we are shifting in by a multiple of 128.
-            (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
+            (carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64)
-                                      (cmove $I64 (CC.Z) (RegMem.Reg zero) carry)))
+                                                     (gpr_mem_imm_new (RegMemImm.Imm 127))
+                                                     amt)
+                                               (cmove $I64
+                                                      (CC.Z)
+                                                      (gpr_to_gpr_mem zero)
+                                                      carry))))
             ;; Add the carry into the high half.
-            (hi_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg hi_shifted))))
+            (hi_shifted_ Gpr (or $I64 carry_ (gpr_to_gpr_mem_imm hi_shifted))))
         ;; Combine the two shifted halves. However, if we are shifting by >= 64
         ;; (modulo 128), then the low bits are zero and the high bits are our
         ;; low bits.
-        (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+        (with_flags_2 (test (OperandSize.Size64)
-                      (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted) zero)
+                            (gpr_mem_imm_new (RegMemImm.Imm 64))
-                      (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted_) lo_shifted))))
+                            amt)
+                      (cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted) zero)
+                      (cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted_) lo_shifted))))
 
 (rule (lower (has_type $I128 (ishl src amt)))
       ;; NB: Only the low bits of `amt` matter since we logically mask the shift
       ;; amount to the value's bit width.
-      (let ((amt_ Reg (lo_reg amt)))
+      (let ((amt_ Gpr (lo_gpr amt)))
         (shl_i128 (put_in_regs src) amt_)))
 
 ;; SSE.
@@ -613,10 +624,12 @@
 (rule (ishl_i8x16_mask (RegMemImm.Reg amt))
       (let ((mask_table SyntheticAmode (ishl_i8x16_mask_table))
             (base_mask_addr Gpr (lea mask_table))
-            (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4))))
+            (mask_offset Gpr (shl $I64
+                                  (gpr_new amt)
+                                  (imm8_to_imm8_gpr 4))))
         (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0
                                                            base_mask_addr
-                                                           (gpr_new mask_offset)
+                                                           mask_offset
                                                            0))))
 (rule (ishl_i8x16_mask (RegMemImm.Mem amt))
       (ishl_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None)))))
@@ -640,38 +653,49 @@
 ;; `i64` and smaller.
 
 (rule (lower (has_type (fits_in_64 ty) (ushr src amt)))
-      (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Zero))))
+      (let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Zero))))
-        (value_reg (shr ty src_ (put_masked_in_imm8_reg amt ty)))))
+        (value_gpr (shr ty src_ (put_masked_in_imm8_gpr amt ty)))))
 
 ;; `i128`.
 
-(decl shr_i128 (ValueRegs Reg) ValueRegs)
+(decl shr_i128 (ValueRegs Gpr) ValueRegs)
 (rule (shr_i128 src amt)
       ;; Unpack the lo/hi halves of `src`.
-      (let ((src_lo Reg (value_regs_get src 0))
+      (let ((src_lo Gpr (value_regs_get_gpr src 0))
-            (src_hi Reg (value_regs_get src 1))
+            (src_hi Gpr (value_regs_get_gpr src 1))
             ;; Do a shift on each half.
-            (lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt)))
+            (lo_shifted Gpr (shr $I64 src_lo (gpr_to_imm8_gpr amt)))
-            (hi_shifted Reg (shr $I64 src_hi (Imm8Reg.Reg amt)))
+            (hi_shifted Gpr (shr $I64 src_hi (gpr_to_imm8_gpr amt)))
             ;; `src_hi << (64 - amt)` are the bits to carry over from the hi
             ;; into the lo.
-            (carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
+            (carry Gpr (shl $I64
+                            src_hi
+                            (gpr_to_imm8_gpr (sub $I64
+                                                  (gpr_new (imm $I64 64))
+                                                  (gpr_to_gpr_mem_imm amt)))))
             ;; Nullify the carry if we are shifting by a multiple of 128.
-            (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
+            (carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64)
-                                      (cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry)))
+                                                     (gpr_mem_imm_new (RegMemImm.Imm 127))
+                                                     amt)
+                                               (cmove $I64
+                                                      (CC.Z)
+                                                      (gpr_to_gpr_mem (gpr_new (imm $I64 0)))
+                                                      carry))))
             ;; Add the carry bits into the lo.
-            (lo_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg lo_shifted))))
+            (lo_shifted_ Gpr (or $I64 carry_ (gpr_to_gpr_mem_imm lo_shifted))))
         ;; Combine the two shifted halves. However, if we are shifting by >= 64
         ;; (modulo 128), then the hi bits are zero and the lo bits are what
         ;; would otherwise be our hi bits.
-        (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+        (with_flags_2 (test (OperandSize.Size64)
-                      (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted)
+                            (gpr_mem_imm_new (RegMemImm.Imm 64))
-                      (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) (imm $I64 0)))))
+                            amt)
+                      (cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted_) hi_shifted)
+                      (cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted) (gpr_new (imm $I64 0))))))
 
 (rule (lower (has_type $I128 (ushr src amt)))
       ;; NB: Only the low bits of `amt` matter since we logically mask the shift
       ;; amount to the value's bit width.
-      (let ((amt_ Reg (lo_reg amt)))
+      (let ((amt_ Gpr (lo_gpr amt)))
         (shr_i128 (put_in_regs src) amt_)))
 
 ;; SSE.
@@ -712,10 +736,12 @@
 (rule (ushr_i8x16_mask (RegMemImm.Reg amt))
       (let ((mask_table SyntheticAmode (ushr_i8x16_mask_table))
             (base_mask_addr Gpr (lea mask_table))
-            (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4))))
+            (mask_offset Gpr (shl $I64
+                                  (gpr_new amt)
+                                  (imm8_to_imm8_gpr 4))))
         (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0
                                                            base_mask_addr
-                                                           (gpr_new mask_offset)
+                                                           mask_offset
                                                            0))))
 (rule (ushr_i8x16_mask (RegMemImm.Mem amt))
       (ushr_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None)))))
@@ -739,41 +765,52 @@
 ;; `i64` and smaller.
 
 (rule (lower (has_type (fits_in_64 ty) (sshr src amt)))
-      (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Sign))))
+      (let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Sign))))
-        (value_reg (sar ty src_ (put_masked_in_imm8_reg amt ty)))))
+        (value_gpr (sar ty src_ (put_masked_in_imm8_gpr amt ty)))))
 
 ;; `i128`.
 
-(decl sar_i128 (ValueRegs Reg) ValueRegs)
+(decl sar_i128 (ValueRegs Gpr) ValueRegs)
 (rule (sar_i128 src amt)
       ;; Unpack the low/high halves of `src`.
-      (let ((src_lo Reg (value_regs_get src 0))
+      (let ((src_lo Gpr (value_regs_get_gpr src 0))
-            (src_hi Reg (value_regs_get src 1))
+            (src_hi Gpr (value_regs_get_gpr src 1))
             ;; Do a shift of each half. NB: the low half uses an unsigned shift
             ;; because its MSB is not a sign bit.
-            (lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt)))
+            (lo_shifted Gpr (shr $I64 src_lo (gpr_to_imm8_gpr amt)))
-            (hi_shifted Reg (sar $I64 src_hi (Imm8Reg.Reg amt)))
+            (hi_shifted Gpr (sar $I64 src_hi (gpr_to_imm8_gpr amt)))
             ;; `src_hi << (64 - amt)` are the bits to carry over from the low
             ;; half to the high half.
-            (carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
+            (carry Gpr (shl $I64
+                            src_hi
+                            (gpr_to_imm8_gpr (sub $I64
+                                                  (gpr_new (imm $I64 64))
+                                                  (gpr_to_gpr_mem_imm amt)))))
             ;; Nullify the carry if we are shifting by a multiple of 128.
-            (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
+            (carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64)
-                                      (cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry)))
+                                                     (gpr_mem_imm_new (RegMemImm.Imm 127))
+                                                     amt)
+                                               (cmove $I64
+                                                      (CC.Z)
+                                                      (gpr_to_gpr_mem (gpr_new (imm $I64 0)))
+                                                      carry))))
             ;; Add the carry into the low half.
-            (lo_shifted_ Reg (or $I64 lo_shifted (RegMemImm.Reg carry_)))
+            (lo_shifted_ Gpr (or $I64 lo_shifted (gpr_to_gpr_mem_imm carry_)))
             ;; Get all sign bits.
-            (sign_bits Reg (sar $I64 src_hi (Imm8Reg.Imm8 63))))
+            (sign_bits Gpr (sar $I64 src_hi (imm8_to_imm8_gpr 63))))
         ;; Combine the two shifted halves. However, if we are shifting by >= 64
         ;; (modulo 128), then the hi bits are all sign bits and the lo bits are
         ;; what would otherwise be our hi bits.
-        (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+        (with_flags_2 (test (OperandSize.Size64)
-                      (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted)
+                            (gpr_mem_imm_new (RegMemImm.Imm 64))
-                      (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) sign_bits))))
+                            amt)
+                      (cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted_) hi_shifted)
+                      (cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted) sign_bits))))
 
 (rule (lower (has_type $I128 (sshr src amt)))
       ;; NB: Only the low bits of `amt` matter since we logically mask the shift
       ;; amount to the value's bit width.
-      (let ((amt_ Reg (lo_reg amt)))
+      (let ((amt_ Gpr (lo_gpr amt)))
         (sar_i128 (put_in_regs src) amt_)))
 
 ;; SSE.
@@ -807,9 +844,13 @@
 (rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i))
       (xmm_mem_imm_new (RegMemImm.Imm (u32_add i 8))))
 (rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r))
-      (mov_rmi_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8)))))
+      (mov_rmi_to_xmm (RegMemImm.Reg (gpr_to_reg (add ty
+                                                      (gpr_new r)
+                                                      (gpr_mem_imm_new (RegMemImm.Imm 8)))))))
 (rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m))
-      (mov_rmi_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi))))
+      (mov_rmi_to_xmm (RegMemImm.Reg (gpr_to_reg (add ty
+                                                      (gpr_new (imm ty 8))
+                                                      (gpr_mem_imm_new rmi))))))
 
 ;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure
 ;; that if the shift amount is in a register, it is in an XMM register.
@@ -834,11 +875,11 @@
       (let ((src_ Xmm (put_in_xmm src))
             (lo Gpr (pextrd $I64 src_ 0))
             (hi Gpr (pextrd $I64 src_ 1))
-            (amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64))
+            (amt_ Imm8Gpr (put_masked_in_imm8_gpr amt $I64))
-            (shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_))
+            (shifted_lo Gpr (sar $I64 lo amt_))
-            (shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_)))
+            (shifted_hi Gpr (sar $I64 hi amt_)))
-        (value_xmm (make_i64x2_from_lanes (reg_mem_to_gpr_mem (RegMem.Reg shifted_lo))
+        (value_xmm (make_i64x2_from_lanes (gpr_to_gpr_mem shifted_lo)
-                                          (reg_mem_to_gpr_mem (RegMem.Reg shifted_hi))))))
+                                          (gpr_to_gpr_mem shifted_hi)))))
 
 ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -846,13 +887,13 @@
 ;; constant.
 
 (rule (lower (has_type (ty_8_or_16 ty) (rotl src amt)))
-      (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero))))
+      (let ((amt_ Gpr (extend_to_gpr amt $I32 (ExtendKind.Zero))))
-        (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
+        (value_gpr (x64_rotl ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
 
 (rule (lower (has_type (ty_8_or_16 ty)
                        (rotl src (u64_from_iconst amt))))
-      (value_reg (x64_rotl ty
+      (value_gpr (x64_rotl ty
-                           (put_in_reg src)
+                           (put_in_gpr src)
                            (const_to_type_masked_imm8 amt ty))))
 
 ;; `i64` and `i32`: we can rely on x86's rotate-amount masking since
@@ -861,13 +902,13 @@
 (rule (lower (has_type (ty_32_or_64 ty) (rotl src amt)))
       ;; NB: Only the low bits of `amt` matter since we logically mask the
       ;; shift amount to the value's bit width.
-      (let ((amt_ Reg (lo_reg amt)))
+      (let ((amt_ Gpr (lo_gpr amt)))
-        (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
+        (value_gpr (x64_rotl ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
 
 (rule (lower (has_type (ty_32_or_64 ty)
                        (rotl src (u64_from_iconst amt))))
-      (value_reg (x64_rotl ty
+      (value_gpr (x64_rotl ty
-                           (put_in_reg src)
+                           (put_in_gpr src)
                            (const_to_type_masked_imm8 amt ty))))
 
 ;; `i128`.
@@ -876,9 +917,11 @@
       (let ((src_ ValueRegs (put_in_regs src))
             ;; NB: Only the low bits of `amt` matter since we logically mask the
             ;; rotation amount to the value's bit width.
-            (amt_ Reg (lo_reg amt)))
+            (amt_ Gpr (lo_gpr amt)))
         (or_i128 (shl_i128 src_ amt_)
-                 (shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_))))))
+                 (shr_i128 src_ (sub $I64
+                                     (gpr_new (imm $I64 128))
+                                     (gpr_to_gpr_mem_imm amt_))))))
 
 ;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -886,13 +929,13 @@
 ;; constant.
 
 (rule (lower (has_type (ty_8_or_16 ty) (rotr src amt)))
-      (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero))))
+      (let ((amt_ Gpr (extend_to_gpr amt $I32 (ExtendKind.Zero))))
-        (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
+        (value_gpr (x64_rotr ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
 
 (rule (lower (has_type (ty_8_or_16 ty)
                        (rotr src (u64_from_iconst amt))))
-      (value_reg (x64_rotr ty
+      (value_gpr (x64_rotr ty
-                           (put_in_reg src)
+                           (put_in_gpr src)
                            (const_to_type_masked_imm8 amt ty))))
 
 ;; `i64` and `i32`: we can rely on x86's rotate-amount masking since
@@ -901,13 +944,13 @@
 (rule (lower (has_type (ty_32_or_64 ty) (rotr src amt)))
       ;; NB: Only the low bits of `amt` matter since we logically mask the
       ;; shift amount to the value's bit width.
-      (let ((amt_ Reg (lo_reg amt)))
+      (let ((amt_ Gpr (lo_gpr amt)))
-        (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
+        (value_gpr (x64_rotr ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
 
 (rule (lower (has_type (ty_32_or_64 ty)
                        (rotr src (u64_from_iconst amt))))
-      (value_reg (x64_rotr ty
+      (value_gpr (x64_rotr ty
-                           (put_in_reg src)
+                           (put_in_gpr src)
                            (const_to_type_masked_imm8 amt ty))))
 
 ;; `i128`.
@@ -916,9 +959,11 @@
       (let ((src_ ValueRegs (put_in_regs src))
             ;; NB: Only the low bits of `amt` matter since we logically mask the
             ;; rotation amount to the value's bit width.
-            (amt_ Reg (lo_reg amt)))
+            (amt_ Gpr (lo_gpr amt)))
         (or_i128 (shr_i128 src_ amt_)
-                 (shl_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_))))))
+                 (shl_i128 src_ (sub $I64
+                                     (gpr_new (imm $I64 128))
+                                     (gpr_to_gpr_mem_imm amt_))))))
 
 ;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -961,33 +1006,33 @@
 
 ;; Multiply two registers.
 (rule (lower (has_type (fits_in_64 ty) (imul x y)))
-      (value_reg (mul ty
+      (value_gpr (mul ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (RegMemImm.Reg (put_in_reg y)))))
+                      (put_in_gpr_mem_imm y))))
 
 ;; Multiply a register and an immediate.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (imul x (simm32_from_value y))))
-      (value_reg (mul ty (put_in_reg x) y)))
+      (value_gpr (mul ty (put_in_gpr x) y)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (imul (simm32_from_value x) y)))
-      (value_reg (mul ty (put_in_reg y) x)))
+      (value_gpr (mul ty (put_in_gpr y) x)))
 
 ;; Multiply a register and a memory load.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (imul x (sinkable_load y))))
-      (value_reg (mul ty
+      (value_gpr (mul ty
-                      (put_in_reg x)
+                      (put_in_gpr x)
-                      (sink_load y))))
+                      (sink_load_to_gpr_mem_imm y))))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (imul (sinkable_load x) y)))
-      (value_reg (mul ty
+      (value_gpr (mul ty
-                      (put_in_reg y)
+                      (put_in_gpr y)
-                      (sink_load x))))
+                      (sink_load_to_gpr_mem_imm x))))
 
 ;; `i128`.
 
@@ -1007,25 +1052,25 @@
 (rule (lower (has_type $I128 (imul x y)))
       ;; Put `x` into registers and unpack its hi/lo halves.
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Reg (value_regs_get x_regs 0))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Reg (value_regs_get x_regs 1))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1))
             ;; Put `y` into registers and unpack its hi/lo halves.
             (y_regs ValueRegs (put_in_regs y))
-            (y_lo Reg (value_regs_get y_regs 0))
+            (y_lo Gpr (value_regs_get_gpr y_regs 0))
-            (y_hi Reg (value_regs_get y_regs 1))
+            (y_hi Gpr (value_regs_get_gpr y_regs 1))
             ;; lo_hi = mul x_lo, y_hi
-            (lo_hi Reg (mul $I64 x_lo (RegMemImm.Reg y_hi)))
+            (lo_hi Gpr (mul $I64 x_lo (gpr_to_gpr_mem_imm y_hi)))
             ;; hi_lo = mul x_hi, y_lo
-            (hi_lo Reg (mul $I64 x_hi (RegMemImm.Reg y_lo)))
+            (hi_lo Gpr (mul $I64 x_hi (gpr_to_gpr_mem_imm y_lo)))
             ;; hilo_hilo = add lo_hi, hi_lo
-            (hilo_hilo Reg (add $I64 lo_hi (RegMemImm.Reg hi_lo)))
+            (hilo_hilo Gpr (add $I64 lo_hi (gpr_to_gpr_mem_imm hi_lo)))
             ;; dst_lo:hi_lolo = mulhi_u x_lo, y_lo
-            (mul_regs ValueRegs (mulhi_u $I64 x_lo (RegMem.Reg y_lo)))
+            (mul_regs ValueRegs (mulhi_u $I64 x_lo (gpr_to_gpr_mem y_lo)))
-            (dst_lo Reg (value_regs_get mul_regs 0))
+            (dst_lo Gpr (value_regs_get_gpr mul_regs 0))
-            (hi_lolo Reg (value_regs_get mul_regs 1))
+            (hi_lolo Gpr (value_regs_get_gpr mul_regs 1))
             ;; dst_hi = add hilo_hilo, hi_lolo
-            (dst_hi Reg (add $I64 hilo_hilo (RegMemImm.Reg hi_lolo))))
+            (dst_hi Gpr (add $I64 hilo_hilo (gpr_to_gpr_mem_imm hi_lolo))))
-        (value_regs dst_lo dst_hi)))
+        (value_gprs dst_lo dst_hi)))
 
 ;; SSE.
 
@@ -1310,8 +1355,8 @@
 (decl i128_not (Value) ValueRegs)
 (rule (i128_not x)
       (let ((x_regs ValueRegs (put_in_regs x))
-            (x_lo Gpr (gpr_new (value_regs_get x_regs 0)))
+            (x_lo Gpr (value_regs_get_gpr x_regs 0))
-            (x_hi Gpr (gpr_new (value_regs_get x_regs 1))))
+            (x_hi Gpr (value_regs_get_gpr x_regs 1)))
         (value_gprs (not $I64 x_lo)
                     (not $I64 x_hi))))
 
@@ -1420,11 +1465,11 @@
 
 (decl cmp_and_choose (Type CC Value Value) ValueRegs)
 (rule (cmp_and_choose (fits_in_64 ty) cc x y)
-      (let ((x_reg Reg (put_in_reg x))
+      (let ((x_reg Gpr (put_in_gpr x))
-            (y_reg Reg (put_in_reg y))
+            (y_reg Gpr (put_in_gpr y))
             (size OperandSize (raw_operand_size_of_type ty)))
-           (value_reg (with_flags_1 (cmp size (RegMemImm.Reg x_reg) y_reg)
+           (value_reg (with_flags_1 (cmp size (gpr_to_gpr_mem_imm x_reg) y_reg)
-                                    (cmove ty cc (RegMem.Reg y_reg) x_reg)))))
+                                    (cmove ty cc (gpr_to_gpr_mem y_reg) x_reg)))))
 
 (rule (lower (has_type (fits_in_64 ty) (umin x y)))
       (cmp_and_choose ty (CC.B) x y))

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -79,7 +79,7 @@ where
 
         if let Some(c) = inputs.constant {
             if let Some(imm) = to_simm32(c as i64) {
-                return imm;
+                return imm.to_reg_mem_imm();
             }
 
             // Generate constants fresh at each use to minimize long-range
@@ -120,21 +120,23 @@ where
         RegMem::reg(self.put_in_reg(val))
     }
 
-    fn put_masked_in_imm8_reg(&mut self, val: Value, ty: Type) -> Imm8Reg {
+    fn put_masked_in_imm8_gpr(&mut self, val: Value, ty: Type) -> Imm8Gpr {
         let inputs = self.lower_ctx.get_value_as_source_or_const(val);
 
         if let Some(c) = inputs.constant {
             let mask = 1_u64
                 .checked_shl(ty.bits() as u32)
                 .map_or(u64::MAX, |x| x - 1);
-            return Imm8Reg::Imm8 {
+            return Imm8Gpr::new(Imm8Reg::Imm8 {
                 imm: (c & mask) as u8,
-            };
+            })
+            .unwrap();
         }
 
-        Imm8Reg::Reg {
+        Imm8Gpr::new(Imm8Reg::Reg {
             reg: self.put_in_regs(val).regs()[0],
-        }
+        })
+        .unwrap()
     }
 
     #[inline]
@@ -178,17 +180,18 @@ where
     }
 
     #[inline]
-    fn const_to_type_masked_imm8(&mut self, c: u64, ty: Type) -> Imm8Reg {
+    fn const_to_type_masked_imm8(&mut self, c: u64, ty: Type) -> Imm8Gpr {
         let mask = 1_u64
             .checked_shl(ty.bits() as u32)
             .map_or(u64::MAX, |x| x - 1);
-        Imm8Reg::Imm8 {
+        Imm8Gpr::new(Imm8Reg::Imm8 {
             imm: (c & mask) as u8,
-        }
+        })
+        .unwrap()
     }
 
     #[inline]
-    fn simm32_from_value(&mut self, val: Value) -> Option<RegMemImm> {
+    fn simm32_from_value(&mut self, val: Value) -> Option<GprMemImm> {
         let inst = self.lower_ctx.dfg().value_def(val).inst()?;
         let constant: u64 = self.lower_ctx.get_constant(inst)?;
         let constant = constant as i64;
@@ -196,7 +199,7 @@ where
     }
 
     #[inline]
-    fn simm32_from_imm64(&mut self, imm: Imm64) -> Option<RegMemImm> {
+    fn simm32_from_imm64(&mut self, imm: Imm64) -> Option<GprMemImm> {
         to_simm32(imm.bits())
     }
 
@@ -412,6 +415,31 @@ where
     fn reg_to_gpr_mem(&mut self, r: Reg) -> GprMem {
         GprMem::new(RegMem::reg(r)).unwrap()
     }
+
+    #[inline]
+    fn imm8_reg_to_imm8_gpr(&mut self, ir: &Imm8Reg) -> Imm8Gpr {
+        Imm8Gpr::new(ir.clone()).unwrap()
+    }
+
+    #[inline]
+    fn gpr_to_gpr_mem(&mut self, gpr: Gpr) -> GprMem {
+        GprMem::from(gpr)
+    }
+
+    #[inline]
+    fn gpr_to_gpr_mem_imm(&mut self, gpr: Gpr) -> GprMemImm {
+        GprMemImm::from(gpr)
+    }
+
+    #[inline]
+    fn gpr_to_imm8_gpr(&mut self, gpr: Gpr) -> Imm8Gpr {
+        Imm8Gpr::from(gpr)
+    }
+
+    #[inline]
+    fn imm8_to_imm8_gpr(&mut self, imm: u8) -> Imm8Gpr {
+        Imm8Gpr::new(Imm8Reg::Imm8 { imm }).unwrap()
+    }
 }
 
 // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we
@@ -446,11 +474,14 @@ const I8X16_USHR_MASKS: [u8; 128] = [
 ];
 
 #[inline]
-fn to_simm32(constant: i64) -> Option<RegMemImm> {
+fn to_simm32(constant: i64) -> Option<GprMemImm> {
     if constant == ((constant << 32) >> 32) {
-        Some(RegMemImm::Imm {
+        Some(
-            simm32: constant as u32,
+            GprMemImm::new(RegMemImm::Imm {
-        })
+                simm32: constant as u32,
+            })
+            .unwrap(),
+        )
     } else {
         None
     }

cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
 src/prelude.isle 73285cd431346d53
-src/isa/x64/inst.isle 7513533d16948249
+src/isa/x64/inst.isle 301db31d5f1118ae
-src/isa/x64/lower.isle 802b6e750d407100
+src/isa/x64/lower.isle cdc94aec26c0bc5b

cranelift/codegen/src/machinst/isle.rs

@@ -15,7 +15,6 @@ pub type ValueSlice<'a> = &'a [Value];
 pub type ValueArray2 = [Value; 2];
 pub type ValueArray3 = [Value; 3];
 pub type WritableReg = Writable<Reg>;
-pub type OptionWritableReg = Option<WritableReg>;
 pub type VecReg = Vec<Reg>;
 pub type VecWritableReg = Vec<WritableReg>;
 pub type ValueRegs = crate::machinst::ValueRegs<Reg>;