ISLE: Lowering of multi-output instructions (#3783)

This changes the output of the `lower` constructor from a
`ValueRegs` to a new `InstOutput` type, which is a vector
of `ValueRegs`.

Code in `lower_common` is updated to use this new type to
handle instructions with multiple outputs.  All back-ends
are updated to use the new type.

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2,7 +2,7 @@
 
 ;; The main lowering constructor term: takes a clif `Inst` and returns the
 ;; register(s) within which the lowered instruction's result values live.
-(decl lower (Inst) ValueRegs)
+(decl lower (Inst) InstOutput)
 
 ;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 8bf92e18323e7041
+src/prelude.isle 957023853b23dacb
 src/isa/aarch64/inst.isle 3678d0a37bdb4cff
-src/isa/aarch64/lower.isle 1bc1f817a4721801
+src/isa/aarch64/lower.isle 90accbfcadaea46d

cranelift/codegen/src/isa/s390x/lower.isle

@@ -2,12 +2,12 @@
 
 ;; The main lowering constructor term: takes a clif `Inst` and returns the
 ;; register(s) within which the lowered instruction's result values live.
-(decl lower (Inst) ValueRegs)
+(decl lower (Inst) InstOutput)
 
 ;; A variant of the main lowering constructor term, used for branches.
 ;; The only difference is that it gets an extra argument holding a vector
 ;; of branch targets to be used.
-(decl lower_branch (Inst VecMachLabel) ValueRegs)
+(decl lower_branch (Inst VecMachLabel) InstOutput)
 
 
 ;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -148,39 +148,39 @@
 ;; containing the carry result, but we do not support the `iflags` mechanism.
 ;; However, the only actual use case is where `iadd_ifcout` feeds into `trapif`,
 ;; which is implemented by explicitly matching on the flags producer.  So we can
-;; get away with just allocating a second temp so that the reg-renaming code
+;; get away with just using an invalid second output, and the reg-renaming code
 ;; does the right thing, for now.
-(decl value_regs_ifcout (Reg) ValueRegs)
+(decl output_ifcout (Reg) InstOutput)
-(rule (value_regs_ifcout reg)
+(rule (output_ifcout reg)
-      (value_regs reg (writable_reg_to_reg (temp_writable_reg $I64))))
+      (output_pair reg (value_regs_invalid)))
 
 ;; Add two registers.
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x y)))
-      (value_regs_ifcout (add_logical_reg ty x y)))
+      (output_ifcout (add_logical_reg ty x y)))
 
 ;; Add a register and a zero-extended register.
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (zext32_value y))))
-      (value_regs_ifcout (add_logical_reg_zext32 ty x y)))
+      (output_ifcout (add_logical_reg_zext32 ty x y)))
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (zext32_value x) y)))
-      (value_regs_ifcout (add_logical_reg_zext32 ty y x)))
+      (output_ifcout (add_logical_reg_zext32 ty y x)))
 
 ;; Add a register and an immediate.
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (u32_from_value y))))
-      (value_regs_ifcout (add_logical_zimm32 ty x y)))
+      (output_ifcout (add_logical_zimm32 ty x y)))
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (u32_from_value x) y)))
-      (value_regs_ifcout (add_logical_zimm32 ty y x)))
+      (output_ifcout (add_logical_zimm32 ty y x)))
 
 ;; Add a register and memory (32/64-bit types).
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (sinkable_load_32_64 y))))
-      (value_regs_ifcout (add_logical_mem ty x (sink_load y))))
+      (output_ifcout (add_logical_mem ty x (sink_load y))))
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (sinkable_load_32_64 x) y)))
-      (value_regs_ifcout (add_logical_mem ty y (sink_load x))))
+      (output_ifcout (add_logical_mem ty y (sink_load x))))
 
 ;; Add a register and zero-extended memory.
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (sinkable_uload32 y))))
-      (value_regs_ifcout (add_logical_mem_zext32 ty x (sink_uload32 y))))
+      (output_ifcout (add_logical_mem_zext32 ty x (sink_uload32 y))))
 (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (sinkable_uload32 x) y)))
-      (value_regs_ifcout (add_logical_mem_zext32 ty y (sink_uload32 x))))
+      (output_ifcout (add_logical_mem_zext32 ty y (sink_uload32 x))))
 
 
 ;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1352,66 +1352,66 @@
 
 ;; Store 8-bit integer type, main lowering entry point.
 (rule (lower (store flags val @ (value_type $I8) addr offset))
-      (value_regs_none (istore8_impl flags val addr offset)))
+      (side_effect (istore8_impl flags val addr offset)))
 
 ;; Store 16-bit integer type, main lowering entry point.
 (rule (lower (store flags val @ (value_type $I16) addr offset))
-      (value_regs_none (istore16_impl flags val addr offset)))
+      (side_effect (istore16_impl flags val addr offset)))
 
 ;; Store 32-bit integer type, main lowering entry point.
 (rule (lower (store flags val @ (value_type $I32) addr offset))
-      (value_regs_none (istore32_impl flags val addr offset)))
+      (side_effect (istore32_impl flags val addr offset)))
 
 ;; Store 64-bit integer type, main lowering entry point.
 (rule (lower (store flags val @ (value_type $I64) addr offset))
-      (value_regs_none (istore64_impl flags val addr offset)))
+      (side_effect (istore64_impl flags val addr offset)))
 
 ;; Store 64-bit reference type, main lowering entry point.
 (rule (lower (store flags val @ (value_type $R64) addr offset))
-      (value_regs_none (istore64_impl flags val addr offset)))
+      (side_effect (istore64_impl flags val addr offset)))
 
 ;; Store 32-bit big-endian floating-point type.
 (rule (lower (store flags @ (bigendian)
                     val @ (value_type $F32) addr offset))
-      (value_regs_none (fpu_store32 (put_in_reg val)
+      (side_effect (fpu_store32 (put_in_reg val)
-                                    (lower_address flags addr offset))))
+                                (lower_address flags addr offset))))
 
 ;; Store 32-bit little-endian floating-point type (z15 instruction).
 (rule (lower (store flags @ (littleendian)
                     val @ (value_type (and $F32 (vxrs_ext2_enabled))) addr offset))
-      (value_regs_none (fpu_storerev32 (put_in_reg val)
+      (side_effect (fpu_storerev32 (put_in_reg val)
-                                       (lower_address flags addr offset))))
+                                   (lower_address flags addr offset))))
 
 ;; Store 32-bit little-endian floating-point type (via GPR on z14).
 (rule (lower (store flags @ (littleendian)
                     val @ (value_type (and $F32 (vxrs_ext2_disabled))) addr offset))
       (let ((gpr Reg (lshr_imm $I64 (mov_from_fpr (put_in_reg val)) 32)))
-        (value_regs_none (storerev32 gpr (lower_address flags addr offset)))))
+        (side_effect (storerev32 gpr (lower_address flags addr offset)))))
 
 ;; Store 64-bit big-endian floating-point type.
 (rule (lower (store flags @ (bigendian)
                     val @ (value_type $F64) addr offset))
-      (value_regs_none (fpu_store64 (put_in_reg val)
+      (side_effect (fpu_store64 (put_in_reg val)
-                                    (lower_address flags addr offset))))
+                                (lower_address flags addr offset))))
 
 ;; Store 64-bit little-endian floating-point type (z15 instruction).
 (rule (lower (store flags @ (littleendian)
                     val @ (value_type (and $F64 (vxrs_ext2_enabled))) addr offset))
-      (value_regs_none (fpu_storerev64 (put_in_reg val)
+      (side_effect (fpu_storerev64 (put_in_reg val)
-                                       (lower_address flags addr offset))))
+                                   (lower_address flags addr offset))))
 
 ;; Store 64-bit little-endian floating-point type (via GPR on z14).
 (rule (lower (store flags @ (littleendian)
                     val @ (value_type (and $F64 (vxrs_ext2_disabled))) addr offset))
       (let ((gpr Reg (mov_from_fpr (put_in_reg val))))
-        (value_regs_none (storerev64 gpr (lower_address flags addr offset)))))
+        (side_effect (storerev64 gpr (lower_address flags addr offset)))))
 
 
 ;;;; Rules for 8-bit integer stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Main `istore8` lowering entry point, dispatching to the helper.
 (rule (lower (istore8 flags val addr offset))
-      (value_regs_none (istore8_impl flags val addr offset)))
+      (side_effect (istore8_impl flags val addr offset)))
 
 ;; Helper to store 8-bit integer types.
 (decl istore8_impl (MemFlags Value Value Offset32) SideEffectNoResult)
@@ -1429,7 +1429,7 @@
 
 ;; Main `istore16` lowering entry point, dispatching to the helper.
 (rule (lower (istore16 flags val addr offset))
-      (value_regs_none (istore16_impl flags val addr offset)))
+      (side_effect (istore16_impl flags val addr offset)))
 
 ;; Helper to store 16-bit integer types.
 (decl istore16_impl (MemFlags Value Value Offset32) SideEffectNoResult)
@@ -1455,7 +1455,7 @@
 
 ;; Main `istore32` lowering entry point, dispatching to the helper.
 (rule (lower (istore32 flags val addr offset))
-      (value_regs_none (istore32_impl flags val addr offset)))
+      (side_effect (istore32_impl flags val addr offset)))
 
 ;; Helper to store 32-bit integer types.
 (decl istore32_impl (MemFlags Value Value Offset32) SideEffectNoResult)
@@ -1854,10 +1854,10 @@
 ;;;; Rules for `atomic_store` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Atomic stores can be implemented via regular stores followed by a fence.
-(decl atomic_store_impl (SideEffectNoResult) ValueRegs)
+(decl atomic_store_impl (SideEffectNoResult) InstOutput)
 (rule (atomic_store_impl store)
-      (let ((_ ValueRegs (value_regs_none store)))
+      (let ((_ InstOutput (side_effect store)))
-         (value_regs_none (fence_impl))))
+         (side_effect (fence_impl))))
 
 ;; 8-bit atomic store.
 (rule (lower (atomic_store flags val @ (value_type $I8) addr))
@@ -1880,7 +1880,7 @@
 
 ;; Fence to ensure sequential consistency.
 (rule (lower (fence))
-      (value_regs_none (fence_impl)))
+      (side_effect (fence_impl)))
 
 
 ;;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2066,7 +2066,7 @@
 ;; Unconditional branch.  The target is found as first (and only) element in
 ;; the list of the current block's branch targets passed as `targets`.
 (rule (lower_branch (jump _ _) targets)
-      (value_regs_none (jump_impl (vec_element targets 0))))
+      (side_effect (jump_impl (vec_element targets 0))))
 
 
 ;;;; Rules for `br_table` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2082,8 +2082,8 @@
             (cond ProducesBool
               (bool (icmpu_uimm32 $I64 idx (vec_length_minus1 targets))
                     (intcc_as_cond (IntCC.UnsignedGreaterThanOrEqual))))
-            (_ ValueRegs (value_regs_none (oneway_cond_br_bool cond
+            (_ InstOutput (side_effect (oneway_cond_br_bool cond
-                                            (vec_element targets 0)))))
+                                         (vec_element targets 0)))))
         ;; Scale the index by the element size, and then emit the
         ;; compound instruction that does:
         ;;
@@ -2098,7 +2098,7 @@
         ;; PC-rel offset to the jumptable would be incorrect.
         ;; (The alternative is to introduce a relocation pass
         ;; for inlined jumptables, which is much worse, IMHO.)
-        (value_regs_none (jt_sequence (lshl_imm $I64 idx 2) targets))))
+        (side_effect (jt_sequence (lshl_imm $I64 idx 2) targets))))
 
 
 ;;;; Rules for `brz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2107,9 +2107,9 @@
 ;; - element 0: target if the condition is true (i.e. value is zero)
 ;; - element 1: target if the condition is false (i.e. value is nonzero)
 (rule (lower_branch (brz val_cond _ _) targets)
-      (value_regs_none (cond_br_bool (invert_bool (value_nonzero val_cond))
+      (side_effect (cond_br_bool (invert_bool (value_nonzero val_cond))
-                                     (vec_element targets 0)
+                                 (vec_element targets 0)
-                                     (vec_element targets 1))))
+                                 (vec_element targets 1))))
 
 
 ;;;; Rules for `brnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2118,9 +2118,9 @@
 ;; - element 0: target if the condition is true (i.e. value is nonzero)
 ;; - element 1: target if the condition is false (i.e. value is zero)
 (rule (lower_branch (brnz val_cond _ _) targets)
-      (value_regs_none (cond_br_bool (value_nonzero val_cond)
+      (side_effect (cond_br_bool (value_nonzero val_cond)
-                                     (vec_element targets 0)
+                                 (vec_element targets 0)
-                                     (vec_element targets 1))))
+                                 (vec_element targets 1))))
 
 
 ;;;; Rules for `brif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2129,9 +2129,9 @@
 ;; generated by common code here.  Others will fail to lower.
 
 (rule (lower_branch (brif int_cc (ifcmp x y) _ _) targets)
-      (value_regs_none (cond_br_bool (icmp_val $false int_cc x y)
+      (side_effect (cond_br_bool (icmp_val $false int_cc x y)
-                                     (vec_element targets 0)
+                                 (vec_element targets 0)
-                                     (vec_element targets 1))))
+                                 (vec_element targets 1))))
 
 
 ;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2167,7 +2167,7 @@
 ;;;; Rules for `debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (debugtrap))
-      (value_regs_none (debugtrap_impl)))
+      (side_effect (debugtrap_impl)))
 
 
 ;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2205,6 +2205,6 @@
 
 (rule (lower (trapif (IntCC.UnsignedGreaterThan)
                      (iadd_ifcout x y) trap_code))
-      (value_regs_none (trap_if_impl (mask_as_cond 3) trap_code)))
+      (side_effect (trap_if_impl (mask_as_cond 3) trap_code)))
 
 

cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 8bf92e18323e7041
+src/prelude.isle 957023853b23dacb
 src/isa/s390x/inst.isle d91a16074ab186a8
-src/isa/s390x/lower.isle 5626c0ef61594d61
+src/isa/s390x/lower.isle 1cc5a12adc8c75f9

cranelift/codegen/src/isa/x64/inst.isle

@@ -1009,20 +1009,20 @@
 (rule (put_in_xmm_mem_imm val)
       (xmm_mem_imm_new (put_in_reg_mem_imm val)))
 
-;; Construct a `ValueRegs` out of a single GPR register.
+;; Construct an `InstOutput` out of a single GPR register.
-(decl value_gpr (Gpr) ValueRegs)
+(decl output_gpr (Gpr) InstOutput)
-(rule (value_gpr x)
+(rule (output_gpr x)
-      (value_reg (gpr_to_reg x)))
+      (output_reg (gpr_to_reg x)))
 
 ;; Construct a `ValueRegs` out of two GPR registers.
 (decl value_gprs (Gpr Gpr) ValueRegs)
 (rule (value_gprs x y)
       (value_regs (gpr_to_reg x) (gpr_to_reg y)))
 
-;; Construct a `ValueRegs` out of a single XMM register.
+;; Construct an `InstOutput` out of a single XMM register.
-(decl value_xmm (Xmm) ValueRegs)
+(decl output_xmm (Xmm) InstOutput)
-(rule (value_xmm x)
+(rule (output_xmm x)
-      (value_reg (xmm_to_reg x)))
+      (output_reg (xmm_to_reg x)))
 
 ;; Get the `n`th reg in a `ValueRegs` and construct a GPR from it.
 ;;
@@ -2223,7 +2223,7 @@
 
 ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(convert Gpr ValueRegs value_gpr)
+(convert Gpr InstOutput output_gpr)
 (convert Value Gpr put_in_gpr)
 (convert Value GprMem put_in_gpr_mem)
 (convert Value GprMemImm put_in_gpr_mem_imm)
@@ -2242,7 +2242,7 @@
 (convert WritableGpr WritableReg writable_gpr_to_reg)
 (convert WritableGpr Reg writable_gpr_to_r_reg)
 
-(convert Xmm ValueRegs value_xmm)
+(convert Xmm InstOutput output_xmm)
 (convert Value Xmm put_in_xmm)
 (convert Value XmmMem put_in_xmm_mem)
 (convert Value XmmMemImm put_in_xmm_mem_imm)

cranelift/codegen/src/isa/x64/lower.isle

@@ -2,7 +2,7 @@
 
 ;; The main lowering constructor term: takes a clif `Inst` and returns the
 ;; register(s) within which the lowered instruction's result values live.
-(decl lower (Inst) ValueRegs)
+(decl lower (Inst) InstOutput)
 
 ;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -148,48 +148,39 @@
 ;; replace this with a bool carry flag, and all consumers of `iflags`
 ;; remain in the handwritten pattern-matching code and explicitly
 ;; match on the flags producer. So we can get away with just
-;; allocating a second temp so that the reg-renaming code does the
+;; using an invalid second output, and the reg-renaming code does the
 ;; right thing, for now. For safety, we assert elsewhere that no one
 ;; actually uses the register assigned to the SSA `iflags`-typed
 ;; `Value`.
 
-(decl unused_iflags () Gpr)
+(decl output_ifcout (Reg) InstOutput)
-(rule (unused_iflags)
+(rule (output_ifcout reg)
-      (temp_writable_gpr))
+      (output_pair reg (value_regs_invalid)))
 
 ;; Add two registers.
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout x y)))
-      (value_gprs (add ty x y)
+      (output_ifcout (add ty x y)))
-                  (unused_iflags)))
 
 ;; Add a register and an immediate.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout x (simm32_from_value y))))
-      (value_gprs (add ty x y)
+      (output_ifcout (add ty x y)))
-                  (unused_iflags)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout (simm32_from_value x) y)))
-      (value_gprs (add ty y x)
+      (output_ifcout (add ty y x)))
-                  (unused_iflags)))
 
 ;; Add a register and memory.
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout x (sinkable_load y))))
-      (value_gprs (add ty
+      (output_ifcout (add ty x (sink_load_to_gpr_mem_imm y))))
-                       x
-                       (sink_load_to_gpr_mem_imm y))
-                  (unused_iflags)))
 
 (rule (lower (has_type (fits_in_64 ty)
                        (iadd_ifcout (sinkable_load x) y)))
-      (value_gprs (add ty
+      (output_ifcout (add ty y (sink_load_to_gpr_mem_imm x))))
-                       y
-                       (sink_load_to_gpr_mem_imm x))
-                  (unused_iflags)))
 
 ;; (No `iadd_ifcout` for `i128`.)
 

cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 8bf92e18323e7041
+src/prelude.isle 957023853b23dacb
-src/isa/x64/inst.isle 1948445a25530d71
+src/isa/x64/inst.isle 5ee89205e6e9a46b
-src/isa/x64/lower.isle d1ee574941be387
+src/isa/x64/lower.isle 348a808ea5de4cdb

cranelift/codegen/src/machinst/isle.rs

@@ -1,9 +1,10 @@
-use crate::ir::{Inst, Value};
+use crate::ir::{types, Inst, Value};
 use crate::machinst::{get_output_reg, InsnOutput, LowerCtx, MachInst, RegRenamer};
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use regalloc::{Reg, Writable};
 use smallvec::SmallVec;
+use std::cell::Cell;
 
 pub use super::MachLabel;
 pub use crate::ir::{ExternalName, FuncRef, GlobalValue, SigRef};
@@ -18,6 +19,8 @@ pub type WritableReg = Writable<Reg>;
 pub type VecReg = Vec<Reg>;
 pub type VecWritableReg = Vec<WritableReg>;
 pub type ValueRegs = crate::machinst::ValueRegs<Reg>;
+pub type InstOutput = SmallVec<[ValueRegs; 2]>;
+pub type InstOutputBuilder = Cell<InstOutput>;
 pub type VecMachLabel = Vec<MachLabel>;
 pub type BoxExternalName = Box<ExternalName>;
 
@@ -64,6 +67,38 @@ macro_rules! isle_prelude_methods {
             ValueRegs::invalid()
         }
 
+        #[inline]
+        fn output_none(&mut self) -> InstOutput {
+            smallvec::smallvec![]
+        }
+
+        #[inline]
+        fn output(&mut self, regs: ValueRegs) -> InstOutput {
+            smallvec::smallvec![regs]
+        }
+
+        #[inline]
+        fn output_pair(&mut self, r1: ValueRegs, r2: ValueRegs) -> InstOutput {
+            smallvec::smallvec![r1, r2]
+        }
+
+        #[inline]
+        fn output_builder_new(&mut self) -> InstOutputBuilder {
+            std::cell::Cell::new(InstOutput::new())
+        }
+
+        #[inline]
+        fn output_builder_push(&mut self, builder: &InstOutputBuilder, regs: ValueRegs) -> Unit {
+            let mut vec = builder.take();
+            vec.push(regs);
+            builder.set(vec);
+        }
+
+        #[inline]
+        fn output_builder_finish(&mut self, builder: &InstOutputBuilder) -> InstOutput {
+            builder.take()
+        }
+
         #[inline]
         fn temp_writable_reg(&mut self, ty: Type) -> WritableReg {
             let value_regs = self.lower_ctx.alloc_tmp(ty);
@@ -363,7 +398,7 @@ pub(crate) fn lower_common<C, F, I, IF, const N: usize>(
 where
     C: LowerCtx,
     [(C::I, bool); N]: smallvec::Array<Item = (C::I, bool)>,
-    IF: Fn(&mut IsleContext<'_, C, F, I, N>, Inst) -> Option<ValueRegs>,
+    IF: Fn(&mut IsleContext<'_, C, F, I, N>, Inst) -> Option<InstOutput>,
 {
     // TODO: reuse the ISLE context across lowerings so we can reuse its
     // internal heap allocations.
@@ -375,22 +410,17 @@ where
     };
 
     let temp_regs = isle_lower(&mut isle_ctx, inst).ok_or(())?;
-    let mut temp_regs = temp_regs.regs().iter();
 
     #[cfg(debug_assertions)]
     {
-        let all_dsts_len = outputs
-            .iter()
-            .map(|out| get_output_reg(isle_ctx.lower_ctx, *out).len())
-            .sum();
         debug_assert_eq!(
             temp_regs.len(),
-            all_dsts_len,
+            outputs.len(),
-            "the number of temporary registers and destination registers do \
+            "the number of temporary values and destination values do \
          not match ({} != {}); ensure the correct registers are being \
          returned.",
             temp_regs.len(),
-            all_dsts_len,
+            outputs.len(),
         );
     }
 
@@ -399,12 +429,22 @@ where
     // registers they were assigned when their value was used as an operand in
     // earlier lowerings.
     let mut renamer = RegRenamer::default();
-    for output in outputs {
+    for i in 0..outputs.len() {
-        let dsts = get_output_reg(isle_ctx.lower_ctx, *output);
+        let regs = temp_regs[i];
-        let ty = isle_ctx.lower_ctx.output_ty(output.insn, output.output);
+        let dsts = get_output_reg(isle_ctx.lower_ctx, outputs[i]);
-        let (_, tys) = <C::I>::rc_for_type(ty).unwrap();
+        let ty = isle_ctx
-        for ((temp, dst), ty) in temp_regs.by_ref().zip(dsts.regs()).zip(tys) {
+            .lower_ctx
-            renamer.add_rename(*temp, dst.to_reg(), *ty);
+            .output_ty(outputs[i].insn, outputs[i].output);
+        if ty == types::IFLAGS || ty == types::FFLAGS {
+            // Flags values do not occupy any registers.
+            assert!(regs.len() == 0);
+        } else {
+            let (_, tys) = <C::I>::rc_for_type(ty).unwrap();
+            assert!(regs.len() == tys.len());
+            assert!(regs.len() == dsts.len());
+            for ((dst, temp), ty) in dsts.regs().iter().zip(regs.regs().iter()).zip(tys) {
+                renamer.add_rename(*temp, dst.to_reg(), *ty);
+            }
         }
     }
     for (inst, _) in isle_ctx.emitted_insts.iter_mut() {

cranelift/codegen/src/prelude.isle

@@ -38,6 +38,11 @@
 (type ValueList (primitive ValueList))
 (type ValueRegs (primitive ValueRegs))
 
+;; Instruction lowering result: a vector of `ValueRegs`.
+(type InstOutput (primitive InstOutput))
+;; (Mutable) builder to incrementally construct an `InstOutput`.
+(type InstOutputBuilder extern (enum))
+
 (decl u32_add (u32 u32) u32)
 (extern constructor u32_add u32_add)
 
@@ -64,6 +69,38 @@
 (decl value_regs_invalid () ValueRegs)
 (extern constructor value_regs_invalid value_regs_invalid)
 
+;; Construct an empty `InstOutput`.
+(decl output_none () InstOutput)
+(extern constructor output_none output_none)
+
+;; Construct a single-element `InstOutput`.
+(decl output (ValueRegs) InstOutput)
+(extern constructor output output)
+
+;; Construct a two-element `InstOutput`.
+(decl output_pair (ValueRegs ValueRegs) InstOutput)
+(extern constructor output_pair output_pair)
+
+;; Construct a single-element `InstOutput` from a single register.
+(decl output_reg (Reg) InstOutput)
+(rule (output_reg reg) (output (value_reg reg)))
+
+;; Construct a single-element `InstOutput` from a value.
+(decl output_value (Value) InstOutput)
+(rule (output_value val) (output (put_in_regs val)))
+
+;; Initially empty `InstOutput` builder.
+(decl output_builder_new () InstOutputBuilder)
+(extern constructor output_builder_new output_builder_new)
+
+;; Append a `ValueRegs` to an `InstOutput` under construction.
+(decl output_builder_push (InstOutputBuilder ValueRegs) Unit)
+(extern constructor output_builder_push output_builder_push)
+
+;; Finish building an `InstOutput` incrementally.
+(decl output_builder_finish (InstOutputBuilder) InstOutput)
+(extern constructor output_builder_finish output_builder_finish)
+
 ;; Get a temporary register for writing.
 (decl temp_writable_reg (Type) WritableReg)
 (extern constructor temp_writable_reg temp_writable_reg)
@@ -307,18 +344,18 @@
 
 (type SideEffectNoResult (enum (Inst (inst MInst))))
 
-;; Create an empty `ValueRegs`, but do emit the given side-effectful
+;; Create an empty `InstOutput`, but do emit the given side-effectful
 ;; instruction.
-(decl value_regs_none (SideEffectNoResult) ValueRegs)
+(decl side_effect (SideEffectNoResult) InstOutput)
-(rule (value_regs_none (SideEffectNoResult.Inst inst))
+(rule (side_effect (SideEffectNoResult.Inst inst))
       (let ((_ Unit (emit inst)))
-        (value_regs_invalid)))
+        (output_none)))
 
 ;; Similarly, but emit the side-effectful instruction as a safepoint.
-(decl safepoint (SideEffectNoResult) ValueRegs)
+(decl safepoint (SideEffectNoResult) InstOutput)
 (rule (safepoint (SideEffectNoResult.Inst inst))
       (let ((_ Unit (emit_safepoint inst)))
-        (value_regs_invalid)))
+        (output_none)))
 
 ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -438,3 +475,6 @@
 (convert Value Reg put_in_reg)
 (convert Value ValueRegs put_in_regs)
 (convert WritableReg Reg writable_reg_to_reg)
+(convert ValueRegs InstOutput output)
+(convert Reg InstOutput output_reg)
+(convert Value InstOutput output_value)