Make regalloc visit fallthrough_return instructions.

Add an explicit "is_ghost" property to selected instructions, and use
that to determine whether reload and coloring should visit instructions.
This allows them to visit fallthrough_return instructions and insert
fills and register moves as needed.

cranelift/filetests/regalloc/fallthrough-return.clif

@@ -0,0 +1,23 @@
+test regalloc
+target x86_64
+
+; Test that fallthrough returns are visited by reload and coloring.
+
+function %foo() -> f64 {
+  fn0 = %bar()
+
+ebb0:
+  v0 = f64const 0.0
+  call fn0()
+  fallthrough_return v0
+}
+; check: fill v0
+
+function %foo() -> f64 {
+  fn0 = %bar() -> f64, f64
+
+ebb0:
+  v0, v1 = call fn0()
+  fallthrough_return v1
+}
+; check: regmove v1, %xmm1 -> %xmm0

lib/codegen/meta-python/base/instructions.py

@@ -844,7 +844,7 @@ vsplit = Instruction(
         the lanes from ``x``. The result may be two scalars if ``x`` only had
         two lanes.
         """,
-        ins=x, outs=(lo, hi))
+        ins=x, outs=(lo, hi), is_ghost=True)
 
 Any128 = TypeVar(
         'Any128', 'Any scalar or vector type with as most 128 lanes',
@@ -864,7 +864,7 @@ vconcat = Instruction(
 
         It is possible to form a vector by concatenating two scalars.
         """,
-        ins=(x, y), outs=a)
+        ins=(x, y), outs=a, is_ghost=True)
 
 c = Operand('c', TxN.as_bool(), doc='Controlling vector')
 x = Operand('x', TxN, doc='Value to use where `c` is true')
@@ -2009,7 +2009,7 @@ isplit = Instruction(
         Returns the low half of `x` and the high half of `x` as two independent
         values.
         """,
-        ins=x, outs=(lo, hi))
+        ins=x, outs=(lo, hi), is_ghost=True)
 
 
 NarrowInt = TypeVar(
@@ -2029,6 +2029,6 @@ iconcat = Instruction(
         the same number of lanes as the inputs, but the lanes are twice the
         size.
         """,
-        ins=(lo, hi), outs=a)
+        ins=(lo, hi), outs=a, is_ghost=True)
 
 GROUP.close()

lib/codegen/meta-python/cdsl/instructions.py

@@ -92,6 +92,7 @@ class Instruction(object):
     :param is_indirect_branch: This is an indirect branch instruction.
     :param is_call: This is a call instruction.
     :param is_return: This is a return instruction.
+    :param is_ghost: This is a ghost instruction.
     :param can_trap: This instruction can trap.
     :param can_load: This instruction can load from memory.
     :param can_store: This instruction can store to memory.
@@ -108,6 +109,7 @@ class Instruction(object):
             'True for all indirect branch or jump instructions.',
             'is_call': 'Is this a call instruction?',
             'is_return': 'Is this a return instruction?',
+            'is_ghost': 'Is this a ghost instruction?',
             'can_load': 'Can this instruction read from memory?',
             'can_store': 'Can this instruction write to memory?',
             'can_trap': 'Can this instruction cause a trap?',

lib/codegen/src/regalloc/coloring.rs

@@ -165,8 +165,9 @@ impl<'a> Context<'a> {
         self.cur.goto_top(ebb);
         while let Some(inst) = self.cur.next_inst() {
             self.cur.use_srcloc(inst);
+            if !self.cur.func.dfg[inst].opcode().is_ghost() {
                 let enc = self.cur.func.encodings[inst];
-            if let Some(constraints) = self.encinfo.operand_constraints(enc) {
+                let constraints = self.encinfo.operand_constraints(enc);
                 if self.visit_inst(inst, constraints, tracker, &mut regs) {
                     self.replace_global_defines(inst, tracker);
                     // Restore cursor location after `replace_global_defines` moves it.
@@ -290,7 +291,7 @@ impl<'a> Context<'a> {
     fn visit_inst(
         &mut self,
         inst: Inst,
-        constraints: &RecipeConstraints,
+        constraints: Option<&RecipeConstraints>,
         tracker: &mut LiveValueTracker,
         regs: &mut AvailableRegs,
     ) -> bool {
@@ -306,7 +307,9 @@ impl<'a> Context<'a> {
 
         // Program the solver with register constraints for the input side.
         self.solver.reset(&regs.input);
+        if let Some(constraints) = constraints {
             self.program_input_constraints(inst, constraints.ins);
+        }
         let call_sig = self.cur.func.dfg.call_signature(inst);
         if let Some(sig) = call_sig {
             program_input_abi(
@@ -390,6 +393,7 @@ impl<'a> Context<'a> {
         // Program the fixed output constraints before the general defines. This allows us to
         // detect conflicts between fixed outputs and tied operands where the input value hasn't
         // been converted to a solver variable.
+        if let Some(constraints) = constraints {
             if constraints.fixed_outs {
                 self.program_fixed_outputs(
                     constraints.outs,
@@ -399,6 +403,7 @@ impl<'a> Context<'a> {
                     &regs.global,
                 );
             }
+        }
         if let Some(sig) = call_sig {
             self.program_output_abi(
                 sig,
@@ -408,6 +413,7 @@ impl<'a> Context<'a> {
                 &regs.global,
             );
         }
+        if let Some(constraints) = constraints {
             self.program_output_constraints(
                 inst,
                 constraints.outs,
@@ -415,6 +421,7 @@ impl<'a> Context<'a> {
                 &mut replace_global_defines,
                 &regs.global,
             );
+        }
 
         // Finally, we've fully programmed the constraint solver.
         // We expect a quick solution in most cases.
@@ -440,6 +447,7 @@ impl<'a> Context<'a> {
 
         // Tied defs are not part of the solution above.
         // Copy register assignments from tied inputs to tied outputs.
+        if let Some(constraints) = constraints {
             if constraints.tied_ops {
                 for (op, lv) in constraints.outs.iter().zip(defs) {
                     if let ConstraintKind::Tied(num) = op.kind {
@@ -449,6 +457,7 @@ impl<'a> Context<'a> {
                     }
                 }
             }
+        }
 
         // Update `regs` for the next instruction.
         regs.input = output_regs;

lib/codegen/src/regalloc/reload.rs

@@ -125,8 +125,8 @@ impl<'a> Context<'a> {
 
         // visit_ebb_header() places us at the first interesting instruction in the EBB.
         while let Some(inst) = self.cur.current_inst() {
+            if !self.cur.func.dfg[inst].opcode().is_ghost() {
                 let encoding = self.cur.func.encodings[inst];
-            if encoding.is_legal() {
                 self.visit_inst(ebb, inst, encoding, tracker);
                 tracker.drop_dead(inst);
             } else {
@@ -200,10 +200,7 @@ impl<'a> Context<'a> {
         self.cur.use_srcloc(inst);
 
         // Get the operand constraints for `inst` that we are trying to satisfy.
-        let constraints = self
+        let constraints = self.encinfo.operand_constraints(encoding);
-            .encinfo
-            .operand_constraints(encoding)
-            .expect("Missing instruction encoding");
 
         // Identify reload candidates.
         debug_assert!(self.candidates.is_empty());
@@ -240,6 +237,7 @@ impl<'a> Context<'a> {
         // v2 = spill v7
         //
         // That way, we don't need to rewrite all future uses of v2.
+        if let Some(constraints) = constraints {
             for (lv, op) in defs.iter().zip(constraints.outs) {
                 if lv.affinity.is_stack() && op.kind != ConstraintKind::Stack {
                     if let InstructionData::Unary {
@@ -258,9 +256,13 @@ impl<'a> Context<'a> {
                     }
                 }
             }
+        }
 
         // Same thing for spilled call return values.
-        let retvals = &defs[constraints.outs.len()..];
+        let retvals = &defs[self.cur.func.dfg[inst]
+                                .opcode()
+                                .constraints()
+                                .fixed_results()..];
         if !retvals.is_empty() {
             let sig = self
                 .cur
@@ -342,9 +344,10 @@ impl<'a> Context<'a> {
     // Find reload candidates for `inst` and add them to `self.candidates`.
     //
     // These are uses of spilled values where the operand constraint requires a register.
-    fn find_candidates(&mut self, inst: Inst, constraints: &RecipeConstraints) {
+    fn find_candidates(&mut self, inst: Inst, constraints: Option<&RecipeConstraints>) {
         let args = self.cur.func.dfg.inst_args(inst);
 
+        if let Some(constraints) = constraints {
             for (argidx, (op, &arg)) in constraints.ins.iter().zip(args).enumerate() {
                 if op.kind != ConstraintKind::Stack && self.liveness[arg].affinity.is_stack() {
                     self.candidates.push(ReloadCandidate {
@@ -354,9 +357,13 @@ impl<'a> Context<'a> {
                     })
                 }
             }
+        }
 
         // If we only have the fixed arguments, we're done now.
-        let offset = constraints.ins.len();
+        let offset = self.cur.func.dfg[inst]
+            .opcode()
+            .constraints()
+            .fixed_value_arguments();
         if args.len() == offset {
             return;
         }