Update the reload pass to replace copies with fill/spill instructions.

cranelift/filetests/regalloc/reload.clif

@@ -17,3 +17,30 @@ ebb0:
     ; check: $(reload=$V) = fill v0
     ; check: return $reload
 }
+
+; Check that copies where the arg has been spilled are replaced with fills.
+;
+; RV32E has 6 registers for function arguments so the 7th, v6, will be placed
+; on the stack.
+function %spilled_copy_arg(i32, i32, i32, i32, i32, i32, i32) -> i32 {
+
+ebb0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32):
+    ; not: copy
+    ; check: v10 = fill v6
+    v10 = copy v6
+    return v10
+}
+
+; Check that copies where the result has been spilled are replaced with spills.
+;
+; v1 is live across a call so it will be spilled.
+function %spilled_copy_result(i32) -> i32 {
+    fn0 = %foo(i32)
+
+ebb0(v0: i32):
+    ; not: copy
+    ; check: v1 = spill v0
+    v1 = copy v0
+    call fn0(v1)
+    return v1
+}

lib/codegen/src/regalloc/reload.rs

@@ -13,7 +13,7 @@ use cursor::{Cursor, EncCursor};
 use dominator_tree::DominatorTree;
 use entity::{SparseMap, SparseMapValue};
 use ir::{AbiParam, ArgumentLoc, InstBuilder};
-use ir::{Ebb, Function, Inst, Value};
+use ir::{Ebb, Function, Inst, InstructionData, Opcode, Value};
 use isa::RegClass;
 use isa::{ConstraintKind, EncInfo, Encoding, RecipeConstraints, TargetIsa};
 use regalloc::affinity::Affinity;
@@ -209,6 +209,84 @@ impl<'a> Context<'a> {
         debug_assert!(self.candidates.is_empty());
         self.find_candidates(inst, constraints);
 
+        if let InstructionData::Unary {
+            opcode: Opcode::Copy,
+            ..
+        } = self.cur.func.dfg[inst]
+        {
+            self.reload_copy_candidates(inst);
+        } else {
+            self.reload_inst_candidates(ebb, inst);
+        }
+
+        // TODO: Reuse reloads for future instructions.
+        self.reloads.clear();
+
+        let (_throughs, _kills, defs) =
+            tracker.process_inst(inst, &self.cur.func.dfg, self.liveness);
+
+        // Advance to the next instruction so we can insert any spills after the instruction.
+        self.cur.next_inst();
+
+        // Rewrite register defs that need to be spilled.
+        //
+        // Change:
+        //
+        // v2 = inst ...
+        //
+        // Into:
+        //
+        // v7 = inst ...
+        // v2 = spill v7
+        //
+        // That way, we don't need to rewrite all future uses of v2.
+        for (lv, op) in defs.iter().zip(constraints.outs) {
+            if lv.affinity.is_stack() && op.kind != ConstraintKind::Stack {
+                if let InstructionData::Unary {
+                    opcode: Opcode::Copy,
+                    arg,
+                } = self.cur.func.dfg[inst]
+                {
+                    self.cur.func.dfg.replace(inst).spill(arg);
+                    let ok = self.cur.func.update_encoding(inst, self.cur.isa).is_ok();
+                    debug_assert!(ok);
+                } else {
+                    let value_type = self.cur.func.dfg.value_type(lv.value);
+                    let reg = self.cur.func.dfg.replace_result(lv.value, value_type);
+                    self.liveness.create_dead(reg, inst, Affinity::new(op));
+                    self.insert_spill(ebb, lv.value, reg);
+                }
+            }
+        }
+
+        // Same thing for spilled call return values.
+        let retvals = &defs[constraints.outs.len()..];
+        if !retvals.is_empty() {
+            let sig = self
+                .cur
+                .func
+                .dfg
+                .call_signature(inst)
+                .expect("Extra results on non-call instruction");
+            for (i, lv) in retvals.iter().enumerate() {
+                let abi = self.cur.func.dfg.signatures[sig].returns[i];
+                debug_assert!(
+                    abi.location.is_reg(),
+                    "expected reg; got {:?}",
+                    abi.location
+                );
+                if lv.affinity.is_stack() {
+                    let reg = self.cur.func.dfg.replace_result(lv.value, abi.value_type);
+                    self.liveness
+                        .create_dead(reg, inst, Affinity::abi(&abi, self.cur.isa));
+                    self.insert_spill(ebb, lv.value, reg);
+                }
+            }
+        }
+    }
+
+    // Reload the current candidates for the given `inst`.
+    fn reload_inst_candidates(&mut self, ebb: Ebb, inst: Inst) {
         // Insert fill instructions before `inst` and replace `cand.value` with the filled value.
         for cand in self.candidates.iter_mut() {
             if let Some(reload) = self.reloads.get(cand.value) {
@@ -244,60 +322,20 @@ impl<'a> Context<'a> {
                 args[cand.argidx] = cand.value;
             }
         }
+    }
 
-        // TODO: Reuse reloads for future instructions.
+    // Reload the current candidates for the given copy `inst`.
-        self.reloads.clear();
+    //
+    // As an optimization, replace a copy instruction where the argument has been spilled with
+    // a fill instruction.
+    fn reload_copy_candidates(&mut self, inst: Inst) {
+        // Copy instructions can only have one argument.
+        debug_assert!(self.candidates.is_empty() || self.candidates.len() == 1);
 
-        let (_throughs, _kills, defs) =
+        if let Some(cand) = self.candidates.pop() {
-            tracker.process_inst(inst, &self.cur.func.dfg, self.liveness);
+            self.cur.func.dfg.replace(inst).fill(cand.value);
-
+            let ok = self.cur.func.update_encoding(inst, self.cur.isa).is_ok();
-        // Advance to the next instruction so we can insert any spills after the instruction.
+            debug_assert!(ok);
-        self.cur.next_inst();
-
-        // Rewrite register defs that need to be spilled.
-        //
-        // Change:
-        //
-        // v2 = inst ...
-        //
-        // Into:
-        //
-        // v7 = inst ...
-        // v2 = spill v7
-        //
-        // That way, we don't need to rewrite all future uses of v2.
-        for (lv, op) in defs.iter().zip(constraints.outs) {
-            if lv.affinity.is_stack() && op.kind != ConstraintKind::Stack {
-                let value_type = self.cur.func.dfg.value_type(lv.value);
-                let reg = self.cur.func.dfg.replace_result(lv.value, value_type);
-                self.liveness.create_dead(reg, inst, Affinity::new(op));
-                self.insert_spill(ebb, lv.value, reg);
-            }
-        }
-
-        // Same thing for spilled call return values.
-        let retvals = &defs[constraints.outs.len()..];
-        if !retvals.is_empty() {
-            let sig = self
-                .cur
-                .func
-                .dfg
-                .call_signature(inst)
-                .expect("Extra results on non-call instruction");
-            for (i, lv) in retvals.iter().enumerate() {
-                let abi = self.cur.func.dfg.signatures[sig].returns[i];
-                debug_assert!(
-                    abi.location.is_reg(),
-                    "expected reg; got {:?}",
-                    abi.location
-                );
-                if lv.affinity.is_stack() {
-                    let reg = self.cur.func.dfg.replace_result(lv.value, abi.value_type);
-                    self.liveness
-                        .create_dead(reg, inst, Affinity::abi(&abi, self.cur.isa));
-                    self.insert_spill(ebb, lv.value, reg);
-                }
-            }
         }
     }