Basic spilling implementation.

Add a spilling pass which lowers register pressure by assigning SSA
values to the stack. Important missing features:

- Resolve conflicts where an instruction uses the same value more than
  once in incompatible ways.
- Deal with EBB arguments.

Fix bugs in the reload pass exposed by the first test case:

- Create live ranges for temporary registers.
- Set encodings on created spill and fill instructions.

cranelift/filetests/regalloc/spill.cton

@@ -0,0 +1,44 @@
+test regalloc
+
+; Test the spiler on an ISA with few registers.
+; RV32E has 16 registers, where:
+; - %x0 is hardwired to zero.
+; - %x1 is the return address.
+; - %x2 is the stack pointer.
+; - %x3 is the global pointer.
+; - %x4 is the thread pointer.
+; - %x10-%x15 are function arguments.
+;
+; regex: V=v\d+
+
+isa riscv enable_e
+
+; In straight-line code, the first value defined is spilled.
+; That is the argument.
+function %pyramid(i32) -> i32 {
+ebb0(v1: i32):
+; check: $v1 = spill $(rv1=$V)
+    v2 = iadd_imm v1, 12
+    v3 = iadd_imm v2, 12
+    v4 = iadd_imm v3, 12
+    v5 = iadd_imm v4, 12
+    v6 = iadd_imm v5, 12
+    v7 = iadd_imm v6, 12
+    v8 = iadd_imm v7, 12
+    v9 = iadd_imm v8, 12
+    v10 = iadd_imm v9, 12
+    v11 = iadd_imm v10, 12
+    v12 = iadd_imm v11, 12
+    v31 = iadd v11, v12
+    v30 = iadd v31, v10
+    v29 = iadd v30, v9
+    v28 = iadd v29, v8
+    v27 = iadd v28, v7
+    v26 = iadd v27, v6
+    v25 = iadd v26, v5
+    v24 = iadd v25, v4
+    v23 = iadd v24, v3
+    v22 = iadd v23, v2
+    v21 = iadd v22, v1
+    return v21
+}

lib/cretonne/src/regalloc/context.rs

@@ -12,6 +12,7 @@ use regalloc::coloring::Coloring;
 use regalloc::live_value_tracker::LiveValueTracker;
 use regalloc::liveness::Liveness;
 use regalloc::reload::Reload;
+use regalloc::spilling::Spilling;
 use result::CtonResult;
 use topo_order::TopoOrder;
 use verifier::{verify_context, verify_liveness};
@@ -21,6 +22,7 @@ pub struct Context {
     liveness: Liveness,
     topo: TopoOrder,
     tracker: LiveValueTracker,
+    spilling: Spilling,
     reload: Reload,
     coloring: Coloring,
 }
@@ -35,6 +37,7 @@ impl Context {
             liveness: Liveness::new(),
             topo: TopoOrder::new(),
             tracker: LiveValueTracker::new(),
+            spilling: Spilling::new(),
             reload: Reload::new(),
             coloring: Coloring::new(),
         }
@@ -62,7 +65,19 @@ impl Context {
             verify_liveness(isa, func, cfg, &self.liveness)?;
         }
 
-        // TODO: Second pass: Spilling.
+        // Second pass: Spilling.
+        self.spilling
+            .run(isa,
+                 func,
+                 domtree,
+                 &mut self.liveness,
+                 &mut self.topo,
+                 &mut self.tracker);
+
+        if isa.flags().enable_verifier() {
+            verify_context(func, cfg, domtree, Some(isa))?;
+            verify_liveness(isa, func, cfg, &self.liveness)?;
+        }
 
         // Third pass: Reload.
         self.reload

lib/cretonne/src/regalloc/live_value_tracker.rs

@@ -12,7 +12,6 @@ use partition_slice::partition_slice;
 use regalloc::affinity::Affinity;
 use regalloc::liveness::Liveness;
 use regalloc::liverange::LiveRange;
-
 use std::collections::HashMap;
 
 type ValueList = EntityList<Value>;
@@ -299,6 +298,20 @@ impl LiveValueTracker {
         self.live.remove_dead_values();
     }
 
+    /// Process new spills.
+    ///
+    /// Any values where `f` returns true are spilled and will be treated as if their affinity was
+    /// `Stack`.
+    pub fn process_spills<F>(&mut self, mut f: F)
+        where F: FnMut(Value) -> bool
+    {
+        for lv in &mut self.live.values {
+            if f(lv.value) {
+                lv.affinity = Affinity::Stack;
+            }
+        }
+    }
+
     /// Save the current set of live values so it is associated with `idom`.
     fn save_idom_live_set(&mut self, idom: Inst) {
         let values = self.live.values.iter().map(|lv| lv.value);

lib/cretonne/src/regalloc/liveness.rs

@@ -182,6 +182,7 @@ use isa::{TargetIsa, EncInfo};
 use regalloc::affinity::Affinity;
 use regalloc::liverange::LiveRange;
 use sparse_map::SparseMap;
+use std::mem;
 
 /// A set of live ranges, indexed by value number.
 type LiveRangeSet = SparseMap<Value, LiveRange>;
@@ -338,6 +339,13 @@ impl Liveness {
         &mut lr.affinity
     }
 
+    /// Change the affinity of `value` to `Stack` and return the previous affinity.
+    pub fn spill(&mut self, value: Value) -> Affinity {
+        let mut lr = self.ranges.get_mut(value).expect("Value has no live range");
+        mem::replace(&mut lr.affinity, Affinity::Stack)
+    }
+
+
     /// Compute the live ranges of all SSA values used in `func`.
     /// This clears out any existing analysis stored in this data structure.
     pub fn compute(&mut self, isa: &TargetIsa, func: &Function, cfg: &ControlFlowGraph) {

lib/cretonne/src/regalloc/mod.rs

@@ -14,6 +14,7 @@ mod diversion;
 mod pressure;
 mod reload;
 mod solver;
+mod spilling;
 
 pub use self::allocatable_set::AllocatableSet;
 pub use self::context::Context;

lib/cretonne/src/regalloc/pressure.rs

@@ -39,6 +39,7 @@
 use isa::registers::{RegInfo, MAX_TOPRCS, RegClass, RegClassMask};
 use regalloc::AllocatableSet;
 use std::cmp::min;
+use std::fmt;
 use std::iter::ExactSizeIterator;
 
 /// Information per top-level register class.
@@ -225,6 +226,16 @@ impl Pressure {
     }
 }
 
+impl fmt::Display for Pressure {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Pressure[")?;
+        for rc in &self.toprc {
+            write!(f, " {}+{}/{}", rc.base_count, rc.transient_count, rc.limit)?;
+        }
+        write!(f, " ]")
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use isa::{TargetIsa, RegClass};

lib/cretonne/src/regalloc/reload.rs

@@ -10,6 +10,7 @@
 //! pressure limits to be exceeded.
 
 use dominator_tree::DominatorTree;
+use entity_map::EntityMap;
 use ir::{Ebb, Inst, Value, Function, DataFlowGraph};
 use ir::layout::{Cursor, CursorPosition};
 use ir::{InstBuilder, ArgumentLoc};
@@ -29,6 +30,7 @@ pub struct Reload {
 
 /// Context data structure that gets instantiated once per pass.
 struct Context<'a> {
+    isa: &'a TargetIsa,
     // Cached ISA information.
     // We save it here to avoid frequent virtual function calls on the `TargetIsa` trait object.
     encinfo: EncInfo,
@@ -60,6 +62,7 @@ impl Reload {
                topo: &mut TopoOrder,
                tracker: &mut LiveValueTracker) {
         let mut ctx = Context {
+            isa,
             encinfo: isa.encoding_info(),
             domtree,
             liveness,
@@ -112,7 +115,13 @@ impl<'a> Context<'a> {
         while let Some(inst) = pos.current_inst() {
             let encoding = func.encodings[inst];
             if encoding.is_legal() {
-                self.visit_inst(ebb, inst, encoding, &mut pos, &mut func.dfg, tracker);
+                self.visit_inst(ebb,
+                                inst,
+                                encoding,
+                                &mut pos,
+                                &mut func.dfg,
+                                &mut func.encodings,
+                                tracker);
                 tracker.drop_dead(inst);
             } else {
                 pos.next_inst();
@@ -121,7 +130,7 @@ impl<'a> Context<'a> {
     }
 
     /// Process the EBB parameters. Return the next instruction in the EBB to be processed
-    fn visit_ebb_header(&self,
+    fn visit_ebb_header(&mut self,
                         ebb: Ebb,
                         func: &mut Function,
                         tracker: &mut LiveValueTracker)
@@ -139,7 +148,7 @@ impl<'a> Context<'a> {
 
     /// Visit the arguments to the entry block.
     /// These values have ABI constraints from the function signature.
-    fn visit_entry_args(&self,
+    fn visit_entry_args(&mut self,
                         ebb: Ebb,
                         func: &mut Function,
                         args: &[LiveValue])
@@ -157,7 +166,15 @@ impl<'a> Context<'a> {
                         // with a temporary register value that is immediately spilled.
                         let reg = func.dfg.replace_ebb_arg(arg.value, abi.value_type);
                         func.dfg.ins(&mut pos).with_result(arg.value).spill(reg);
-                        // TODO: Update live ranges.
+                        let spill = func.dfg.value_def(arg.value).unwrap_inst();
+                        *func.encodings.ensure(spill) = self.isa
+                            .encode(&func.dfg, &func.dfg[spill], abi.value_type)
+                            .expect("Can't encode spill");
+                        // Update live ranges.
+                        self.liveness.move_def_locally(arg.value, spill);
+                        let affinity = Affinity::abi(abi, self.isa);
+                        self.liveness.create_dead(reg, ebb, affinity);
+                        self.liveness.extend_locally(reg, ebb, spill, pos.layout);
                     }
                 }
                 ArgumentLoc::Stack(_) => {
@@ -184,6 +201,7 @@ impl<'a> Context<'a> {
                   encoding: Encoding,
                   pos: &mut Cursor,
                   dfg: &mut DataFlowGraph,
+                  encodings: &mut EntityMap<Inst, Encoding>,
                   tracker: &mut LiveValueTracker) {
         // Get the operand constraints for `inst` that we are trying to satisfy.
         let constraints = self.encinfo
@@ -213,6 +231,11 @@ impl<'a> Context<'a> {
             }
 
             let reg = dfg.ins(pos).fill(cand.value);
+            let fill = dfg.value_def(reg).unwrap_inst();
+            *encodings.ensure(fill) = self.isa
+                .encode(dfg, &dfg[fill], dfg.value_type(reg))
+                .expect("Can't encode fill");
+
             self.reloads
                 .insert(ReloadedValue {
                             stack: cand.value,

lib/cretonne/src/regalloc/spilling.rs

@@ -0,0 +1,259 @@
+//! Spilling pass.
+//!
+//! The spilling pass is the first to run after the liveness analysis. Its primary function is to
+//! ensure that the register pressure never exceeds the number of available registers by moving
+//! some SSA values to spill slots on the stack. This is encoded in the affinity of the value's
+//! live range.
+
+use dominator_tree::DominatorTree;
+use ir::{DataFlowGraph, Layout, Cursor};
+use ir::{Function, Ebb, Inst, Value};
+use isa::{TargetIsa, RegInfo, EncInfo, RecipeConstraints, ConstraintKind};
+use isa::registers::RegClassMask;
+use regalloc::affinity::Affinity;
+use regalloc::live_value_tracker::{LiveValue, LiveValueTracker};
+use regalloc::liveness::Liveness;
+use regalloc::pressure::Pressure;
+use topo_order::TopoOrder;
+
+/// Persistent data structures for the spilling pass.
+pub struct Spilling {
+    spills: Vec<Value>,
+}
+
+/// Context data structure that gets instantiated once per pass.
+struct Context<'a> {
+    // Cached ISA information.
+    reginfo: RegInfo,
+    encinfo: EncInfo,
+
+    // References to contextual data structures we need.
+    domtree: &'a DominatorTree,
+    liveness: &'a mut Liveness,
+    topo: &'a mut TopoOrder,
+
+    // Current register pressure.
+    pressure: Pressure,
+
+    // Values spilled for the current instruction. These values have already been removed from the
+    // pressure tracker, but they are still present in the live value tracker and their affinity
+    // hasn't been changed yet.
+    spills: &'a mut Vec<Value>,
+}
+
+impl Spilling {
+    /// Create a new spilling data structure.
+    pub fn new() -> Spilling {
+        Spilling { spills: Vec::new() }
+    }
+
+    /// Run the spilling algorithm over `func`.
+    pub fn run(&mut self,
+               isa: &TargetIsa,
+               func: &mut Function,
+               domtree: &DominatorTree,
+               liveness: &mut Liveness,
+               topo: &mut TopoOrder,
+               tracker: &mut LiveValueTracker) {
+        dbg!("Spilling for:\n{}", func.display(isa));
+        let reginfo = isa.register_info();
+        let usable_regs = isa.allocatable_registers(func);
+        let mut ctx = Context {
+            reginfo: isa.register_info(),
+            encinfo: isa.encoding_info(),
+            domtree,
+            liveness,
+            topo,
+            pressure: Pressure::new(&reginfo, &usable_regs),
+            spills: &mut self.spills,
+        };
+        ctx.run(func, tracker)
+    }
+}
+
+impl<'a> Context<'a> {
+    fn run(&mut self, func: &mut Function, tracker: &mut LiveValueTracker) {
+        self.topo.reset(func.layout.ebbs());
+        while let Some(ebb) = self.topo.next(&func.layout, self.domtree) {
+            self.visit_ebb(ebb, func, tracker);
+        }
+    }
+
+    fn visit_ebb(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) {
+        dbg!("Spilling {}:", ebb);
+        self.visit_ebb_header(ebb, func, tracker);
+        tracker.drop_dead_args();
+
+        let mut pos = Cursor::new(&mut func.layout);
+        pos.goto_top(ebb);
+        while let Some(inst) = pos.next_inst() {
+            if let Some(constraints) = self.encinfo.operand_constraints(func.encodings[inst]) {
+                self.visit_inst(inst, constraints, &mut pos, &mut func.dfg, tracker);
+                tracker.drop_dead(inst);
+                self.process_spills(tracker);
+            }
+        }
+    }
+
+    // Take all live registers in `regs` from the pressure set.
+    // This doesn't cause any spilling, it is assumed there are enough registers.
+    fn take_live_regs(&mut self, regs: &[LiveValue]) {
+        for lv in regs {
+            if !lv.is_dead {
+                if let Affinity::Reg(rci) = lv.affinity {
+                    let rc = self.reginfo.rc(rci);
+                    self.pressure.take(rc);
+                }
+            }
+        }
+    }
+
+    // Free all registers in `kills` from the pressure set.
+    fn free_regs(&mut self, kills: &[LiveValue]) {
+        for lv in kills {
+            if let Affinity::Reg(rci) = lv.affinity {
+                let rc = self.reginfo.rc(rci);
+                self.pressure.free(rc);
+            }
+        }
+    }
+
+    fn visit_ebb_header(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) {
+        let (liveins, args) =
+            tracker.ebb_top(ebb, &func.dfg, self.liveness, &func.layout, self.domtree);
+
+        // Count the live-in registers. These should already fit in registers; they did at the
+        // dominator.
+        self.pressure.reset();
+        self.take_live_regs(liveins);
+
+        // TODO: Process and count EBB arguments. Some may need spilling.
+        self.take_live_regs(args);
+    }
+
+    fn visit_inst(&mut self,
+                  inst: Inst,
+                  constraints: &RecipeConstraints,
+                  pos: &mut Cursor,
+                  dfg: &DataFlowGraph,
+                  tracker: &mut LiveValueTracker) {
+        dbg!("Inst {}, {}", dfg.display_inst(inst), self.pressure);
+        // TODO: Repair constraint violations by copying input values.
+        //
+        // - Tied use of value that is not killed.
+        // - Inconsistent uses of the same value.
+        //
+        // Each inserted copy may increase register pressure. Fix by spilling something not used by
+        // the instruction.
+        //
+        // Count pressure for register uses of spilled values too.
+        //
+        // Finally, reset pressure state to level from before the input adjustments, minus spills.
+        //
+        // Spills should be removed from tracker. Otherwise they could be double-counted by
+        // free_regs below.
+
+        // Update the live value tracker with this instruction.
+        let (throughs, kills, defs) = tracker.process_inst(inst, dfg, self.liveness);
+
+
+        // Remove kills from the pressure tracker.
+        self.free_regs(kills);
+
+        // Make sure we have enough registers for the register defs.
+        // Dead defs are included here. They need a register too.
+        // No need to process call return values, they are in fixed registers.
+        for op in constraints.outs {
+            if op.kind != ConstraintKind::Stack {
+                // Add register def to pressure, spill if needed.
+                while let Err(mask) = self.pressure.take_transient(op.regclass) {
+                    dbg!("Need {} reg from {} throughs", op.regclass, throughs.len());
+                    self.spill_from(mask, throughs, dfg, &pos.layout);
+                }
+            }
+        }
+        self.pressure.reset_transient();
+
+        // Restore pressure state, compute pressure with affinities from `defs`.
+        // Exclude dead defs. Includes call return values.
+        // This won't cause spilling.
+        self.take_live_regs(defs);
+    }
+
+    // Spill a candidate from `candidates` whose top-level register class is in `mask`.
+    fn spill_from<'ii, II>(&mut self,
+                           mask: RegClassMask,
+                           candidates: II,
+                           dfg: &DataFlowGraph,
+                           layout: &Layout)
+        where II: IntoIterator<Item = &'ii LiveValue>
+    {
+        // Find the best viable spill candidate.
+        //
+        // The very simple strategy implemented here is to spill the value with the earliest def in
+        // the reverse post-order. This strategy depends on a good reload pass to generate good
+        // code.
+        //
+        // We know that all candidate defs dominate the current instruction, so one of them will
+        // dominate the others. That is the earliest def.
+        let best = candidates
+            .into_iter()
+            .filter_map(|lv| {
+                // Viable candidates are registers in one of the `mask` classes, and not already in
+                // the spill set.
+                if let Affinity::Reg(rci) = lv.affinity {
+                    let rc = self.reginfo.rc(rci);
+                    if (mask & (1 << rc.toprc)) != 0 && !self.spills.contains(&lv.value) {
+                        // Here, `lv` is a viable spill candidate.
+                        return Some(lv.value);
+                    }
+                }
+                None
+            })
+            .min_by(|&a, &b| {
+                        // Find the minimum candidate according to the RPO of their defs.
+                        self.domtree
+                            .rpo_cmp(dfg.value_def(a), dfg.value_def(b), layout)
+                    });
+
+        if let Some(value) = best {
+            // Found a spill candidate.
+            self.spill_reg(value);
+        } else {
+            panic!("Ran out of registers for mask={}", mask);
+        }
+    }
+
+    /// Spill `value` immediately by
+    ///
+    /// 1. Changing its affinity to `Stack` which marks the spill.
+    /// 2. Removing the value from the pressure tracker.
+    /// 3. Adding the value to `self.spills` for later reference by `process_spills`.
+    ///
+    /// Note that this does not update the cached affinity in the live value tracker. Call
+    /// `process_spills` to do that.
+    fn spill_reg(&mut self, value: Value) {
+        if let Affinity::Reg(rci) = self.liveness.spill(value) {
+            let rc = self.reginfo.rc(rci);
+            self.pressure.free(rc);
+            self.spills.push(value);
+            dbg!("Spilled {}:{} -> {}", value, rc, self.pressure);
+        } else {
+            panic!("Cannot spill {} that was already on the stack", value);
+        }
+    }
+
+    /// Process any pending spills in the `self.spills` vector.
+    ///
+    /// It is assumed that spills are removed from the pressure tracker immediately, see
+    /// `spill_from` above.
+    ///
+    /// We also need to update the live range affinity and remove spilled values from the live
+    /// value tracker.
+    fn process_spills(&mut self, tracker: &mut LiveValueTracker) {
+        if !self.spills.is_empty() {
+            tracker.process_spills(|v| self.spills.contains(&v));
+            self.spills.clear()
+        }
+    }
+}