diff --git a/filetests/regalloc/spill.cton b/filetests/regalloc/spill.cton new file mode 100644 index 0000000000..a6565b987e --- /dev/null +++ b/filetests/regalloc/spill.cton @@ -0,0 +1,44 @@ +test regalloc + +; Test the spiler on an ISA with few registers. +; RV32E has 16 registers, where: +; - %x0 is hardwired to zero. +; - %x1 is the return address. +; - %x2 is the stack pointer. +; - %x3 is the global pointer. +; - %x4 is the thread pointer. +; - %x10-%x15 are function arguments. +; +; regex: V=v\d+ + +isa riscv enable_e + +; In straight-line code, the first value defined is spilled. +; That is the argument. +function %pyramid(i32) -> i32 { +ebb0(v1: i32): +; check: $v1 = spill $(rv1=$V) + v2 = iadd_imm v1, 12 + v3 = iadd_imm v2, 12 + v4 = iadd_imm v3, 12 + v5 = iadd_imm v4, 12 + v6 = iadd_imm v5, 12 + v7 = iadd_imm v6, 12 + v8 = iadd_imm v7, 12 + v9 = iadd_imm v8, 12 + v10 = iadd_imm v9, 12 + v11 = iadd_imm v10, 12 + v12 = iadd_imm v11, 12 + v31 = iadd v11, v12 + v30 = iadd v31, v10 + v29 = iadd v30, v9 + v28 = iadd v29, v8 + v27 = iadd v28, v7 + v26 = iadd v27, v6 + v25 = iadd v26, v5 + v24 = iadd v25, v4 + v23 = iadd v24, v3 + v22 = iadd v23, v2 + v21 = iadd v22, v1 + return v21 +} diff --git a/lib/cretonne/src/regalloc/context.rs b/lib/cretonne/src/regalloc/context.rs index 3cf2b5dd81..b404602a34 100644 --- a/lib/cretonne/src/regalloc/context.rs +++ b/lib/cretonne/src/regalloc/context.rs @@ -12,6 +12,7 @@ use regalloc::coloring::Coloring; use regalloc::live_value_tracker::LiveValueTracker; use regalloc::liveness::Liveness; use regalloc::reload::Reload; +use regalloc::spilling::Spilling; use result::CtonResult; use topo_order::TopoOrder; use verifier::{verify_context, verify_liveness}; @@ -21,6 +22,7 @@ pub struct Context { liveness: Liveness, topo: TopoOrder, tracker: LiveValueTracker, + spilling: Spilling, reload: Reload, coloring: Coloring, } @@ -35,6 +37,7 @@ impl Context { liveness: Liveness::new(), topo: TopoOrder::new(), tracker: LiveValueTracker::new(), + spilling: Spilling::new(), reload: Reload::new(), coloring: Coloring::new(), } @@ -62,7 +65,19 @@ impl Context { verify_liveness(isa, func, cfg, &self.liveness)?; } - // TODO: Second pass: Spilling. + // Second pass: Spilling. + self.spilling + .run(isa, + func, + domtree, + &mut self.liveness, + &mut self.topo, + &mut self.tracker); + + if isa.flags().enable_verifier() { + verify_context(func, cfg, domtree, Some(isa))?; + verify_liveness(isa, func, cfg, &self.liveness)?; + } // Third pass: Reload. self.reload diff --git a/lib/cretonne/src/regalloc/live_value_tracker.rs b/lib/cretonne/src/regalloc/live_value_tracker.rs index a5ee8197e8..ccaf6c9209 100644 --- a/lib/cretonne/src/regalloc/live_value_tracker.rs +++ b/lib/cretonne/src/regalloc/live_value_tracker.rs @@ -12,7 +12,6 @@ use partition_slice::partition_slice; use regalloc::affinity::Affinity; use regalloc::liveness::Liveness; use regalloc::liverange::LiveRange; - use std::collections::HashMap; type ValueList = EntityList; @@ -299,6 +298,20 @@ impl LiveValueTracker { self.live.remove_dead_values(); } + /// Process new spills. + /// + /// Any values where `f` returns true are spilled and will be treated as if their affinity was + /// `Stack`. + pub fn process_spills(&mut self, mut f: F) + where F: FnMut(Value) -> bool + { + for lv in &mut self.live.values { + if f(lv.value) { + lv.affinity = Affinity::Stack; + } + } + } + /// Save the current set of live values so it is associated with `idom`. fn save_idom_live_set(&mut self, idom: Inst) { let values = self.live.values.iter().map(|lv| lv.value); diff --git a/lib/cretonne/src/regalloc/liveness.rs b/lib/cretonne/src/regalloc/liveness.rs index 3ced8ebc00..afccbffaa3 100644 --- a/lib/cretonne/src/regalloc/liveness.rs +++ b/lib/cretonne/src/regalloc/liveness.rs @@ -182,6 +182,7 @@ use isa::{TargetIsa, EncInfo}; use regalloc::affinity::Affinity; use regalloc::liverange::LiveRange; use sparse_map::SparseMap; +use std::mem; /// A set of live ranges, indexed by value number. type LiveRangeSet = SparseMap; @@ -338,6 +339,13 @@ impl Liveness { &mut lr.affinity } + /// Change the affinity of `value` to `Stack` and return the previous affinity. + pub fn spill(&mut self, value: Value) -> Affinity { + let mut lr = self.ranges.get_mut(value).expect("Value has no live range"); + mem::replace(&mut lr.affinity, Affinity::Stack) + } + + /// Compute the live ranges of all SSA values used in `func`. /// This clears out any existing analysis stored in this data structure. pub fn compute(&mut self, isa: &TargetIsa, func: &Function, cfg: &ControlFlowGraph) { diff --git a/lib/cretonne/src/regalloc/mod.rs b/lib/cretonne/src/regalloc/mod.rs index 4a86abbe93..59eb341eb0 100644 --- a/lib/cretonne/src/regalloc/mod.rs +++ b/lib/cretonne/src/regalloc/mod.rs @@ -14,6 +14,7 @@ mod diversion; mod pressure; mod reload; mod solver; +mod spilling; pub use self::allocatable_set::AllocatableSet; pub use self::context::Context; diff --git a/lib/cretonne/src/regalloc/pressure.rs b/lib/cretonne/src/regalloc/pressure.rs index 6ddf6bdf4e..1398bff8d6 100644 --- a/lib/cretonne/src/regalloc/pressure.rs +++ b/lib/cretonne/src/regalloc/pressure.rs @@ -39,6 +39,7 @@ use isa::registers::{RegInfo, MAX_TOPRCS, RegClass, RegClassMask}; use regalloc::AllocatableSet; use std::cmp::min; +use std::fmt; use std::iter::ExactSizeIterator; /// Information per top-level register class. @@ -225,6 +226,16 @@ impl Pressure { } } +impl fmt::Display for Pressure { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Pressure[")?; + for rc in &self.toprc { + write!(f, " {}+{}/{}", rc.base_count, rc.transient_count, rc.limit)?; + } + write!(f, " ]") + } +} + #[cfg(test)] mod tests { use isa::{TargetIsa, RegClass}; diff --git a/lib/cretonne/src/regalloc/reload.rs b/lib/cretonne/src/regalloc/reload.rs index 533dd45595..1342625380 100644 --- a/lib/cretonne/src/regalloc/reload.rs +++ b/lib/cretonne/src/regalloc/reload.rs @@ -10,6 +10,7 @@ //! pressure limits to be exceeded. use dominator_tree::DominatorTree; +use entity_map::EntityMap; use ir::{Ebb, Inst, Value, Function, DataFlowGraph}; use ir::layout::{Cursor, CursorPosition}; use ir::{InstBuilder, ArgumentLoc}; @@ -29,6 +30,7 @@ pub struct Reload { /// Context data structure that gets instantiated once per pass. struct Context<'a> { + isa: &'a TargetIsa, // Cached ISA information. // We save it here to avoid frequent virtual function calls on the `TargetIsa` trait object. encinfo: EncInfo, @@ -60,6 +62,7 @@ impl Reload { topo: &mut TopoOrder, tracker: &mut LiveValueTracker) { let mut ctx = Context { + isa, encinfo: isa.encoding_info(), domtree, liveness, @@ -112,7 +115,13 @@ impl<'a> Context<'a> { while let Some(inst) = pos.current_inst() { let encoding = func.encodings[inst]; if encoding.is_legal() { - self.visit_inst(ebb, inst, encoding, &mut pos, &mut func.dfg, tracker); + self.visit_inst(ebb, + inst, + encoding, + &mut pos, + &mut func.dfg, + &mut func.encodings, + tracker); tracker.drop_dead(inst); } else { pos.next_inst(); @@ -121,7 +130,7 @@ impl<'a> Context<'a> { } /// Process the EBB parameters. Return the next instruction in the EBB to be processed - fn visit_ebb_header(&self, + fn visit_ebb_header(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) @@ -139,7 +148,7 @@ impl<'a> Context<'a> { /// Visit the arguments to the entry block. /// These values have ABI constraints from the function signature. - fn visit_entry_args(&self, + fn visit_entry_args(&mut self, ebb: Ebb, func: &mut Function, args: &[LiveValue]) @@ -157,7 +166,15 @@ impl<'a> Context<'a> { // with a temporary register value that is immediately spilled. let reg = func.dfg.replace_ebb_arg(arg.value, abi.value_type); func.dfg.ins(&mut pos).with_result(arg.value).spill(reg); - // TODO: Update live ranges. + let spill = func.dfg.value_def(arg.value).unwrap_inst(); + *func.encodings.ensure(spill) = self.isa + .encode(&func.dfg, &func.dfg[spill], abi.value_type) + .expect("Can't encode spill"); + // Update live ranges. + self.liveness.move_def_locally(arg.value, spill); + let affinity = Affinity::abi(abi, self.isa); + self.liveness.create_dead(reg, ebb, affinity); + self.liveness.extend_locally(reg, ebb, spill, pos.layout); } } ArgumentLoc::Stack(_) => { @@ -184,6 +201,7 @@ impl<'a> Context<'a> { encoding: Encoding, pos: &mut Cursor, dfg: &mut DataFlowGraph, + encodings: &mut EntityMap, tracker: &mut LiveValueTracker) { // Get the operand constraints for `inst` that we are trying to satisfy. let constraints = self.encinfo @@ -213,6 +231,11 @@ impl<'a> Context<'a> { } let reg = dfg.ins(pos).fill(cand.value); + let fill = dfg.value_def(reg).unwrap_inst(); + *encodings.ensure(fill) = self.isa + .encode(dfg, &dfg[fill], dfg.value_type(reg)) + .expect("Can't encode fill"); + self.reloads .insert(ReloadedValue { stack: cand.value, diff --git a/lib/cretonne/src/regalloc/spilling.rs b/lib/cretonne/src/regalloc/spilling.rs new file mode 100644 index 0000000000..8938944952 --- /dev/null +++ b/lib/cretonne/src/regalloc/spilling.rs @@ -0,0 +1,259 @@ +//! Spilling pass. +//! +//! The spilling pass is the first to run after the liveness analysis. Its primary function is to +//! ensure that the register pressure never exceeds the number of available registers by moving +//! some SSA values to spill slots on the stack. This is encoded in the affinity of the value's +//! live range. + +use dominator_tree::DominatorTree; +use ir::{DataFlowGraph, Layout, Cursor}; +use ir::{Function, Ebb, Inst, Value}; +use isa::{TargetIsa, RegInfo, EncInfo, RecipeConstraints, ConstraintKind}; +use isa::registers::RegClassMask; +use regalloc::affinity::Affinity; +use regalloc::live_value_tracker::{LiveValue, LiveValueTracker}; +use regalloc::liveness::Liveness; +use regalloc::pressure::Pressure; +use topo_order::TopoOrder; + +/// Persistent data structures for the spilling pass. +pub struct Spilling { + spills: Vec, +} + +/// Context data structure that gets instantiated once per pass. +struct Context<'a> { + // Cached ISA information. + reginfo: RegInfo, + encinfo: EncInfo, + + // References to contextual data structures we need. + domtree: &'a DominatorTree, + liveness: &'a mut Liveness, + topo: &'a mut TopoOrder, + + // Current register pressure. + pressure: Pressure, + + // Values spilled for the current instruction. These values have already been removed from the + // pressure tracker, but they are still present in the live value tracker and their affinity + // hasn't been changed yet. + spills: &'a mut Vec, +} + +impl Spilling { + /// Create a new spilling data structure. + pub fn new() -> Spilling { + Spilling { spills: Vec::new() } + } + + /// Run the spilling algorithm over `func`. + pub fn run(&mut self, + isa: &TargetIsa, + func: &mut Function, + domtree: &DominatorTree, + liveness: &mut Liveness, + topo: &mut TopoOrder, + tracker: &mut LiveValueTracker) { + dbg!("Spilling for:\n{}", func.display(isa)); + let reginfo = isa.register_info(); + let usable_regs = isa.allocatable_registers(func); + let mut ctx = Context { + reginfo: isa.register_info(), + encinfo: isa.encoding_info(), + domtree, + liveness, + topo, + pressure: Pressure::new(®info, &usable_regs), + spills: &mut self.spills, + }; + ctx.run(func, tracker) + } +} + +impl<'a> Context<'a> { + fn run(&mut self, func: &mut Function, tracker: &mut LiveValueTracker) { + self.topo.reset(func.layout.ebbs()); + while let Some(ebb) = self.topo.next(&func.layout, self.domtree) { + self.visit_ebb(ebb, func, tracker); + } + } + + fn visit_ebb(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) { + dbg!("Spilling {}:", ebb); + self.visit_ebb_header(ebb, func, tracker); + tracker.drop_dead_args(); + + let mut pos = Cursor::new(&mut func.layout); + pos.goto_top(ebb); + while let Some(inst) = pos.next_inst() { + if let Some(constraints) = self.encinfo.operand_constraints(func.encodings[inst]) { + self.visit_inst(inst, constraints, &mut pos, &mut func.dfg, tracker); + tracker.drop_dead(inst); + self.process_spills(tracker); + } + } + } + + // Take all live registers in `regs` from the pressure set. + // This doesn't cause any spilling, it is assumed there are enough registers. + fn take_live_regs(&mut self, regs: &[LiveValue]) { + for lv in regs { + if !lv.is_dead { + if let Affinity::Reg(rci) = lv.affinity { + let rc = self.reginfo.rc(rci); + self.pressure.take(rc); + } + } + } + } + + // Free all registers in `kills` from the pressure set. + fn free_regs(&mut self, kills: &[LiveValue]) { + for lv in kills { + if let Affinity::Reg(rci) = lv.affinity { + let rc = self.reginfo.rc(rci); + self.pressure.free(rc); + } + } + } + + fn visit_ebb_header(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) { + let (liveins, args) = + tracker.ebb_top(ebb, &func.dfg, self.liveness, &func.layout, self.domtree); + + // Count the live-in registers. These should already fit in registers; they did at the + // dominator. + self.pressure.reset(); + self.take_live_regs(liveins); + + // TODO: Process and count EBB arguments. Some may need spilling. + self.take_live_regs(args); + } + + fn visit_inst(&mut self, + inst: Inst, + constraints: &RecipeConstraints, + pos: &mut Cursor, + dfg: &DataFlowGraph, + tracker: &mut LiveValueTracker) { + dbg!("Inst {}, {}", dfg.display_inst(inst), self.pressure); + // TODO: Repair constraint violations by copying input values. + // + // - Tied use of value that is not killed. + // - Inconsistent uses of the same value. + // + // Each inserted copy may increase register pressure. Fix by spilling something not used by + // the instruction. + // + // Count pressure for register uses of spilled values too. + // + // Finally, reset pressure state to level from before the input adjustments, minus spills. + // + // Spills should be removed from tracker. Otherwise they could be double-counted by + // free_regs below. + + // Update the live value tracker with this instruction. + let (throughs, kills, defs) = tracker.process_inst(inst, dfg, self.liveness); + + + // Remove kills from the pressure tracker. + self.free_regs(kills); + + // Make sure we have enough registers for the register defs. + // Dead defs are included here. They need a register too. + // No need to process call return values, they are in fixed registers. + for op in constraints.outs { + if op.kind != ConstraintKind::Stack { + // Add register def to pressure, spill if needed. + while let Err(mask) = self.pressure.take_transient(op.regclass) { + dbg!("Need {} reg from {} throughs", op.regclass, throughs.len()); + self.spill_from(mask, throughs, dfg, &pos.layout); + } + } + } + self.pressure.reset_transient(); + + // Restore pressure state, compute pressure with affinities from `defs`. + // Exclude dead defs. Includes call return values. + // This won't cause spilling. + self.take_live_regs(defs); + } + + // Spill a candidate from `candidates` whose top-level register class is in `mask`. + fn spill_from<'ii, II>(&mut self, + mask: RegClassMask, + candidates: II, + dfg: &DataFlowGraph, + layout: &Layout) + where II: IntoIterator + { + // Find the best viable spill candidate. + // + // The very simple strategy implemented here is to spill the value with the earliest def in + // the reverse post-order. This strategy depends on a good reload pass to generate good + // code. + // + // We know that all candidate defs dominate the current instruction, so one of them will + // dominate the others. That is the earliest def. + let best = candidates + .into_iter() + .filter_map(|lv| { + // Viable candidates are registers in one of the `mask` classes, and not already in + // the spill set. + if let Affinity::Reg(rci) = lv.affinity { + let rc = self.reginfo.rc(rci); + if (mask & (1 << rc.toprc)) != 0 && !self.spills.contains(&lv.value) { + // Here, `lv` is a viable spill candidate. + return Some(lv.value); + } + } + None + }) + .min_by(|&a, &b| { + // Find the minimum candidate according to the RPO of their defs. + self.domtree + .rpo_cmp(dfg.value_def(a), dfg.value_def(b), layout) + }); + + if let Some(value) = best { + // Found a spill candidate. + self.spill_reg(value); + } else { + panic!("Ran out of registers for mask={}", mask); + } + } + + /// Spill `value` immediately by + /// + /// 1. Changing its affinity to `Stack` which marks the spill. + /// 2. Removing the value from the pressure tracker. + /// 3. Adding the value to `self.spills` for later reference by `process_spills`. + /// + /// Note that this does not update the cached affinity in the live value tracker. Call + /// `process_spills` to do that. + fn spill_reg(&mut self, value: Value) { + if let Affinity::Reg(rci) = self.liveness.spill(value) { + let rc = self.reginfo.rc(rci); + self.pressure.free(rc); + self.spills.push(value); + dbg!("Spilled {}:{} -> {}", value, rc, self.pressure); + } else { + panic!("Cannot spill {} that was already on the stack", value); + } + } + + /// Process any pending spills in the `self.spills` vector. + /// + /// It is assumed that spills are removed from the pressure tracker immediately, see + /// `spill_from` above. + /// + /// We also need to update the live range affinity and remove spilled values from the live + /// value tracker. + fn process_spills(&mut self, tracker: &mut LiveValueTracker) { + if !self.spills.is_empty() { + tracker.process_spills(|v| self.spills.contains(&v)); + self.spills.clear() + } + } +}