diff --git a/cranelift/filetests/regalloc/spill.cton b/cranelift/filetests/regalloc/spill.cton index eaea391504..f163c478b3 100644 --- a/cranelift/filetests/regalloc/spill.cton +++ b/cranelift/filetests/regalloc/spill.cton @@ -75,3 +75,32 @@ ebb0(v1: i32): ; check: call $fn0 return } + +; The same value used for two function arguments. +function %doubleuse(i32) { + fn0 = function %xx(i32, i32) +ebb0(v0: i32): + ; check: $(c=$V) = copy $v0 + call fn0(v0, v0) + ; check: call $fn0($v0, $c) + return +} + +; The same value used as indirect callee and argument. +function %doubleuse_icall1(i32) { + sig0 = signature(i32) +ebb0(v0: i32): + ; not:copy + call_indirect sig0, v0(v0) + return +} + +; The same value used as indirect callee and two arguments. +function %doubleuse_icall2(i32) { + sig0 = signature(i32, i32) +ebb0(v0: i32): + ; check: $(c=$V) = copy $v0 + call_indirect sig0, v0(v0, v0) + ; check: call_indirect $sig0, $v0($v0, $c) + return +} diff --git a/lib/cretonne/src/ir/entities.rs b/lib/cretonne/src/ir/entities.rs index 813ec978c2..04b54e0488 100644 --- a/lib/cretonne/src/ir/entities.rs +++ b/lib/cretonne/src/ir/entities.rs @@ -74,7 +74,7 @@ impl Ebb { } /// An opaque reference to an SSA value. -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] pub struct Value(u32); entity_impl!(Value, "v"); diff --git a/lib/cretonne/src/regalloc/spilling.rs b/lib/cretonne/src/regalloc/spilling.rs index da0453f549..23d4799dad 100644 --- a/lib/cretonne/src/regalloc/spilling.rs +++ b/lib/cretonne/src/regalloc/spilling.rs @@ -4,12 +4,23 @@ //! ensure that the register pressure never exceeds the number of available registers by moving //! some SSA values to spill slots on the stack. This is encoded in the affinity of the value's //! live range. +//! +//! Some instruction operand constraints may require additional registers to resolve. Since this +//! can cause spilling, the spilling pass is also responsible for resolving those constraints by +//! inserting copies. The extra constraints are: +//! +//! 1. A value used by a tied operand must be killed by the instruction. This is resolved by +//! inserting a copy to a temporary value when necessary. +//! 2. When the same value is used more than once by an instruction, the operand constraints must +//! be compatible. Otherwise, the value must be copied into a new register for some of the +//! operands. use dominator_tree::DominatorTree; -use ir::{DataFlowGraph, Layout, Cursor}; -use ir::{Function, Ebb, Inst, Value}; -use isa::{TargetIsa, RegInfo, EncInfo, RecipeConstraints, ConstraintKind}; -use isa::registers::RegClassMask; +use entity_map::EntityMap; +use ir::{DataFlowGraph, Layout, Cursor, InstBuilder}; +use ir::{Function, Ebb, Inst, Value, SigRef}; +use isa::registers::{RegClass, RegClassMask}; +use isa::{TargetIsa, RegInfo, EncInfo, Encoding, RecipeConstraints, ConstraintKind}; use regalloc::affinity::Affinity; use regalloc::live_value_tracker::{LiveValue, LiveValueTracker}; use regalloc::liveness::Liveness; @@ -19,14 +30,19 @@ use topo_order::TopoOrder; /// Persistent data structures for the spilling pass. pub struct Spilling { spills: Vec, + reg_uses: Vec, } /// Context data structure that gets instantiated once per pass. struct Context<'a> { + isa: &'a TargetIsa, // Cached ISA information. reginfo: RegInfo, encinfo: EncInfo, + // References to parts of the current function. + encodings: &'a mut EntityMap, + // References to contextual data structures we need. domtree: &'a DominatorTree, liveness: &'a mut Liveness, @@ -39,12 +55,18 @@ struct Context<'a> { // pressure tracker, but they are still present in the live value tracker and their affinity // hasn't been changed yet. spills: &'a mut Vec, + + // Uses of register values in the current instruction. + reg_uses: &'a mut Vec, } impl Spilling { /// Create a new spilling data structure. pub fn new() -> Spilling { - Spilling { spills: Vec::new() } + Spilling { + spills: Vec::new(), + reg_uses: Vec::new(), + } } /// Run the spilling algorithm over `func`. @@ -59,36 +81,46 @@ impl Spilling { let reginfo = isa.register_info(); let usable_regs = isa.allocatable_registers(func); let mut ctx = Context { + isa, reginfo: isa.register_info(), encinfo: isa.encoding_info(), + encodings: &mut func.encodings, domtree, liveness, topo, pressure: Pressure::new(®info, &usable_regs), spills: &mut self.spills, + reg_uses: &mut self.reg_uses, }; - ctx.run(func, tracker) + ctx.run(&mut func.layout, &mut func.dfg, tracker) } } impl<'a> Context<'a> { - fn run(&mut self, func: &mut Function, tracker: &mut LiveValueTracker) { - self.topo.reset(func.layout.ebbs()); - while let Some(ebb) = self.topo.next(&func.layout, self.domtree) { - self.visit_ebb(ebb, func, tracker); + fn run(&mut self, + layout: &mut Layout, + dfg: &mut DataFlowGraph, + tracker: &mut LiveValueTracker) { + self.topo.reset(layout.ebbs()); + while let Some(ebb) = self.topo.next(layout, self.domtree) { + self.visit_ebb(ebb, layout, dfg, tracker); } } - fn visit_ebb(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) { + fn visit_ebb(&mut self, + ebb: Ebb, + layout: &mut Layout, + dfg: &mut DataFlowGraph, + tracker: &mut LiveValueTracker) { dbg!("Spilling {}:", ebb); - self.visit_ebb_header(ebb, func, tracker); + self.visit_ebb_header(ebb, layout, dfg, tracker); tracker.drop_dead_args(); - let mut pos = Cursor::new(&mut func.layout); + let mut pos = Cursor::new(layout); pos.goto_top(ebb); while let Some(inst) = pos.next_inst() { - if let Some(constraints) = self.encinfo.operand_constraints(func.encodings[inst]) { - self.visit_inst(inst, constraints, &mut pos, &mut func.dfg, tracker); + if let Some(constraints) = self.encinfo.operand_constraints(self.encodings[inst]) { + self.visit_inst(inst, constraints, &mut pos, dfg, tracker); tracker.drop_dead(inst); self.process_spills(tracker); } @@ -118,9 +150,12 @@ impl<'a> Context<'a> { } } - fn visit_ebb_header(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) { - let (liveins, args) = - tracker.ebb_top(ebb, &func.dfg, self.liveness, &func.layout, self.domtree); + fn visit_ebb_header(&mut self, + ebb: Ebb, + layout: &mut Layout, + dfg: &mut DataFlowGraph, + tracker: &mut LiveValueTracker) { + let (liveins, args) = tracker.ebb_top(ebb, dfg, self.liveness, layout, self.domtree); // Count the live-in registers. These should already fit in registers; they did at the // dominator. @@ -135,24 +170,32 @@ impl<'a> Context<'a> { inst: Inst, constraints: &RecipeConstraints, pos: &mut Cursor, - dfg: &DataFlowGraph, + dfg: &mut DataFlowGraph, tracker: &mut LiveValueTracker) { dbg!("Inst {}, {}", dfg.display_inst(inst), self.pressure); // TODO: Repair constraint violations by copying input values. // // - Tied use of value that is not killed. - // - Inconsistent uses of the same value. - // - // Each inserted copy may increase register pressure. Fix by spilling something not used by - // the instruction. - // - // Count pressure for register uses of spilled values too. - // - // Finally, reset pressure state to level from before the input adjustments, minus spills. - // - // Spills should be removed from tracker. Otherwise they could be double-counted by - // free_regs below. + // - Count pressure for register uses of spilled values too. + + assert!(self.reg_uses.is_empty()); + + // If the instruction has any fixed register operands, we may need to resolve register + // constraints. + if constraints.fixed_ins { + self.collect_reg_uses(inst, constraints, dfg); + } + + // Calls usually have fixed register uses. let call_sig = dfg.call_signature(inst); + if let Some(sig) = call_sig { + self.collect_abi_reg_uses(inst, sig, dfg); + } + + if !self.reg_uses.is_empty() { + self.process_reg_uses(inst, pos, dfg, tracker); + } + // Update the live value tracker with this instruction. let (throughs, kills, defs) = tracker.process_inst(inst, dfg, self.liveness); @@ -190,6 +233,85 @@ impl<'a> Context<'a> { // This won't cause spilling. self.take_live_regs(defs); } + // Collect register uses from the fixed input constraints. + // + // We are assuming here that if a value is used both by a fixed register operand and a register + // class operand, they two are compatible. We are also assuming that two register class + // operands are always compatible. + fn collect_reg_uses(&mut self, + inst: Inst, + constraints: &RecipeConstraints, + dfg: &DataFlowGraph) { + let args = dfg.inst_args(inst); + for (idx, (op, &arg)) in constraints.ins.iter().zip(args).enumerate() { + match op.kind { + ConstraintKind::FixedReg(_) => { + self.reg_uses.push(RegUse::new(arg, idx)); + } + _ => {} + } + } + } + + // Collect register uses from the ABI input constraints. + fn collect_abi_reg_uses(&mut self, inst: Inst, sig: SigRef, dfg: &DataFlowGraph) { + let fixed_args = dfg[inst].opcode().constraints().fixed_value_arguments(); + let args = dfg.inst_variable_args(inst); + for (idx, (abi, &arg)) in + dfg.signatures[sig] + .argument_types + .iter() + .zip(args) + .enumerate() { + if abi.location.is_reg() { + self.reg_uses.push(RegUse::new(arg, fixed_args + idx)); + } + } + } + + // Process multiple register uses to resolve potential conflicts. + // + // Look for multiple uses of the same value in `self.reg_uses` and insert copies as necessary. + // Trigger spilling if any of the temporaries cause the register pressure to become too high. + // + // Leave `self.reg_uses` empty. + fn process_reg_uses(&mut self, + inst: Inst, + pos: &mut Cursor, + dfg: &mut DataFlowGraph, + tracker: &LiveValueTracker) { + // We're looking for multiple uses of the same value, so start by sorting by value. The + // secondary `opidx` key makes it possible to use an unstable sort once that is available + // outside nightly Rust. + self.reg_uses.sort_by_key(|u| (u.value, u.opidx)); + + // We are assuming that `reg_uses` has an entry per fixed register operand, and that any + // non-fixed register operands are compatible with one of the fixed uses of the value. + for i in 1..self.reg_uses.len() { + let ru = self.reg_uses[i]; + if self.reg_uses[i - 1].value != ru.value { + continue; + } + + // We have two fixed uses of the same value. Make a copy. + let (copy, rc) = self.insert_copy(ru.value, pos, dfg); + dfg.inst_args_mut(inst)[ru.opidx as usize] = copy; + + // Make sure the new copy doesn't blow the register pressure. + while let Err(mask) = self.pressure.take_transient(rc) { + dbg!("Copy of {} reg causes spill", rc); + // Spill a live register that is *not* used by the current instruction. + // Spilling a use wouldn't help. + let args = dfg.inst_args(inst); + self.spill_from(mask, + tracker.live().iter().filter(|lv| !args.contains(&lv.value)), + dfg, + &pos.layout); + } + } + self.pressure.reset_transient(); + self.reg_uses.clear() + } // Spill a candidate from `candidates` whose top-level register class is in `mask`. fn spill_from<'ii, II>(&mut self, @@ -267,4 +389,57 @@ impl<'a> Context<'a> { self.spills.clear() } } + + /// Insert a `copy value` before `pos` and give it a live range extending to `pos`. + /// + /// Returns the new local value created and its register class. + fn insert_copy(&mut self, + value: Value, + pos: &mut Cursor, + dfg: &mut DataFlowGraph) + -> (Value, RegClass) { + let copy = dfg.ins(pos).copy(value); + let inst = dfg.value_def(copy).unwrap_inst(); + let ty = dfg.value_type(copy); + + // Give it an encoding. + let encoding = self.isa + .encode(dfg, &dfg[inst], ty) + .expect("Can't encode copy"); + *self.encodings.ensure(inst) = encoding; + + // Update live ranges. + let rc = self.encinfo + .operand_constraints(encoding) + .expect("Bad copy encoding") + .outs + [0] + .regclass; + self.liveness + .create_dead(copy, inst, Affinity::Reg(rc.into())); + self.liveness + .extend_locally(copy, + pos.layout.pp_ebb(inst), + pos.current_inst().expect("must be at an instruction"), + pos.layout); + + (copy, rc) + } +} + +// Struct representing a register use of a value. +// Used to detect multiple uses of the same value with incompatible register constraints. +#[derive(Clone, Copy)] +struct RegUse { + value: Value, + opidx: u16, +} + +impl RegUse { + fn new(value: Value, idx: usize) -> RegUse { + RegUse { + value, + opidx: idx as u16, + } + } }