From feaea238bccb03dd6c99469832c6af89991ee94d Mon Sep 17 00:00:00 2001 From: Jakob Stoklund Olesen Date: Mon, 4 Dec 2017 13:43:10 -0800 Subject: [PATCH] Use bforest::Map for representing live ranges. Get rid of the per-value Vec in the LiveRange data type and use a bforest::Map instead to represent the live-in intervals for non-local live ranges. This has some advantages: - The memory footprint of a local live range is reduced from 40 to 20 bytes, and - Clearing the Liveness data structure is now a constant time operation which doesn't call free(). - The potentially quadratic behavior when computing large live ranges is controlled by the logarithmic B-tree operations. --- lib/cretonne/src/regalloc/coalescing.rs | 12 +- lib/cretonne/src/regalloc/coloring.rs | 19 +- .../src/regalloc/live_value_tracker.rs | 3 +- lib/cretonne/src/regalloc/liveness.rs | 29 +- lib/cretonne/src/regalloc/liverange.rs | 402 ++++++++++-------- lib/cretonne/src/regalloc/spilling.rs | 5 +- lib/cretonne/src/verifier/cssa.rs | 3 +- lib/cretonne/src/verifier/liveness.rs | 17 +- lib/cretonne/src/verifier/locations.rs | 4 +- 9 files changed, 279 insertions(+), 215 deletions(-) diff --git a/lib/cretonne/src/regalloc/coalescing.rs b/lib/cretonne/src/regalloc/coalescing.rs index 224e97840a..31228f680a 100644 --- a/lib/cretonne/src/regalloc/coalescing.rs +++ b/lib/cretonne/src/regalloc/coalescing.rs @@ -157,13 +157,11 @@ impl DomForest { layout, domtree, }; + let ctx = liveness.context(layout); for node in merged { if let Some(parent) = self.push_node(node, layout, domtree) { // Check if `parent` live range contains `node.def`. - let lr = liveness.get(parent).expect( - "No live range for parent value", - ); - if lr.overlaps_def(node.def, layout.pp_ebb(node.def), layout) { + if liveness[parent].overlaps_def(node.def, layout.pp_ebb(node.def), ctx) { // Interference detected. Get the `(a, b)` order right in the error. return Err(if node.set == 0 { (node.value, parent) @@ -447,7 +445,11 @@ impl<'a> Context<'a> { if self.liveness .get(a) .expect("No live range for interfering value") - .reaches_use(pred_inst, pred_ebb, &self.func.layout) + .reaches_use( + pred_inst, + pred_ebb, + self.liveness.context(&self.func.layout), + ) { // Splitting at `pred_inst` wouldn't resolve the interference, so we need to // start over. diff --git a/lib/cretonne/src/regalloc/coloring.rs b/lib/cretonne/src/regalloc/coloring.rs index 23cc3e27c2..25b271d17e 100644 --- a/lib/cretonne/src/regalloc/coloring.rs +++ b/lib/cretonne/src/regalloc/coloring.rs @@ -44,7 +44,7 @@ use cursor::{Cursor, EncCursor}; use dominator_tree::DominatorTree; -use ir::{Ebb, Inst, Value, Function, ValueLoc, SigRef}; +use ir::{Ebb, Inst, Value, Function, Layout, ValueLoc, SigRef}; use ir::{InstBuilder, AbiParam, ArgumentLoc, ValueDef}; use isa::{RegUnit, RegClass, RegInfo, regs_overlap}; use isa::{TargetIsa, EncInfo, RecipeConstraints, OperandConstraint, ConstraintKind}; @@ -54,7 +54,7 @@ use regalloc::affinity::Affinity; use regalloc::allocatable_set::AllocatableSet; use regalloc::live_value_tracker::{LiveValue, LiveValueTracker}; use regalloc::liveness::Liveness; -use regalloc::liverange::LiveRange; +use regalloc::liverange::{LiveRange, LiveRangeContext}; use regalloc::solver::{Solver, SolverError}; use std::mem; @@ -542,8 +542,8 @@ impl<'a> Context<'a> { if op.regclass.contains(cur_reg) { // This code runs after calling `solver.inputs_done()` so we must identify // the new variable as killed or live-through. - let layout = &self.cur.func.layout; - if self.liveness[value].killed_at(inst, layout.pp_ebb(inst), layout) { + let ctx = self.liveness.context(&self.cur.func.layout); + if self.liveness[value].killed_at(inst, ctx.order.pp_ebb(inst), ctx) { self.solver.add_killed_var(value, op.regclass, cur_reg); } else { self.solver.add_through_var(value, op.regclass, cur_reg); @@ -572,7 +572,7 @@ impl<'a> Context<'a> { // // Values with a global live range that are not live in to `dest` could appear as branch // arguments, so they can't always be un-diverted. - self.undivert_regs(|lr, func| lr.is_livein(dest, &func.layout)); + self.undivert_regs(|lr, ctx| lr.is_livein(dest, ctx)); // Now handle the EBB arguments. let br_args = self.cur.func.dfg.inst_variable_args(inst); @@ -642,13 +642,13 @@ impl<'a> Context<'a> { /// are reallocated to their global register assignments. fn undivert_regs(&mut self, mut pred: Pred) where - Pred: FnMut(&LiveRange, &Function) -> bool, + Pred: FnMut(&LiveRange, LiveRangeContext) -> bool, { for rdiv in self.divert.all() { let lr = self.liveness.get(rdiv.value).expect( "Missing live range for diverted register", ); - if pred(lr, &self.cur.func) { + if pred(lr, self.liveness.context(&self.cur.func.layout)) { if let Affinity::Reg(rci) = lr.affinity { let rc = self.reginfo.rc(rci); // Stack diversions should not be possible here. The only live transiently @@ -882,17 +882,18 @@ impl<'a> Context<'a> { use ir::instructions::BranchInfo::*; let inst = self.cur.current_inst().expect("Not on an instruction"); + let ctx = self.liveness.context(&self.cur.func.layout); match self.cur.func.dfg[inst].analyze_branch(&self.cur.func.dfg.value_lists) { NotABranch => false, SingleDest(ebb, _) => { let lr = &self.liveness[value]; - lr.is_livein(ebb, &self.cur.func.layout) + lr.is_livein(ebb, ctx) } Table(jt) => { let lr = &self.liveness[value]; !lr.is_local() && self.cur.func.jump_tables[jt].entries().any(|(_, ebb)| { - lr.is_livein(ebb, &self.cur.func.layout) + lr.is_livein(ebb, ctx) }) } } diff --git a/lib/cretonne/src/regalloc/live_value_tracker.rs b/lib/cretonne/src/regalloc/live_value_tracker.rs index cb57683661..e6f53151e2 100644 --- a/lib/cretonne/src/regalloc/live_value_tracker.rs +++ b/lib/cretonne/src/regalloc/live_value_tracker.rs @@ -189,6 +189,7 @@ impl LiveValueTracker { let idom_live_list = self.idom_sets.get(&idom).expect( "No stored live set for dominator", ); + let ctx = liveness.context(layout); // Get just the values that are live-in to `ebb`. for &value in idom_live_list.as_slice(&self.idom_pool) { let lr = liveness.get(value).expect( @@ -196,7 +197,7 @@ impl LiveValueTracker { ); // Check if this value is live-in here. - if let Some(endpoint) = lr.livein_local_end(ebb, layout) { + if let Some(endpoint) = lr.livein_local_end(ebb, ctx) { self.live.push(value, endpoint, lr); } } diff --git a/lib/cretonne/src/regalloc/liveness.rs b/lib/cretonne/src/regalloc/liveness.rs index e152cfaa32..20208e36a1 100644 --- a/lib/cretonne/src/regalloc/liveness.rs +++ b/lib/cretonne/src/regalloc/liveness.rs @@ -181,7 +181,7 @@ use ir::dfg::ValueDef; use ir::{Function, Value, Inst, Ebb, Layout, ProgramPoint}; use isa::{TargetIsa, EncInfo}; use regalloc::affinity::Affinity; -use regalloc::liverange::LiveRange; +use regalloc::liverange::{LiveRange, LiveRangeForest, LiveRangeContext}; use std::mem; use std::ops::Index; @@ -247,13 +247,14 @@ fn extend_to_use( worklist: &mut Vec, func: &Function, cfg: &ControlFlowGraph, + forest: &mut LiveRangeForest, ) { // This is our scratch working space, and we'll leave it empty when we return. assert!(worklist.is_empty()); // Extend the range locally in `ebb`. // If there already was a live interval in that block, we're done. - if lr.extend_in_ebb(ebb, to, &func.layout) { + if lr.extend_in_ebb(ebb, to, &func.layout, forest) { worklist.push(ebb); } @@ -270,7 +271,7 @@ fn extend_to_use( // We've learned that the value needs to be live-in to the `livein` EBB. // Make sure it is also live at all predecessor branches to `livein`. for (pred, branch) in cfg.pred_iter(livein) { - if lr.extend_in_ebb(pred, branch, &func.layout) { + if lr.extend_in_ebb(pred, branch, &func.layout, forest) { // This predecessor EBB also became live-in. We need to process it later. worklist.push(pred); } @@ -285,6 +286,9 @@ pub struct Liveness { /// The live ranges that have been computed so far. ranges: LiveRangeSet, + /// Memory pool for the live ranges. + forest: LiveRangeForest, + /// Working space for the `extend_to_use` algorithm. /// This vector is always empty, except for inside that function. /// It lives here to avoid repeated allocation of scratch memory. @@ -302,14 +306,21 @@ impl Liveness { pub fn new() -> Self { Self { ranges: LiveRangeSet::new(), + forest: LiveRangeForest::new(), worklist: Vec::new(), ebb_params: Vec::new(), } } + /// Get a context needed for working with a `LiveRange`. + pub fn context<'a>(&'a self, layout: &'a Layout) -> LiveRangeContext<'a, Layout> { + LiveRangeContext::new(layout, &self.forest) + } + /// Clear all data structures in this liveness analysis. pub fn clear(&mut self) { self.ranges.clear(); + self.forest.clear(); self.worklist.clear(); self.ebb_params.clear(); } @@ -359,7 +370,7 @@ impl Liveness { ) -> &mut Affinity { debug_assert_eq!(Some(ebb), layout.inst_ebb(user)); let lr = self.ranges.get_mut(value).expect("Value has no live range"); - let livein = lr.extend_in_ebb(ebb, user, layout); + let livein = lr.extend_in_ebb(ebb, user, layout, &mut self.forest); assert!(!livein, "{} should already be live in {}", value, ebb); &mut lr.affinity } @@ -416,7 +427,15 @@ impl Liveness { let lr = get_or_create(&mut self.ranges, arg, isa, func, &enc_info); // Extend the live range to reach this use. - extend_to_use(lr, ebb, inst, &mut self.worklist, func, cfg); + extend_to_use( + lr, + ebb, + inst, + &mut self.worklist, + func, + cfg, + &mut self.forest, + ); // Apply operand constraint, ignoring any variable arguments after the fixed // operands described by `operand_constraints`. Variable arguments are either diff --git a/lib/cretonne/src/regalloc/liverange.rs b/lib/cretonne/src/regalloc/liverange.rs index 840c882223..60079b3705 100644 --- a/lib/cretonne/src/regalloc/liverange.rs +++ b/lib/cretonne/src/regalloc/liverange.rs @@ -107,11 +107,11 @@ //! of coalescing, so we would need to roll our own. //! +use bforest; use entity::SparseMapValue; use ir::{Inst, Ebb, Value, Layout, ProgramPoint, ExpandedProgramPoint, ProgramOrder}; use regalloc::affinity::Affinity; use std::cmp::Ordering; -use std::marker::PhantomData; /// Global live range of a single SSA value. /// @@ -165,52 +165,54 @@ pub struct GenLiveRange { /// Additional live-in intervals sorted in program order. /// - /// This vector is empty for most values which are only used in one EBB. + /// This map is empty for most values which are only used in one EBB. /// - /// Invariants: + /// A map entry `ebb -> inst` means that the live range is live-in to `ebb`, continuing up to + /// `inst` which may belong to a later EBB in the program order. /// - /// - Sorted, disjoint: For all `i < j`: `liveins[i].end < liveins[j].begin`. - /// - Not overlapping defining EBB: For all `i`: - /// `liveins[i].end < def_begin` or `liveins[i].begin > def_end`. - liveins: Vec, - - unused: PhantomData, + /// The entries are non-overlapping, and none of them overlap the EBB where the value is + /// defined. + liveins: bforest::Map, } -/// An additional contiguous interval of a global live range. -/// -/// This represents a live-in interval for a single EBB, or a coalesced set of live-in intervals -/// for contiguous EBBs where all but the last live-in interval covers the whole EBB. -/// -#[derive(Copy, Clone)] -pub struct Interval { - /// Interval starting point. - /// - /// Since this interval does not represent the def of the value, it must begin at an EBB header - /// where the value is live-in. - pub begin: Ebb, - - /// Interval end point. - /// - /// The only interval end point that can be an EBB header is `def_end` above in a dead def - /// live range for an unused EBB argument. All other intervals must end at an instruction -- - /// either the last use in the EBB or the last branch/jump that can reach a use. - /// - /// When this represents multiple contiguous live-in intervals, this is the end point of the - /// last interval. The other intervals end at the terminator instructions of their respective - /// EBB. - pub end: Inst, +/// Context information needed to query a `LiveRange`. +pub struct LiveRangeContext<'a, PO: 'a + ProgramOrder> { + /// Ordering of EBBs. + pub order: &'a PO, + /// Memory pool. + pub forest: &'a bforest::MapForest, } -impl Interval { - /// Extend the interval end point to reach `to`, but only if it would make the interval longer. - fn extend_to(&mut self, to: Inst, order: &PO) { - if order.cmp(to, self.end) == Ordering::Greater { - self.end = to; +impl<'a, PO: ProgramOrder> LiveRangeContext<'a, PO> { + /// Make a new context. + pub fn new( + order: &'a PO, + forest: &'a bforest::MapForest, + ) -> LiveRangeContext<'a, PO> { + LiveRangeContext { order, forest } + } +} + +impl<'a, PO: ProgramOrder> Clone for LiveRangeContext<'a, PO> { + fn clone(&self) -> Self { + LiveRangeContext { + order: self.order, + forest: self.forest, } } } +impl<'a, PO: ProgramOrder> Copy for LiveRangeContext<'a, PO> {} + +/// Forest of B-trees used for storing live ranges. +pub type LiveRangeForest = bforest::MapForest; + +impl bforest::Comparator for PO { + fn cmp(&self, a: Ebb, b: Ebb) -> Ordering { + self.cmp(a, b) + } +} + impl GenLiveRange { /// Create a new live range for `value` defined at `def`. /// @@ -221,28 +223,10 @@ impl GenLiveRange { affinity, def_begin: def, def_end: def, - liveins: Vec::new(), - unused: PhantomData, + liveins: bforest::Map::new(), } } - /// Find the live-in interval containing `ebb`, if any. - /// - /// Return `Ok(n)` if `liveins[n]` already contains `ebb`. - /// Otherwise, return `Err(n)` with the index where such an interval should be inserted. - fn find_ebb_interval(&self, ebb: Ebb, order: &PO) -> Result { - self.liveins - .binary_search_by(|intv| order.cmp(intv.begin, ebb)) - .or_else(|n| { - // The interval at `n-1` may cover `ebb`. - if n > 0 && order.cmp(self.liveins[n - 1].end, ebb) == Ordering::Greater { - Ok(n - 1) - } else { - Err(n) - } - }) - } - /// Extend the local interval for `ebb` so it reaches `to` which must belong to `ebb`. /// Create a live-in interval if necessary. /// @@ -254,7 +238,13 @@ impl GenLiveRange { /// /// The return value can be used to detect if we just learned that the value is live-in to /// `ebb`. This can trigger recursive extensions in `ebb`'s CFG predecessor blocks. - pub fn extend_in_ebb(&mut self, ebb: Ebb, to: Inst, order: &PO) -> bool { + pub fn extend_in_ebb( + &mut self, + ebb: Ebb, + to: Inst, + order: &PO, + forest: &mut bforest::MapForest, + ) -> bool { // First check if we're extending the def interval. // // We're assuming here that `to` never precedes `def_begin` in the same EBB, but we can't @@ -275,68 +265,60 @@ impl GenLiveRange { } // Now check if we're extending any of the existing live-in intervals. - match self.find_ebb_interval(ebb, order) { - Ok(n) => { - // We have an interval that contains `ebb`, so we can simply extend it. - self.liveins[n].extend_to(to, order); + let mut c = self.liveins.cursor(forest, order); + let first_time_livein; - // If `to` is the terminator and the value lives in the successor EBB, - // coalesce the two intervals. - if let Some(next) = self.liveins.get(n + 1).cloned() { - if order.is_ebb_gap(to, next.begin) { - self.liveins[n].extend_to(next.end, order); - self.liveins.remove(n + 1); - } - } - - false + if let Some(end) = c.goto(ebb) { + // There's an interval beginning at `ebb`. See if it extends. + first_time_livein = false; + if order.cmp(end, to) == Ordering::Less { + *c.value_mut().unwrap() = to; + } else { + return first_time_livein; } - Err(n) => { - // Insert a new live-in interval at `n`, or coalesce to predecessor or successor - // if possible. - - // Determine if the new live-in range touches the predecessor or successor range - // and can therefore be coalesced to them. - let (coalesce_prev, coalesce_next) = { - let prev = n.checked_sub(1).and_then(|i| self.liveins.get(i)); - let next = self.liveins.get(n); - - ( - prev.map_or(false, |prev| order.is_ebb_gap(prev.end, ebb)), - next.map_or(false, |next| order.is_ebb_gap(to, next.begin)), - ) - }; - - match (coalesce_prev, coalesce_next) { - // Extend predecessor interval to cover new and successor intervals - (true, true) => { - let end = self.liveins[n].end; - self.liveins[n - 1].extend_to(end, order); - self.liveins.remove(n); + } else { + // There's no interval beginning at `ebb`, but we could still be live-in at `ebb` with + // a coalesced interval that begins before and ends after. + if let Some((_, end)) = c.prev() { + if order.cmp(end, ebb) == Ordering::Greater { + // Yep, the previous interval overlaps `ebb`. + first_time_livein = false; + if order.cmp(end, to) == Ordering::Less { + *c.value_mut().unwrap() = to; + } else { + return first_time_livein; } - // Extend predecessor interval to cover new interval - (true, false) => { - self.liveins[n - 1].extend_to(to, order); - } - // Extend successor interval to cover new interval - (false, true) => { - self.liveins[n].begin = ebb; - } - // Cannot coalesce; insert new interval - (false, false) => { - self.liveins.insert( - n, - Interval { - begin: ebb, - end: to, - }, - ); + } else { + first_time_livein = true; + // The current interval does not overlap `ebb`, but it may still be possible to + // coalesce with it. + if order.is_ebb_gap(end, ebb) { + *c.value_mut().unwrap() = to; + } else { + c.insert(ebb, to); } } - - true + } else { + // There is no existing interval before `ebb`. + first_time_livein = true; + c.insert(ebb, to); } } + + // Now `c` to left pointing at an interval that ends in `to`. + debug_assert_eq!(c.value(), Some(to)); + + // See if it can be coalesced with the following interval. + if let Some((next_ebb, next_end)) = c.next() { + if order.is_ebb_gap(to, next_ebb) { + // Remove this interval and extend the previous end point to `next_end`. + c.remove(); + c.prev(); + *c.value_mut().unwrap() = next_end; + } + } + + first_time_livein } /// Is this the live range of a dead value? @@ -387,59 +369,77 @@ impl GenLiveRange { /// If the live range is live through all of `ebb`, the terminator of `ebb` is a correct /// answer, but it is also possible that an even later program point is returned. So don't /// depend on the returned `Inst` to belong to `ebb`. - pub fn livein_local_end(&self, ebb: Ebb, order: &PO) -> Option { - self.find_ebb_interval(ebb, order).ok().map(|n| { - self.liveins[n].end - }) + pub fn livein_local_end(&self, ebb: Ebb, ctx: LiveRangeContext) -> Option { + self.liveins + .get_or_less(ebb, ctx.forest, ctx.order) + .and_then(|(_, inst)| { + // We have an entry that ends at `inst`. + if ctx.order.cmp(inst, ebb) == Ordering::Greater { + Some(inst) + } else { + None + } + }) } /// Is this value live-in to `ebb`? /// /// An EBB argument is not considered to be live in. - pub fn is_livein(&self, ebb: Ebb, order: &PO) -> bool { - self.livein_local_end(ebb, order).is_some() + pub fn is_livein(&self, ebb: Ebb, ctx: LiveRangeContext) -> bool { + self.livein_local_end(ebb, ctx).is_some() } /// Get all the live-in intervals. - pub fn liveins(&self) -> &[Interval] { - &self.liveins + /// + /// Note that the intervals are stored in a compressed form so each entry may span multiple + /// EBBs where the value is live in. + pub fn liveins<'a>( + &'a self, + ctx: LiveRangeContext<'a, PO>, + ) -> bforest::MapIter<'a, Ebb, Inst, PO> { + self.liveins.iter(ctx.forest) } /// Check if this live range overlaps a definition in `ebb`. - pub fn overlaps_def(&self, def: ExpandedProgramPoint, ebb: Ebb, order: &PO) -> bool { + pub fn overlaps_def( + &self, + def: ExpandedProgramPoint, + ebb: Ebb, + ctx: LiveRangeContext, + ) -> bool { // Check for an overlap with the local range. - if order.cmp(def, self.def_begin) != Ordering::Less && - order.cmp(def, self.def_end) == Ordering::Less + if ctx.order.cmp(def, self.def_begin) != Ordering::Less && + ctx.order.cmp(def, self.def_end) == Ordering::Less { return true; } // Check for an overlap with a live-in range. - match self.livein_local_end(ebb, order) { - Some(inst) => order.cmp(def, inst) == Ordering::Less, + match self.livein_local_end(ebb, ctx) { + Some(inst) => ctx.order.cmp(def, inst) == Ordering::Less, None => false, } } /// Check if this live range reaches a use at `user` in `ebb`. - pub fn reaches_use(&self, user: Inst, ebb: Ebb, order: &PO) -> bool { + pub fn reaches_use(&self, user: Inst, ebb: Ebb, ctx: LiveRangeContext) -> bool { // Check for an overlap with the local range. - if order.cmp(user, self.def_begin) == Ordering::Greater && - order.cmp(user, self.def_end) != Ordering::Greater + if ctx.order.cmp(user, self.def_begin) == Ordering::Greater && + ctx.order.cmp(user, self.def_end) != Ordering::Greater { return true; } // Check for an overlap with a live-in range. - match self.livein_local_end(ebb, order) { - Some(inst) => order.cmp(user, inst) != Ordering::Greater, + match self.livein_local_end(ebb, ctx) { + Some(inst) => ctx.order.cmp(user, inst) != Ordering::Greater, None => false, } } /// Check if this live range is killed at `user` in `ebb`. - pub fn killed_at(&self, user: Inst, ebb: Ebb, order: &PO) -> bool { - self.def_local_end() == user.into() || self.livein_local_end(ebb, order) == Some(user) + pub fn killed_at(&self, user: Inst, ebb: Ebb, ctx: LiveRangeContext) -> bool { + self.def_local_end() == user.into() || self.livein_local_end(ebb, ctx) == Some(user) } } @@ -452,7 +452,8 @@ impl SparseMapValue for GenLiveRange { #[cfg(test)] mod tests { - use super::GenLiveRange; + use super::{GenLiveRange, LiveRangeContext}; + use bforest; use ir::{Inst, Ebb, Value}; use entity::EntityRef; use ir::{ProgramOrder, ExpandedProgramPoint}; @@ -503,7 +504,11 @@ mod tests { } // Validate the live range invariants. - fn validate(&self, lr: &GenLiveRange) { + fn validate( + &self, + lr: &GenLiveRange, + forest: &bforest::MapForest, + ) { // The def interval must cover a single EBB. let def_ebb = self.pp_ebb(lr.def_begin); assert_eq!(def_ebb, self.pp_ebb(lr.def_end)); @@ -519,20 +524,20 @@ mod tests { // Check the live-in intervals. let mut prev_end = None; - for li in &lr.liveins { - assert_eq!(self.cmp(li.begin, li.end), Ordering::Less); + for (begin, end) in lr.liveins.iter(forest) { + assert_eq!(self.cmp(begin, end), Ordering::Less); if let Some(e) = prev_end { - assert_eq!(self.cmp(e, li.begin), Ordering::Less); + assert_eq!(self.cmp(e, begin), Ordering::Less); } assert!( - self.cmp(lr.def_end, li.begin) == Ordering::Less || - self.cmp(lr.def_begin, li.end) == Ordering::Greater, + self.cmp(lr.def_end, begin) == Ordering::Less || + self.cmp(lr.def_begin, end) == Ordering::Greater, "Interval can't overlap the def EBB" ); // Save for next round. - prev_end = Some(li.end); + prev_end = Some(end); } } @@ -547,12 +552,14 @@ mod tests { let i1 = Inst::new(1); let e2 = Ebb::new(2); let lr = GenLiveRange::new(v0, i1.into(), Default::default()); + let forest = &bforest::MapForest::new(); + let ctx = LiveRangeContext::new(PO, forest); assert!(lr.is_dead()); assert!(lr.is_local()); assert_eq!(lr.def(), i1.into()); assert_eq!(lr.def_local_end(), i1.into()); - assert_eq!(lr.livein_local_end(e2, PO), None); - PO.validate(&lr); + assert_eq!(lr.livein_local_end(e2, ctx), None); + PO.validate(&lr, ctx.forest); } #[test] @@ -560,13 +567,15 @@ mod tests { let v0 = Value::new(0); let e2 = Ebb::new(2); let lr = GenLiveRange::new(v0, e2.into(), Default::default()); + let forest = &bforest::MapForest::new(); + let ctx = LiveRangeContext::new(PO, forest); assert!(lr.is_dead()); assert!(lr.is_local()); assert_eq!(lr.def(), e2.into()); assert_eq!(lr.def_local_end(), e2.into()); // The def interval of an EBB argument does not count as live-in. - assert_eq!(lr.livein_local_end(e2, PO), None); - PO.validate(&lr); + assert_eq!(lr.livein_local_end(e2, ctx), None); + PO.validate(&lr, ctx.forest); } #[test] @@ -577,17 +586,18 @@ mod tests { let i12 = Inst::new(12); let i13 = Inst::new(13); let mut lr = GenLiveRange::new(v0, i11.into(), Default::default()); + let forest = &mut bforest::MapForest::new(); - assert_eq!(lr.extend_in_ebb(e10, i13, PO), false); - PO.validate(&lr); + assert_eq!(lr.extend_in_ebb(e10, i13, PO, forest), false); + PO.validate(&lr, forest); assert!(!lr.is_dead()); assert!(lr.is_local()); assert_eq!(lr.def(), i11.into()); assert_eq!(lr.def_local_end(), i13.into()); // Extending to an already covered inst should not change anything. - assert_eq!(lr.extend_in_ebb(e10, i12, PO), false); - PO.validate(&lr); + assert_eq!(lr.extend_in_ebb(e10, i12, PO, forest), false); + PO.validate(&lr, forest); assert_eq!(lr.def(), i11.into()); assert_eq!(lr.def_local_end(), i13.into()); } @@ -600,25 +610,26 @@ mod tests { let i12 = Inst::new(12); let i13 = Inst::new(13); let mut lr = GenLiveRange::new(v0, e10.into(), Default::default()); + let forest = &mut bforest::MapForest::new(); // Extending a dead EBB argument in its own block should not indicate that a live-in // interval was created. - assert_eq!(lr.extend_in_ebb(e10, i12, PO), false); - PO.validate(&lr); + assert_eq!(lr.extend_in_ebb(e10, i12, PO, forest), false); + PO.validate(&lr, forest); assert!(!lr.is_dead()); assert!(lr.is_local()); assert_eq!(lr.def(), e10.into()); assert_eq!(lr.def_local_end(), i12.into()); // Extending to an already covered inst should not change anything. - assert_eq!(lr.extend_in_ebb(e10, i11, PO), false); - PO.validate(&lr); + assert_eq!(lr.extend_in_ebb(e10, i11, PO, forest), false); + PO.validate(&lr, forest); assert_eq!(lr.def(), e10.into()); assert_eq!(lr.def_local_end(), i12.into()); // Extending further. - assert_eq!(lr.extend_in_ebb(e10, i13, PO), false); - PO.validate(&lr); + assert_eq!(lr.extend_in_ebb(e10, i13, PO, forest), false); + PO.validate(&lr, forest); assert_eq!(lr.def(), e10.into()); assert_eq!(lr.def_local_end(), i13.into()); } @@ -634,22 +645,32 @@ mod tests { let i22 = Inst::new(22); let i23 = Inst::new(23); let mut lr = GenLiveRange::new(v0, i11.into(), Default::default()); + let forest = &mut bforest::MapForest::new(); - assert_eq!(lr.extend_in_ebb(e10, i12, PO), false); + assert_eq!(lr.extend_in_ebb(e10, i12, PO, forest), false); // Adding a live-in interval. - assert_eq!(lr.extend_in_ebb(e20, i22, PO), true); - PO.validate(&lr); - assert_eq!(lr.livein_local_end(e20, PO), Some(i22)); + assert_eq!(lr.extend_in_ebb(e20, i22, PO, forest), true); + PO.validate(&lr, forest); + assert_eq!( + lr.livein_local_end(e20, LiveRangeContext::new(PO, forest)), + Some(i22) + ); // Non-extending the live-in. - assert_eq!(lr.extend_in_ebb(e20, i21, PO), false); - assert_eq!(lr.livein_local_end(e20, PO), Some(i22)); + assert_eq!(lr.extend_in_ebb(e20, i21, PO, forest), false); + assert_eq!( + lr.livein_local_end(e20, LiveRangeContext::new(PO, forest)), + Some(i22) + ); // Extending the existing live-in. - assert_eq!(lr.extend_in_ebb(e20, i23, PO), false); - PO.validate(&lr); - assert_eq!(lr.livein_local_end(e20, PO), Some(i23)); + assert_eq!(lr.extend_in_ebb(e20, i23, PO, forest), false); + PO.validate(&lr, forest); + assert_eq!( + lr.livein_local_end(e20, LiveRangeContext::new(PO, forest)), + Some(i23) + ); } #[test] @@ -663,35 +684,54 @@ mod tests { let e40 = Ebb::new(40); let i41 = Inst::new(41); let mut lr = GenLiveRange::new(v0, i11.into(), Default::default()); + let forest = &mut bforest::MapForest::new(); - assert_eq!(lr.extend_in_ebb(e30, i31, PO), true); - assert_eq!(lr.liveins.len(), 1); + assert_eq!(lr.extend_in_ebb(e30, i31, PO, forest), true); + assert_eq!( + lr.liveins(LiveRangeContext::new(PO, forest)) + .collect::>(), + [(e30, i31)] + ); // Coalesce to previous - assert_eq!(lr.extend_in_ebb(e40, i41, PO), true); - assert_eq!(lr.liveins.len(), 1); - assert_eq!(lr.liveins[0].begin, e30); - assert_eq!(lr.liveins[0].end, i41); + assert_eq!(lr.extend_in_ebb(e40, i41, PO, forest), true); + assert_eq!( + lr.liveins(LiveRangeContext::new(PO, forest)) + .collect::>(), + [(e30, i41)] + ); // Coalesce to next - assert_eq!(lr.extend_in_ebb(e20, i21, PO), true); - assert_eq!(lr.liveins.len(), 1); - assert_eq!(lr.liveins[0].begin, e20); - assert_eq!(lr.liveins[0].end, i41); + assert_eq!(lr.extend_in_ebb(e20, i21, PO, forest), true); + assert_eq!( + lr.liveins(LiveRangeContext::new(PO, forest)) + .collect::>(), + [(e20, i41)] + ); let mut lr = GenLiveRange::new(v0, i11.into(), Default::default()); - assert_eq!(lr.extend_in_ebb(e40, i41, PO), true); - assert_eq!(lr.liveins.len(), 1); + assert_eq!(lr.extend_in_ebb(e40, i41, PO, forest), true); + assert_eq!( + lr.liveins(LiveRangeContext::new(PO, forest)) + .collect::>(), + [(e40, i41)] + ); - assert_eq!(lr.extend_in_ebb(e20, i21, PO), true); - assert_eq!(lr.liveins.len(), 2); + assert_eq!(lr.extend_in_ebb(e20, i21, PO, forest), true); + assert_eq!( + lr.liveins(LiveRangeContext::new(PO, forest)) + .collect::>(), + [(e20, i21), (e40, i41)] + ); // Coalesce to previous and next - assert_eq!(lr.extend_in_ebb(e30, i31, PO), true); - assert_eq!(lr.liveins.len(), 1); - assert_eq!(lr.liveins[0].begin, e20); - assert_eq!(lr.liveins[0].end, i41); + assert_eq!(lr.extend_in_ebb(e30, i31, PO, forest), true); + assert_eq!( + lr.liveins(LiveRangeContext::new(PO, forest)) + .collect::>(), + [(e20, i41)] + ); } // TODO: Add more tests that exercise the binary search algorithm. diff --git a/lib/cretonne/src/regalloc/spilling.rs b/lib/cretonne/src/regalloc/spilling.rs index 253b96c25a..0bbb4c4225 100644 --- a/lib/cretonne/src/regalloc/spilling.rs +++ b/lib/cretonne/src/regalloc/spilling.rs @@ -300,16 +300,17 @@ impl<'a> Context<'a> { for (idx, (op, &arg)) in constraints.ins.iter().zip(args).enumerate() { let mut reguse = RegUse::new(arg, idx, op.regclass.into()); let lr = &self.liveness[arg]; + let ctx = self.liveness.context(&self.cur.func.layout); match op.kind { ConstraintKind::Stack => continue, ConstraintKind::FixedReg(_) => reguse.fixed = true, ConstraintKind::Tied(_) => { // A tied operand must kill the used value. - reguse.tied = !lr.killed_at(inst, ebb, &self.cur.func.layout); + reguse.tied = !lr.killed_at(inst, ebb, ctx); } ConstraintKind::FixedTied(_) => { reguse.fixed = true; - reguse.tied = !lr.killed_at(inst, ebb, &self.cur.func.layout); + reguse.tied = !lr.killed_at(inst, ebb, ctx); } ConstraintKind::Reg => {} } diff --git a/lib/cretonne/src/verifier/cssa.rs b/lib/cretonne/src/verifier/cssa.rs index 1010b45d0f..853f27251c 100644 --- a/lib/cretonne/src/verifier/cssa.rs +++ b/lib/cretonne/src/verifier/cssa.rs @@ -89,7 +89,8 @@ impl<'a> CssaVerifier<'a> { // Knowing that values are in RPO order, we can check for interference this // way. - if self.liveness[prev_val].overlaps_def(def, def_ebb, &self.func.layout) { + let ctx = self.liveness.context(&self.func.layout); + if self.liveness[prev_val].overlaps_def(def, def_ebb, ctx) { return err!(val, "Value def in {} interferes with {}", vreg, prev_val); } } diff --git a/lib/cretonne/src/verifier/liveness.rs b/lib/cretonne/src/verifier/liveness.rs index 4e72d59224..78eb3e17ca 100644 --- a/lib/cretonne/src/verifier/liveness.rs +++ b/lib/cretonne/src/verifier/liveness.rs @@ -129,18 +129,18 @@ impl<'a> LivenessVerifier<'a> { /// Is `lr` live at the use `inst`? fn live_at_use(&self, lr: &LiveRange, inst: Inst) -> bool { - let l = &self.func.layout; + let ctx = self.liveness.context(&self.func.layout); // Check if `inst` is in the def range, not including the def itself. - if l.cmp(lr.def(), inst) == Ordering::Less && - l.cmp(inst, lr.def_local_end()) != Ordering::Greater + if ctx.order.cmp(lr.def(), inst) == Ordering::Less && + ctx.order.cmp(inst, lr.def_local_end()) != Ordering::Greater { return true; } // Otherwise see if `inst` is in one of the live-in ranges. - match lr.livein_local_end(l.inst_ebb(inst).unwrap(), l) { - Some(end) => l.cmp(inst, end) != Ordering::Greater, + match lr.livein_local_end(ctx.order.inst_ebb(inst).unwrap(), ctx) { + Some(end) => ctx.order.cmp(inst, end) != Ordering::Greater, None => false, } } @@ -205,12 +205,11 @@ impl<'a> LivenessVerifier<'a> { } // Now check the live-in intervals against the CFG. - for &livein in lr.liveins() { - let mut ebb = livein.begin; + for (mut ebb, end) in lr.liveins(self.liveness.context(l)) { if !l.is_ebb_inserted(ebb) { return err!(loc, "{} livein at {} which is not in the layout", val, ebb); } - let end_ebb = match l.inst_ebb(livein.end) { + let end_ebb = match l.inst_ebb(end) { Some(e) => e, None => { return err!( @@ -218,7 +217,7 @@ impl<'a> LivenessVerifier<'a> { "{} livein for {} ends at {} which is not in the layout", val, ebb, - livein.end + end ) } }; diff --git a/lib/cretonne/src/verifier/locations.rs b/lib/cretonne/src/verifier/locations.rs index 94958c2f89..8df40ba924 100644 --- a/lib/cretonne/src/verifier/locations.rs +++ b/lib/cretonne/src/verifier/locations.rs @@ -282,7 +282,7 @@ impl<'a> LocationVerifier<'a> { SingleDest(ebb, _) => { for d in divert.all() { let lr = &liveness[d.value]; - if lr.is_livein(ebb, &self.func.layout) { + if lr.is_livein(ebb, liveness.context(&self.func.layout)) { return err!( inst, "{} is diverted to {} and live in to {}", @@ -297,7 +297,7 @@ impl<'a> LocationVerifier<'a> { for d in divert.all() { let lr = &liveness[d.value]; for (_, ebb) in self.func.jump_tables[jt].entries() { - if lr.is_livein(ebb, &self.func.layout) { + if lr.is_livein(ebb, liveness.context(&self.func.layout)) { return err!( inst, "{} is diverted to {} and live in to {}",