T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use a sorted array for (Ebb, Inst) interval again (fixes #1084);

Browse Source
This commit is contained in:
Benjamin Bouvier
2019-09-13 18:58:50 +02:00
parent 79784dfaf6
commit 4e3cb25983
10 changed files with 212 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
cranelift/codegen/src/regalloc/coalescing.rs
View File

@@ -196,12 +196,7 @@ impl<'a> Context<'a> {
// to be live at the use. // to be live at the use.
for i in 0..num_params { for i in 0..num_params {
let param = self.func.dfg.ebb_params(ebb)[i]; let param = self.func.dfg.ebb_params(ebb)[i];
if self.liveness[param].reaches_use( if self.liveness[param].reaches_use(pred_inst, pred_ebb, &self.func.layout) {
pred_inst,
pred_ebb,
self.liveness.forest(),
&self.func.layout,
) {
self.isolate_param(ebb, param); self.isolate_param(ebb, param);
} }
} }
@@ -255,7 +250,7 @@ impl<'a> Context<'a> {
); );
// The only other possibility is that `arg` is live-in to `ebb`. // The only other possibility is that `arg` is live-in to `ebb`.
lr.is_livein(ebb, self.liveness.forest(), &self.func.layout) lr.is_livein(ebb, &self.func.layout)
}; };
if interference { if interference {
@@ -435,12 +430,7 @@ impl<'a> Context<'a> {
// Check for interference between `parent` and `value`. Since `parent` dominates // Check for interference between `parent` and `value`. Since `parent` dominates
// `value`, we only have to check if it overlaps the definition. // `value`, we only have to check if it overlaps the definition.
if self.liveness[parent.value].overlaps_def( if self.liveness[parent.value].overlaps_def(node.def, node.ebb, &self.func.layout) {
node.def,
node.ebb,
self.liveness.forest(),
&self.func.layout,
) {
// The two values are interfering, so they can't be in the same virtual register. // The two values are interfering, so they can't be in the same virtual register.
debug!("-> interference: {} overlaps def of {}", parent, value); debug!("-> interference: {} overlaps def of {}", parent, value);
return false; return false;
@@ -626,12 +616,7 @@ impl<'a> Context<'a> {
// Check if the parent value interferes with the virtual copy. // Check if the parent value interferes with the virtual copy.
let inst = node.def.unwrap_inst(); let inst = node.def.unwrap_inst();
if node.set_id != parent.set_id if node.set_id != parent.set_id
&& self.liveness[parent.value].reaches_use( && self.liveness[parent.value].reaches_use(inst, node.ebb, &self.func.layout)
inst,
node.ebb,
self.liveness.forest(),
&self.func.layout,
)
{ {
debug!( debug!(
" - interference: {} overlaps vcopy at {}:{}", " - interference: {} overlaps vcopy at {}:{}",
@@ -655,12 +640,7 @@ impl<'a> Context<'a> {
// Both node and parent are values, so check for interference. // Both node and parent are values, so check for interference.
debug_assert!(node.is_value() && parent.is_value()); debug_assert!(node.is_value() && parent.is_value());
if node.set_id != parent.set_id if node.set_id != parent.set_id
&& self.liveness[parent.value].overlaps_def( && self.liveness[parent.value].overlaps_def(node.def, node.ebb, &self.func.layout)
node.def,
node.ebb,
self.liveness.forest(),
&self.func.layout,
)
{ {
// The two values are interfering. // The two values are interfering.
debug!(" - interference: {} overlaps def of {}", parent, node.value); debug!(" - interference: {} overlaps def of {}", parent, node.value);

24
cranelift/codegen/src/regalloc/coloring.rs
View File

@@ -54,7 +54,7 @@ use crate::regalloc::affinity::Affinity;
use crate::regalloc::diversion::RegDiversions; use crate::regalloc::diversion::RegDiversions;
use crate::regalloc::live_value_tracker::{LiveValue, LiveValueTracker}; use crate::regalloc::live_value_tracker::{LiveValue, LiveValueTracker};
use crate::regalloc::liveness::Liveness; use crate::regalloc::liveness::Liveness;
use crate::regalloc::liverange::{LiveRange, LiveRangeForest}; use crate::regalloc::liverange::LiveRange;
use crate::regalloc::register_set::RegisterSet; use crate::regalloc::register_set::RegisterSet;
use crate::regalloc::solver::{Solver, SolverError}; use crate::regalloc::solver::{Solver, SolverError};
use crate::timing; use crate::timing;
@@ -461,7 +461,7 @@ impl<'a> Context<'a> {
"Can't handle EBB arguments: {}", "Can't handle EBB arguments: {}",
self.cur.display_inst(inst) self.cur.display_inst(inst)
); );
self.undivert_regs(|lr, _, _| !lr.is_local()); self.undivert_regs(|lr, _| !lr.is_local());
} }
} }
@@ -726,12 +726,7 @@ impl<'a> Context<'a> {
// the new variable as killed or live-through. Always special-case the // the new variable as killed or live-through. Always special-case the
// pinned register as a through variable. // pinned register as a through variable.
let layout = &self.cur.func.layout; let layout = &self.cur.func.layout;
if self.liveness[value].killed_at( if self.liveness[value].killed_at(inst, layout.pp_ebb(inst), layout) {
inst,
layout.pp_ebb(inst),
self.liveness.forest(),
layout,
) {
self.solver.add_killed_var(value, op.regclass, cur_reg); self.solver.add_killed_var(value, op.regclass, cur_reg);
} else { } else {
self.solver.add_through_var(value, op.regclass, cur_reg); self.solver.add_through_var(value, op.regclass, cur_reg);
@@ -760,7 +755,7 @@ impl<'a> Context<'a> {
// //
// Values with a global live range that are not live in to `dest` could appear as branch // Values with a global live range that are not live in to `dest` could appear as branch
// arguments, so they can't always be un-diverted. // arguments, so they can't always be un-diverted.
self.undivert_regs(|lr, forest, layout| lr.is_livein(dest, forest, layout)); self.undivert_regs(|lr, layout| lr.is_livein(dest, layout));
// Now handle the EBB arguments. // Now handle the EBB arguments.
let br_args = self.cur.func.dfg.inst_variable_args(inst); let br_args = self.cur.func.dfg.inst_variable_args(inst);
@@ -830,14 +825,14 @@ impl<'a> Context<'a> {
/// are reallocated to their global register assignments. /// are reallocated to their global register assignments.
fn undivert_regs<Pred>(&mut self, mut pred: Pred) fn undivert_regs<Pred>(&mut self, mut pred: Pred)
where where
Pred: FnMut(&LiveRange, &LiveRangeForest, &Layout) -> bool, Pred: FnMut(&LiveRange, &Layout) -> bool,
{ {
for (&value, rdiv) in self.divert.iter() { for (&value, rdiv) in self.divert.iter() {
let lr = self let lr = self
.liveness .liveness
.get(value) .get(value)
.expect("Missing live range for diverted register"); .expect("Missing live range for diverted register");
if pred(lr, self.liveness.forest(), &self.cur.func.layout) { if pred(lr, &self.cur.func.layout) {
if let Affinity::Reg(rci) = lr.affinity { if let Affinity::Reg(rci) = lr.affinity {
let rc = self.reginfo.rc(rci); let rc = self.reginfo.rc(rci);
// Stack diversions should not be possible here. They only live transiently // Stack diversions should not be possible here. They only live transiently
@@ -1086,20 +1081,19 @@ impl<'a> Context<'a> {
let inst = self.cur.current_inst().expect("Not on an instruction"); let inst = self.cur.current_inst().expect("Not on an instruction");
let layout = &self.cur.func.layout; let layout = &self.cur.func.layout;
let forest = self.liveness.forest();
match self.cur.func.dfg.analyze_branch(inst) { match self.cur.func.dfg.analyze_branch(inst) {
NotABranch => false, NotABranch => false,
SingleDest(ebb, _) => { SingleDest(ebb, _) => {
let lr = &self.liveness[value]; let lr = &self.liveness[value];
lr.is_livein(ebb, forest, layout) lr.is_livein(ebb, layout)
} }
Table(jt, ebb) => { Table(jt, ebb) => {
let lr = &self.liveness[value]; let lr = &self.liveness[value];
!lr.is_local() !lr.is_local()
&& (ebb.map_or(false, |ebb| lr.is_livein(ebb, forest, layout)) && (ebb.map_or(false, |ebb| lr.is_livein(ebb, layout))
|| self.cur.func.jump_tables[jt] || self.cur.func.jump_tables[jt]
.iter() .iter()
.any(|ebb| lr.is_livein(*ebb, forest, layout))) .any(|ebb| lr.is_livein(*ebb, layout)))
} }
} }
} }

2
cranelift/codegen/src/regalloc/live_value_tracker.rs
View File

@@ -198,7 +198,7 @@ impl LiveValueTracker {
.expect("Immediate dominator value has no live range"); .expect("Immediate dominator value has no live range");
// Check if this value is live-in here. // Check if this value is live-in here.
if let Some(endpoint) = lr.livein_local_end(ebb, liveness.forest(), layout) { if let Some(endpoint) = lr.livein_local_end(ebb, layout) {
self.live.push(value, endpoint, lr); self.live.push(value, endpoint, lr);
} }
} }

29
cranelift/codegen/src/regalloc/liveness.rs
View File

@@ -181,7 +181,7 @@ use crate::ir::dfg::ValueDef;
use crate::ir::{Ebb, Function, Inst, Layout, ProgramPoint, Value}; use crate::ir::{Ebb, Function, Inst, Layout, ProgramPoint, Value};
use crate::isa::{EncInfo, OperandConstraint, TargetIsa}; use crate::isa::{EncInfo, OperandConstraint, TargetIsa};
use crate::regalloc::affinity::Affinity; use crate::regalloc::affinity::Affinity;
use crate::regalloc::liverange::{LiveRange, LiveRangeForest}; use crate::regalloc::liverange::LiveRange;
use crate::timing; use crate::timing;
use core::mem; use core::mem;
use core::ops::Index; use core::ops::Index;
@@ -249,14 +249,13 @@ fn extend_to_use(
worklist: &mut Vec<Ebb>, worklist: &mut Vec<Ebb>,
func: &Function, func: &Function,
cfg: &ControlFlowGraph, cfg: &ControlFlowGraph,
forest: &mut LiveRangeForest,
) { ) {
// This is our scratch working space, and we'll leave it empty when we return. // This is our scratch working space, and we'll leave it empty when we return.
debug_assert!(worklist.is_empty()); debug_assert!(worklist.is_empty());
// Extend the range locally in `ebb`. // Extend the range locally in `ebb`.
// If there already was a live interval in that block, we're done. // If there already was a live interval in that block, we're done.
if lr.extend_in_ebb(ebb, to, &func.layout, forest) { if lr.extend_in_ebb(ebb, to, &func.layout) {
worklist.push(ebb); worklist.push(ebb);
} }
@@ -277,7 +276,7 @@ fn extend_to_use(
inst: branch, inst: branch,
} in cfg.pred_iter(livein) } in cfg.pred_iter(livein)
{ {
if lr.extend_in_ebb(pred, branch, &func.layout, forest) { if lr.extend_in_ebb(pred, branch, &func.layout) {
// This predecessor EBB also became live-in. We need to process it later. // This predecessor EBB also became live-in. We need to process it later.
worklist.push(pred); worklist.push(pred);
} }
@@ -292,9 +291,6 @@ pub struct Liveness {
/// The live ranges that have been computed so far. /// The live ranges that have been computed so far.
ranges: LiveRangeSet, ranges: LiveRangeSet,
/// Memory pool for the live ranges.
forest: LiveRangeForest,
/// Working space for the `extend_to_use` algorithm. /// Working space for the `extend_to_use` algorithm.
/// This vector is always empty, except for inside that function. /// This vector is always empty, except for inside that function.
/// It lives here to avoid repeated allocation of scratch memory. /// It lives here to avoid repeated allocation of scratch memory.
@@ -309,16 +305,10 @@ impl Liveness {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
ranges: LiveRangeSet::new(), ranges: LiveRangeSet::new(),
forest: LiveRangeForest::new(),
worklist: Vec::new(), worklist: Vec::new(),
} }
} }
/// Current forest storage.
pub fn forest(&self) -> &LiveRangeForest {
&self.forest
}
/// Current live ranges. /// Current live ranges.
pub fn ranges(&self) -> &LiveRangeSet { pub fn ranges(&self) -> &LiveRangeSet {
&self.ranges &self.ranges
@@ -327,7 +317,6 @@ impl Liveness {
/// Clear all data structures in this liveness analysis. /// Clear all data structures in this liveness analysis.
pub fn clear(&mut self) { pub fn clear(&mut self) {
self.ranges.clear(); self.ranges.clear();
self.forest.clear();
self.worklist.clear(); self.worklist.clear();
} }
@@ -376,7 +365,7 @@ impl Liveness {
) -> &mut Affinity { ) -> &mut Affinity {
debug_assert_eq!(Some(ebb), layout.inst_ebb(user)); debug_assert_eq!(Some(ebb), layout.inst_ebb(user));
let lr = self.ranges.get_mut(value).expect("Value has no live range"); let lr = self.ranges.get_mut(value).expect("Value has no live range");
let livein = lr.extend_in_ebb(ebb, user, layout, &mut self.forest); let livein = lr.extend_in_ebb(ebb, user, layout);
debug_assert!(!livein, "{} should already be live in {}", value, ebb); debug_assert!(!livein, "{} should already be live in {}", value, ebb);
&mut lr.affinity &mut lr.affinity
} }
@@ -431,15 +420,7 @@ impl Liveness {
let lr = get_or_create(&mut self.ranges, arg, isa, func, &encinfo); let lr = get_or_create(&mut self.ranges, arg, isa, func, &encinfo);
// Extend the live range to reach this use. // Extend the live range to reach this use.
extend_to_use( extend_to_use(lr, ebb, inst, &mut self.worklist, func, cfg);
lr,
ebb,
inst,
&mut self.worklist,
func,
cfg,
&mut self.forest,
);
// Apply operand constraint, ignoring any variable arguments after the fixed // Apply operand constraint, ignoring any variable arguments after the fixed
// operands described by `operand_constraints`. Variable arguments are either // operands described by `operand_constraints`. Variable arguments are either

342
cranelift/codegen/src/regalloc/liverange.rs
View File

@@ -63,11 +63,11 @@
//! //!
//! ## Current representation //! ## Current representation
//! //!
//! Our current implementation uses a B-tree map with the necessary interface for an efficient //! Our current implementation uses a sorted array of compressed intervals, represented by their
//! implementation of coalescing, implemented as a generic data-structure bforest::Map. //! boundaries (Ebb, Inst), sorted by Ebb. This is a simple data structure, enables coalescing of
//! //! intervals easily, and shows some nice performance behavior. See
//! A `BTreeMap<Ebb, Inst>` could have been used for the live-in intervals, but it doesn't provide //! https://github.com/CraneStation/cranelift/issues/1084 for benchmarks against using a
//! the necessary API to make coalescing easy, nor does it optimize for our types' sizes. //! bforest::Map<Ebb, Inst>.
//! //!
//! ## EBB ordering //! ## EBB ordering
//! //!
@@ -107,13 +107,19 @@
//! It is more complicated to work with, though, so it is probably not worth it. The performance //! It is more complicated to work with, though, so it is probably not worth it. The performance
//! benefits of switching to a numerical EBB order only appears if the binary search is doing //! benefits of switching to a numerical EBB order only appears if the binary search is doing
//! EBB-EBB comparisons. //! EBB-EBB comparisons.
//!
//! A `BTreeMap<Ebb, Inst>` could have been used for the live-in intervals, but it doesn't provide
//! the necessary API to make coalescing easy, nor does it optimize for our types' sizes.
//!
//! Even the specialized `bforest::Map<Ebb, Inst>` implementation is slower than a plain sorted
//! array, see https://github.com/CraneStation/cranelift/issues/1084 for details.
use crate::bforest;
use crate::entity::SparseMapValue; use crate::entity::SparseMapValue;
use crate::ir::{Ebb, ExpandedProgramPoint, Inst, Layout, ProgramOrder, ProgramPoint, Value}; use crate::ir::{Ebb, ExpandedProgramPoint, Inst, Layout, ProgramOrder, ProgramPoint, Value};
use crate::regalloc::affinity::Affinity; use crate::regalloc::affinity::Affinity;
use core::cmp::Ordering; use core::cmp::Ordering;
use core::marker::PhantomData; use core::marker::PhantomData;
use smallvec::SmallVec;
/// Global live range of a single SSA value. /// Global live range of a single SSA value.
/// ///
@@ -144,6 +150,12 @@ use core::marker::PhantomData;
/// branch and jump instructions. /// branch and jump instructions.
pub type LiveRange = GenericLiveRange<Layout>; pub type LiveRange = GenericLiveRange<Layout>;
// See comment of liveins below.
pub struct Interval {
begin: Ebb,
end: Inst,
}
/// Generic live range implementation. /// Generic live range implementation.
/// ///
/// The intended generic parameter is `PO=Layout`, but tests are simpler with a mock order. /// The intended generic parameter is `PO=Layout`, but tests are simpler with a mock order.
@@ -167,29 +179,18 @@ pub struct GenericLiveRange<PO: ProgramOrder> {
/// Additional live-in intervals sorted in program order. /// Additional live-in intervals sorted in program order.
/// ///
/// This map is empty for most values which are only used in one EBB. /// This vector is empty for most values which are only used in one EBB.
/// ///
/// A map entry `ebb -> inst` means that the live range is live-in to `ebb`, continuing up to /// An entry `ebb -> inst` means that the live range is live-in to `ebb`, continuing up to
/// `inst` which may belong to a later EBB in the program order. /// `inst` which may belong to a later EBB in the program order.
/// ///
/// The entries are non-overlapping, and none of them overlap the EBB where the value is /// The entries are non-overlapping, and none of them overlap the EBB where the value is
/// defined. /// defined.
liveins: bforest::Map<Ebb, Inst>, liveins: SmallVec<[Interval; 2]>,
po: PhantomData<*const PO>, po: PhantomData<*const PO>,
} }
/// Forest of B-trees used for storing live ranges.
pub type LiveRangeForest = bforest::MapForest<Ebb, Inst>;
struct Cmp<'a, PO: ProgramOrder + 'a>(&'a PO);
impl<'a, PO: ProgramOrder> bforest::Comparator<Ebb> for Cmp<'a, PO> {
fn cmp(&self, a: Ebb, b: Ebb) -> Ordering {
self.0.cmp(a, b)
}
}
/// A simple helper macro to make comparisons more natural to read. /// A simple helper macro to make comparisons more natural to read.
macro_rules! cmp { macro_rules! cmp {
($order:ident, $a:ident > $b:expr) => { ($order:ident, $a:ident > $b:expr) => {
@@ -216,11 +217,26 @@ impl<PO: ProgramOrder> GenericLiveRange<PO> {
affinity, affinity,
def_begin: def, def_begin: def,
def_end: def, def_end: def,
liveins: bforest::Map::new(), liveins: SmallVec::new(),
po: PhantomData, po: PhantomData,
} }
} }
/// Finds an entry in the compressed set of live-in intervals that contains `ebb`, or return
/// the position where to insert such a new entry.
fn lookup_entry_containing_ebb(&self, ebb: Ebb, order: &PO) -> Result<usize, usize> {
self.liveins
.binary_search_by(|interval| order.cmp(interval.begin, ebb))
.or_else(|n| {
// The previous interval's end might cover the searched ebb.
if n > 0 && cmp!(order, ebb <= self.liveins[n - 1].end) {
Ok(n - 1)
} else {
Err(n)
}
})
}
/// Extend the local interval for `ebb` so it reaches `to` which must belong to `ebb`. /// Extend the local interval for `ebb` so it reaches `to` which must belong to `ebb`.
/// Create a live-in interval if necessary. /// Create a live-in interval if necessary.
/// ///
@@ -232,83 +248,101 @@ impl<PO: ProgramOrder> GenericLiveRange<PO> {
/// ///
/// The return value can be used to detect if we just learned that the value is live-in to /// The return value can be used to detect if we just learned that the value is live-in to
/// `ebb`. This can trigger recursive extensions in `ebb`'s CFG predecessor blocks. /// `ebb`. This can trigger recursive extensions in `ebb`'s CFG predecessor blocks.
pub fn extend_in_ebb( pub fn extend_in_ebb(&mut self, ebb: Ebb, inst: Inst, order: &PO) -> bool {
&mut self,
ebb: Ebb,
to: Inst,
order: &PO,
forest: &mut bforest::MapForest<Ebb, Inst>,
) -> bool {
// First check if we're extending the def interval. // First check if we're extending the def interval.
// //
// We're assuming here that `to` never precedes `def_begin` in the same EBB, but we can't // We're assuming here that `inst` never precedes `def_begin` in the same EBB, but we can't
// check it without a method for getting `to`'s EBB. // check it without a method for getting `inst`'s EBB.
if cmp!(order, ebb <= self.def_end) && cmp!(order, to >= self.def_begin) { if cmp!(order, ebb <= self.def_end) && cmp!(order, inst >= self.def_begin) {
let to_pp = to.into(); let inst_pp = inst.into();
debug_assert_ne!( debug_assert_ne!(
to_pp, self.def_begin, inst_pp, self.def_begin,
"Can't use value in the defining instruction." "Can't use value in the defining instruction."
); );
if cmp!(order, to > self.def_end) { if cmp!(order, inst > self.def_end) {
self.def_end = to_pp; self.def_end = inst_pp;
} }
return false; return false;
} }
// Now check if we're extending any of the existing live-in intervals. // Now check if we're extending any of the existing live-in intervals.
let cmp = Cmp(order); match self.lookup_entry_containing_ebb(ebb, order) {
let mut c = self.liveins.cursor(forest, &cmp); Ok(n) => {
let first_time_livein; // We found one interval and might need to extend it.
if cmp!(order, inst <= self.liveins[n].end) {
if let Some(end) = c.goto(ebb) { // Both interval parts are already included in a compressed interval.
// There's an interval beginning at `ebb`. See if it extends. return false;
first_time_livein = false;
if cmp!(order, end < to) {
*c.value_mut().unwrap() = to;
} else {
return first_time_livein;
}
} else if let Some((_, end)) = c.prev() {
// There's no interval beginning at `ebb`, but we could still be live-in at `ebb` with
// a coalesced interval that begins before and ends after.
if cmp!(order, end > ebb) {
// Yep, the previous interval overlaps `ebb`.
first_time_livein = false;
if cmp!(order, end < to) {
*c.value_mut().unwrap() = to;
} else {
return first_time_livein;
} }
} else {
first_time_livein = true; // If the instruction at the end is the last instruction before the next block,
// The current interval does not overlap `ebb`, but it may still be possible to // coalesce the two intervals:
// coalesce with it. // [ival.begin; ival.end] + [next.begin; next.end] = [ival.begin; next.end]
if order.is_ebb_gap(end, ebb) { if let Some(next) = &self.liveins.get(n + 1) {
*c.value_mut().unwrap() = to; if order.is_ebb_gap(inst, next.begin) {
} else { // At this point we can choose to remove the current interval or the next
c.insert(ebb, to); // one; remove the next one to avoid one memory move.
let next_end = next.end;
debug_assert!(cmp!(order, next_end > self.liveins[n].end));
self.liveins[n].end = next_end;
self.liveins.remove(n + 1);
return false;
}
} }
// We can't coalesce, just extend the interval.
self.liveins[n].end = inst;
false
} }
} else {
// There is no existing interval before `ebb`.
first_time_livein = true;
c.insert(ebb, to);
}
// Now `c` is left pointing at an interval that ends in `to`. Err(n) => {
debug_assert_eq!(c.value(), Some(to)); // No interval was found containing the current EBB: we need to insert a new one,
// unless there's a coalescing opportunity with the previous or next one.
let coalesce_next = self
.liveins
.get(n)
.filter(|next| order.is_ebb_gap(inst, next.begin))
.is_some();
let coalesce_prev = self
.liveins
.get(n.wrapping_sub(1))
.filter(|prev| order.is_ebb_gap(prev.end, ebb))
.is_some();
// See if it can be coalesced with the following interval. match (coalesce_prev, coalesce_next) {
if let Some((next_ebb, next_end)) = c.next() { // The new interval is the missing hole between prev and next: we can merge
if order.is_ebb_gap(to, next_ebb) { // them all together.
// Remove this interval and extend the previous end point to `next_end`. (true, true) => {
c.remove(); let prev_end = self.liveins[n - 1].end;
c.prev(); debug_assert!(cmp!(order, prev_end <= self.liveins[n].end));
*c.value_mut().unwrap() = next_end; self.liveins[n - 1].end = self.liveins[n].end;
self.liveins.remove(n);
}
// Coalesce only with the previous or next one.
(true, false) => {
debug_assert!(cmp!(order, inst >= self.liveins[n - 1].end));
self.liveins[n - 1].end = inst;
}
(false, true) => {
debug_assert!(cmp!(order, ebb <= self.liveins[n].begin));
self.liveins[n].begin = ebb;
}
(false, false) => {
// No coalescing opportunity, we have to insert.
self.liveins.insert(
n,
Interval {
begin: ebb,
end: inst,
},
);
}
}
true
} }
} }
first_time_livein
} }
/// Is this the live range of a dead value? /// Is this the live range of a dead value?
@@ -359,43 +393,39 @@ impl<PO: ProgramOrder> GenericLiveRange<PO> {
/// If the live range is live through all of `ebb`, the terminator of `ebb` is a correct /// If the live range is live through all of `ebb`, the terminator of `ebb` is a correct
/// answer, but it is also possible that an even later program point is returned. So don't /// answer, but it is also possible that an even later program point is returned. So don't
/// depend on the returned `Inst` to belong to `ebb`. /// depend on the returned `Inst` to belong to `ebb`.
pub fn livein_local_end(&self, ebb: Ebb, forest: &LiveRangeForest, order: &PO) -> Option<Inst> { pub fn livein_local_end(&self, ebb: Ebb, order: &PO) -> Option<Inst> {
let cmp = Cmp(order); self.lookup_entry_containing_ebb(ebb, order)
self.liveins .and_then(|i| {
.get_or_less(ebb, forest, &cmp) let inst = self.liveins[i].end;
.and_then(|(_, inst)| { if cmp!(order, ebb < inst) {
// We have an entry that ends at `inst`. Ok(inst)
if cmp!(order, inst > ebb) {
Some(inst)
} else { } else {
None // Can be any error type, really, since it's discarded by ok().
Err(i)
} }
}) })
.ok()
} }
/// Is this value live-in to `ebb`? /// Is this value live-in to `ebb`?
/// ///
/// An EBB argument is not considered to be live in. /// An EBB argument is not considered to be live in.
pub fn is_livein(&self, ebb: Ebb, forest: &LiveRangeForest, order: &PO) -> bool { pub fn is_livein(&self, ebb: Ebb, order: &PO) -> bool {
self.livein_local_end(ebb, forest, order).is_some() self.livein_local_end(ebb, order).is_some()
} }
/// Get all the live-in intervals. /// Get all the live-in intervals.
/// ///
/// Note that the intervals are stored in a compressed form so each entry may span multiple /// Note that the intervals are stored in a compressed form so each entry may span multiple
/// EBBs where the value is live in. /// EBBs where the value is live in.
pub fn liveins<'a>(&'a self, forest: &'a LiveRangeForest) -> bforest::MapIter<'a, Ebb, Inst> { pub fn liveins<'a>(&'a self) -> impl Iterator<Item = (Ebb, Inst)> + 'a {
self.liveins.iter(forest) self.liveins
.iter()
.map(|interval| (interval.begin, interval.end))
} }
/// Check if this live range overlaps a definition in `ebb`. /// Check if this live range overlaps a definition in `ebb`.
pub fn overlaps_def( pub fn overlaps_def(&self, def: ExpandedProgramPoint, ebb: Ebb, order: &PO) -> bool {
&self,
def: ExpandedProgramPoint,
ebb: Ebb,
forest: &LiveRangeForest,
order: &PO,
) -> bool {
// Two defs at the same program point always overlap, even if one is dead. // Two defs at the same program point always overlap, even if one is dead.
if def == self.def_begin.into() { if def == self.def_begin.into() {
return true; return true;
@@ -407,30 +437,29 @@ impl<PO: ProgramOrder> GenericLiveRange<PO> {
} }
// Check for an overlap with a live-in range. // Check for an overlap with a live-in range.
match self.livein_local_end(ebb, forest, order) { match self.livein_local_end(ebb, order) {
Some(inst) => cmp!(order, def < inst), Some(inst) => cmp!(order, def < inst),
None => false, None => false,
} }
} }
/// Check if this live range reaches a use at `user` in `ebb`. /// Check if this live range reaches a use at `user` in `ebb`.
pub fn reaches_use(&self, user: Inst, ebb: Ebb, forest: &LiveRangeForest, order: &PO) -> bool { pub fn reaches_use(&self, user: Inst, ebb: Ebb, order: &PO) -> bool {
// Check for an overlap with the local range. // Check for an overlap with the local range.
if cmp!(order, user > self.def_begin) && cmp!(order, user <= self.def_end) { if cmp!(order, user > self.def_begin) && cmp!(order, user <= self.def_end) {
return true; return true;
} }
// Check for an overlap with a live-in range. // Check for an overlap with a live-in range.
match self.livein_local_end(ebb, forest, order) { match self.livein_local_end(ebb, order) {
Some(inst) => cmp!(order, user <= inst), Some(inst) => cmp!(order, user <= inst),
None => false, None => false,
} }
} }
/// Check if this live range is killed at `user` in `ebb`. /// Check if this live range is killed at `user` in `ebb`.
pub fn killed_at(&self, user: Inst, ebb: Ebb, forest: &LiveRangeForest, order: &PO) -> bool { pub fn killed_at(&self, user: Inst, ebb: Ebb, order: &PO) -> bool {
self.def_local_end() == user.into() self.def_local_end() == user.into() || self.livein_local_end(ebb, order) == Some(user)
|| self.livein_local_end(ebb, forest, order) == Some(user)
} }
} }
@@ -443,8 +472,7 @@ impl<PO: ProgramOrder> SparseMapValue<Value> for GenericLiveRange<PO> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::GenericLiveRange; use super::{GenericLiveRange, Interval};
use crate::bforest;
use crate::entity::EntityRef; use crate::entity::EntityRef;
use crate::ir::{Ebb, Inst, Value}; use crate::ir::{Ebb, Inst, Value};
use crate::ir::{ExpandedProgramPoint, ProgramOrder}; use crate::ir::{ExpandedProgramPoint, ProgramOrder};
@@ -496,11 +524,7 @@ mod tests {
} }
// Validate the live range invariants. // Validate the live range invariants.
fn validate( fn validate(&self, lr: &GenericLiveRange<Self>) {
&self,
lr: &GenericLiveRange<ProgOrder>,
forest: &bforest::MapForest<Ebb, Inst>,
) {
// The def interval must cover a single EBB. // The def interval must cover a single EBB.
let def_ebb = self.pp_ebb(lr.def_begin); let def_ebb = self.pp_ebb(lr.def_begin);
assert_eq!(def_ebb, self.pp_ebb(lr.def_end)); assert_eq!(def_ebb, self.pp_ebb(lr.def_end));
@@ -516,7 +540,10 @@ mod tests {
// Check the live-in intervals. // Check the live-in intervals.
let mut prev_end = None; let mut prev_end = None;
for (begin, end) in lr.liveins.iter(forest) { for Interval { begin, end } in lr.liveins.iter() {
let begin = *begin;
let end = *end;
assert_eq!(self.cmp(begin, end), Ordering::Less); assert_eq!(self.cmp(begin, end), Ordering::Less);
if let Some(e) = prev_end { if let Some(e) = prev_end {
assert_eq!(self.cmp(e, begin), Ordering::Less); assert_eq!(self.cmp(e, begin), Ordering::Less);
@@ -545,18 +572,17 @@ mod tests {
let i2 = Inst::new(2); let i2 = Inst::new(2);
let e2 = Ebb::new(2); let e2 = Ebb::new(2);
let lr = GenericLiveRange::new(v0, i1.into(), Default::default()); let lr = GenericLiveRange::new(v0, i1.into(), Default::default());
let forest = &bforest::MapForest::new();
assert!(lr.is_dead()); assert!(lr.is_dead());
assert!(lr.is_local()); assert!(lr.is_local());
assert_eq!(lr.def(), i1.into()); assert_eq!(lr.def(), i1.into());
assert_eq!(lr.def_local_end(), i1.into()); assert_eq!(lr.def_local_end(), i1.into());
assert_eq!(lr.livein_local_end(e2, forest, PO), None); assert_eq!(lr.livein_local_end(e2, PO), None);
PO.validate(&lr, forest); PO.validate(&lr);
// A dead live range overlaps its own def program point. // A dead live range overlaps its own def program point.
assert!(lr.overlaps_def(i1.into(), e0, forest, PO)); assert!(lr.overlaps_def(i1.into(), e0, PO));
assert!(!lr.overlaps_def(i2.into(), e0, forest, PO)); assert!(!lr.overlaps_def(i2.into(), e0, PO));
assert!(!lr.overlaps_def(e0.into(), e0, forest, PO)); assert!(!lr.overlaps_def(e0.into(), e0, PO));
} }
#[test] #[test]
@@ -564,14 +590,13 @@ mod tests {
let v0 = Value::new(0); let v0 = Value::new(0);
let e2 = Ebb::new(2); let e2 = Ebb::new(2);
let lr = GenericLiveRange::new(v0, e2.into(), Default::default()); let lr = GenericLiveRange::new(v0, e2.into(), Default::default());
let forest = &bforest::MapForest::new();
assert!(lr.is_dead()); assert!(lr.is_dead());
assert!(lr.is_local()); assert!(lr.is_local());
assert_eq!(lr.def(), e2.into()); assert_eq!(lr.def(), e2.into());
assert_eq!(lr.def_local_end(), e2.into()); assert_eq!(lr.def_local_end(), e2.into());
// The def interval of an EBB argument does not count as live-in. // The def interval of an EBB argument does not count as live-in.
assert_eq!(lr.livein_local_end(e2, forest, PO), None); assert_eq!(lr.livein_local_end(e2, PO), None);
PO.validate(&lr, forest); PO.validate(&lr);
} }
#[test] #[test]
@@ -582,18 +607,17 @@ mod tests {
let i12 = Inst::new(12); let i12 = Inst::new(12);
let i13 = Inst::new(13); let i13 = Inst::new(13);
let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default()); let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default());
let forest = &mut bforest::MapForest::new();
assert_eq!(lr.extend_in_ebb(e10, i13, PO, forest), false); assert_eq!(lr.extend_in_ebb(e10, i13, PO), false);
PO.validate(&lr, forest); PO.validate(&lr);
assert!(!lr.is_dead()); assert!(!lr.is_dead());
assert!(lr.is_local()); assert!(lr.is_local());
assert_eq!(lr.def(), i11.into()); assert_eq!(lr.def(), i11.into());
assert_eq!(lr.def_local_end(), i13.into()); assert_eq!(lr.def_local_end(), i13.into());
// Extending to an already covered inst should not change anything. // Extending to an already covered inst should not change anything.
assert_eq!(lr.extend_in_ebb(e10, i12, PO, forest), false); assert_eq!(lr.extend_in_ebb(e10, i12, PO), false);
PO.validate(&lr, forest); PO.validate(&lr);
assert_eq!(lr.def(), i11.into()); assert_eq!(lr.def(), i11.into());
assert_eq!(lr.def_local_end(), i13.into()); assert_eq!(lr.def_local_end(), i13.into());
} }
@@ -606,26 +630,25 @@ mod tests {
let i12 = Inst::new(12); let i12 = Inst::new(12);
let i13 = Inst::new(13); let i13 = Inst::new(13);
let mut lr = GenericLiveRange::new(v0, e10.into(), Default::default()); let mut lr = GenericLiveRange::new(v0, e10.into(), Default::default());
let forest = &mut bforest::MapForest::new();
// Extending a dead EBB argument in its own block should not indicate that a live-in // Extending a dead EBB argument in its own block should not indicate that a live-in
// interval was created. // interval was created.
assert_eq!(lr.extend_in_ebb(e10, i12, PO, forest), false); assert_eq!(lr.extend_in_ebb(e10, i12, PO), false);
PO.validate(&lr, forest); PO.validate(&lr);
assert!(!lr.is_dead()); assert!(!lr.is_dead());
assert!(lr.is_local()); assert!(lr.is_local());
assert_eq!(lr.def(), e10.into()); assert_eq!(lr.def(), e10.into());
assert_eq!(lr.def_local_end(), i12.into()); assert_eq!(lr.def_local_end(), i12.into());
// Extending to an already covered inst should not change anything. // Extending to an already covered inst should not change anything.
assert_eq!(lr.extend_in_ebb(e10, i11, PO, forest), false); assert_eq!(lr.extend_in_ebb(e10, i11, PO), false);
PO.validate(&lr, forest); PO.validate(&lr);
assert_eq!(lr.def(), e10.into()); assert_eq!(lr.def(), e10.into());
assert_eq!(lr.def_local_end(), i12.into()); assert_eq!(lr.def_local_end(), i12.into());
// Extending further. // Extending further.
assert_eq!(lr.extend_in_ebb(e10, i13, PO, forest), false); assert_eq!(lr.extend_in_ebb(e10, i13, PO), false);
PO.validate(&lr, forest); PO.validate(&lr);
assert_eq!(lr.def(), e10.into()); assert_eq!(lr.def(), e10.into());
assert_eq!(lr.def_local_end(), i13.into()); assert_eq!(lr.def_local_end(), i13.into());
} }
@@ -641,23 +664,22 @@ mod tests {
let i22 = Inst::new(22); let i22 = Inst::new(22);
let i23 = Inst::new(23); let i23 = Inst::new(23);
let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default()); let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default());
let forest = &mut bforest::MapForest::new();
assert_eq!(lr.extend_in_ebb(e10, i12, PO, forest), false); assert_eq!(lr.extend_in_ebb(e10, i12, PO), false);
// Adding a live-in interval. // Adding a live-in interval.
assert_eq!(lr.extend_in_ebb(e20, i22, PO, forest), true); assert_eq!(lr.extend_in_ebb(e20, i22, PO), true);
PO.validate(&lr, forest); PO.validate(&lr);
assert_eq!(lr.livein_local_end(e20, forest, PO), Some(i22)); assert_eq!(lr.livein_local_end(e20, PO), Some(i22));
// Non-extending the live-in. // Non-extending the live-in.
assert_eq!(lr.extend_in_ebb(e20, i21, PO, forest), false); assert_eq!(lr.extend_in_ebb(e20, i21, PO), false);
assert_eq!(lr.livein_local_end(e20, forest, PO), Some(i22)); assert_eq!(lr.livein_local_end(e20, PO), Some(i22));
// Extending the existing live-in. // Extending the existing live-in.
assert_eq!(lr.extend_in_ebb(e20, i23, PO, forest), false); assert_eq!(lr.extend_in_ebb(e20, i23, PO), false);
PO.validate(&lr, forest); PO.validate(&lr);
assert_eq!(lr.livein_local_end(e20, forest, PO), Some(i23)); assert_eq!(lr.livein_local_end(e20, PO), Some(i23));
} }
#[test] #[test]
@@ -671,32 +693,28 @@ mod tests {
let e40 = Ebb::new(40); let e40 = Ebb::new(40);
let i41 = Inst::new(41); let i41 = Inst::new(41);
let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default()); let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default());
let forest = &mut bforest::MapForest::new();
assert_eq!(lr.extend_in_ebb(e30, i31, PO, forest), true); assert_eq!(lr.extend_in_ebb(e30, i31, PO,), true);
assert_eq!(lr.liveins(forest).collect::<Vec<_>>(), [(e30, i31)]); assert_eq!(lr.liveins().collect::<Vec<_>>(), [(e30, i31)]);
// Coalesce to previous // Coalesce to previous
assert_eq!(lr.extend_in_ebb(e40, i41, PO, forest), true); assert_eq!(lr.extend_in_ebb(e40, i41, PO,), true);
assert_eq!(lr.liveins(forest).collect::<Vec<_>>(), [(e30, i41)]); assert_eq!(lr.liveins().collect::<Vec<_>>(), [(e30, i41)]);
// Coalesce to next // Coalesce to next
assert_eq!(lr.extend_in_ebb(e20, i21, PO, forest), true); assert_eq!(lr.extend_in_ebb(e20, i21, PO,), true);
assert_eq!(lr.liveins(forest).collect::<Vec<_>>(), [(e20, i41)]); assert_eq!(lr.liveins().collect::<Vec<_>>(), [(e20, i41)]);
let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default()); let mut lr = GenericLiveRange::new(v0, i11.into(), Default::default());
assert_eq!(lr.extend_in_ebb(e40, i41, PO, forest), true); assert_eq!(lr.extend_in_ebb(e40, i41, PO,), true);
assert_eq!(lr.liveins(forest).collect::<Vec<_>>(), [(e40, i41)]); assert_eq!(lr.liveins().collect::<Vec<_>>(), [(e40, i41)]);
assert_eq!(lr.extend_in_ebb(e20, i21, PO, forest), true); assert_eq!(lr.extend_in_ebb(e20, i21, PO,), true);
assert_eq!( assert_eq!(lr.liveins().collect::<Vec<_>>(), [(e20, i21), (e40, i41)]);
lr.liveins(forest).collect::<Vec<_>>(),
[(e20, i21), (e40, i41)]
);
// Coalesce to previous and next // Coalesce to previous and next
assert_eq!(lr.extend_in_ebb(e30, i31, PO, forest), true); assert_eq!(lr.extend_in_ebb(e30, i31, PO,), true);
assert_eq!(lr.liveins(forest).collect::<Vec<_>>(), [(e20, i41)]); assert_eq!(lr.liveins().collect::<Vec<_>>(), [(e20, i41)]);
} }
} }

6
cranelift/codegen/src/regalloc/spilling.rs
View File

@@ -324,13 +324,11 @@ impl<'a> Context<'a> {
ConstraintKind::FixedReg(_) => reguse.fixed = true, ConstraintKind::FixedReg(_) => reguse.fixed = true,
ConstraintKind::Tied(_) => { ConstraintKind::Tied(_) => {
// A tied operand must kill the used value. // A tied operand must kill the used value.
reguse.tied = reguse.tied = !lr.killed_at(inst, ebb, &self.cur.func.layout);
!lr.killed_at(inst, ebb, self.liveness.forest(), &self.cur.func.layout);
} }
ConstraintKind::FixedTied(_) => { ConstraintKind::FixedTied(_) => {
reguse.fixed = true; reguse.fixed = true;
reguse.tied = reguse.tied = !lr.killed_at(inst, ebb, &self.cur.func.layout);
!lr.killed_at(inst, ebb, self.liveness.forest(), &self.cur.func.layout);
} }
ConstraintKind::Reg => {} ConstraintKind::Reg => {}
} }

8
cranelift/codegen/src/value_label.rs
View File

@@ -98,7 +98,6 @@ where
let encinfo = isa.encoding_info(); let encinfo = isa.encoding_info();
let values_locations = &func.locations; let values_locations = &func.locations;
let liveness_ranges = regalloc.liveness().ranges(); let liveness_ranges = regalloc.liveness().ranges();
let liveness_forest = regalloc.liveness().forest();
let mut ranges = HashMap::new(); let mut ranges = HashMap::new();
let mut add_range = |label, range: (u32, u32), loc: ValueLoc| { let mut add_range = |label, range: (u32, u32), loc: ValueLoc| {
@@ -127,10 +126,7 @@ where
// Remove killed values. // Remove killed values.
tracked_values.retain(|(x, label, start_offset, last_loc)| { tracked_values.retain(|(x, label, start_offset, last_loc)| {
let range = liveness_ranges.get(*x); let range = liveness_ranges.get(*x);
if range if range.expect("value").killed_at(inst, ebb, &func.layout) {
.expect("value")
.killed_at(inst, ebb, &liveness_forest, &func.layout)
{
add_range(*label, (*start_offset, end_offset), *last_loc); add_range(*label, (*start_offset, end_offset), *last_loc);
return false; return false;
} }
@@ -177,7 +173,7 @@ where
// Ignore dead/inactive Values. // Ignore dead/inactive Values.
let range = liveness_ranges.get(*v); let range = liveness_ranges.get(*v);
match range { match range {
Some(r) => r.reaches_use(inst, ebb, &liveness_forest, &func.layout), Some(r) => r.reaches_use(inst, ebb, &func.layout),
None => false, None => false,
} }
}); });

7
cranelift/codegen/src/verifier/cssa.rs
View File

@@ -118,12 +118,7 @@ impl<'a> CssaVerifier<'a> {
if self.preorder.dominates(prev_ebb, def_ebb) if self.preorder.dominates(prev_ebb, def_ebb)
&& self.domtree.dominates(prev_def, def, &self.func.layout) && self.domtree.dominates(prev_def, def, &self.func.layout)
{ {
if self.liveness[prev_val].overlaps_def( if self.liveness[prev_val].overlaps_def(def, def_ebb, &self.func.layout) {
def,
def_ebb,
self.liveness.forest(),
&self.func.layout,
) {
return fatal!( return fatal!(
errors, errors,
val, val,

6
cranelift/codegen/src/verifier/liveness.rs
View File

@@ -107,7 +107,7 @@ impl<'a> LivenessVerifier<'a> {
}; };
debug_assert!(self.func.layout.inst_ebb(inst).unwrap() == ebb); debug_assert!(self.func.layout.inst_ebb(inst).unwrap() == ebb);
if !lr.reaches_use(inst, ebb, self.liveness.forest(), &self.func.layout) { if !lr.reaches_use(inst, ebb, &self.func.layout) {
return fatal!(errors, inst, "{} is not live at this use", val); return fatal!(errors, inst, "{} is not live at this use", val);
} }
@@ -179,7 +179,7 @@ impl<'a> LivenessVerifier<'a> {
} }
// Now check the live-in intervals against the CFG. // Now check the live-in intervals against the CFG.
for (mut ebb, end) in lr.liveins(self.liveness.forest()) { for (mut ebb, end) in lr.liveins() {
if !l.is_ebb_inserted(ebb) { if !l.is_ebb_inserted(ebb) {
return fatal!( return fatal!(
errors, errors,
@@ -207,7 +207,7 @@ impl<'a> LivenessVerifier<'a> {
loop { loop {
// If `val` is live-in at `ebb`, it must be live at all the predecessors. // If `val` is live-in at `ebb`, it must be live at all the predecessors.
for BasicBlock { inst: pred, ebb } in self.cfg.pred_iter(ebb) { for BasicBlock { inst: pred, ebb } in self.cfg.pred_iter(ebb) {
if !lr.reaches_use(pred, ebb, self.liveness.forest(), &self.func.layout) { if !lr.reaches_use(pred, ebb, &self.func.layout) {
return fatal!( return fatal!(
errors, errors,
pred, pred,

8
cranelift/codegen/src/verifier/locations.rs
View File

@@ -334,10 +334,10 @@ impl<'a> LocationVerifier<'a> {
let lr = &liveness[value]; let lr = &liveness[value];
if is_after_branch && unique_predecessor { if is_after_branch && unique_predecessor {
// Forward diversions based on the targeted branch. // Forward diversions based on the targeted branch.
if !lr.is_livein(ebb, liveness.forest(), &self.func.layout) { if !lr.is_livein(ebb, &self.func.layout) {
val_to_remove.push(value) val_to_remove.push(value)
} }
} else if lr.is_livein(ebb, liveness.forest(), &self.func.layout) { } else if lr.is_livein(ebb, &self.func.layout) {
return fatal!( return fatal!(
errors, errors,
inst, inst,
@@ -359,7 +359,7 @@ impl<'a> LocationVerifier<'a> {
for (&value, d) in divert.iter() { for (&value, d) in divert.iter() {
let lr = &liveness[value]; let lr = &liveness[value];
if let Some(ebb) = ebb { if let Some(ebb) = ebb {
if lr.is_livein(ebb, liveness.forest(), &self.func.layout) { if lr.is_livein(ebb, &self.func.layout) {
return fatal!( return fatal!(
errors, errors,
inst, inst,
@@ -371,7 +371,7 @@ impl<'a> LocationVerifier<'a> {
} }
} }
for ebb in self.func.jump_tables[jt].iter() { for ebb in self.func.jump_tables[jt].iter() {
if lr.is_livein(*ebb, liveness.forest(), &self.func.layout) { if lr.is_livein(*ebb, &self.func.layout) {
return fatal!( return fatal!(
errors, errors,
inst, inst,