T0b1/regalloc2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Struct-of-array transform: pull LiveRangeHot out of LiveRange with just range and next-in-bundle link

Browse Source
This commit is contained in:
Chris Fallin
2021-05-07 20:41:33 -07:00
parent 4185eab441
commit e41b0101a8
1 changed files with 145 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
src/ion/mod.rs
View File

@@ -113,8 +113,13 @@ define_index!(SpillSlotIndex);
type LiveBundleVec = SmallVec<[LiveBundleIndex; 4]>; type LiveBundleVec = SmallVec<[LiveBundleIndex; 4]>;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
struct LiveRange { struct LiveRangeHot {
range: CodeRange, range: CodeRange,
next_in_bundle: LiveRangeIndex,
}
#[derive(Clone, Debug)]
struct LiveRange {
vreg: VRegIndex, vreg: VRegIndex,
bundle: LiveBundleIndex, bundle: LiveBundleIndex,
uses_spill_weight_and_flags: u32, uses_spill_weight_and_flags: u32,
@@ -122,7 +127,6 @@ struct LiveRange {
first_use: UseIndex, first_use: UseIndex,
last_use: UseIndex, last_use: UseIndex,
next_in_bundle: LiveRangeIndex,
next_in_reg: LiveRangeIndex, next_in_reg: LiveRangeIndex,
merged_into: LiveRangeIndex, merged_into: LiveRangeIndex,
@@ -171,11 +175,11 @@ impl Use {
#[inline(always)] #[inline(always)]
fn new(operand: Operand, pos: ProgPoint, next_use: UseIndex, slot: u8) -> Self { fn new(operand: Operand, pos: ProgPoint, next_use: UseIndex, slot: u8) -> Self {
debug_assert!(next_use.is_invalid() || next_use.index() < ((1 << 24) - 1)); debug_assert!(next_use.is_invalid() || next_use.index() < ((1 << 24) - 1));
let next_use = next_use.0 & 0x00ff_ffff; let next_use = (next_use.0 as usize) & 0x00ff_ffff;
Self { Self {
operand, operand,
pos, pos,
next_use_and_slot: next_use | ((slot as u32) << 24), next_use_and_slot: (next_use as u32) | ((slot as u32) << 24),
} }
} }
#[inline(always)] #[inline(always)]
@@ -183,7 +187,7 @@ impl Use {
let val = self.next_use_and_slot & 0x00ff_ffff; let val = self.next_use_and_slot & 0x00ff_ffff;
// Sign-extend 0x00ff_ffff to INVALID (0xffff_ffff). // Sign-extend 0x00ff_ffff to INVALID (0xffff_ffff).
let val = ((val as i32) << 8) >> 8; let val = ((val as i32) << 8) >> 8;
UseIndex(val as u32) UseIndex::new(val as usize)
} }
#[inline(always)] #[inline(always)]
fn slot(&self) -> u8 { fn slot(&self) -> u8 {
@@ -192,8 +196,8 @@ impl Use {
#[inline(always)] #[inline(always)]
fn set_next_use(&mut self, u: UseIndex) { fn set_next_use(&mut self, u: UseIndex) {
debug_assert!(u.is_invalid() || u.index() < ((1 << 24) - 1)); debug_assert!(u.is_invalid() || u.index() < ((1 << 24) - 1));
let u = u.0 & 0x00ff_ffff; let u = (u.0 as usize) & 0x00ff_ffff;
self.next_use_and_slot = (self.next_use_and_slot & 0xff00_0000) | u; self.next_use_and_slot = (self.next_use_and_slot & 0xff00_0000) | (u as u32);
} }
} }
@@ -371,6 +375,7 @@ struct Env<'a, F: Function> {
blockparam_allocs: Vec<(Block, u32, VRegIndex, Allocation)>, blockparam_allocs: Vec<(Block, u32, VRegIndex, Allocation)>,
ranges: Vec<LiveRange>, ranges: Vec<LiveRange>,
ranges_hot: Vec<LiveRangeHot>,
range_ranges: Vec<CodeRange>, range_ranges: Vec<CodeRange>,
bundles: Vec<LiveBundle>, bundles: Vec<LiveBundle>,
spillsets: Vec<SpillSet>, spillsets: Vec<SpillSet>,
@@ -770,6 +775,7 @@ impl<'a, F: Function> Env<'a, F> {
blockparam_allocs: vec![], blockparam_allocs: vec![],
bundles: Vec::with_capacity(n), bundles: Vec::with_capacity(n),
ranges: Vec::with_capacity(4 * n), ranges: Vec::with_capacity(4 * n),
ranges_hot: Vec::with_capacity(4 * n),
range_ranges: Vec::with_capacity(4 * n), range_ranges: Vec::with_capacity(4 * n),
spillsets: Vec::with_capacity(n), spillsets: Vec::with_capacity(n),
uses: Vec::with_capacity(4 * n), uses: Vec::with_capacity(4 * n),
@@ -850,8 +856,12 @@ impl<'a, F: Function> Env<'a, F> {
fn create_liverange(&mut self, range: CodeRange) -> LiveRangeIndex { fn create_liverange(&mut self, range: CodeRange) -> LiveRangeIndex {
let idx = self.ranges.len(); let idx = self.ranges.len();
self.ranges.push(LiveRange { self.ranges_hot.push(LiveRangeHot {
range, range,
next_in_bundle: LiveRangeIndex::invalid(),
});
self.ranges.push(LiveRange {
vreg: VRegIndex::invalid(), vreg: VRegIndex::invalid(),
bundle: LiveBundleIndex::invalid(), bundle: LiveBundleIndex::invalid(),
uses_spill_weight_and_flags: 0, uses_spill_weight_and_flags: 0,
@@ -859,7 +869,6 @@ impl<'a, F: Function> Env<'a, F> {
first_use: UseIndex::invalid(), first_use: UseIndex::invalid(),
last_use: UseIndex::invalid(), last_use: UseIndex::invalid(),
next_in_bundle: LiveRangeIndex::invalid(),
next_in_reg: LiveRangeIndex::invalid(), next_in_reg: LiveRangeIndex::invalid(),
merged_into: LiveRangeIndex::invalid(), merged_into: LiveRangeIndex::invalid(),
@@ -889,40 +898,40 @@ impl<'a, F: Function> Env<'a, F> {
let mut iter = self.vregs[vreg.index()].first_range; let mut iter = self.vregs[vreg.index()].first_range;
let mut prev = LiveRangeIndex::invalid(); let mut prev = LiveRangeIndex::invalid();
while iter.is_valid() { while iter.is_valid() {
let existing = &mut self.ranges[iter.index()]; log::debug!(" -> existing range: {:?}", self.ranges[iter.index()]);
log::debug!(" -> existing range: {:?}", existing); if range.from >= self.ranges_hot[iter.index()].range.to && *num_ranges < COALESCE_LIMIT
if range.from >= existing.range.to && *num_ranges < COALESCE_LIMIT { {
// New range comes fully after this one -- record it as a lower bound. // New range comes fully after this one -- record it as a lower bound.
insert_after = iter; insert_after = iter;
prev = iter; prev = iter;
iter = existing.next_in_reg; iter = self.ranges[iter.index()].next_in_reg;
log::debug!(" -> lower bound"); log::debug!(" -> lower bound");
continue; continue;
} }
if range.to <= existing.range.from { if range.to <= self.ranges_hot[iter.index()].range.from {
// New range comes fully before this one -- we're found our spot. // New range comes fully before this one -- we're found our spot.
log::debug!(" -> upper bound (break search loop)"); log::debug!(" -> upper bound (break search loop)");
break; break;
} }
// If we're here, then we overlap with at least one endpoint of the range. // If we're here, then we overlap with at least one endpoint of the range.
log::debug!(" -> must overlap"); log::debug!(" -> must overlap");
debug_assert!(range.overlaps(&existing.range)); debug_assert!(range.overlaps(&self.ranges_hot[iter.index()].range));
if merged.is_invalid() { if merged.is_invalid() {
// This is the first overlapping range. Extend to simply cover the new range. // This is the first overlapping range. Extend to simply cover the new range.
merged = iter; merged = iter;
if range.from < existing.range.from { if range.from < self.ranges_hot[iter.index()].range.from {
existing.range.from = range.from; self.ranges_hot[iter.index()].range.from = range.from;
} }
if range.to > existing.range.to { if range.to > self.ranges_hot[iter.index()].range.to {
existing.range.to = range.to; self.ranges_hot[iter.index()].range.to = range.to;
} }
log::debug!( log::debug!(
" -> extended range of existing range to {:?}", " -> extended range of existing range to {:?}",
existing.range self.ranges_hot[iter.index()].range
); );
// Continue; there may be more ranges to merge with. // Continue; there may be more ranges to merge with.
prev = iter; prev = iter;
iter = existing.next_in_reg; iter = self.ranges[iter.index()].next_in_reg;
continue; continue;
} }
// We overlap but we've already extended the first overlapping existing liverange, so // We overlap but we've already extended the first overlapping existing liverange, so
@@ -934,10 +943,11 @@ impl<'a, F: Function> Env<'a, F> {
self.ranges[merged.index()] self.ranges[merged.index()]
); );
debug_assert!( debug_assert!(
self.ranges[iter.index()].range.from >= self.ranges[merged.index()].range.from self.ranges_hot[iter.index()].range.from
>= self.ranges_hot[merged.index()].range.from
); // Because we see LRs in order. ); // Because we see LRs in order.
if self.ranges[iter.index()].range.to > self.ranges[merged.index()].range.to { if self.ranges_hot[iter.index()].range.to > self.ranges_hot[merged.index()].range.to {
self.ranges[merged.index()].range.to = self.ranges[iter.index()].range.to; self.ranges_hot[merged.index()].range.to = self.ranges_hot[iter.index()].range.to;
} }
self.distribute_liverange_uses(iter, merged); self.distribute_liverange_uses(iter, merged);
log::debug!( log::debug!(
@@ -983,7 +993,7 @@ impl<'a, F: Function> Env<'a, F> {
self.ranges[from.index()].vreg, self.ranges[from.index()].vreg,
self.ranges[into.index()].vreg self.ranges[into.index()].vreg
); );
let into_range = self.ranges[into.index()].range; let into_range = self.ranges_hot[into.index()].range;
// For every use in `from`... // For every use in `from`...
let mut prev = UseIndex::invalid(); let mut prev = UseIndex::invalid();
let mut iter = self.ranges[from.index()].first_use; let mut iter = self.ranges[from.index()].first_use;
@@ -1095,7 +1105,7 @@ impl<'a, F: Function> Env<'a, F> {
) -> Option<LiveRangeIndex> { ) -> Option<LiveRangeIndex> {
let mut range = self.vregs[vreg.index()].first_range; let mut range = self.vregs[vreg.index()].first_range;
while range.is_valid() { while range.is_valid() {
if self.ranges[range.index()].range.contains_point(pos) { if self.ranges_hot[range.index()].range.contains_point(pos) {
return Some(range); return Some(range);
} }
range = self.ranges[range.index()].next_in_reg; range = self.ranges[range.index()].next_in_reg;
@@ -1306,11 +1316,11 @@ impl<'a, F: Function> Env<'a, F> {
} else { } else {
log::debug!(" -> has existing LR {:?}", dst_lr); log::debug!(" -> has existing LR {:?}", dst_lr);
} }
if self.ranges[dst_lr.index()].range.from if self.ranges_hot[dst_lr.index()].range.from
== self.cfginfo.block_entry[block.index()] == self.cfginfo.block_entry[block.index()]
{ {
log::debug!(" -> started at block start; trimming to {:?}", pos); log::debug!(" -> started at block start; trimming to {:?}", pos);
self.ranges[dst_lr.index()].range.from = pos; self.ranges_hot[dst_lr.index()].range.from = pos;
} }
live.set(dst.vreg(), false); live.set(dst.vreg(), false);
vreg_ranges[dst.vreg()] = LiveRangeIndex::invalid(); vreg_ranges[dst.vreg()] = LiveRangeIndex::invalid();
@@ -1393,7 +1403,7 @@ impl<'a, F: Function> Env<'a, F> {
match operand.kind() { match operand.kind() {
OperandKind::Def | OperandKind::Mod => { OperandKind::Def | OperandKind::Mod => {
// Create the use object. // Create the use object.
let u = UseIndex(self.uses.len() as u32); let u = UseIndex::new(self.uses.len());
self.uses.push(Use::new( self.uses.push(Use::new(
operand, operand,
pos, pos,
@@ -1433,14 +1443,14 @@ impl<'a, F: Function> Env<'a, F> {
// start of this block (i.e. was not // start of this block (i.e. was not
// merged into some larger LiveRange due // merged into some larger LiveRange due
// to out-of-order blocks). // to out-of-order blocks).
if self.ranges[lr.index()].range.from if self.ranges_hot[lr.index()].range.from
== self.cfginfo.block_entry[block.index()] == self.cfginfo.block_entry[block.index()]
{ {
log::debug!( log::debug!(
" -> started at block start; trimming to {:?}", " -> started at block start; trimming to {:?}",
pos pos
); );
self.ranges[lr.index()].range.from = pos; self.ranges_hot[lr.index()].range.from = pos;
} }
// Remove from live-set. // Remove from live-set.
@@ -1450,7 +1460,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
OperandKind::Use => { OperandKind::Use => {
// Create the use object. // Create the use object.
let u = UseIndex(self.uses.len() as u32); let u = UseIndex::new(self.uses.len());
self.uses.push(Use::new( self.uses.push(Use::new(
operand, operand,
pos, pos,
@@ -1537,8 +1547,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut iter = self.vregs[vreg.index()].first_range; let mut iter = self.vregs[vreg.index()].first_range;
let mut safepoint_idx = 0; let mut safepoint_idx = 0;
while iter.is_valid() { while iter.is_valid() {
let rangedata = &self.ranges[iter.index()]; let range = self.ranges_hot[iter.index()].range;
let range = rangedata.range;
while safepoint_idx < self.safepoints.len() while safepoint_idx < self.safepoints.len()
&& ProgPoint::before(self.safepoints[safepoint_idx]) < range.from && ProgPoint::before(self.safepoints[safepoint_idx]) < range.from
{ {
@@ -1557,7 +1566,7 @@ impl<'a, F: Function> Env<'a, F> {
); );
// Create the actual use object. // Create the actual use object.
let u = UseIndex(self.uses.len() as u32); let u = UseIndex::new(self.uses.len());
self.uses self.uses
.push(Use::new(operand, pos, UseIndex::invalid(), SLOT_NONE)); .push(Use::new(operand, pos, UseIndex::invalid(), SLOT_NONE));
@@ -1781,13 +1790,13 @@ impl<'a, F: Function> Env<'a, F> {
while iter.is_valid() { while iter.is_valid() {
let vreg = self.ranges[iter.index()].vreg; let vreg = self.ranges[iter.index()].vreg;
assert_eq!(rc, self.vregs[vreg.index()].reg.class()); assert_eq!(rc, self.vregs[vreg.index()].reg.class());
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
let mut iter = self.bundles[to.index()].first_range; let mut iter = self.bundles[to.index()].first_range;
while iter.is_valid() { while iter.is_valid() {
let vreg = self.ranges[iter.index()].vreg; let vreg = self.ranges[iter.index()].vreg;
assert_eq!(rc, self.vregs[vreg.index()].reg.class()); assert_eq!(rc, self.vregs[vreg.index()].reg.class());
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
} }
@@ -1802,10 +1811,13 @@ impl<'a, F: Function> Env<'a, F> {
return false; return false;
} }
if self.ranges[iter0.index()].range.from >= self.ranges[iter1.index()].range.to { if self.ranges_hot[iter0.index()].range.from >= self.ranges_hot[iter1.index()].range.to
iter1 = self.ranges[iter1.index()].next_in_bundle; {
} else if self.ranges[iter1.index()].range.from >= self.ranges[iter0.index()].range.to { iter1 = self.ranges_hot[iter1.index()].next_in_bundle;
iter0 = self.ranges[iter0.index()].next_in_bundle; } else if self.ranges_hot[iter1.index()].range.from
>= self.ranges_hot[iter0.index()].range.to
{
iter0 = self.ranges_hot[iter0.index()].next_in_bundle;
} else { } else {
// Overlap -- cannot merge. // Overlap -- cannot merge.
return false; return false;
@@ -1832,7 +1844,7 @@ impl<'a, F: Function> Env<'a, F> {
self.bundles[from.index()].last_range = LiveRangeIndex::invalid(); self.bundles[from.index()].last_range = LiveRangeIndex::invalid();
while iter0.is_valid() { while iter0.is_valid() {
self.ranges[iter0.index()].bundle = from; self.ranges[iter0.index()].bundle = from;
iter0 = self.ranges[iter0.index()].next_in_bundle; iter0 = self.ranges_hot[iter0.index()].next_in_bundle;
} }
return true; return true;
} }
@@ -1844,8 +1856,8 @@ impl<'a, F: Function> Env<'a, F> {
// Pick the next range. // Pick the next range.
let next_range_iter = if iter0.is_valid() { let next_range_iter = if iter0.is_valid() {
if iter1.is_valid() { if iter1.is_valid() {
if self.ranges[iter0.index()].range.from if self.ranges_hot[iter0.index()].range.from
<= self.ranges[iter1.index()].range.from <= self.ranges_hot[iter1.index()].range.from
{ {
&mut iter0 &mut iter0
} else { } else {
@@ -1858,11 +1870,11 @@ impl<'a, F: Function> Env<'a, F> {
&mut iter1 &mut iter1
}; };
let next = *next_range_iter; let next = *next_range_iter;
*next_range_iter = self.ranges[next.index()].next_in_bundle; *next_range_iter = self.ranges_hot[next.index()].next_in_bundle;
// link from prev. // link from prev.
if prev.is_valid() { if prev.is_valid() {
self.ranges[prev.index()].next_in_bundle = next; self.ranges_hot[prev.index()].next_in_bundle = next;
} else { } else {
self.bundles[to.index()].first_range = next; self.bundles[to.index()].first_range = next;
} }
@@ -1882,42 +1894,42 @@ impl<'a, F: Function> Env<'a, F> {
lr, lr,
bundle bundle
); );
self.ranges[lr.index()].next_in_bundle = LiveRangeIndex::invalid(); self.ranges_hot[lr.index()].next_in_bundle = LiveRangeIndex::invalid();
self.ranges[lr.index()].bundle = bundle; self.ranges[lr.index()].bundle = bundle;
if self.bundles[bundle.index()].first_range.is_invalid() { if self.bundles[bundle.index()].first_range.is_invalid() {
// Empty bundle. // Empty bundle.
self.bundles[bundle.index()].first_range = lr; self.bundles[bundle.index()].first_range = lr;
self.bundles[bundle.index()].last_range = lr; self.bundles[bundle.index()].last_range = lr;
} else if self.ranges[self.bundles[bundle.index()].first_range.index()] } else if self.ranges_hot[self.bundles[bundle.index()].first_range.index()]
.range .range
.to .to
<= self.ranges[lr.index()].range.from <= self.ranges_hot[lr.index()].range.from
{ {
// After last range in bundle. // After last range in bundle.
let last = self.bundles[bundle.index()].last_range; let last = self.bundles[bundle.index()].last_range;
self.ranges[last.index()].next_in_bundle = lr; self.ranges_hot[last.index()].next_in_bundle = lr;
self.bundles[bundle.index()].last_range = lr; self.bundles[bundle.index()].last_range = lr;
} else { } else {
// Find location to insert. // Find location to insert.
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
let mut insert_after = LiveRangeIndex::invalid(); let mut insert_after = LiveRangeIndex::invalid();
let insert_range = self.ranges[lr.index()].range; let insert_range = self.ranges_hot[lr.index()].range;
while iter.is_valid() { while iter.is_valid() {
debug_assert!(!self.ranges[iter.index()].range.overlaps(&insert_range)); debug_assert!(!self.ranges_hot[iter.index()].range.overlaps(&insert_range));
if self.ranges[iter.index()].range.to <= insert_range.from { if self.ranges_hot[iter.index()].range.to <= insert_range.from {
break; break;
} }
insert_after = iter; insert_after = iter;
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
if insert_after.is_valid() { if insert_after.is_valid() {
self.ranges[insert_after.index()].next_in_bundle = lr; self.ranges_hot[insert_after.index()].next_in_bundle = lr;
if self.bundles[bundle.index()].last_range == insert_after { if self.bundles[bundle.index()].last_range == insert_after {
self.bundles[bundle.index()].last_range = lr; self.bundles[bundle.index()].last_range = lr;
} }
} else { } else {
let next = self.bundles[bundle.index()].first_range; let next = self.bundles[bundle.index()].first_range;
self.ranges[lr.index()].next_in_bundle = next; self.ranges_hot[lr.index()].next_in_bundle = next;
self.bundles[bundle.index()].first_range = lr; self.bundles[bundle.index()].first_range = lr;
} }
} }
@@ -1945,13 +1957,13 @@ impl<'a, F: Function> Env<'a, F> {
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
while iter.is_valid() { while iter.is_valid() {
let range = self.ranges[iter.index()].range; let range = self.ranges_hot[iter.index()].range;
// Create a new LiveRange for the PReg // Create a new LiveRange for the PReg
// reservation, unaffiliated with the VReg, to // reservation, unaffiliated with the VReg, to
// reserve it (like a clobber) without the // reserve it (like a clobber) without the
// possibility of eviction. // possibility of eviction.
self.add_liverange_to_preg(range, preg); self.add_liverange_to_preg(range, preg);
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
continue; continue;
} }
@@ -2039,16 +2051,16 @@ impl<'a, F: Function> Env<'a, F> {
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
let start_idx = self.range_ranges.len(); let start_idx = self.range_ranges.len();
let start_pos = if iter.is_valid() { let start_pos = if iter.is_valid() {
self.ranges[iter.index()].range.from self.ranges_hot[iter.index()].range.from
} else { } else {
ProgPoint::from_index(0) ProgPoint::from_index(0)
}; };
let mut end_pos = start_pos; let mut end_pos = start_pos;
while iter.is_valid() { while iter.is_valid() {
let range = self.ranges[iter.index()].range; let range = self.ranges_hot[iter.index()].range;
end_pos = range.to; end_pos = range.to;
self.range_ranges.push(range); self.range_ranges.push(range);
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
let end_idx = self.range_ranges.len(); let end_idx = self.range_ranges.len();
let bound = CodeRange { let bound = CodeRange {
@@ -2080,8 +2092,8 @@ impl<'a, F: Function> Env<'a, F> {
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
let mut total = 0; let mut total = 0;
while iter.is_valid() { while iter.is_valid() {
total += self.ranges[iter.index()].range.len() as u32; total += self.ranges_hot[iter.index()].range.len() as u32;
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
total total
} }
@@ -2136,7 +2148,7 @@ impl<'a, F: Function> Env<'a, F> {
log::debug!("vreg{}: first_range={:?}", i, v.first_range,); log::debug!("vreg{}: first_range={:?}", i, v.first_range,);
} }
log::debug!("Ranges:"); log::debug!("Ranges:");
for (i, (r, rc)) in self.ranges.iter().zip(self.ranges.iter()).enumerate() { for (i, (r, rh)) in self.ranges.iter().zip(self.ranges_hot.iter()).enumerate() {
log::debug!( log::debug!(
concat!( concat!(
"range{}: range={:?} vreg={:?} bundle={:?} ", "range{}: range={:?} vreg={:?} bundle={:?} ",
@@ -2144,14 +2156,14 @@ impl<'a, F: Function> Env<'a, F> {
"next_in_bundle={:?} next_in_reg={:?}" "next_in_bundle={:?} next_in_reg={:?}"
), ),
i, i,
r.range, rh.range,
rc.vreg, r.vreg,
rc.bundle, r.bundle,
r.uses_spill_weight(), r.uses_spill_weight(),
r.first_use, r.first_use,
rc.last_use, r.last_use,
r.next_in_bundle, rh.next_in_bundle,
rc.next_in_reg r.next_in_reg
); );
} }
log::debug!("Uses:"); log::debug!("Uses:");
@@ -2184,8 +2196,9 @@ impl<'a, F: Function> Env<'a, F> {
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
while iter.is_valid() { while iter.is_valid() {
let range_hot = &self.ranges_hot[iter.index()];
let range = &self.ranges[iter.index()]; let range = &self.ranges[iter.index()];
log::debug!(" -> range {:?}", range.range); log::debug!(" -> range {:?}", range_hot.range);
let mut use_iter = range.first_use; let mut use_iter = range.first_use;
while use_iter.is_valid() { while use_iter.is_valid() {
let usedata = &self.uses[use_iter.index()]; let usedata = &self.uses[use_iter.index()];
@@ -2196,7 +2209,7 @@ impl<'a, F: Function> Env<'a, F> {
log::debug!(" -> needed {:?}", needed); log::debug!(" -> needed {:?}", needed);
use_iter = usedata.next_use(); use_iter = usedata.next_use();
} }
iter = range.next_in_bundle; iter = range_hot.next_in_bundle;
} }
log::debug!(" -> final needed: {:?}", needed); log::debug!(" -> final needed: {:?}", needed);
@@ -2210,8 +2223,8 @@ impl<'a, F: Function> Env<'a, F> {
return None; return None;
} }
Some(CodeRange { Some(CodeRange {
from: self.ranges[first_range.index()].range.from, from: self.ranges_hot[first_range.index()].range.from,
to: self.ranges[last_range.index()].range.to, to: self.ranges_hot[last_range.index()].range.to,
}) })
} }
@@ -2287,7 +2300,7 @@ impl<'a, F: Function> Env<'a, F> {
self.bundles[bundle.index()].allocation = Allocation::reg(preg); self.bundles[bundle.index()].allocation = Allocation::reg(preg);
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
while iter.is_valid() { while iter.is_valid() {
let range = &self.ranges[iter.index()]; let range = &self.ranges_hot[iter.index()];
self.pregs[reg.index()] self.pregs[reg.index()]
.allocations .allocations
.btree .btree
@@ -2322,8 +2335,10 @@ impl<'a, F: Function> Env<'a, F> {
self.pregs[preg_idx.index()] self.pregs[preg_idx.index()]
.allocations .allocations
.btree .btree
.remove(&LiveRangeKey::from_range(&self.ranges[iter.index()].range)); .remove(&LiveRangeKey::from_range(
iter = self.ranges[iter.index()].next_in_bundle; &self.ranges_hot[iter.index()].range,
));
iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
let prio = self.bundles[bundle.index()].prio; let prio = self.bundles[bundle.index()].prio;
log::debug!(" -> prio {}; back into queue", prio); log::debug!(" -> prio {}; back into queue", prio);
@@ -2354,6 +2369,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut fixed = false; let mut fixed = false;
let bundledata = &self.bundles[bundle.index()]; let bundledata = &self.bundles[bundle.index()];
let first_range = &self.ranges[bundledata.first_range.index()]; let first_range = &self.ranges[bundledata.first_range.index()];
let first_range_hot = &self.ranges_hot[bundledata.first_range.index()];
log::debug!("recompute bundle properties: bundle {:?}", bundle); log::debug!("recompute bundle properties: bundle {:?}", bundle);
@@ -2376,13 +2392,13 @@ impl<'a, F: Function> Env<'a, F> {
// the range covers only one instruction. Note that it // the range covers only one instruction. Note that it
// could cover just one ProgPoint, i.e. X.Before..X.After, // could cover just one ProgPoint, i.e. X.Before..X.After,
// or two ProgPoints, i.e. X.Before..X+1.Before. // or two ProgPoints, i.e. X.Before..X+1.Before.
log::debug!(" -> first range has range {:?}", first_range.range); log::debug!(" -> first range has range {:?}", first_range_hot.range);
log::debug!( log::debug!(
" -> first range has next in bundle {:?}", " -> first range has next in bundle {:?}",
first_range.next_in_bundle first_range_hot.next_in_bundle
); );
minimal = first_range.next_in_bundle.is_invalid() minimal = first_range_hot.next_in_bundle.is_invalid()
&& first_range.range.from.inst() == first_range.range.to.prev().inst(); && first_range_hot.range.from.inst() == first_range_hot.range.to.prev().inst();
log::debug!(" -> minimal: {}", minimal); log::debug!(" -> minimal: {}", minimal);
} }
@@ -2404,7 +2420,7 @@ impl<'a, F: Function> Env<'a, F> {
range_data.uses_spill_weight() range_data.uses_spill_weight()
); );
total += range_data.uses_spill_weight(); total += range_data.uses_spill_weight();
range = range_data.next_in_bundle; range = self.ranges_hot[range.index()].next_in_bundle;
} }
if self.bundles[bundle.index()].prio > 0 { if self.bundles[bundle.index()].prio > 0 {
@@ -2472,12 +2488,12 @@ impl<'a, F: Function> Env<'a, F> {
let (conflict_from, conflict_to) = if conflicting.is_valid() { let (conflict_from, conflict_to) = if conflicting.is_valid() {
( (
Some( Some(
self.ranges[self.bundles[conflicting.index()].first_range.index()] self.ranges_hot[self.bundles[conflicting.index()].first_range.index()]
.range .range
.from, .from,
), ),
Some( Some(
self.ranges[self.bundles[conflicting.index()].last_range.index()] self.ranges_hot[self.bundles[conflicting.index()].last_range.index()]
.range .range
.to, .to,
), ),
@@ -2487,14 +2503,14 @@ impl<'a, F: Function> Env<'a, F> {
}; };
let bundle_start = if self.bundles[bundle.index()].first_range.is_valid() { let bundle_start = if self.bundles[bundle.index()].first_range.is_valid() {
self.ranges[self.bundles[bundle.index()].first_range.index()] self.ranges_hot[self.bundles[bundle.index()].first_range.index()]
.range .range
.from .from
} else { } else {
ProgPoint::before(Inst::new(0)) ProgPoint::before(Inst::new(0))
}; };
let bundle_end = if self.bundles[bundle.index()].last_range.is_valid() { let bundle_end = if self.bundles[bundle.index()].last_range.is_valid() {
self.ranges[self.bundles[bundle.index()].last_range.index()] self.ranges_hot[self.bundles[bundle.index()].last_range.index()]
.range .range
.to .to
} else { } else {
@@ -2505,7 +2521,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut clobberidx = 0; let mut clobberidx = 0;
while our_iter.is_valid() { while our_iter.is_valid() {
// Probe the hot-code tree. // Probe the hot-code tree.
let our_range = self.ranges[our_iter.index()].range; let our_range = self.ranges_hot[our_iter.index()].range;
log::debug!(" -> range {:?}", our_range); log::debug!(" -> range {:?}", our_range);
if let Some(hot_range_idx) = self if let Some(hot_range_idx) = self
.hot_code .hot_code
@@ -2516,7 +2532,7 @@ impl<'a, F: Function> Env<'a, F> {
// There may be cold code in our range on either side of the hot // There may be cold code in our range on either side of the hot
// range. Record the transition points if so. // range. Record the transition points if so.
let hot_range = self.ranges[hot_range_idx.index()].range; let hot_range = self.ranges_hot[hot_range_idx.index()].range;
log::debug!(" -> overlaps with hot-code range {:?}", hot_range); log::debug!(" -> overlaps with hot-code range {:?}", hot_range);
let start_cold = our_range.from < hot_range.from; let start_cold = our_range.from < hot_range.from;
let end_cold = our_range.to > hot_range.to; let end_cold = our_range.to > hot_range.to;
@@ -2585,7 +2601,7 @@ impl<'a, F: Function> Env<'a, F> {
use_idx = use_data.next_use(); use_idx = use_data.next_use();
} }
our_iter = self.ranges[our_iter.index()].next_in_bundle; our_iter = self.ranges_hot[our_iter.index()].next_in_bundle;
} }
log::debug!( log::debug!(
" -> first use/def after conflict range: {:?}", " -> first use/def after conflict range: {:?}",
@@ -2634,7 +2650,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
log::debug!("finding all use/def splits for {:?}", bundle); log::debug!("finding all use/def splits for {:?}", bundle);
let bundle_start = if iter.is_valid() { let bundle_start = if iter.is_valid() {
self.ranges[iter.index()].range.from self.ranges_hot[iter.index()].range.from
} else { } else {
ProgPoint::before(Inst::new(0)) ProgPoint::before(Inst::new(0))
}; };
@@ -2644,9 +2660,12 @@ impl<'a, F: Function> Env<'a, F> {
// the middle* of an instruction, because we would not be able // the middle* of an instruction, because we would not be able
// to insert moves to reify such an assignment. // to insert moves to reify such an assignment.
while iter.is_valid() { while iter.is_valid() {
let rangedata = &self.ranges[iter.index()]; log::debug!(
log::debug!(" -> range {:?}: {:?}", iter, rangedata.range); " -> range {:?}: {:?}",
let mut use_idx = rangedata.first_use; iter,
self.ranges_hot[iter.index()].range
);
let mut use_idx = self.ranges[iter.index()].first_use;
while use_idx.is_valid() { while use_idx.is_valid() {
let use_data = &self.uses[use_idx.index()]; let use_data = &self.uses[use_idx.index()];
log::debug!(" -> use: {:?}", use_data); log::debug!(" -> use: {:?}", use_data);
@@ -2674,7 +2693,7 @@ impl<'a, F: Function> Env<'a, F> {
use_idx = use_data.next_use(); use_idx = use_data.next_use();
} }
iter = rangedata.next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
splits.sort_unstable(); splits.sort_unstable();
log::debug!(" -> final splits: {:?}", splits); log::debug!(" -> final splits: {:?}", splits);
@@ -2741,7 +2760,7 @@ impl<'a, F: Function> Env<'a, F> {
// at the start of the first range in the bundle. // at the start of the first range in the bundle.
let first_range = self.bundles[bundle.index()].first_range; let first_range = self.bundles[bundle.index()].first_range;
let bundle_start = if first_range.is_valid() { let bundle_start = if first_range.is_valid() {
self.ranges[first_range.index()].range.from self.ranges_hot[first_range.index()].range.from
} else { } else {
ProgPoint::before(Inst::new(0)) ProgPoint::before(Inst::new(0))
}; };
@@ -2757,10 +2776,10 @@ impl<'a, F: Function> Env<'a, F> {
let mut range_summary_idx = self.bundles[bundle.index()].range_summary.from; let mut range_summary_idx = self.bundles[bundle.index()].range_summary.from;
while iter.is_valid() { while iter.is_valid() {
// Read `next` link now and then clear it -- we rebuild the list below. // Read `next` link now and then clear it -- we rebuild the list below.
let next = self.ranges[iter.index()].next_in_bundle; let next = self.ranges_hot[iter.index()].next_in_bundle;
self.ranges[iter.index()].next_in_bundle = LiveRangeIndex::invalid(); self.ranges_hot[iter.index()].next_in_bundle = LiveRangeIndex::invalid();
let mut range = self.ranges[iter.index()].range; let mut range = self.ranges_hot[iter.index()].range;
log::debug!(" -> has range {:?} (LR {:?})", range, iter); log::debug!(" -> has range {:?} (LR {:?})", range, iter);
// If any splits occur before this range, create a new // If any splits occur before this range, create a new
@@ -2784,8 +2803,8 @@ impl<'a, F: Function> Env<'a, F> {
// Link into current bundle. // Link into current bundle.
self.ranges[iter.index()].bundle = cur_bundle; self.ranges[iter.index()].bundle = cur_bundle;
if self.bundles[cur_bundle.index()].first_range.is_valid() { if self.bundles[cur_bundle.index()].first_range.is_valid() {
self.ranges[self.bundles[cur_bundle.index()].last_range.index()].next_in_bundle = self.ranges_hot[self.bundles[cur_bundle.index()].last_range.index()]
iter; .next_in_bundle = iter;
} else { } else {
self.bundles[cur_bundle.index()].first_range = iter; self.bundles[cur_bundle.index()].first_range = iter;
} }
@@ -2825,14 +2844,14 @@ impl<'a, F: Function> Env<'a, F> {
debug_assert!(range.from < split_point && split_point < range.to); debug_assert!(range.from < split_point && split_point < range.to);
let rest_range = CodeRange { let rest_range = CodeRange {
from: split_point, from: split_point,
to: self.ranges[iter.index()].range.to, to: self.ranges_hot[iter.index()].range.to,
}; };
self.ranges[iter.index()].range.to = split_point; self.ranges_hot[iter.index()].range.to = split_point;
range = rest_range; range = rest_range;
log::debug!( log::debug!(
" -> range of {:?} now {:?}", " -> range of {:?} now {:?}",
iter, iter,
self.ranges[iter.index()].range self.ranges_hot[iter.index()].range
); );
// Create the rest-range and insert it into the vreg's // Create the rest-range and insert it into the vreg's
@@ -2898,7 +2917,7 @@ impl<'a, F: Function> Env<'a, F> {
log::debug!( log::debug!(
" -> range {:?} next-in-bundle is {:?}", " -> range {:?} next-in-bundle is {:?}",
iter, iter,
self.ranges[iter.index()].next_in_bundle self.ranges_hot[iter.index()].next_in_bundle
); );
// Create a new bundle to hold the rest-range. // Create a new bundle to hold the rest-range.
@@ -2943,12 +2962,12 @@ impl<'a, F: Function> Env<'a, F> {
fn fixup_range_summary_bound(&mut self, bundle: LiveBundleIndex) { fn fixup_range_summary_bound(&mut self, bundle: LiveBundleIndex) {
let bundledata = &mut self.bundles[bundle.index()]; let bundledata = &mut self.bundles[bundle.index()];
let from = if bundledata.first_range.is_valid() { let from = if bundledata.first_range.is_valid() {
self.ranges[bundledata.first_range.index()].range.from self.ranges_hot[bundledata.first_range.index()].range.from
} else { } else {
ProgPoint::from_index(0) ProgPoint::from_index(0)
}; };
let to = if bundledata.last_range.is_valid() { let to = if bundledata.last_range.is_valid() {
self.ranges[bundledata.last_range.index()].range.to self.ranges_hot[bundledata.last_range.index()].range.to
} else { } else {
ProgPoint::from_index(0) ProgPoint::from_index(0)
}; };
@@ -2963,8 +2982,11 @@ impl<'a, F: Function> Env<'a, F> {
.range_summary .range_summary
.iter(&self.range_ranges[..]); .iter(&self.range_ranges[..]);
while iter.is_valid() { while iter.is_valid() {
assert_eq!(summary_iter.next(), Some(self.ranges[iter.index()].range)); assert_eq!(
iter = self.ranges[iter.index()].next_in_bundle; summary_iter.next(),
Some(self.ranges_hot[iter.index()].range)
);
iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
assert_eq!(summary_iter.next(), None); assert_eq!(summary_iter.next(), None);
} }
@@ -3032,11 +3054,12 @@ impl<'a, F: Function> Env<'a, F> {
// location in the code and by the bundle we're // location in the code and by the bundle we're
// considering. This has the effect of spreading // considering. This has the effect of spreading
// demand more evenly across registers. // demand more evenly across registers.
let scan_offset = self.ranges[self.bundles[bundle.index()].first_range.index()] let scan_offset = self.ranges_hot
.range [self.bundles[bundle.index()].first_range.index()]
.from .range
.inst() .from
.index() .inst()
.index()
+ bundle.index(); + bundle.index();
// If the bundle is more than one range, see if we // If the bundle is more than one range, see if we
@@ -3235,7 +3258,7 @@ impl<'a, F: Function> Env<'a, F> {
for &bundle in &self.spillsets[spillset.index()].bundles { for &bundle in &self.spillsets[spillset.index()].bundles {
let mut iter = self.bundles[bundle.index()].first_range; let mut iter = self.bundles[bundle.index()].first_range;
while iter.is_valid() { while iter.is_valid() {
let range = self.ranges[iter.index()].range; let range = self.ranges_hot[iter.index()].range;
if self.spillslots[spillslot.index()] if self.spillslots[spillslot.index()]
.ranges .ranges
.btree .btree
@@ -3243,7 +3266,7 @@ impl<'a, F: Function> Env<'a, F> {
{ {
return false; return false;
} }
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
} }
true true
@@ -3271,14 +3294,14 @@ impl<'a, F: Function> Env<'a, F> {
spillslot, spillslot,
iter, iter,
bundle, bundle,
self.ranges[iter.index()].range self.ranges_hot[iter.index()].range
); );
let range = self.ranges[iter.index()].range; let range = self.ranges_hot[iter.index()].range;
self.spillslots[spillslot.index()] self.spillslots[spillslot.index()]
.ranges .ranges
.btree .btree
.insert(LiveRangeKey::from_range(&range), iter); .insert(LiveRangeKey::from_range(&range), iter);
iter = self.ranges[iter.index()].next_in_bundle; iter = self.ranges_hot[iter.index()].next_in_bundle;
} }
} }
} }
@@ -3512,7 +3535,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut prev = LiveRangeIndex::invalid(); let mut prev = LiveRangeIndex::invalid();
while iter.is_valid() { while iter.is_valid() {
let alloc = self.get_alloc_for_range(iter); let alloc = self.get_alloc_for_range(iter);
let range = self.ranges[iter.index()].range; let range = self.ranges_hot[iter.index()].range;
log::debug!( log::debug!(
"apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?}", "apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?}",
vreg, vreg,
@@ -3565,7 +3588,7 @@ impl<'a, F: Function> Env<'a, F> {
// instruction). // instruction).
if prev.is_valid() { if prev.is_valid() {
let prev_alloc = self.get_alloc_for_range(prev); let prev_alloc = self.get_alloc_for_range(prev);
let prev_range = self.ranges[prev.index()].range; let prev_range = self.ranges_hot[prev.index()].range;
let first_use = self.ranges[iter.index()].first_use; let first_use = self.ranges[iter.index()].first_use;
let first_is_def = if first_use.is_valid() { let first_is_def = if first_use.is_valid() {
self.uses[first_use.index()].operand.kind() == OperandKind::Def self.uses[first_use.index()].operand.kind() == OperandKind::Def
@@ -4235,8 +4258,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut safepoint_idx = 0; let mut safepoint_idx = 0;
let mut iter = self.vregs[vreg.index()].first_range; let mut iter = self.vregs[vreg.index()].first_range;
while iter.is_valid() { while iter.is_valid() {
let rangedata = &self.ranges[iter.index()]; let range = self.ranges_hot[iter.index()].range;
let range = rangedata.range;
let alloc = self.get_alloc_for_range(iter); let alloc = self.get_alloc_for_range(iter);
log::debug!(" -> range {:?}: alloc {}", range, alloc); log::debug!(" -> range {:?}: alloc {}", range, alloc);
while safepoint_idx < safepoints.len() && safepoints[safepoint_idx] < range.to { while safepoint_idx < safepoints.len() && safepoints[safepoint_idx] < range.to {
@@ -4252,7 +4274,7 @@ impl<'a, F: Function> Env<'a, F> {
self.safepoint_slots.push((safepoints[safepoint_idx], slot)); self.safepoint_slots.push((safepoints[safepoint_idx], slot));
safepoint_idx += 1; safepoint_idx += 1;
} }
iter = rangedata.next_in_reg; iter = self.ranges[iter.index()].next_in_reg;
} }
} }