diff --git a/src/ion/mod.rs b/src/ion/mod.rs index d19f759..1c558ba 100644 --- a/src/ion/mod.rs +++ b/src/ion/mod.rs @@ -244,6 +244,7 @@ struct SpillSet { slot: SpillSlotIndex, reg_hint: PReg, class: RegClass, + spill_bundle: LiveBundleIndex, size: u8, } @@ -1858,6 +1859,7 @@ impl<'a, F: Function> Env<'a, F> { size, class: reg.class(), reg_hint: PReg::invalid(), + spill_bundle: LiveBundleIndex::invalid(), }); self.bundles[bundle.index()].spillset = ssidx; } @@ -2626,6 +2628,12 @@ impl<'a, F: Function> Env<'a, F> { let spillset = self.bundles[bundle.index()].spillset; + // Get the spill bundle, which is the single bundle into which + // we place empty ranges (those with no uses). This may be + // `LiveBundleIndex::invalid()`, in which case we'll create it + // when we first need it. + let mut spill_bundle = self.spillsets[spillset.index()].spill_bundle; + let mut split_idx = 0; // Fast-forward past any splits that occur before or exactly @@ -2680,13 +2688,41 @@ impl<'a, F: Function> Env<'a, F> { log::debug!(" -> use at {:?}", u.pos); self.ranges[cur_lr.index()].uses.push(u); } - self.ranges[cur_lr.index()].bundle = cur_bundle; - self.bundles[cur_bundle.index()] - .ranges - .push(LiveRangeListEntry { - range: cur_range, - index: cur_lr, - }); + if self.ranges[cur_lr.index()].uses.len() > 0 { + self.ranges[cur_lr.index()].bundle = cur_bundle; + self.bundles[cur_bundle.index()] + .ranges + .push(LiveRangeListEntry { + range: cur_range, + index: cur_lr, + }); + } else { + if spill_bundle.is_invalid() { + spill_bundle = self.create_bundle(); + log::debug!( + " -> allocating new spill-bundle for empty ranges: bundle{}", + spill_bundle.index() + ); + self.bundles[spill_bundle.index()].spillset = spillset; + self.spillsets[spillset.index()].spill_bundle = spill_bundle; + self.spilled_bundles.push(spill_bundle); + } + // Range lists in empty-range bundles are not + // sorted until later (when we try to allocate + // regs or spillslots for them). + log::debug!( + " -> no uses in range{}; placing in empty-range spill-bundle bundle{}", + cur_lr.index(), + spill_bundle.index() + ); + self.ranges[cur_lr.index()].bundle = spill_bundle; + self.bundles[spill_bundle.index()] + .ranges + .push(LiveRangeListEntry { + range: cur_range, + index: cur_lr, + }); + } break; } @@ -2746,13 +2782,7 @@ impl<'a, F: Function> Env<'a, F> { self.ranges[cur_lr.index()].range = existing_range; self.ranges[new_lr.index()].vreg = self.ranges[cur_lr.index()].vreg; self.ranges[new_lr.index()].bundle = new_bundle; - self.ranges[cur_lr.index()].bundle = cur_bundle; - self.bundles[cur_bundle.index()] - .ranges - .push(LiveRangeListEntry { - range: existing_range, - index: cur_lr, - }); + while let Some(u) = cur_uses.peek() { if u.pos >= split { break; @@ -2763,6 +2793,44 @@ impl<'a, F: Function> Env<'a, F> { cur_uses.next(); } + if self.ranges[cur_lr.index()].uses.len() > 0 { + log::debug!( + " -> adding current LR {:?} to current bundle {:?}", + cur_lr, + cur_bundle + ); + self.ranges[cur_lr.index()].bundle = cur_bundle; + self.bundles[cur_bundle.index()] + .ranges + .push(LiveRangeListEntry { + range: existing_range, + index: cur_lr, + }); + } else { + if spill_bundle.is_invalid() { + spill_bundle = self.create_bundle(); + log::debug!( + " -> allocating new spill-bundle for empty ranges: bundle{}", + spill_bundle.index() + ); + self.bundles[spill_bundle.index()].spillset = spillset; + self.spillsets[spillset.index()].spill_bundle = spill_bundle; + self.spilled_bundles.push(spill_bundle); + } + log::debug!( + " -> no uses in range{}; placing in empty-range spill-bundle bundle{}", + cur_lr.index(), + spill_bundle.index() + ); + self.ranges[cur_lr.index()].bundle = spill_bundle; + self.bundles[spill_bundle.index()] + .ranges + .push(LiveRangeListEntry { + range: cur_range, + index: cur_lr, + }); + } + if self.annotations_enabled && log::log_enabled!(log::Level::Debug) { self.annotate( existing_range.to, @@ -2798,15 +2866,19 @@ impl<'a, F: Function> Env<'a, F> { // Recompute weights and priorities of all bundles, and // enqueue all split-bundles on the allocation queue. - let prio = self.compute_bundle_prio(bundle); - self.bundles[bundle.index()].prio = prio; - self.recompute_bundle_properties(bundle); - self.allocation_queue.insert(bundle, prio as usize); + if self.bundles[bundle.index()].ranges.len() > 0 { + let prio = self.compute_bundle_prio(bundle); + self.bundles[bundle.index()].prio = prio; + self.recompute_bundle_properties(bundle); + self.allocation_queue.insert(bundle, prio as usize); + } for &b in &new_bundles { - let prio = self.compute_bundle_prio(b); - self.bundles[b.index()].prio = prio; - self.recompute_bundle_properties(b); - self.allocation_queue.insert(b, prio as usize); + if self.bundles[b.index()].ranges.len() > 0 { + let prio = self.compute_bundle_prio(b); + self.bundles[b.index()].prio = prio; + self.recompute_bundle_properties(b); + self.allocation_queue.insert(b, prio as usize); + } } } @@ -3032,11 +3104,15 @@ impl<'a, F: Function> Env<'a, F> { log::debug!("allocating regs for spilled bundles"); for i in 0..self.spilled_bundles.len() { let bundle = self.spilled_bundles[i]; // don't borrow self - let any_vreg = self.vreg_regs[self.ranges - [self.bundles[bundle.index()].ranges[0].index.index()] - .vreg - .index()]; - let class = any_vreg.class(); + + let class = self.spillsets[self.bundles[bundle.index()].spillset.index()].class; + + // This may be an empty-range bundle whose ranges are not + // sorted; sort all range-lists again here. + self.bundles[bundle.index()] + .ranges + .sort_unstable_by_key(|entry| entry.range.from); + let mut success = false; self.stats.spill_bundle_reg_probes += 1; for preg in RegTraversalIter::new( @@ -3385,11 +3461,12 @@ impl<'a, F: Function> Env<'a, F> { .unwrap_or_else(|| self.get_alloc_for_range(entry.index)); let range = entry.range; log::debug!( - "apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?}", + "apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?} (pinned {:?})", vreg, entry.index, range, - alloc + alloc, + pinned_alloc, ); debug_assert!(alloc != Allocation::none());