diff --git a/src/ion/data_structures.rs b/src/ion/data_structures.rs index 5c986bb..01c0db1 100644 --- a/src/ion/data_structures.rs +++ b/src/ion/data_structures.rs @@ -252,8 +252,11 @@ pub struct SpillSet { pub spill_bundle: LiveBundleIndex, pub required: bool, pub size: u8, + pub splits: u8, } +pub(crate) const MAX_SPLITS_PER_SPILLSET: u8 = 10; + #[derive(Clone, Debug)] pub struct VRegData { pub ranges: LiveRangeList, diff --git a/src/ion/merge.rs b/src/ion/merge.rs index ef83b93..c3a0216 100644 --- a/src/ion/merge.rs +++ b/src/ion/merge.rs @@ -291,6 +291,7 @@ impl<'a, F: Function> Env<'a, F> { class: reg.class(), reg_hint: PReg::invalid(), spill_bundle: LiveBundleIndex::invalid(), + splits: 0, }); self.bundles[bundle.index()].spillset = ssidx; } diff --git a/src/ion/process.rs b/src/ion/process.rs index 3274391..2d7c15b 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -15,18 +15,18 @@ use super::{ spill_weight_from_constraint, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, LiveRangeIndex, LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, - Requirement, SpillWeight, UseList, + Requirement, SpillWeight, UseList, VRegIndex, }; use crate::{ ion::data_structures::{ - CodeRange, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MINIMAL_BUNDLE_SPILL_WEIGHT, - MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT, + CodeRange, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MAX_SPLITS_PER_SPILLSET, + MINIMAL_BUNDLE_SPILL_WEIGHT, MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT, }, Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint, RegAllocError, }; use fxhash::FxHashSet; -use smallvec::smallvec; +use smallvec::{smallvec, SmallVec}; use std::fmt::Debug; #[derive(Clone, Debug, PartialEq, Eq)] @@ -410,6 +410,16 @@ impl<'a, F: Function> Env<'a, F> { let spillset = self.bundles[bundle.index()].spillset; + // Have we reached the maximum split count? If so, fall back + // to a "minimal bundles and spill bundle" setup for this + // bundle. See the doc-comment on + // `split_into_minimal_bundles()` above for more. + if self.spillsets[spillset.index()].splits >= MAX_SPLITS_PER_SPILLSET { + self.split_into_minimal_bundles(bundle, reg_hint); + return; + } + self.spillsets[spillset.index()].splits += 1; + debug_assert!(!self.bundles[bundle.index()].ranges.is_empty()); // Split point *at* start is OK; this means we peel off // exactly one use to create a minimal bundle. @@ -511,6 +521,7 @@ impl<'a, F: Function> Env<'a, F> { self.bundles[bundle.index()] .ranges .truncate(last_lr_in_old_bundle_idx + 1); + self.bundles[bundle.index()].ranges.shrink_to_fit(); // If the first entry in `new_lr_list` is a LR that is split // down the middle, replace it with a new LR and chop off the @@ -537,6 +548,7 @@ impl<'a, F: Function> Env<'a, F> { .collect(); self.ranges[new_lr.index()].uses = rest_uses; self.ranges[orig_lr.index()].uses.truncate(first_use); + self.ranges[orig_lr.index()].uses.shrink_to_fit(); self.recompute_range_properties(orig_lr); self.recompute_range_properties(new_lr); new_lr_list[0].index = new_lr; @@ -728,6 +740,247 @@ impl<'a, F: Function> Env<'a, F> { } } + /// Splits the given bundle into minimal bundles per Use, falling + /// back onto the spill bundle. This must work for any bundle no + /// matter how many conflicts. + /// + /// This is meant to solve a quadratic-cost problem that exists + /// with "normal" splitting as implemented above. With that + /// procedure, , splitting a bundle produces two + /// halves. Furthermore, it has cost linear in the length of the + /// bundle, because the resulting half-bundles have their + /// requirements recomputed with a new scan, and because we copy + /// half the use-list over to the tail end sub-bundle. + /// + /// This works fine when a bundle has a handful of splits overall, + /// but not when an input has a systematic pattern of conflicts + /// that will require O(|bundle|) splits (e.g., every Use is + /// constrained to a different fixed register than the last + /// one). In such a case, we get quadratic behavior. + /// + /// This method implements a direct split into minimal bundles + /// along the whole length of the bundle, putting the regions + /// without uses in the spill bundle. We do this once the number + /// of splits in an original bundle (tracked by spillset) reaches + /// a pre-determined limit. + /// + /// This basically approximates what a non-splitting allocator + /// would do: it "spills" the whole bundle to possibly a + /// stackslot, or a second-chance register allocation at best, via + /// the spill bundle; and then does minimal reservations of + /// registers just at uses/defs and moves the "spilled" value + /// into/out of them immediately. + pub fn split_into_minimal_bundles(&mut self, bundle: LiveBundleIndex, reg_hint: PReg) { + let mut removed_lrs: FxHashSet = FxHashSet::default(); + let mut removed_lrs_vregs: FxHashSet = FxHashSet::default(); + let mut new_lrs: SmallVec<[(VRegIndex, LiveRangeIndex); 16]> = smallvec![]; + let mut new_bundles: SmallVec<[LiveBundleIndex; 16]> = smallvec![]; + + let spill = self + .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) + .unwrap(); + + trace!( + "Splitting bundle {:?} into minimal bundles with reg hint {}", + bundle, + reg_hint + ); + + let mut last_lr: Option = None; + let mut last_bundle: Option = None; + let mut last_inst: Option = None; + let mut last_vreg: Option = None; + + for entry_idx in 0..self.bundles[bundle.index()].ranges.len() { + // Iterate manually; don't borrow `self`. + let entry = self.bundles[bundle.index()].ranges[entry_idx]; + + removed_lrs.insert(entry.index); + let vreg = self.ranges[entry.index.index()].vreg; + removed_lrs_vregs.insert(vreg); + trace!(" -> removing old LR {:?} for vreg {:?}", entry.index, vreg); + + let mut last_live_pos = entry.range.from; + for use_idx in 0..self.ranges[entry.index.index()].uses.len() { + let u = self.ranges[entry.index.index()].uses[use_idx]; + trace!(" -> use {:?} (last_live_pos {:?})", u, last_live_pos); + + // If we just created a LR for this inst at the last + // pos, add this use to the same LR. + if Some(u.pos.inst()) == last_inst && Some(vreg) == last_vreg { + self.ranges[last_lr.unwrap().index()].uses.push(u); + trace!(" -> appended to last LR {:?}", last_lr.unwrap()); + continue; + } + + // If the last bundle was at the same inst, add a new + // LR to the same bundle; otherwise, create a LR and a + // new bundle. + if Some(u.pos.inst()) == last_inst { + let cr = CodeRange { + from: u.pos, + to: ProgPoint::before(u.pos.inst().next()), + }; + let lr = self.create_liverange(cr); + new_lrs.push((vreg, lr)); + self.ranges[lr.index()].uses.push(u); + self.ranges[lr.index()].vreg = vreg; + + trace!( + " -> created new LR {:?} but adding to existing bundle {:?}", + lr, + last_bundle.unwrap() + ); + // Edit the previous LR to end mid-inst. + self.bundles[last_bundle.unwrap().index()] + .ranges + .last_mut() + .unwrap() + .range + .to = u.pos; + self.ranges[last_lr.unwrap().index()].range.to = u.pos; + // Add this LR to the bundle. + self.bundles[last_bundle.unwrap().index()] + .ranges + .push(LiveRangeListEntry { + range: cr, + index: lr, + }); + self.ranges[lr.index()].bundle = last_bundle.unwrap(); + last_live_pos = ProgPoint::before(u.pos.inst().next()); + continue; + } + + // Otherwise, create a new LR. + let pos = ProgPoint::before(u.pos.inst()); + let cr = CodeRange { + from: pos, + to: ProgPoint::before(u.pos.inst().next()), + }; + let lr = self.create_liverange(cr); + new_lrs.push((vreg, lr)); + self.ranges[lr.index()].uses.push(u); + self.ranges[lr.index()].vreg = vreg; + + // Create a new bundle that contains only this LR. + let new_bundle = self.create_bundle(); + self.ranges[lr.index()].bundle = new_bundle; + self.bundles[new_bundle.index()].spillset = self.bundles[bundle.index()].spillset; + self.bundles[new_bundle.index()] + .ranges + .push(LiveRangeListEntry { + range: cr, + index: lr, + }); + new_bundles.push(new_bundle); + + // If this use was a Def, set the StartsAtDef flag for + // the new LR. (N.B.: *not* Mod, only Def, because Mod + // needs an input. This flag specifically indicates + // that the LR does not require the value to be moved + // into location at start because it (re)defines the + // value.) + if u.operand.kind() == OperandKind::Def { + self.ranges[lr.index()].set_flag(LiveRangeFlag::StartsAtDef); + } + + trace!( + " -> created new LR {:?} range {:?} with new bundle {:?} for this use", + lr, + cr, + new_bundle + ); + + // If there was any intervening range in the LR not + // covered by the minimal new LR above, add it to the + // spillset. + if pos > last_live_pos { + let cr = CodeRange { + from: last_live_pos, + to: pos, + }; + let spill_lr = self.create_liverange(cr); + self.ranges[spill_lr.index()].vreg = vreg; + self.ranges[spill_lr.index()].bundle = spill; + new_lrs.push((vreg, spill_lr)); + self.bundles[spill.index()].ranges.push(LiveRangeListEntry { + range: cr, + index: spill_lr, + }); + self.ranges[spill_lr.index()].bundle = spill; + trace!( + " -> put intervening range {:?} in new LR {:?} in spill bundle {:?}", + cr, + spill_lr, + spill + ); + } + last_live_pos = ProgPoint::before(u.pos.inst().next()); + + last_lr = Some(lr); + last_bundle = Some(new_bundle); + last_inst = Some(u.pos.inst()); + last_vreg = Some(vreg); + } + + // Clear the use-list from the original LR. + self.ranges[entry.index.index()].uses = Default::default(); + + // If there is space from the last use to the end of the + // LR, put that in the spill bundle too. + if entry.range.to > last_live_pos { + let cr = CodeRange { + from: last_live_pos, + to: entry.range.to, + }; + let spill_lr = self.create_liverange(cr); + self.ranges[spill_lr.index()].vreg = vreg; + self.ranges[spill_lr.index()].bundle = spill; + new_lrs.push((vreg, spill_lr)); + self.bundles[spill.index()].ranges.push(LiveRangeListEntry { + range: cr, + index: spill_lr, + }); + self.ranges[spill_lr.index()].bundle = spill; + trace!( + " -> put trailing range {:?} in new LR {:?} in spill bundle {:?}", + cr, + spill_lr, + spill + ); + } + } + + // Clear the LR list in the original bundle. + self.bundles[bundle.index()].ranges.clear(); + self.bundles[bundle.index()].ranges.shrink_to_fit(); + + // Remove all of the removed LRs from respective vregs' lists. + for vreg in removed_lrs_vregs { + self.vregs[vreg.index()] + .ranges + .retain(|entry| !removed_lrs.contains(&entry.index)); + } + + // Add the new LRs to their respective vreg lists. + for (vreg, lr) in new_lrs { + let range = self.ranges[lr.index()].range; + let entry = LiveRangeListEntry { range, index: lr }; + self.vregs[vreg.index()].ranges.push(entry); + } + + // Recompute bundle properties for all new bundles and enqueue + // them. + for bundle in new_bundles { + if self.bundles[bundle.index()].ranges.len() > 0 { + self.recompute_bundle_properties(bundle); + let prio = self.bundles[bundle.index()].prio; + self.allocation_queue + .insert(bundle, prio as usize, reg_hint); + } + } + } + pub fn process_bundle( &mut self, bundle: LiveBundleIndex, diff --git a/src/ion/spill.rs b/src/ion/spill.rs index 45a2716..121d3f7 100644 --- a/src/ion/spill.rs +++ b/src/ion/spill.rs @@ -25,6 +25,10 @@ impl<'a, F: Function> Env<'a, F> { for i in 0..self.spilled_bundles.len() { let bundle = self.spilled_bundles[i]; // don't borrow self + if self.bundles[bundle.index()].ranges.is_empty() { + continue; + } + let class = self.spillsets[self.bundles[bundle.index()].spillset.index()].class; let hint = self.spillsets[self.bundles[bundle.index()].spillset.index()].reg_hint;