Add all empty LRs to a single "spill bundle", to avoid many small bundles and excessive moves

src/ion/mod.rs

@@ -244,6 +244,7 @@ struct SpillSet {
     slot: SpillSlotIndex,
     reg_hint: PReg,
     class: RegClass,
+    spill_bundle: LiveBundleIndex,
     size: u8,
 }
 
@@ -1858,6 +1859,7 @@ impl<'a, F: Function> Env<'a, F> {
                 size,
                 class: reg.class(),
                 reg_hint: PReg::invalid(),
+                spill_bundle: LiveBundleIndex::invalid(),
             });
             self.bundles[bundle.index()].spillset = ssidx;
         }
@@ -2626,6 +2628,12 @@ impl<'a, F: Function> Env<'a, F> {
 
         let spillset = self.bundles[bundle.index()].spillset;
 
+        // Get the spill bundle, which is the single bundle into which
+        // we place empty ranges (those with no uses). This may be
+        // `LiveBundleIndex::invalid()`, in which case we'll create it
+        // when we first need it.
+        let mut spill_bundle = self.spillsets[spillset.index()].spill_bundle;
+
         let mut split_idx = 0;
 
         // Fast-forward past any splits that occur before or exactly
@@ -2680,13 +2688,41 @@ impl<'a, F: Function> Env<'a, F> {
                         log::debug!("  -> use at {:?}", u.pos);
                         self.ranges[cur_lr.index()].uses.push(u);
                     }
-                    self.ranges[cur_lr.index()].bundle = cur_bundle;
+                    if self.ranges[cur_lr.index()].uses.len() > 0 {
-                    self.bundles[cur_bundle.index()]
+                        self.ranges[cur_lr.index()].bundle = cur_bundle;
-                        .ranges
+                        self.bundles[cur_bundle.index()]
-                        .push(LiveRangeListEntry {
+                            .ranges
-                            range: cur_range,
+                            .push(LiveRangeListEntry {
-                            index: cur_lr,
+                                range: cur_range,
-                        });
+                                index: cur_lr,
+                            });
+                    } else {
+                        if spill_bundle.is_invalid() {
+                            spill_bundle = self.create_bundle();
+                            log::debug!(
+                                " -> allocating new spill-bundle for empty ranges: bundle{}",
+                                spill_bundle.index()
+                            );
+                            self.bundles[spill_bundle.index()].spillset = spillset;
+                            self.spillsets[spillset.index()].spill_bundle = spill_bundle;
+                            self.spilled_bundles.push(spill_bundle);
+                        }
+                        // Range lists in empty-range bundles are not
+                        // sorted until later (when we try to allocate
+                        // regs or spillslots for them).
+                        log::debug!(
+                            " -> no uses in range{}; placing in empty-range spill-bundle bundle{}",
+                            cur_lr.index(),
+                            spill_bundle.index()
+                        );
+                        self.ranges[cur_lr.index()].bundle = spill_bundle;
+                        self.bundles[spill_bundle.index()]
+                            .ranges
+                            .push(LiveRangeListEntry {
+                                range: cur_range,
+                                index: cur_lr,
+                            });
+                    }
                     break;
                 }
 
@@ -2746,13 +2782,7 @@ impl<'a, F: Function> Env<'a, F> {
                 self.ranges[cur_lr.index()].range = existing_range;
                 self.ranges[new_lr.index()].vreg = self.ranges[cur_lr.index()].vreg;
                 self.ranges[new_lr.index()].bundle = new_bundle;
-                self.ranges[cur_lr.index()].bundle = cur_bundle;
+
-                self.bundles[cur_bundle.index()]
-                    .ranges
-                    .push(LiveRangeListEntry {
-                        range: existing_range,
-                        index: cur_lr,
-                    });
                 while let Some(u) = cur_uses.peek() {
                     if u.pos >= split {
                         break;
@@ -2763,6 +2793,44 @@ impl<'a, F: Function> Env<'a, F> {
                     cur_uses.next();
                 }
 
+                if self.ranges[cur_lr.index()].uses.len() > 0 {
+                    log::debug!(
+                        " -> adding current LR {:?} to current bundle {:?}",
+                        cur_lr,
+                        cur_bundle
+                    );
+                    self.ranges[cur_lr.index()].bundle = cur_bundle;
+                    self.bundles[cur_bundle.index()]
+                        .ranges
+                        .push(LiveRangeListEntry {
+                            range: existing_range,
+                            index: cur_lr,
+                        });
+                } else {
+                    if spill_bundle.is_invalid() {
+                        spill_bundle = self.create_bundle();
+                        log::debug!(
+                            " -> allocating new spill-bundle for empty ranges: bundle{}",
+                            spill_bundle.index()
+                        );
+                        self.bundles[spill_bundle.index()].spillset = spillset;
+                        self.spillsets[spillset.index()].spill_bundle = spill_bundle;
+                        self.spilled_bundles.push(spill_bundle);
+                    }
+                    log::debug!(
+                        " -> no uses in range{}; placing in empty-range spill-bundle bundle{}",
+                        cur_lr.index(),
+                        spill_bundle.index()
+                    );
+                    self.ranges[cur_lr.index()].bundle = spill_bundle;
+                    self.bundles[spill_bundle.index()]
+                        .ranges
+                        .push(LiveRangeListEntry {
+                            range: cur_range,
+                            index: cur_lr,
+                        });
+                }
+
                 if self.annotations_enabled && log::log_enabled!(log::Level::Debug) {
                     self.annotate(
                         existing_range.to,
@@ -2798,15 +2866,19 @@ impl<'a, F: Function> Env<'a, F> {
 
         // Recompute weights and priorities of all bundles, and
         // enqueue all split-bundles on the allocation queue.
-        let prio = self.compute_bundle_prio(bundle);
+        if self.bundles[bundle.index()].ranges.len() > 0 {
-        self.bundles[bundle.index()].prio = prio;
+            let prio = self.compute_bundle_prio(bundle);
-        self.recompute_bundle_properties(bundle);
+            self.bundles[bundle.index()].prio = prio;
-        self.allocation_queue.insert(bundle, prio as usize);
+            self.recompute_bundle_properties(bundle);
+            self.allocation_queue.insert(bundle, prio as usize);
+        }
         for &b in &new_bundles {
-            let prio = self.compute_bundle_prio(b);
+            if self.bundles[b.index()].ranges.len() > 0 {
-            self.bundles[b.index()].prio = prio;
+                let prio = self.compute_bundle_prio(b);
-            self.recompute_bundle_properties(b);
+                self.bundles[b.index()].prio = prio;
-            self.allocation_queue.insert(b, prio as usize);
+                self.recompute_bundle_properties(b);
+                self.allocation_queue.insert(b, prio as usize);
+            }
         }
     }
 
@@ -3032,11 +3104,15 @@ impl<'a, F: Function> Env<'a, F> {
         log::debug!("allocating regs for spilled bundles");
         for i in 0..self.spilled_bundles.len() {
             let bundle = self.spilled_bundles[i]; // don't borrow self
-            let any_vreg = self.vreg_regs[self.ranges
+
-                [self.bundles[bundle.index()].ranges[0].index.index()]
+            let class = self.spillsets[self.bundles[bundle.index()].spillset.index()].class;
-            .vreg
+
-            .index()];
+            // This may be an empty-range bundle whose ranges are not
-            let class = any_vreg.class();
+            // sorted; sort all range-lists again here.
+            self.bundles[bundle.index()]
+                .ranges
+                .sort_unstable_by_key(|entry| entry.range.from);
+
             let mut success = false;
             self.stats.spill_bundle_reg_probes += 1;
             for preg in RegTraversalIter::new(
@@ -3385,11 +3461,12 @@ impl<'a, F: Function> Env<'a, F> {
                     .unwrap_or_else(|| self.get_alloc_for_range(entry.index));
                 let range = entry.range;
                 log::debug!(
-                    "apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?}",
+                    "apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?} (pinned {:?})",
                     vreg,
                     entry.index,
                     range,
-                    alloc
+                    alloc,
+                    pinned_alloc,
                 );
                 debug_assert!(alloc != Allocation::none());