T0b1/regalloc2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework inner allocation-loop code: choose more wisely between splitting and evicting based on costs (and unify the fixed-reg-constraint case)

Browse Source
This commit is contained in:
Chris Fallin
2021-05-28 17:36:06 -07:00
parent 43d7095cbd
commit 789651f947
1 changed files with 237 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

454
src/ion/mod.rs
View File

@@ -691,6 +691,8 @@ struct RegTraversalIter<'a> {
non_pref_idx: usize, non_pref_idx: usize,
offset_pref: usize, offset_pref: usize,
offset_non_pref: usize, offset_non_pref: usize,
is_fixed: bool,
fixed: Option<PReg>,
} }
impl<'a> RegTraversalIter<'a> { impl<'a> RegTraversalIter<'a> {
@@ -700,6 +702,7 @@ impl<'a> RegTraversalIter<'a> {
hint_reg: PReg, hint_reg: PReg,
hint2_reg: PReg, hint2_reg: PReg,
offset: usize, offset: usize,
fixed: Option<PReg>,
) -> Self { ) -> Self {
let mut hint_reg = if hint_reg != PReg::invalid() { let mut hint_reg = if hint_reg != PReg::invalid() {
Some(hint_reg) Some(hint_reg)
@@ -737,6 +740,8 @@ impl<'a> RegTraversalIter<'a> {
non_pref_idx: 0, non_pref_idx: 0,
offset_pref, offset_pref,
offset_non_pref, offset_non_pref,
is_fixed: fixed.is_some(),
fixed,
} }
} }
} }
@@ -745,6 +750,12 @@ impl<'a> std::iter::Iterator for RegTraversalIter<'a> {
type Item = PReg; type Item = PReg;
fn next(&mut self) -> Option<PReg> { fn next(&mut self) -> Option<PReg> {
if self.is_fixed {
let ret = self.fixed;
self.fixed = None;
return ret;
}
fn wrap(idx: usize, limit: usize) -> usize { fn wrap(idx: usize, limit: usize) -> usize {
if idx >= limit { if idx >= limit {
idx - limit idx - limit
@@ -2939,138 +2950,27 @@ impl<'a, F: Function> Env<'a, F> {
}; };
log::debug!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,); log::debug!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,);
if let Requirement::Conflict = req {
// We have to split right away.
let bundle_start = self.bundles[bundle.index()].ranges[0].range.from;
self.split_and_requeue_bundle(
bundle,
/* split_at_point = */ bundle_start,
reg_hint,
);
return Ok(());
}
// Try to allocate! // Try to allocate!
let mut attempts = 0; let mut attempts = 0;
let mut split_at_point = self.bundles[bundle.index()].ranges[0].range.from;
let mut requeue_with_reg = PReg::invalid();
loop { loop {
attempts += 1; attempts += 1;
log::debug!("attempt {}, req {:?}", attempts, req); log::debug!("attempt {}, req {:?}", attempts, req);
debug_assert!(attempts < 100 * self.func.insts()); debug_assert!(attempts < 100 * self.func.insts());
let (conflicting_bundles, first_conflict_point, first_conflict_reg) = match req { let (class, fixed_preg) = match req {
Requirement::Fixed(preg) => { Requirement::Fixed(preg) => (preg.class(), Some(preg)),
let preg_idx = PRegIndex::new(preg.index()); Requirement::Register(class) => (class, None),
self.stats.process_bundle_reg_probes_fixed += 1;
log::debug!("trying fixed reg {:?}", preg_idx);
match self.try_to_allocate_bundle_to_reg(bundle, preg_idx, None) {
AllocRegResult::Allocated(alloc) => {
self.stats.process_bundle_reg_success_fixed += 1;
log::debug!(" -> allocated to fixed {:?}", preg_idx);
self.spillsets[self.bundles[bundle.index()].spillset.index()]
.reg_hint = alloc.as_reg().unwrap();
return Ok(());
}
AllocRegResult::Conflict(bundles) => {
log::debug!(" -> conflict with bundles {:?}", bundles);
let first_bundle = bundles[0];
(
bundles,
self.bundles[first_bundle.index()].ranges[0].range.from,
preg,
)
}
AllocRegResult::ConflictWithFixed(_, point) => {
log::debug!(" -> conflict with fixed alloc");
(smallvec![], point, preg)
}
AllocRegResult::ConflictHighCost => unreachable!(),
}
}
Requirement::Register(class) => {
// Scan all pregs and attempt to allocate.
let mut lowest_cost_conflict_set: Option<LiveBundleVec> = None;
let mut lowest_cost_conflict_cost: Option<u32> = None;
let mut lowest_cost_conflict_point = ProgPoint::before(Inst::new(0));
let mut lowest_cost_conflict_reg = PReg::invalid();
// Heuristic: start the scan for an available
// register at an offset influenced both by our
// location in the code and by the bundle we're
// considering. This has the effect of spreading
// demand more evenly across registers.
let scan_offset = self.ranges
[self.bundles[bundle.index()].ranges[0].index.index()]
.range
.from
.inst()
.index()
+ bundle.index();
self.stats.process_bundle_reg_probe_start_any += 1;
for preg in RegTraversalIter::new(
self.env,
class,
hint_reg,
PReg::invalid(),
scan_offset,
) {
self.stats.process_bundle_reg_probes_any += 1;
let preg_idx = PRegIndex::new(preg.index());
log::debug!("trying preg {:?}", preg_idx);
match self.try_to_allocate_bundle_to_reg(
bundle,
preg_idx,
lowest_cost_conflict_cost,
) {
AllocRegResult::Allocated(alloc) => {
self.stats.process_bundle_reg_success_any += 1;
log::debug!(" -> allocated to any {:?}", preg_idx);
self.spillsets[self.bundles[bundle.index()].spillset.index()]
.reg_hint = alloc.as_reg().unwrap();
return Ok(());
}
AllocRegResult::Conflict(bundles) => {
log::debug!(" -> conflict with bundles {:?}", bundles);
let first_conflict_point =
self.bundles[bundles[0].index()].ranges[0].range.from;
let cost = self.maximum_spill_weight_in_bundle_set(&bundles);
if lowest_cost_conflict_cost.is_none()
|| cost < lowest_cost_conflict_cost.unwrap()
{
lowest_cost_conflict_cost = Some(cost);
lowest_cost_conflict_set = Some(bundles);
lowest_cost_conflict_point = first_conflict_point;
lowest_cost_conflict_reg = preg;
}
}
AllocRegResult::ConflictWithFixed(max_cost, point) => {
log::debug!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
if lowest_cost_conflict_cost.is_none()
|| max_cost < lowest_cost_conflict_cost.unwrap()
{
lowest_cost_conflict_cost = Some(max_cost);
lowest_cost_conflict_set = Some(smallvec![]);
lowest_cost_conflict_point = point;
lowest_cost_conflict_reg = preg;
}
}
AllocRegResult::ConflictHighCost => {
// Simply don't consider -- we already have
// a lower-cost conflict bundle option
// to evict.
continue;
}
}
}
// Otherwise, we *require* a register, but didn't fit into
// any with current bundle assignments. Hence, we will need
// to either split or attempt to evict some bundles. Return
// the conflicting bundles to evict and retry. Empty list
// means nothing to try (due to fixed conflict) so we must
// split instead.
(
lowest_cost_conflict_set.unwrap_or(smallvec![]),
lowest_cost_conflict_point,
lowest_cost_conflict_reg,
)
}
Requirement::Stack(_) => { Requirement::Stack(_) => {
// If we must be on the stack, mark our spillset // If we must be on the stack, mark our spillset
// as required immediately. // as required immediately.
@@ -3086,114 +2986,233 @@ impl<'a, F: Function> Env<'a, F> {
return Ok(()); return Ok(());
} }
Requirement::Conflict => { Requirement::Conflict => unreachable!(),
break;
}
}; };
// Scan all pregs, or the one fixed preg, and attempt to allocate.
log::debug!(" -> conflict set {:?}", conflicting_bundles); let mut lowest_cost_evict_conflict_set: Option<LiveBundleVec> = None;
log::debug!( let mut lowest_cost_evict_conflict_cost: Option<u32> = None;
" -> first conflict {:?} with reg {:?}",
first_conflict_point,
first_conflict_reg
);
// If we have already tried evictions once before and are let mut lowest_cost_split_conflict_cost: Option<u32> = None;
// still unsuccessful, give up and move on to splitting as let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0));
// long as this is not a minimal bundle. let mut lowest_cost_split_conflict_reg = PReg::invalid();
if attempts >= 2 && !self.minimal_bundle(bundle) {
break;
}
let bundle_start = self.bundles[bundle.index()].ranges[0].range.from; // Heuristic: start the scan for an available
split_at_point = std::cmp::max(first_conflict_point, bundle_start); // register at an offset influenced both by our
requeue_with_reg = first_conflict_reg; // location in the code and by the bundle we're
// considering. This has the effect of spreading
// demand more evenly across registers.
let scan_offset = self.ranges[self.bundles[bundle.index()].ranges[0].index.index()]
.range
.from
.inst()
.index()
+ bundle.index();
// Adjust `split_at_point` if it is within a deeper loop self.stats.process_bundle_reg_probe_start_any += 1;
// than the bundle start -- hoist it to just before the for preg in RegTraversalIter::new(
// first loop header it encounters. self.env,
let bundle_start_depth = self.cfginfo.approx_loop_depth class,
[self.cfginfo.insn_block[bundle_start.inst().index()].index()]; hint_reg,
let split_at_depth = self.cfginfo.approx_loop_depth PReg::invalid(),
[self.cfginfo.insn_block[split_at_point.inst().index()].index()]; scan_offset,
if split_at_depth > bundle_start_depth { fixed_preg,
for block in (self.cfginfo.insn_block[bundle_start.inst().index()].index() + 1) ) {
..=self.cfginfo.insn_block[split_at_point.inst().index()].index() self.stats.process_bundle_reg_probes_any += 1;
{ let preg_idx = PRegIndex::new(preg.index());
if self.cfginfo.approx_loop_depth[block] > bundle_start_depth { log::debug!("trying preg {:?}", preg_idx);
split_at_point = self.cfginfo.block_entry[block];
break; let scan_limit_cost = match (
lowest_cost_evict_conflict_cost,
lowest_cost_split_conflict_cost,
) {
(Some(a), Some(b)) => Some(std::cmp::max(a, b)),
_ => None,
};
match self.try_to_allocate_bundle_to_reg(bundle, preg_idx, scan_limit_cost) {
AllocRegResult::Allocated(alloc) => {
self.stats.process_bundle_reg_success_any += 1;
log::debug!(" -> allocated to any {:?}", preg_idx);
self.spillsets[self.bundles[bundle.index()].spillset.index()].reg_hint =
alloc.as_reg().unwrap();
return Ok(());
} }
} AllocRegResult::Conflict(bundles) => {
} log::debug!(" -> conflict with bundles {:?}", bundles);
// If we hit a fixed conflict, give up and move on to splitting. let first_conflict_point =
if conflicting_bundles.is_empty() { self.bundles[bundles[0].index()].ranges[0].range.from;
break;
}
// If the maximum spill weight in the conflicting-bundles set is >= this bundle's spill let conflict_cost = self.maximum_spill_weight_in_bundle_set(&bundles);
// weight, then don't evict.
let max_spill_weight = self.maximum_spill_weight_in_bundle_set(&conflicting_bundles);
log::debug!(
" -> max_spill_weight = {}; our spill weight {}",
max_spill_weight,
self.bundle_spill_weight(bundle)
);
if max_spill_weight >= self.bundle_spill_weight(bundle) {
log::debug!(" -> we're already the cheapest bundle to spill -- going to split");
break;
}
// Evict all bundles in `conflicting bundles` and try again. if lowest_cost_evict_conflict_cost.is_none()
self.stats.evict_bundle_event += 1; || conflict_cost < lowest_cost_evict_conflict_cost.unwrap()
for &bundle in &conflicting_bundles { {
log::debug!(" -> evicting {:?}", bundle); lowest_cost_evict_conflict_cost = Some(conflict_cost);
self.evict_bundle(bundle); lowest_cost_evict_conflict_set = Some(bundles);
self.stats.evict_bundle_count += 1; }
}
}
// A minimal bundle cannot be split. let loop_depth = self.cfginfo.approx_loop_depth
if self.minimal_bundle(bundle) { [self.cfginfo.insn_block[first_conflict_point.inst().index()].index()];
if let Requirement::Register(class) = req { let move_cost = spill_weight_from_policy(
// Check if this is a too-many-live-registers situation. OperandPolicy::Reg,
let range = self.bundles[bundle.index()].ranges[0].range; loop_depth as usize,
let mut min_bundles_assigned = 0; /* is_def = */ true,
let mut fixed_assigned = 0; );
let mut total_regs = 0; if lowest_cost_split_conflict_cost.is_none()
for preg in self.env.preferred_regs_by_class[class as u8 as usize] || (conflict_cost + move_cost)
.iter() < lowest_cost_split_conflict_cost.unwrap()
.chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter()) {
{ lowest_cost_split_conflict_cost = Some(conflict_cost + move_cost);
if let Some(&lr) = self.pregs[preg.index()] lowest_cost_split_conflict_point = first_conflict_point;
.allocations lowest_cost_split_conflict_reg = preg;
.btree
.get(&LiveRangeKey::from_range(&range))
{
if lr.is_valid() {
if self.minimal_bundle(self.ranges[lr.index()].bundle) {
min_bundles_assigned += 1;
}
} else {
fixed_assigned += 1;
} }
} }
total_regs += 1; AllocRegResult::ConflictWithFixed(max_cost, point) => {
log::debug!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
let loop_depth = self.cfginfo.approx_loop_depth
[self.cfginfo.insn_block[point.inst().index()].index()];
let move_cost = spill_weight_from_policy(
OperandPolicy::Reg,
loop_depth as usize,
/* is_def = */ true,
);
if lowest_cost_split_conflict_cost.is_none()
|| (max_cost + move_cost) < lowest_cost_split_conflict_cost.unwrap()
{
lowest_cost_split_conflict_cost = Some(max_cost + move_cost);
lowest_cost_split_conflict_point = point;
lowest_cost_split_conflict_reg = preg;
}
}
AllocRegResult::ConflictHighCost => {
// Simply don't consider -- we already have
// a lower-cost conflict bundle option
// to evict.
continue;
}
} }
if min_bundles_assigned + fixed_assigned == total_regs { }
return Err(RegAllocError::TooManyLiveRegs);
// Otherwise, we *require* a register, but didn't fit into
// any with current bundle assignments. Hence, we will need
// to either split or attempt to evict some bundles.
log::debug!(
" -> lowest cost evict: set {:?}, cost {:?}",
lowest_cost_evict_conflict_set,
lowest_cost_evict_conflict_cost,
);
log::debug!(
" -> lowest cost split: cost {:?}, point {:?}, reg {:?}",
lowest_cost_split_conflict_cost,
lowest_cost_split_conflict_point,
lowest_cost_split_conflict_reg
);
// If we reach here, we *must* have an option either to split or evict.
assert!(
lowest_cost_split_conflict_cost.is_some()
|| lowest_cost_evict_conflict_cost.is_some()
);
// If our bundle's weight is less than or equal to(*) the
// evict cost, choose to split. Also pick splitting if
// we're on our second or more attempt as long as the
// bundle isn't minimal. Also pick splitting if the
// conflict set is empty, meaning a fixed conflict that
// can't be evicted.
//
// (*) the "equal to" part is very important: it prevents
// an infinite loop where two bundles with equal spill
// cost continually evict each other in an infinite
// allocation loop. In such a case, the first bundle in
// wins, and the other splits.
if (attempts >= 2 && !self.minimal_bundle(bundle))
|| lowest_cost_evict_conflict_cost.is_none()
|| self.bundle_spill_weight(bundle) <= lowest_cost_evict_conflict_cost.unwrap()
{
log::debug!(
" -> deciding to split: our spill weight is {}",
self.bundle_spill_weight(bundle)
);
let bundle_start = self.bundles[bundle.index()].ranges[0].range.from;
let mut split_at_point =
std::cmp::max(lowest_cost_split_conflict_point, bundle_start);
let requeue_with_reg = lowest_cost_split_conflict_reg;
// Adjust `split_at_point` if it is within a deeper loop
// than the bundle start -- hoist it to just before the
// first loop header it encounters.
let bundle_start_depth = self.cfginfo.approx_loop_depth
[self.cfginfo.insn_block[bundle_start.inst().index()].index()];
let split_at_depth = self.cfginfo.approx_loop_depth
[self.cfginfo.insn_block[split_at_point.inst().index()].index()];
if split_at_depth > bundle_start_depth {
for block in (self.cfginfo.insn_block[bundle_start.inst().index()].index() + 1)
..=self.cfginfo.insn_block[split_at_point.inst().index()].index()
{
if self.cfginfo.approx_loop_depth[block] > bundle_start_depth {
split_at_point = self.cfginfo.block_entry[block];
break;
}
}
}
// A minimal bundle cannot be split.
if self.minimal_bundle(bundle) {
if let Requirement::Register(class) = req {
// Check if this is a too-many-live-registers situation.
let range = self.bundles[bundle.index()].ranges[0].range;
let mut min_bundles_assigned = 0;
let mut fixed_assigned = 0;
let mut total_regs = 0;
for preg in self.env.preferred_regs_by_class[class as u8 as usize]
.iter()
.chain(
self.env.non_preferred_regs_by_class[class as u8 as usize].iter(),
)
{
if let Some(&lr) = self.pregs[preg.index()]
.allocations
.btree
.get(&LiveRangeKey::from_range(&range))
{
if lr.is_valid() {
if self.minimal_bundle(self.ranges[lr.index()].bundle) {
min_bundles_assigned += 1;
}
} else {
fixed_assigned += 1;
}
}
total_regs += 1;
}
if min_bundles_assigned + fixed_assigned == total_regs {
return Err(RegAllocError::TooManyLiveRegs);
}
}
}
if self.minimal_bundle(bundle) {
self.dump_state();
}
assert!(!self.minimal_bundle(bundle));
self.split_and_requeue_bundle(bundle, split_at_point, requeue_with_reg);
return Ok(());
} else {
// Evict all bundles in `conflicting bundles` and try again.
self.stats.evict_bundle_event += 1;
for &bundle in &lowest_cost_evict_conflict_set.unwrap() {
log::debug!(" -> evicting {:?}", bundle);
self.evict_bundle(bundle);
self.stats.evict_bundle_count += 1;
} }
} }
} }
if self.minimal_bundle(bundle) {
self.dump_state();
}
assert!(!self.minimal_bundle(bundle));
self.split_and_requeue_bundle(bundle, split_at_point, requeue_with_reg);
Ok(())
} }
fn try_allocating_regs_for_spilled_bundles(&mut self) { fn try_allocating_regs_for_spilled_bundles(&mut self) {
@@ -3217,6 +3236,7 @@ impl<'a, F: Function> Env<'a, F> {
PReg::invalid(), PReg::invalid(),
PReg::invalid(), PReg::invalid(),
bundle.index(), bundle.index(),
None,
) { ) {
log::debug!("trying bundle {:?} to preg {:?}", bundle, preg); log::debug!("trying bundle {:?} to preg {:?}", bundle, preg);
let preg_idx = PRegIndex::new(preg.index()); let preg_idx = PRegIndex::new(preg.index());