T0b1/regalloc2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Some memory-size/bitpacking optimizations

Browse Source
This commit is contained in:
Chris Fallin
2021-05-06 20:47:17 -07:00
parent 07a5a88972
commit 02b6516acd
4 changed files with 92 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/bitvec.rs
View File

@@ -20,6 +20,7 @@ impl AdaptiveMap {
fn new() -> Self { fn new() -> Self {
Self::Small(0, [INVALID, INVALID, INVALID, INVALID], [0, 0, 0, 0]) Self::Small(0, [INVALID, INVALID, INVALID, INVALID], [0, 0, 0, 0])
} }
#[inline(never)]
fn expand(&mut self) { fn expand(&mut self) {
match self { match self {
&mut Self::Small(len, ref keys, ref values) => { &mut Self::Small(len, ref keys, ref values) => {
@@ -32,6 +33,7 @@ impl AdaptiveMap {
_ => {} _ => {}
} }
} }
#[inline(always)]
fn get_or_insert<'a>(&'a mut self, key: u32) -> &'a mut u64 { fn get_or_insert<'a>(&'a mut self, key: u32) -> &'a mut u64 {
let needs_expand = match self { let needs_expand = match self {
&mut Self::Small(len, ref keys, ..) => len == 4 && !keys.iter().any(|k| *k == key), &mut Self::Small(len, ref keys, ..) => len == 4 && !keys.iter().any(|k| *k == key),
@@ -58,6 +60,7 @@ impl AdaptiveMap {
&mut Self::Large(ref mut map) => map.entry(key).or_insert(0), &mut Self::Large(ref mut map) => map.entry(key).or_insert(0),
} }
} }
#[inline(always)]
fn get_mut(&mut self, key: u32) -> Option<&mut u64> { fn get_mut(&mut self, key: u32) -> Option<&mut u64> {
match self { match self {
&mut Self::Small(len, ref keys, ref mut values) => { &mut Self::Small(len, ref keys, ref mut values) => {
@@ -71,6 +74,7 @@ impl AdaptiveMap {
&mut Self::Large(ref mut map) => map.get_mut(&key), &mut Self::Large(ref mut map) => map.get_mut(&key),
} }
} }
#[inline(always)]
fn get(&self, key: u32) -> Option<&u64> { fn get(&self, key: u32) -> Option<&u64> {
match self { match self {
&Self::Small(len, ref keys, ref values) => { &Self::Small(len, ref keys, ref values) => {

2
src/checker.rs
View File

@@ -526,7 +526,7 @@ impl<'a, F: Function> Checker<'a, F> {
let mut safepoint_slots: HashMap<Inst, Vec<SpillSlot>> = HashMap::new(); let mut safepoint_slots: HashMap<Inst, Vec<SpillSlot>> = HashMap::new();
for &(progpoint, slot) in &out.safepoint_slots { for &(progpoint, slot) in &out.safepoint_slots {
safepoint_slots safepoint_slots
.entry(progpoint.inst) .entry(progpoint.inst())
.or_insert_with(|| vec![]) .or_insert_with(|| vec![])
.push(slot); .push(slot);
} }

107
src/ion/mod.rs
View File

@@ -76,7 +76,7 @@ impl CodeRange {
other.to > self.from && other.from < self.to other.to > self.from && other.from < self.to
} }
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.to.inst.index() - self.from.inst.index() self.to.inst().index() - self.from.inst().index()
} }
} }
@@ -123,7 +123,7 @@ struct LiveRange {
next_in_reg: LiveRangeIndex, next_in_reg: LiveRangeIndex,
// if a bundle partly fits, this is used to record LRs that do fit // if a bundle partly fits, this is used to record LRs that do fit
reg_hint: Option<PReg>, reg_hint: PReg,
merged_into: LiveRangeIndex, merged_into: LiveRangeIndex,
} }
@@ -172,12 +172,12 @@ impl LiveRange {
struct Use { struct Use {
operand: Operand, operand: Operand,
pos: ProgPoint, pos: ProgPoint,
slot: usize,
next_use: UseIndex, next_use: UseIndex,
slot: u8,
is_def: bool, is_def: bool,
} }
const SLOT_NONE: usize = usize::MAX; const SLOT_NONE: u8 = u8::MAX;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
struct LiveBundle { struct LiveBundle {
@@ -216,10 +216,10 @@ impl LiveBundle {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
struct SpillSet { struct SpillSet {
bundles: LiveBundleVec, bundles: LiveBundleVec,
size: u32,
class: RegClass,
slot: SpillSlotIndex, slot: SpillSlotIndex,
reg_hint: Option<PReg>, reg_hint: PReg,
class: RegClass,
size: u8,
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
@@ -594,10 +594,21 @@ impl<'a> RegTraversalIter<'a> {
pub fn new( pub fn new(
env: &'a MachineEnv, env: &'a MachineEnv,
class: RegClass, class: RegClass,
mut hint_reg: Option<PReg>, hint_reg: PReg,
mut hint2_reg: Option<PReg>, hint2_reg: PReg,
offset: usize, offset: usize,
) -> Self { ) -> Self {
let mut hint_reg = if hint_reg != PReg::invalid() {
Some(hint_reg)
} else {
None
};
let mut hint2_reg = if hint2_reg != PReg::invalid() {
Some(hint2_reg)
} else {
None
};
if hint_reg.is_none() { if hint_reg.is_none() {
hint_reg = hint2_reg; hint_reg = hint2_reg;
hint2_reg = None; hint2_reg = None;
@@ -744,7 +755,7 @@ impl<'a, F: Function> Env<'a, F> {
last_use: UseIndex::invalid(), last_use: UseIndex::invalid(),
next_in_bundle: LiveRangeIndex::invalid(), next_in_bundle: LiveRangeIndex::invalid(),
next_in_reg: LiveRangeIndex::invalid(), next_in_reg: LiveRangeIndex::invalid(),
reg_hint: None, reg_hint: PReg::invalid(),
merged_into: LiveRangeIndex::invalid(), merged_into: LiveRangeIndex::invalid(),
}); });
LiveRangeIndex::new(idx) LiveRangeIndex::new(idx)
@@ -1200,7 +1211,7 @@ impl<'a, F: Function> Env<'a, F> {
self.uses.push(Use { self.uses.push(Use {
operand, operand,
pos, pos,
slot: i, slot: i as u8,
next_use: UseIndex::invalid(), next_use: UseIndex::invalid(),
is_def: true, is_def: true,
}); });
@@ -1267,7 +1278,7 @@ impl<'a, F: Function> Env<'a, F> {
self.uses.push(Use { self.uses.push(Use {
operand, operand,
pos, pos,
slot: i, slot: i as u8,
next_use: UseIndex::invalid(), next_use: UseIndex::invalid(),
is_def: false, is_def: false,
}); });
@@ -1538,9 +1549,9 @@ impl<'a, F: Function> Env<'a, F> {
log::debug!( log::debug!(
" -> extra clobber {} at inst{}", " -> extra clobber {} at inst{}",
preg, preg,
pos.inst.index() pos.inst().index()
); );
extra_clobbers.push((preg, pos.inst)); extra_clobbers.push((preg, pos.inst()));
} }
} else { } else {
seen_fixed_for_vreg.push(op.vreg()); seen_fixed_for_vreg.push(op.vreg());
@@ -1552,7 +1563,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut use_iter = self.ranges[iter.index()].first_use; let mut use_iter = self.ranges[iter.index()].first_use;
while use_iter.is_valid() { while use_iter.is_valid() {
let pos = self.uses[use_iter.index()].pos; let pos = self.uses[use_iter.index()].pos;
let slot = self.uses[use_iter.index()].slot; let slot = self.uses[use_iter.index()].slot as usize;
fixup_multi_fixed_vregs( fixup_multi_fixed_vregs(
pos, pos,
slot, slot,
@@ -1917,13 +1928,13 @@ impl<'a, F: Function> Env<'a, F> {
// compute its priority, and enqueue it. // compute its priority, and enqueue it.
let ssidx = SpillSetIndex::new(self.spillsets.len()); let ssidx = SpillSetIndex::new(self.spillsets.len());
let reg = self.vregs[vreg.index()].reg; let reg = self.vregs[vreg.index()].reg;
let size = self.func.spillslot_size(reg.class(), reg) as u32; let size = self.func.spillslot_size(reg.class(), reg) as u8;
self.spillsets.push(SpillSet { self.spillsets.push(SpillSet {
bundles: smallvec![], bundles: smallvec![],
slot: SpillSlotIndex::invalid(), slot: SpillSlotIndex::invalid(),
size, size,
class: reg.class(), class: reg.class(),
reg_hint: None, reg_hint: PReg::invalid(),
}); });
self.bundles[bundle.index()].spillset = ssidx; self.bundles[bundle.index()].spillset = ssidx;
let prio = self.compute_bundle_prio(bundle); let prio = self.compute_bundle_prio(bundle);
@@ -2100,7 +2111,7 @@ impl<'a, F: Function> Env<'a, F> {
return AllocRegResult::ConflictWithFixed; return AllocRegResult::ConflictWithFixed;
} }
} else { } else {
self.ranges[iter.index()].reg_hint = Some(self.pregs[reg.index()].reg); self.ranges[iter.index()].reg_hint = self.pregs[reg.index()].reg;
} }
iter = next; iter = next;
} }
@@ -2210,7 +2221,7 @@ impl<'a, F: Function> Env<'a, F> {
first_range.next_in_bundle first_range.next_in_bundle
); );
minimal = first_range.next_in_bundle.is_invalid() minimal = first_range.next_in_bundle.is_invalid()
&& first_range.range.from.inst == first_range.range.to.prev().inst; && first_range.range.from.inst() == first_range.range.to.prev().inst();
log::debug!(" -> minimal: {}", minimal); log::debug!(" -> minimal: {}", minimal);
} }
@@ -2385,7 +2396,7 @@ impl<'a, F: Function> Env<'a, F> {
// Update last-before-conflict and first-before-conflict positions. // Update last-before-conflict and first-before-conflict positions.
let mut update_with_pos = |pos: ProgPoint| { let mut update_with_pos = |pos: ProgPoint| {
let before_inst = ProgPoint::before(pos.inst); let before_inst = ProgPoint::before(pos.inst());
let before_next_inst = before_inst.next().next(); let before_next_inst = before_inst.next().next();
if before_inst > bundle_start if before_inst > bundle_start
&& (conflict_from.is_none() || before_inst < conflict_from.unwrap()) && (conflict_from.is_none() || before_inst < conflict_from.unwrap())
@@ -2398,7 +2409,7 @@ impl<'a, F: Function> Env<'a, F> {
&& (conflict_to.is_none() || pos >= conflict_to.unwrap()) && (conflict_to.is_none() || pos >= conflict_to.unwrap())
&& (first_after_conflict.is_none() || pos > first_after_conflict.unwrap()) && (first_after_conflict.is_none() || pos > first_after_conflict.unwrap())
{ {
first_after_conflict = Some(ProgPoint::before(pos.inst.next())); first_after_conflict = Some(ProgPoint::before(pos.inst().next()));
} }
}; };
@@ -2483,9 +2494,9 @@ impl<'a, F: Function> Env<'a, F> {
} else { } else {
// For an use, split before the instruction -- // For an use, split before the instruction --
// this allows us to insert a move if necessary. // this allows us to insert a move if necessary.
ProgPoint::before(use_data.pos.inst) ProgPoint::before(use_data.pos.inst())
}; };
let after_use_inst = ProgPoint::before(use_data.pos.inst.next()); let after_use_inst = ProgPoint::before(use_data.pos.inst().next());
log::debug!( log::debug!(
" -> splitting before and after use: {:?} and {:?}", " -> splitting before and after use: {:?} and {:?}",
before_use_inst, before_use_inst,
@@ -2767,7 +2778,7 @@ impl<'a, F: Function> Env<'a, F> {
let hint2_reg = if self.bundles[bundle.index()].first_range.is_valid() { let hint2_reg = if self.bundles[bundle.index()].first_range.is_valid() {
self.ranges[self.bundles[bundle.index()].first_range.index()].reg_hint self.ranges[self.bundles[bundle.index()].first_range.index()].reg_hint
} else { } else {
None PReg::invalid()
}; };
log::debug!( log::debug!(
"process_bundle: bundle {:?} requirement {:?} hint {:?} hint2 {:?}", "process_bundle: bundle {:?} requirement {:?} hint {:?} hint2 {:?}",
@@ -2802,7 +2813,7 @@ impl<'a, F: Function> Env<'a, F> {
self.stats.process_bundle_reg_success_fixed += 1; self.stats.process_bundle_reg_success_fixed += 1;
log::debug!(" -> allocated to fixed {:?}", preg_idx); log::debug!(" -> allocated to fixed {:?}", preg_idx);
self.spillsets[self.bundles[bundle.index()].spillset.index()] self.spillsets[self.bundles[bundle.index()].spillset.index()]
.reg_hint = Some(alloc.as_reg().unwrap()); .reg_hint = alloc.as_reg().unwrap();
return; return;
} }
AllocRegResult::Conflict(bundles) => { AllocRegResult::Conflict(bundles) => {
@@ -2829,7 +2840,7 @@ impl<'a, F: Function> Env<'a, F> {
let scan_offset = self.ranges[self.bundles[bundle.index()].first_range.index()] let scan_offset = self.ranges[self.bundles[bundle.index()].first_range.index()]
.range .range
.from .from
.inst .inst()
.index() .index()
+ bundle.index(); + bundle.index();
@@ -2855,7 +2866,7 @@ impl<'a, F: Function> Env<'a, F> {
_ => panic!("Impossible result: {:?}", result), _ => panic!("Impossible result: {:?}", result),
}; };
self.spillsets[self.bundles[bundle.index()].spillset.index()] self.spillsets[self.bundles[bundle.index()].spillset.index()]
.reg_hint = Some(alloc.as_reg().unwrap()); .reg_hint = alloc.as_reg().unwrap();
log::debug!(" -> definitely fits; assigning"); log::debug!(" -> definitely fits; assigning");
return; return;
} }
@@ -2873,7 +2884,7 @@ impl<'a, F: Function> Env<'a, F> {
self.stats.process_bundle_reg_success_any += 1; self.stats.process_bundle_reg_success_any += 1;
log::debug!(" -> allocated to any {:?}", preg_idx); log::debug!(" -> allocated to any {:?}", preg_idx);
self.spillsets[self.bundles[bundle.index()].spillset.index()] self.spillsets[self.bundles[bundle.index()].spillset.index()]
.reg_hint = Some(alloc.as_reg().unwrap()); .reg_hint = alloc.as_reg().unwrap();
return; return;
} }
AllocRegResult::Conflict(bundles) => { AllocRegResult::Conflict(bundles) => {
@@ -2983,7 +2994,13 @@ impl<'a, F: Function> Env<'a, F> {
let class = any_vreg.class(); let class = any_vreg.class();
let mut success = false; let mut success = false;
self.stats.spill_bundle_reg_probes += 1; self.stats.spill_bundle_reg_probes += 1;
for preg in RegTraversalIter::new(self.env, class, None, None, bundle.index()) { for preg in RegTraversalIter::new(
self.env,
class,
PReg::invalid(),
PReg::invalid(),
bundle.index(),
) {
let preg_idx = PRegIndex::new(preg.index()); let preg_idx = PRegIndex::new(preg.index());
if let AllocRegResult::Allocated(_) = if let AllocRegResult::Allocated(_) =
self.try_to_allocate_bundle_to_reg(bundle, preg_idx) self.try_to_allocate_bundle_to_reg(bundle, preg_idx)
@@ -3165,11 +3182,11 @@ impl<'a, F: Function> Env<'a, F> {
} }
fn is_start_of_block(&self, pos: ProgPoint) -> bool { fn is_start_of_block(&self, pos: ProgPoint) -> bool {
let block = self.cfginfo.insn_block[pos.inst.index()]; let block = self.cfginfo.insn_block[pos.inst().index()];
pos == self.cfginfo.block_entry[block.index()] pos == self.cfginfo.block_entry[block.index()]
} }
fn is_end_of_block(&self, pos: ProgPoint) -> bool { fn is_end_of_block(&self, pos: ProgPoint) -> bool {
let block = self.cfginfo.insn_block[pos.inst.index()]; let block = self.cfginfo.insn_block[pos.inst().index()];
pos == self.cfginfo.block_exit[block.index()] pos == self.cfginfo.block_exit[block.index()]
} }
@@ -3365,7 +3382,7 @@ impl<'a, F: Function> Env<'a, F> {
alloc, alloc,
vreg.index() vreg.index()
); );
assert_eq!(range.from.pos, InstPosition::Before); assert_eq!(range.from.pos(), InstPosition::Before);
self.insert_move(range.from, InsertMovePrio::Regular, prev_alloc, alloc); self.insert_move(range.from, InsertMovePrio::Regular, prev_alloc, alloc);
} }
} }
@@ -3375,7 +3392,7 @@ impl<'a, F: Function> Env<'a, F> {
// already in this range (hence guaranteed to have the // already in this range (hence guaranteed to have the
// same allocation) and if the vreg is live, add a // same allocation) and if the vreg is live, add a
// Source half-move. // Source half-move.
let mut block = self.cfginfo.insn_block[range.from.inst.index()]; let mut block = self.cfginfo.insn_block[range.from.inst().index()];
while block.is_valid() && block.index() < self.func.blocks() { while block.is_valid() && block.index() < self.func.blocks() {
if range.to < self.cfginfo.block_exit[block.index()].next() { if range.to < self.cfginfo.block_exit[block.index()].next() {
break; break;
@@ -3456,7 +3473,7 @@ impl<'a, F: Function> Env<'a, F> {
// this range and for which the vreg is live at the // this range and for which the vreg is live at the
// start of the block. For each, for each predecessor, // start of the block. For each, for each predecessor,
// add a Dest half-move. // add a Dest half-move.
let mut block = self.cfginfo.insn_block[range.from.inst.index()]; let mut block = self.cfginfo.insn_block[range.from.inst().index()];
if self.cfginfo.block_entry[block.index()] < range.from { if self.cfginfo.block_entry[block.index()] < range.from {
block = block.next(); block = block.next();
} }
@@ -3559,13 +3576,13 @@ impl<'a, F: Function> Env<'a, F> {
while use_iter.is_valid() { while use_iter.is_valid() {
let usedata = &self.uses[use_iter.index()]; let usedata = &self.uses[use_iter.index()];
debug_assert!(range.contains_point(usedata.pos)); debug_assert!(range.contains_point(usedata.pos));
let inst = usedata.pos.inst; let inst = usedata.pos.inst();
let slot = usedata.slot; let slot = usedata.slot;
let operand = usedata.operand; let operand = usedata.operand;
// Safepoints add virtual uses with no slots; // Safepoints add virtual uses with no slots;
// avoid these. // avoid these.
if slot != SLOT_NONE { if slot != SLOT_NONE {
self.set_alloc(inst, slot, alloc); self.set_alloc(inst, slot as usize, alloc);
} }
if let OperandPolicy::Reuse(_) = operand.policy() { if let OperandPolicy::Reuse(_) = operand.policy() {
reuse_input_insts.push(inst); reuse_input_insts.push(inst);
@@ -3574,15 +3591,15 @@ impl<'a, F: Function> Env<'a, F> {
} }
// Scan over program move srcs/dsts to fill in allocations. // Scan over program move srcs/dsts to fill in allocations.
let move_src_start = if range.from.pos == InstPosition::Before { let move_src_start = if range.from.pos() == InstPosition::Before {
(vreg, range.from.inst) (vreg, range.from.inst())
} else { } else {
(vreg, range.from.inst.next()) (vreg, range.from.inst().next())
}; };
let move_src_end = if range.to.pos == InstPosition::Before { let move_src_end = if range.to.pos() == InstPosition::Before {
(vreg, range.to.inst) (vreg, range.to.inst())
} else { } else {
(vreg, range.to.inst.next()) (vreg, range.to.inst().next())
}; };
log::debug!( log::debug!(
"vreg {:?} range {:?}: looking for program-move sources from {:?} to {:?}", "vreg {:?} range {:?}: looking for program-move sources from {:?} to {:?}",
@@ -3610,8 +3627,8 @@ impl<'a, F: Function> Env<'a, F> {
prog_move_src_idx += 1; prog_move_src_idx += 1;
} }
let move_dst_start = (vreg, range.from.inst); let move_dst_start = (vreg, range.from.inst());
let move_dst_end = (vreg, range.to.inst); let move_dst_end = (vreg, range.to.inst());
log::debug!( log::debug!(
"vreg {:?} range {:?}: looking for program-move dests from {:?} to {:?}", "vreg {:?} range {:?}: looking for program-move dests from {:?} to {:?}",
vreg, vreg,
@@ -3753,7 +3770,7 @@ impl<'a, F: Function> Env<'a, F> {
Allocation::reg(self.pregs[to_preg.index()].reg), Allocation::reg(self.pregs[to_preg.index()].reg),
); );
self.set_alloc( self.set_alloc(
progpoint.inst, progpoint.inst(),
slot, slot,
Allocation::reg(self.pregs[to_preg.index()].reg), Allocation::reg(self.pregs[to_preg.index()].reg),
); );

61
src/lib.rs
View File

@@ -781,64 +781,51 @@ pub enum InstPosition {
/// A program point: a single point before or after a given instruction. /// A program point: a single point before or after a given instruction.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ProgPoint { pub struct ProgPoint {
pub inst: Inst, bits: u32,
pub pos: InstPosition,
} }
impl ProgPoint { impl ProgPoint {
pub fn new(inst: Inst, pos: InstPosition) -> Self {
let bits = ((inst.0 as u32) << 1) | (pos as u8 as u32);
Self { bits }
}
pub fn before(inst: Inst) -> Self { pub fn before(inst: Inst) -> Self {
Self { Self::new(inst, InstPosition::Before)
inst,
pos: InstPosition::Before,
} }
}
pub fn after(inst: Inst) -> Self { pub fn after(inst: Inst) -> Self {
Self { Self::new(inst, InstPosition::After)
inst, }
pos: InstPosition::After, pub fn inst(self) -> Inst {
// Cast to i32 to do an arithmetic right-shift, which will
// preserve an `Inst::invalid()` (which is -1, or all-ones).
Inst::new(((self.bits as i32) >> 1) as usize)
}
pub fn pos(self) -> InstPosition {
match self.bits & 1 {
0 => InstPosition::Before,
1 => InstPosition::After,
_ => unreachable!(),
} }
} }
pub fn next(self) -> ProgPoint { pub fn next(self) -> ProgPoint {
match self.pos { Self {
InstPosition::Before => ProgPoint { bits: self.bits + 1,
inst: self.inst,
pos: InstPosition::After,
},
InstPosition::After => ProgPoint {
inst: self.inst.next(),
pos: InstPosition::Before,
},
} }
} }
pub fn prev(self) -> ProgPoint { pub fn prev(self) -> ProgPoint {
match self.pos { Self {
InstPosition::Before => ProgPoint { bits: self.bits - 1,
inst: self.inst.prev(),
pos: InstPosition::After,
},
InstPosition::After => ProgPoint {
inst: self.inst,
pos: InstPosition::Before,
},
} }
} }
pub fn to_index(self) -> u32 { pub fn to_index(self) -> u32 {
debug_assert!(self.inst.index() <= ((1 << 31) - 1)); self.bits
((self.inst.index() as u32) << 1) | (self.pos as u8 as u32)
} }
pub fn from_index(index: u32) -> Self { pub fn from_index(index: u32) -> Self {
let inst = Inst::new((index >> 1) as usize); Self { bits: index }
let pos = match index & 1 {
0 => InstPosition::Before,
1 => InstPosition::After,
_ => unreachable!(),
};
Self { inst, pos }
} }
} }