diff --git a/src/bitvec.rs b/src/bitvec.rs index df53bb5..46a56ff 100644 --- a/src/bitvec.rs +++ b/src/bitvec.rs @@ -3,34 +3,53 @@ * exception. See `LICENSE` for details. */ -//! Bit vectors. +//! Index sets: sets of integers that represent indices into a space. +//! +//! For historical reasons this is called a `BitVec` but it is no +//! longer a dense bitvector; the chunked adaptive-sparse data +//! structure here has better performance. use fxhash::FxHashMap; +use std::cell::Cell; const SMALL_ELEMS: usize = 12; -/// A hybrid large/small-mode sparse mapping from integer indices to elements. +/// A hybrid large/small-mode sparse mapping from integer indices to +/// elements. +/// +/// The trailing `(u32, u64)` elements in each variant is a one-item +/// cache to allow fast access when streaming through. #[derive(Clone, Debug)] enum AdaptiveMap { - Small(u32, [u32; SMALL_ELEMS], [u64; SMALL_ELEMS]), - Large(FxHashMap), + Small( + u32, + [u32; SMALL_ELEMS], + [u64; SMALL_ELEMS], + Cell<(u32, u64)>, + ), + Large(FxHashMap, Cell<(u32, u64)>), } const INVALID: u32 = 0xffff_ffff; impl AdaptiveMap { fn new() -> Self { - Self::Small(0, [INVALID; SMALL_ELEMS], [0; SMALL_ELEMS]) + Self::Small( + 0, + [INVALID; SMALL_ELEMS], + [0; SMALL_ELEMS], + Cell::new((INVALID, 0)), + ) } #[inline(never)] fn expand(&mut self) { match self { - &mut Self::Small(len, ref keys, ref values) => { + &mut Self::Small(len, ref keys, ref values, ref cache) => { let mut map = FxHashMap::default(); for i in 0..len { map.insert(keys[i as usize], values[i as usize]); } - *self = Self::Large(map); + *self = Self::Large(map, cache.clone()); } _ => {} } @@ -48,7 +67,10 @@ impl AdaptiveMap { } match self { - &mut Self::Small(ref mut len, ref mut keys, ref mut values) => { + &mut Self::Small(ref mut len, ref mut keys, ref mut values, ref cached) => { + if cached.get().0 == key { + cached.set((INVALID, 0)); + } for i in 0..*len { if keys[i as usize] == key { return &mut values[i as usize]; @@ -61,13 +83,21 @@ impl AdaptiveMap { values[idx as usize] = 0; &mut values[idx as usize] } - &mut Self::Large(ref mut map) => map.entry(key).or_insert(0), + &mut Self::Large(ref mut map, ref cached) => { + if cached.get().0 == key { + cached.set((INVALID, 0)); + } + map.entry(key).or_insert(0) + } } } #[inline(always)] fn get_mut(&mut self, key: u32) -> Option<&mut u64> { match self { - &mut Self::Small(len, ref keys, ref mut values) => { + &mut Self::Small(len, ref keys, ref mut values, ref cached) => { + if cached.get().0 == key { + cached.set((INVALID, 0)); + } for i in 0..len { if keys[i as usize] == key { return Some(&mut values[i as usize]); @@ -75,29 +105,48 @@ impl AdaptiveMap { } None } - &mut Self::Large(ref mut map) => map.get_mut(&key), + &mut Self::Large(ref mut map, ref cached) => { + if cached.get().0 == key { + cached.set((INVALID, 0)); + } + map.get_mut(&key) + } } } #[inline(always)] - fn get(&self, key: u32) -> Option<&u64> { + fn get(&self, key: u32) -> Option { match self { - &Self::Small(len, ref keys, ref values) => { + &Self::Small(len, ref keys, ref values, ref cached) => { + if cached.get().0 == key { + return Some(cached.get().1); + } for i in 0..len { if keys[i as usize] == key { - return Some(&values[i as usize]); + let value = values[i as usize]; + cached.set((key, value)); + return Some(value); } } None } - &Self::Large(ref map) => map.get(&key), + &Self::Large(ref map, ref cached) => { + if cached.get().0 == key { + return Some(cached.get().1); + } + let value = map.get(&key).cloned(); + if let Some(value) = value { + cached.set((key, value)); + } + value + } } } fn iter<'a>(&'a self) -> AdaptiveMapIter<'a> { match self { - &Self::Small(len, ref keys, ref values) => { + &Self::Small(len, ref keys, ref values, ..) => { AdaptiveMapIter::Small(&keys[0..len as usize], &values[0..len as usize]) } - &Self::Large(ref map) => AdaptiveMapIter::Large(map.iter()), + &Self::Large(ref map, ..) => AdaptiveMapIter::Large(map.iter()), } } } @@ -155,7 +204,7 @@ impl BitVec { } #[inline(always)] - fn maybe_elem(&self, bit_index: usize) -> Option<&u64> { + fn maybe_elem(&self, bit_index: usize) -> Option { let word_index = (bit_index / BITS_PER_WORD) as u32; self.elems.get(word_index) } @@ -178,7 +227,7 @@ impl BitVec { pub fn get(&self, idx: usize) -> bool { let bit = idx % BITS_PER_WORD; if let Some(word) = self.maybe_elem(idx) { - (*word & (1 << bit)) != 0 + (word & (1 << bit)) != 0 } else { false } diff --git a/src/ion/mod.rs b/src/ion/mod.rs index fa28a76..a93ded7 100644 --- a/src/ion/mod.rs +++ b/src/ion/mod.rs @@ -267,6 +267,7 @@ struct Env<'a, F: Function> { env: &'a MachineEnv, cfginfo: CFGInfo, liveins: Vec, + livein_parents: Vec>, /// Blockparam outputs: from-vreg, (end of) from-block, (start of) /// to-block, to-vreg. The field order is significant: these are sorted so /// that a scan over vregs, then blocks in each range, can scan in @@ -663,6 +664,7 @@ impl<'a, F: Function> Env<'a, F> { cfginfo, liveins: vec![], + livein_parents: vec![], blockparam_outs: vec![], blockparam_ins: vec![], blockparam_allocs: vec![], @@ -1013,10 +1015,24 @@ impl<'a, F: Function> Env<'a, F> { .insert(LiveRangeKey::from_range(&range), lr); } + fn is_live_in(&mut self, block: Block, vreg: VRegIndex) -> bool { + if self.liveins[block.index()].get(vreg.index()) { + return true; + } + for &parent in &self.livein_parents[block.index()] { + if self.liveins[parent.index()].get(vreg.index()) { + self.liveins[block.index()].set(vreg.index(), true); + return true; + } + } + false + } + fn compute_liveness(&mut self) { // Create initial LiveIn bitsets. for _ in 0..self.func.blocks() { self.liveins.push(BitVec::new()); + self.livein_parents.push(vec![]); } let mut num_ranges = 0; @@ -1050,10 +1066,18 @@ impl<'a, F: Function> Env<'a, F> { // Init live-set to union of liveins from successors // (excluding backedges; those are handled below). - let mut live = BitVec::new(); + let mut live = None; for &succ in self.func.block_succs(block) { - live.or(&self.liveins[succ.index()]); + if block_to_postorder[succ.index()].is_none() { + continue; + } + if live.is_none() { + live = Some(self.liveins[succ.index()].clone()); + } else { + live.as_mut().unwrap().or(&self.liveins[succ.index()]); + } } + let mut live = live.unwrap_or(BitVec::new()); // Initially, registers are assumed live for the whole block. for vreg in live.iter() { @@ -1404,7 +1428,7 @@ impl<'a, F: Function> Env<'a, F> { ); log::debug!(" -> loop range {:?}", loop_range); for &loopblock in loop_blocks { - self.liveins[loopblock.index()].or(&live); + self.livein_parents[loopblock.index()].push(block); } for vreg in live.iter() { log::debug!( @@ -3404,7 +3428,7 @@ impl<'a, F: Function> Env<'a, F> { continue; } log::debug!(" -> out of this range, requires half-move if live"); - if self.liveins[succ.index()].get(vreg.index()) { + if self.is_live_in(succ, vreg) { log::debug!(" -> live at input to succ, adding halfmove"); half_moves.push(HalfMove { key: half_move_key(block, succ, vreg, HalfMoveKind::Source), @@ -3524,7 +3548,7 @@ impl<'a, F: Function> Env<'a, F> { blockparam_in_idx += 1; } - if !self.liveins[block.index()].get(vreg.index()) { + if !self.is_live_in(block, vreg) { block = block.next(); continue; }