diff --git a/Cargo.toml b/Cargo.toml index 5410cf1..de7c373 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "regalloc2" version = "0.0.1" -authors = ["Chris Fallin ", "Mozilla SpiderMonkey Developers"] +authors = [ + "Chris Fallin ", + "Mozilla SpiderMonkey Developers", +] edition = "2018" license = "Apache-2.0 WITH LLVM-exception" description = "Backtracking register allocator inspired from IonMonkey" @@ -17,9 +20,20 @@ slice-group-by = "0.3.0" # Keep this in sync with libfuzzer_sys's crate version: arbitrary = { version = "^0.4.6", optional = true } +# When testing regalloc2 by itself, enable debug assertions and overflow checks [profile.release] debug = true +debug-assertions = true +overflow-checks = true [features] default = [] -fuzzing = ["arbitrary"] + +# Enables generation of DefAlloc edits for the checker. +checker = [] + +# Enables detailed logging which can be somewhat expensive. +trace-log = [] + +# Exposes the internal API for fuzzing. +fuzzing = ["arbitrary", "checker", "trace-log"] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 199eb9d..4bf4e17 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -49,3 +49,9 @@ name = "ion_checker" path = "fuzz_targets/ion_checker.rs" test = false doc = false + +# Enable debug assertions and overflow checks when fuzzing +[profile.release] +debug = true +debug-assertions = true +overflow-checks = true diff --git a/src/checker.rs b/src/checker.rs index 9e81e27..323c66b 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -191,7 +191,7 @@ impl CheckerValue { CheckerValue::Reg(r1, ref1) } _ => { - log::trace!("{:?} and {:?} meet to Conflicted", self, other); + trace!("{:?} and {:?} meet to Conflicted", self, other); CheckerValue::Conflicted } } @@ -322,7 +322,7 @@ impl CheckerState { .get(alloc) .cloned() .unwrap_or(Default::default()); - log::trace!( + trace!( "checker: checkinst {:?}: op {:?}, alloc {:?}, checker value {:?}", checkinst, op, @@ -339,7 +339,7 @@ impl CheckerState { .get(&alloc) .cloned() .unwrap_or(Default::default()); - log::trace!( + trace!( "checker: checkinst {:?}: safepoint slot {}, checker value {:?}", checkinst, alloc, @@ -372,7 +372,7 @@ impl CheckerState { .get(&from) .cloned() .unwrap_or(Default::default()); - log::trace!( + trace!( "checker: checkinst {:?} updating: move {:?} -> {:?} val {:?}", checkinst, from, @@ -534,7 +534,7 @@ impl<'a, F: Function> Checker<'a, F> { /// Build the list of checker instructions based on the given func /// and allocation results. pub fn prepare(&mut self, out: &Output) { - log::trace!("checker: out = {:?}", out); + trace!("checker: out = {:?}", out); // Preprocess safepoint stack-maps into per-inst vecs. let mut safepoint_slots: HashMap> = HashMap::new(); for &(progpoint, slot) in &out.safepoint_slots { @@ -550,7 +550,7 @@ impl<'a, F: Function> Checker<'a, F> { for inst_or_edit in out.block_insts_and_edits(self.f, block) { match inst_or_edit { InstOrEdit::Inst(inst) => { - assert!(last_inst.is_none() || inst > last_inst.unwrap()); + debug_assert!(last_inst.is_none() || inst > last_inst.unwrap()); last_inst = Some(inst); self.handle_inst(block, inst, &mut safepoint_slots, out); } @@ -591,25 +591,19 @@ impl<'a, F: Function> Checker<'a, F> { allocs, clobbers, }; - log::trace!("checker: adding inst {:?}", checkinst); + trace!("checker: adding inst {:?}", checkinst); self.bb_insts.get_mut(&block).unwrap().push(checkinst); } } fn handle_edit(&mut self, block: Block, edit: &Edit) { - log::trace!("checker: adding edit {:?}", edit); + trace!("checker: adding edit {:?}", edit); match edit { - &Edit::Move { from, to, to_vreg } => { + &Edit::Move { from, to } => { self.bb_insts .get_mut(&block) .unwrap() .push(CheckerInst::Move { into: to, from }); - if let Some(vreg) = to_vreg { - self.bb_insts - .get_mut(&block) - .unwrap() - .push(CheckerInst::DefAlloc { alloc: to, vreg }); - } } &Edit::DefAlloc { alloc, vreg } => { self.bb_insts @@ -634,10 +628,10 @@ impl<'a, F: Function> Checker<'a, F> { let block = queue.pop_front().unwrap(); queue_set.remove(&block); let mut state = self.bb_in.get(&block).cloned().unwrap(); - log::trace!("analyze: block {} has state {:?}", block.index(), state); + trace!("analyze: block {} has state {:?}", block.index(), state); for inst in self.bb_insts.get(&block).unwrap() { state.update(inst, self); - log::trace!("analyze: inst {:?} -> state {:?}", inst, state); + trace!("analyze: inst {:?} -> state {:?}", inst, state); } for &succ in self.f.block_succs(block) { @@ -646,7 +640,7 @@ impl<'a, F: Function> Checker<'a, F> { new_state.meet_with(cur_succ_in); let changed = &new_state != cur_succ_in; if changed { - log::trace!( + trace!( "analyze: block {} state changed from {:?} to {:?}; pushing onto queue", succ.index(), cur_succ_in, @@ -671,12 +665,12 @@ impl<'a, F: Function> Checker<'a, F> { let mut state = input.clone(); for inst in self.bb_insts.get(block).unwrap() { if let Err(e) = state.check(InstPosition::Before, inst) { - log::trace!("Checker error: {:?}", e); + trace!("Checker error: {:?}", e); errors.push(e); } state.update(inst, self); if let Err(e) = state.check(InstPosition::After, inst) { - log::trace!("Checker error: {:?}", e); + trace!("Checker error: {:?}", e); errors.push(e); } } @@ -695,20 +689,20 @@ impl<'a, F: Function> Checker<'a, F> { self.analyze(); let result = self.find_errors(); - log::trace!("=== CHECKER RESULT ==="); + trace!("=== CHECKER RESULT ==="); fn print_state(state: &CheckerState) { let mut s = vec![]; for (alloc, state) in &state.allocations { s.push(format!("{} := {}", alloc, state)); } - log::trace!(" {{ {} }}", s.join(", ")) + trace!(" {{ {} }}", s.join(", ")) } for vreg in self.f.reftype_vregs() { - log::trace!(" REF: {}", vreg); + trace!(" REF: {}", vreg); } for bb in 0..self.f.num_blocks() { let bb = Block::new(bb); - log::trace!("block{}:", bb.index()); + trace!("block{}:", bb.index()); let insts = self.bb_insts.get(&bb).unwrap(); let mut state = self.bb_in.get(&bb).unwrap().clone(); print_state(&state); @@ -720,7 +714,7 @@ impl<'a, F: Function> Checker<'a, F> { ref allocs, ref clobbers, } => { - log::trace!( + trace!( " inst{}: {:?} ({:?}) clobbers:{:?}", inst.index(), operands, @@ -729,17 +723,17 @@ impl<'a, F: Function> Checker<'a, F> { ); } &CheckerInst::Move { from, into } => { - log::trace!(" {} -> {}", from, into); + trace!(" {} -> {}", from, into); } &CheckerInst::DefAlloc { alloc, vreg } => { - log::trace!(" defalloc: {}:{}", vreg, alloc); + trace!(" defalloc: {}:{}", vreg, alloc); } &CheckerInst::Safepoint { ref allocs, .. } => { let mut slotargs = vec![]; for &slot in allocs { slotargs.push(format!("{}", slot)); } - log::trace!(" safepoint: {}", slotargs.join(", ")); + trace!(" safepoint: {}", slotargs.join(", ")); } } state.update(inst, &self); diff --git a/src/domtree.rs b/src/domtree.rs index 4300e04..1bf4139 100644 --- a/src/domtree.rs +++ b/src/domtree.rs @@ -33,7 +33,7 @@ fn merge_sets( node2 = idom[node2.index()]; } } - assert!(node1 == node2); + debug_assert!(node1 == node2); node1 } diff --git a/src/fuzzing/func.rs b/src/fuzzing/func.rs index cbc9717..89445b4 100644 --- a/src/fuzzing/func.rs +++ b/src/fuzzing/func.rs @@ -79,7 +79,7 @@ impl Function for Func { } fn entry_block(&self) -> Block { - assert!(self.blocks.len() > 0); + debug_assert!(self.blocks.len() > 0); Block::new(0) } @@ -237,7 +237,7 @@ fn choose_dominating_block( allow_self: bool, u: &mut Unstructured, ) -> ArbitraryResult { - assert!(block.is_valid()); + debug_assert!(block.is_valid()); let orig_block = block; loop { if (allow_self || block != orig_block) && bool::arbitrary(u)? { @@ -445,7 +445,7 @@ impl Func { if operands.len() > 1 && opts.reused_inputs && bool::arbitrary(u)? { // Make the def a reused input. let op = operands[0]; - assert_eq!(op.kind(), OperandKind::Def); + debug_assert_eq!(op.kind(), OperandKind::Def); let reused = u.int_in_range(1..=(operands.len() - 1))?; operands[0] = Operand::new( op.vreg(), diff --git a/src/index.rs b/src/index.rs index 21dd976..4c376fa 100644 --- a/src/index.rs +++ b/src/index.rs @@ -10,7 +10,7 @@ macro_rules! define_index { } #[inline(always)] pub fn index(self) -> usize { - assert!(self.is_valid()); + debug_assert!(self.is_valid()); self.0 as usize } #[inline(always)] @@ -27,12 +27,12 @@ macro_rules! define_index { } #[inline(always)] pub fn next(self) -> $ix { - assert!(self.is_valid()); + debug_assert!(self.is_valid()); Self(self.0 + 1) } #[inline(always)] pub fn prev(self) -> $ix { - assert!(self.is_valid()); + debug_assert!(self.is_valid()); Self(self.0 - 1) } @@ -62,19 +62,19 @@ pub struct InstRange(Inst, Inst, bool); impl InstRange { #[inline(always)] pub fn forward(from: Inst, to: Inst) -> Self { - assert!(from.index() <= to.index()); + debug_assert!(from.index() <= to.index()); InstRange(from, to, true) } #[inline(always)] pub fn backward(from: Inst, to: Inst) -> Self { - assert!(from.index() >= to.index()); + debug_assert!(from.index() >= to.index()); InstRange(to, from, false) } #[inline(always)] pub fn first(self) -> Inst { - assert!(self.len() > 0); + debug_assert!(self.len() > 0); if self.is_forward() { self.0 } else { @@ -84,7 +84,7 @@ impl InstRange { #[inline(always)] pub fn last(self) -> Inst { - assert!(self.len() > 0); + debug_assert!(self.len() > 0); if self.is_forward() { self.1.prev() } else { @@ -94,7 +94,7 @@ impl InstRange { #[inline(always)] pub fn rest(self) -> InstRange { - assert!(self.len() > 0); + debug_assert!(self.len() > 0); if self.is_forward() { InstRange::forward(self.0.next(), self.1) } else { @@ -147,13 +147,13 @@ mod test { #[test] fn test_inst_range() { let range = InstRange::forward(Inst::new(0), Inst::new(0)); - assert_eq!(range.len(), 0); + debug_assert_eq!(range.len(), 0); let range = InstRange::forward(Inst::new(0), Inst::new(5)); - assert_eq!(range.first().index(), 0); - assert_eq!(range.last().index(), 4); - assert_eq!(range.len(), 5); - assert_eq!( + debug_assert_eq!(range.first().index(), 0); + debug_assert_eq!(range.last().index(), 4); + debug_assert_eq!(range.len(), 5); + debug_assert_eq!( range.iter().collect::>(), vec![ Inst::new(0), @@ -164,10 +164,10 @@ mod test { ] ); let range = range.rev(); - assert_eq!(range.first().index(), 4); - assert_eq!(range.last().index(), 0); - assert_eq!(range.len(), 5); - assert_eq!( + debug_assert_eq!(range.first().index(), 4); + debug_assert_eq!(range.last().index(), 0); + debug_assert_eq!(range.len(), 5); + debug_assert_eq!( range.iter().collect::>(), vec![ Inst::new(4), diff --git a/src/indexset.rs b/src/indexset.rs index 35d90dd..a5c263a 100644 --- a/src/indexset.rs +++ b/src/indexset.rs @@ -93,7 +93,7 @@ impl AdaptiveMap { }; if needs_expand { - assert!(small_mode_idx.is_none()); + debug_assert!(small_mode_idx.is_none()); self.expand(); } @@ -111,7 +111,7 @@ impl AdaptiveMap { } // Otherwise, the key must not be present; add a new // entry. - assert!(*len < SMALL_ELEMS as u32); + debug_assert!(*len < SMALL_ELEMS as u32); let idx = *len; *len += 1; keys[idx as usize] = key; @@ -344,11 +344,11 @@ mod test { let mut checksum = 0; for bit in vec.iter() { - assert!(bit % 17 == 0); + debug_assert!(bit % 17 == 0); checksum += bit; } - assert_eq!(sum, checksum); + debug_assert_eq!(sum, checksum); } #[test] @@ -362,6 +362,6 @@ mod test { // should still be in small mode. vec.set(64 * 5, false); vec.set(64 * 100, true); - assert!(vec.is_small()); + debug_assert!(vec.is_small()); } } diff --git a/src/ion/data_structures.rs b/src/ion/data_structures.rs index 09f7834..2bc4e3b 100644 --- a/src/ion/data_structures.rs +++ b/src/ion/data_structures.rs @@ -264,7 +264,6 @@ pub struct VRegData { #[derive(Clone, Debug)] pub struct PRegData { - pub reg: PReg, pub allocations: LiveRangeSet, pub is_stack: bool, } @@ -278,6 +277,56 @@ pub struct MultiFixedRegFixup { pub vreg: VRegIndex, } +/// The field order is significant: these are sorted so that a +/// scan over vregs, then blocks in each range, can scan in +/// order through this (sorted) list and add allocs to the +/// half-move list. +/// +/// The fields in this struct are reversed in sort order so that the entire +/// struct can be treated as a u128 for sorting purposes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub struct BlockparamOut { + pub to_vreg: VRegIndex, + pub to_block: Block, + pub from_block: Block, + pub from_vreg: VRegIndex, +} +impl BlockparamOut { + #[inline(always)] + pub fn key(&self) -> u128 { + u128_key( + self.from_vreg.raw_u32(), + self.from_block.raw_u32(), + self.to_block.raw_u32(), + self.to_vreg.raw_u32(), + ) + } +} + +/// As above for `BlockparamIn`, field order is significant. +/// +/// The fields in this struct are reversed in sort order so that the entire +/// struct can be treated as a u128 for sorting purposes. +#[derive(Clone, Debug)] +#[repr(C)] +pub struct BlockparamIn { + pub from_block: Block, + pub to_block: Block, + pub to_vreg: VRegIndex, +} +impl BlockparamIn { + #[inline(always)] + pub fn key(&self) -> u128 { + u128_key( + self.to_vreg.raw_u32(), + self.to_block.raw_u32(), + self.from_block.raw_u32(), + 0, + ) + } +} + #[derive(Clone, Debug)] pub struct Env<'a, F: Function> { pub func: &'a F, @@ -285,16 +334,8 @@ pub struct Env<'a, F: Function> { pub cfginfo: CFGInfo, pub liveins: Vec, pub liveouts: Vec, - /// Blockparam outputs: from-vreg, (end of) from-block, (start of) - /// to-block, to-vreg. The field order is significant: these are sorted so - /// that a scan over vregs, then blocks in each range, can scan in - /// order through this (sorted) list and add allocs to the - /// half-move list. - pub blockparam_outs: Vec<(VRegIndex, Block, Block, VRegIndex)>, - /// Blockparam inputs: to-vreg, (start of) to-block, (end of) - /// from-block. As above for `blockparam_outs`, field order is - /// significant. - pub blockparam_ins: Vec<(VRegIndex, Block, Block)>, + pub blockparam_outs: Vec, + pub blockparam_ins: Vec, /// Blockparam allocs: block, idx, vreg, alloc. Info to describe /// blockparam locations at block entry, for metadata purposes /// (e.g. for the checker). @@ -344,7 +385,7 @@ pub struct Env<'a, F: Function> { pub inserted_moves: Vec, // Output: - pub edits: Vec<(u32, InsertMovePrio, Edit)>, + pub edits: Vec<(PosWithPrio, Edit)>, pub allocs: Vec, pub inst_alloc_offsets: Vec, pub num_spillslots: u32, @@ -488,8 +529,7 @@ impl LiveRangeSet { #[derive(Clone, Debug)] pub struct InsertedMove { - pub pos: ProgPoint, - pub prio: InsertMovePrio, + pub pos_prio: PosWithPrio, pub from_alloc: Allocation, pub to_alloc: Allocation, pub to_vreg: Option, @@ -506,6 +546,21 @@ pub enum InsertMovePrio { OutEdgeMoves, } +/// The fields in this struct are reversed in sort order so that the entire +/// struct can be treated as a u64 for sorting purposes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub struct PosWithPrio { + pub prio: u32, + pub pos: ProgPoint, +} + +impl PosWithPrio { + pub fn key(self) -> u64 { + u64_key(self.pos.to_index(), self.prio) + } +} + #[derive(Clone, Copy, Debug, Default)] pub struct Stats { pub livein_blocks: usize, @@ -544,3 +599,17 @@ pub struct Stats { pub halfmoves_count: usize, pub edits_count: usize, } + +// Helper function for generating sorting keys. The order of arguments is from +// the most significant field to the least significant one. +// +// These work best when the fields are stored in reverse order in memory so that +// they can be loaded with a single u64 load on little-endian machines. +#[inline(always)] +pub fn u64_key(b: u32, a: u32) -> u64 { + a as u64 | (b as u64) << 32 +} +#[inline(always)] +pub fn u128_key(d: u32, c: u32, b: u32, a: u32) -> u128 { + a as u128 | (b as u128) << 32 | (c as u128) << 64 | (d as u128) << 96 +} diff --git a/src/ion/dump.rs b/src/ion/dump.rs index e7a8ed7..d57d59c 100644 --- a/src/ion/dump.rs +++ b/src/ion/dump.rs @@ -5,16 +5,16 @@ use crate::{Block, Function, ProgPoint}; impl<'a, F: Function> Env<'a, F> { pub fn dump_state(&self) { - log::trace!("Bundles:"); + trace!("Bundles:"); for (i, b) in self.bundles.iter().enumerate() { - log::trace!( + trace!( "bundle{}: spillset={:?} alloc={:?}", i, b.spillset, b.allocation ); for entry in &b.ranges { - log::trace!( + trace!( " * range {:?} -- {:?}: range{}", entry.range.from, entry.range.to, @@ -22,11 +22,11 @@ impl<'a, F: Function> Env<'a, F> { ); } } - log::trace!("VRegs:"); + trace!("VRegs:"); for (i, v) in self.vregs.iter().enumerate() { - log::trace!("vreg{}:", i); + trace!("vreg{}:", i); for entry in &v.ranges { - log::trace!( + trace!( " * range {:?} -- {:?}: range{}", entry.range.from, entry.range.to, @@ -34,9 +34,9 @@ impl<'a, F: Function> Env<'a, F> { ); } } - log::trace!("Ranges:"); + trace!("Ranges:"); for (i, r) in self.ranges.iter().enumerate() { - log::trace!( + trace!( "range{}: range={:?} vreg={:?} bundle={:?} weight={:?}", i, r.range, @@ -45,7 +45,7 @@ impl<'a, F: Function> Env<'a, F> { r.uses_spill_weight(), ); for u in &r.uses { - log::trace!(" * use at {:?} (slot {}): {:?}", u.pos, u.slot, u.operand); + trace!(" * use at {:?} (slot {}): {:?}", u.pos, u.slot, u.operand); } } } diff --git a/src/ion/liveranges.rs b/src/ion/liveranges.rs index 404b438..7676a2f 100644 --- a/src/ion/liveranges.rs +++ b/src/ion/liveranges.rs @@ -18,7 +18,7 @@ use super::{ SpillSetIndex, Use, VRegData, VRegIndex, SLOT_NONE, }; use crate::indexset::IndexSet; -use crate::ion::data_structures::MultiFixedRegFixup; +use crate::ion::data_structures::{BlockparamIn, BlockparamOut, MultiFixedRegFixup}; use crate::{ Allocation, Block, Function, Inst, InstPosition, Operand, OperandConstraint, OperandKind, OperandPos, PReg, ProgPoint, RegAllocError, VReg, @@ -102,17 +102,10 @@ impl<'a, F: Function> Env<'a, F> { self.pregs.resize( PReg::NUM_INDEX, PRegData { - reg: PReg::invalid(), allocations: LiveRangeSet::new(), is_stack: false, }, ); - for i in 0..=PReg::MAX { - let preg_int = PReg::new(i, RegClass::Int); - self.pregs[preg_int.index()].reg = preg_int; - let preg_float = PReg::new(i, RegClass::Float); - self.pregs[preg_float.index()].reg = preg_float; - } for &preg in &self.env.fixed_stack_slots { self.pregs[preg.index()].is_stack = true; } @@ -186,7 +179,7 @@ impl<'a, F: Function> Env<'a, F> { /// /// Returns the liverange that contains the given range. pub fn add_liverange_to_vreg(&mut self, vreg: VRegIndex, range: CodeRange) -> LiveRangeIndex { - log::trace!("add_liverange_to_vreg: vreg {:?} range {:?}", vreg, range); + trace!("add_liverange_to_vreg: vreg {:?} range {:?}", vreg, range); // Invariant: as we are building liveness information, we // *always* process instructions bottom-to-top, and as a @@ -199,7 +192,7 @@ impl<'a, F: Function> Env<'a, F> { // array, then reverse them at the end of // `compute_liveness()`. - assert!( + debug_assert!( self.vregs[vreg.index()].ranges.is_empty() || range.to <= self.ranges[self.vregs[vreg.index()] @@ -235,7 +228,7 @@ impl<'a, F: Function> Env<'a, F> { // Is contiguous with previously-added range; just extend // its range and return it. let lr = self.vregs[vreg.index()].ranges.last().unwrap().index; - assert!(range.to == self.ranges[lr.index()].range.from); + debug_assert!(range.to == self.ranges[lr.index()].range.from); self.ranges[lr.index()].range.from = range.from; lr } @@ -253,7 +246,7 @@ impl<'a, F: Function> Env<'a, F> { ); u.weight = weight.to_bits(); - log::trace!( + trace!( "insert use {:?} into lr {:?} with weight {:?}", u, into, @@ -269,7 +262,7 @@ impl<'a, F: Function> Env<'a, F> { // Update stats. let range_weight = self.ranges[into.index()].uses_spill_weight() + weight; self.ranges[into.index()].set_uses_spill_weight(range_weight); - log::trace!( + trace!( " -> now range has weight {:?}", self.ranges[into.index()].uses_spill_weight(), ); @@ -289,7 +282,7 @@ impl<'a, F: Function> Env<'a, F> { } pub fn add_liverange_to_preg(&mut self, range: CodeRange, reg: PReg) { - log::trace!("adding liverange to preg: {:?} to {}", range, reg); + trace!("adding liverange to preg: {:?} to {}", range, reg); let preg_idx = PRegIndex::new(reg.index()); self.pregs[preg_idx.index()] .allocations @@ -323,12 +316,12 @@ impl<'a, F: Function> Env<'a, F> { workqueue_set.remove(&block); let insns = self.func.block_insns(block); - log::trace!("computing liveins for block{}", block.index()); + trace!("computing liveins for block{}", block.index()); self.stats.livein_iterations += 1; let mut live = self.liveouts[block.index()].clone(); - log::trace!(" -> initial liveout set: {:?}", live); + trace!(" -> initial liveout set: {:?}", live); // Include outgoing blockparams in the initial live set. if self.func.is_branch(insns.last()) { @@ -349,7 +342,7 @@ impl<'a, F: Function> Env<'a, F> { for op in self.func.inst_operands(inst) { if op.pos() == *pos { let was_live = live.get(op.vreg().vreg()); - log::trace!("op {:?} was_live = {}", op, was_live); + trace!("op {:?} was_live = {}", op, was_live); match op.kind() { OperandKind::Use | OperandKind::Mod => { live.set(op.vreg().vreg(), true); @@ -375,7 +368,7 @@ impl<'a, F: Function> Env<'a, F> { } } - log::trace!("computed liveins at block{}: {:?}", block.index(), live); + trace!("computed liveins at block{}: {:?}", block.index(), live); self.liveins[block.index()] = live; } @@ -387,7 +380,7 @@ impl<'a, F: Function> Env<'a, F> { .next() .is_some() { - log::trace!( + trace!( "non-empty liveins to entry block: {:?}", self.liveins[self.func.entry_block().index()] ); @@ -437,8 +430,12 @@ impl<'a, F: Function> Env<'a, F> { { let blockparam_out = VRegIndex::new(blockparam_out.vreg()); let blockparam_in = VRegIndex::new(blockparam_in.vreg()); - self.blockparam_outs - .push((blockparam_out, block, succ, blockparam_in)); + self.blockparam_outs.push(BlockparamOut { + to_vreg: blockparam_in, + to_block: succ, + from_block: block, + from_vreg: blockparam_out, + }); // Include outgoing blockparams in the initial live set. live.set(blockparam_out.index(), true); @@ -452,7 +449,7 @@ impl<'a, F: Function> Env<'a, F> { from: self.cfginfo.block_entry[block.index()], to: self.cfginfo.block_exit[block.index()].next(), }; - log::trace!( + trace!( "vreg {:?} live at end of block --> create range {:?}", VRegIndex::new(vreg), range @@ -502,13 +499,13 @@ impl<'a, F: Function> Env<'a, F> { // We can completely skip the move if it is // trivial (vreg to same vreg). if src.vreg() != dst.vreg() { - log::trace!(" -> move inst{}: src {} -> dst {}", inst.index(), src, dst); + trace!(" -> move inst{}: src {} -> dst {}", inst.index(), src, dst); - assert_eq!(src.class(), dst.class()); - assert_eq!(src.kind(), OperandKind::Use); - assert_eq!(src.pos(), OperandPos::Early); - assert_eq!(dst.kind(), OperandKind::Def); - assert_eq!(dst.pos(), OperandPos::Late); + debug_assert_eq!(src.class(), dst.class()); + debug_assert_eq!(src.kind(), OperandKind::Use); + debug_assert_eq!(src.pos(), OperandPos::Early); + debug_assert_eq!(dst.kind(), OperandKind::Def); + debug_assert_eq!(dst.pos(), OperandPos::Late); // If both src and dest are pinned, emit the // move right here, right now. @@ -564,9 +561,7 @@ impl<'a, F: Function> Env<'a, F> { else if self.vregs[src.vreg().vreg()].is_pinned || self.vregs[dst.vreg().vreg()].is_pinned { - log::trace!( - " -> exactly one of src/dst is pinned; converting to ghost use" - ); + trace!(" -> exactly one of src/dst is pinned; converting to ghost use"); let (preg, vreg, pinned_vreg, kind, pos, progpoint) = if self.vregs[src.vreg().vreg()].is_pinned { // Source is pinned: this is a def on the dst with a pinned preg. @@ -592,7 +587,7 @@ impl<'a, F: Function> Env<'a, F> { let constraint = OperandConstraint::FixedReg(preg); let operand = Operand::new(vreg, constraint, kind, pos); - log::trace!( + trace!( concat!( " -> preg {:?} vreg {:?} kind {:?} ", "pos {:?} progpoint {:?} constraint {:?} operand {:?}" @@ -619,9 +614,9 @@ impl<'a, F: Function> Env<'a, F> { VRegIndex::new(vreg.vreg()), CodeRange { from, to }, ); - log::trace!(" -> dead; created LR"); + trace!(" -> dead; created LR"); } - log::trace!(" -> LR {:?}", lr); + trace!(" -> LR {:?}", lr); self.insert_use_into_liverange( lr, @@ -655,7 +650,7 @@ impl<'a, F: Function> Env<'a, F> { // (this is the last use), start it // before. if kind == OperandKind::Def { - log::trace!(" -> src on pinned vreg {:?}", pinned_vreg); + trace!(" -> src on pinned vreg {:?}", pinned_vreg); // The *other* vreg is a def, so the pinned-vreg // mention is a use. If already live, // end the existing LR just *after* @@ -669,7 +664,7 @@ impl<'a, F: Function> Env<'a, F> { if live.get(pinned_vreg.vreg()) { let pinned_lr = vreg_ranges[pinned_vreg.vreg()]; let orig_start = self.ranges[pinned_lr.index()].range.from; - log::trace!( + trace!( " -> live with LR {:?}; truncating to start at {:?}", pinned_lr, progpoint.next() @@ -683,7 +678,7 @@ impl<'a, F: Function> Env<'a, F> { }, ); vreg_ranges[pinned_vreg.vreg()] = new_lr; - log::trace!(" -> created LR {:?} with remaining range from {:?} to {:?}", new_lr, orig_start, progpoint); + trace!(" -> created LR {:?} with remaining range from {:?} to {:?}", new_lr, orig_start, progpoint); // Add an edit right now to indicate that at // this program point, the given @@ -717,10 +712,7 @@ impl<'a, F: Function> Env<'a, F> { ); vreg_ranges[pinned_vreg.vreg()] = new_lr; live.set(pinned_vreg.vreg(), true); - log::trace!( - " -> was not live; created new LR {:?}", - new_lr - ); + trace!(" -> was not live; created new LR {:?}", new_lr); } // Add an edit right now to indicate that at @@ -737,7 +729,7 @@ impl<'a, F: Function> Env<'a, F> { ); } } else { - log::trace!(" -> dst on pinned vreg {:?}", pinned_vreg); + trace!(" -> dst on pinned vreg {:?}", pinned_vreg); // The *other* vreg is a use, so the pinned-vreg // mention is a def. Truncate its LR // just *after* the `progpoint` @@ -745,7 +737,7 @@ impl<'a, F: Function> Env<'a, F> { if live.get(pinned_vreg.vreg()) { let pinned_lr = vreg_ranges[pinned_vreg.vreg()]; self.ranges[pinned_lr.index()].range.from = progpoint.next(); - log::trace!( + trace!( " -> was live with LR {:?}; truncated start to {:?}", pinned_lr, progpoint.next() @@ -845,14 +837,14 @@ impl<'a, F: Function> Env<'a, F> { VRegIndex::new(dst.vreg().vreg()), CodeRange { from, to }, ); - log::trace!(" -> invalid LR for def; created {:?}", dst_lr); + trace!(" -> invalid LR for def; created {:?}", dst_lr); } - log::trace!(" -> has existing LR {:?}", dst_lr); + trace!(" -> has existing LR {:?}", dst_lr); // Trim the LR to start here. if self.ranges[dst_lr.index()].range.from == self.cfginfo.block_entry[block.index()] { - log::trace!(" -> started at block start; trimming to {:?}", pos); + trace!(" -> started at block start; trimming to {:?}", pos); self.ranges[dst_lr.index()].range.from = pos; } self.ranges[dst_lr.index()].set_flag(LiveRangeFlag::StartsAtDef); @@ -879,7 +871,7 @@ impl<'a, F: Function> Env<'a, F> { vreg_ranges[src.vreg().vreg()] }; - log::trace!(" -> src LR {:?}", src_lr); + trace!(" -> src LR {:?}", src_lr); // Add to live-set. let src_is_dead_after_move = !live.get(src.vreg().vreg()); @@ -932,7 +924,7 @@ impl<'a, F: Function> Env<'a, F> { continue; } - log::trace!( + trace!( "processing inst{} operand at {:?}: {:?}", inst.index(), pos, @@ -941,14 +933,14 @@ impl<'a, F: Function> Env<'a, F> { match operand.kind() { OperandKind::Def | OperandKind::Mod => { - log::trace!("Def of {} at {:?}", operand.vreg(), pos); + trace!("Def of {} at {:?}", operand.vreg(), pos); // Fill in vreg's actual data. self.vreg_regs[operand.vreg().vreg()] = operand.vreg(); // Get or create the LiveRange. let mut lr = vreg_ranges[operand.vreg().vreg()]; - log::trace!(" -> has existing LR {:?}", lr); + trace!(" -> has existing LR {:?}", lr); // If there was no liverange (dead def), create a trivial one. if !live.get(operand.vreg().vreg()) { let from = match operand.kind() { @@ -965,7 +957,7 @@ impl<'a, F: Function> Env<'a, F> { VRegIndex::new(operand.vreg().vreg()), CodeRange { from, to }, ); - log::trace!(" -> invalid; created {:?}", lr); + trace!(" -> invalid; created {:?}", lr); vreg_ranges[operand.vreg().vreg()] = lr; live.set(operand.vreg().vreg(), true); } @@ -982,10 +974,7 @@ impl<'a, F: Function> Env<'a, F> { if self.ranges[lr.index()].range.from == self.cfginfo.block_entry[block.index()] { - log::trace!( - " -> started at block start; trimming to {:?}", - pos - ); + trace!(" -> started at block start; trimming to {:?}", pos); self.ranges[lr.index()].range.from = pos; } @@ -1012,9 +1001,9 @@ impl<'a, F: Function> Env<'a, F> { ); vreg_ranges[operand.vreg().vreg()] = lr; } - assert!(lr.is_valid()); + debug_assert!(lr.is_valid()); - log::trace!("Use of {:?} at {:?} -> {:?}", operand, pos, lr,); + trace!("Use of {:?} at {:?} -> {:?}", operand, pos, lr,); self.insert_use_into_liverange(lr, Use::new(operand, pos, i as u8)); @@ -1026,11 +1015,11 @@ impl<'a, F: Function> Env<'a, F> { } if self.func.requires_refs_on_stack(inst) { - log::trace!("inst{} is safepoint", inst.index()); + trace!("inst{} is safepoint", inst.index()); self.safepoints.push(inst); for vreg in live.iter() { if let Some(safepoints) = self.safepoints_per_vreg.get_mut(&vreg) { - log::trace!("vreg v{} live at safepoint inst{}", vreg, inst.index()); + trace!("vreg v{} live at safepoint inst{}", vreg, inst.index()); safepoints.insert(inst); } } @@ -1057,7 +1046,11 @@ impl<'a, F: Function> Env<'a, F> { // add `blockparam_ins` entries. let vreg_idx = VRegIndex::new(vreg.vreg()); for &pred in self.func.block_preds(block) { - self.blockparam_ins.push((vreg_idx, block, pred)); + self.blockparam_ins.push(BlockparamIn { + to_vreg: vreg_idx, + to_block: block, + from_block: pred, + }); } } } @@ -1082,7 +1075,7 @@ impl<'a, F: Function> Env<'a, F> { // need to update with the final range here. entry.range = self.ranges[entry.index.index()].range; // Assert in-order and non-overlapping. - assert!(last.is_none() || last.unwrap() <= entry.range.from); + debug_assert!(last.is_none() || last.unwrap() <= entry.range.from); last = Some(entry.range.to); } } @@ -1123,7 +1116,7 @@ impl<'a, F: Function> Env<'a, F> { OperandPos::Early, ); - log::trace!( + trace!( "Safepoint-induced stack use of {:?} at {:?} -> {:?}", operand, pos, @@ -1148,13 +1141,13 @@ impl<'a, F: Function> Env<'a, F> { } } - self.blockparam_ins.sort_unstable(); - self.blockparam_outs.sort_unstable(); + self.blockparam_ins.sort_unstable_by_key(|x| x.key()); + self.blockparam_outs.sort_unstable_by_key(|x| x.key()); self.prog_move_srcs.sort_unstable_by_key(|(pos, _)| *pos); self.prog_move_dsts.sort_unstable_by_key(|(pos, _)| *pos); - log::trace!("prog_move_srcs = {:?}", self.prog_move_srcs); - log::trace!("prog_move_dsts = {:?}", self.prog_move_dsts); + trace!("prog_move_srcs = {:?}", self.prog_move_srcs); + trace!("prog_move_dsts = {:?}", self.prog_move_dsts); self.stats.initial_liverange_count = self.ranges.len(); self.stats.blockparam_ins_count = self.blockparam_ins.len(); @@ -1176,7 +1169,7 @@ impl<'a, F: Function> Env<'a, F> { for range_idx in 0..self.vregs[vreg].ranges.len() { let entry = self.vregs[vreg].ranges[range_idx]; let range = entry.index; - log::trace!( + trace!( "multi-fixed-reg cleanup: vreg {:?} range {:?}", VRegIndex::new(vreg), range, @@ -1245,7 +1238,7 @@ impl<'a, F: Function> Env<'a, F> { if let OperandConstraint::FixedReg(preg) = u.operand.constraint() { let vreg_idx = VRegIndex::new(u.operand.vreg().vreg()); let preg_idx = PRegIndex::new(preg.index()); - log::trace!( + trace!( "at pos {:?}, vreg {:?} has fixed constraint to preg {:?}", u.pos, vreg_idx, @@ -1264,7 +1257,7 @@ impl<'a, F: Function> Env<'a, F> { continue; } - log::trace!(" -> duplicate; switching to constraint Any"); + trace!(" -> duplicate; switching to constraint Any"); self.multi_fixed_reg_fixups.push(MultiFixedRegFixup { pos: u.pos, from_slot: source_slot, @@ -1278,11 +1271,7 @@ impl<'a, F: Function> Env<'a, F> { u.operand.kind(), u.operand.pos(), ); - log::trace!( - " -> extra clobber {} at inst{}", - preg, - u.pos.inst().index() - ); + trace!(" -> extra clobber {} at inst{}", preg, u.pos.inst().index()); extra_clobbers.push((preg, u.pos.inst())); } } diff --git a/src/ion/merge.rs b/src/ion/merge.rs index a8e1fe5..89ec386 100644 --- a/src/ion/merge.rs +++ b/src/ion/merge.rs @@ -16,7 +16,7 @@ use super::{ Env, LiveBundleIndex, LiveRangeIndex, LiveRangeKey, SpillSet, SpillSetIndex, SpillSlotIndex, VRegIndex, }; -use crate::{Function, Inst, OperandConstraint, PReg}; +use crate::{ion::data_structures::BlockparamOut, Function, Inst, OperandConstraint, PReg}; use smallvec::smallvec; impl<'a, F: Function> Env<'a, F> { @@ -25,7 +25,7 @@ impl<'a, F: Function> Env<'a, F> { // Merge bundle into self -- trivial merge. return true; } - log::trace!( + trace!( "merging from bundle{} to bundle{}", from.index(), to.index() @@ -35,7 +35,7 @@ impl<'a, F: Function> Env<'a, F> { let from_rc = self.spillsets[self.bundles[from.index()].spillset.index()].class; let to_rc = self.spillsets[self.bundles[to.index()].spillset.index()].class; if from_rc != to_rc { - log::trace!(" -> mismatching reg classes"); + trace!(" -> mismatching reg classes"); return false; } @@ -43,7 +43,7 @@ impl<'a, F: Function> Env<'a, F> { if self.bundles[from.index()].allocation.is_some() || self.bundles[to.index()].allocation.is_some() { - log::trace!("one of the bundles is already assigned (pinned)"); + trace!("one of the bundles is already assigned (pinned)"); return false; } @@ -52,11 +52,11 @@ impl<'a, F: Function> Env<'a, F> { // Sanity check: both bundles should contain only ranges with appropriate VReg classes. for entry in &self.bundles[from.index()].ranges { let vreg = self.ranges[entry.index.index()].vreg; - assert_eq!(from_rc, self.vreg_regs[vreg.index()].class()); + debug_assert_eq!(from_rc, self.vreg_regs[vreg.index()].class()); } for entry in &self.bundles[to.index()].ranges { let vreg = self.ranges[entry.index.index()].vreg; - assert_eq!(to_rc, self.vreg_regs[vreg.index()].class()); + debug_assert_eq!(to_rc, self.vreg_regs[vreg.index()].class()); } } @@ -70,7 +70,7 @@ impl<'a, F: Function> Env<'a, F> { while idx_from < ranges_from.len() && idx_to < ranges_to.len() { range_count += 1; if range_count > 200 { - log::trace!( + trace!( "reached merge complexity (range_count = {}); exiting", range_count ); @@ -84,7 +84,7 @@ impl<'a, F: Function> Env<'a, F> { idx_from += 1; } else { // Overlap -- cannot merge. - log::trace!( + trace!( " -> overlap between {:?} and {:?}, exiting", ranges_from[idx_from].index, ranges_to[idx_to].index @@ -100,12 +100,12 @@ impl<'a, F: Function> Env<'a, F> { || self.bundles[to.index()].cached_fixed() { if self.merge_bundle_requirements(from, to).is_err() { - log::trace!(" -> conflicting requirements; aborting merge"); + trace!(" -> conflicting requirements; aborting merge"); return false; } } - log::trace!(" -> committing to merge"); + trace!(" -> committing to merge"); // If we reach here, then the bundles do not overlap -- merge // them! We do this with a merge-sort-like scan over both @@ -113,13 +113,13 @@ impl<'a, F: Function> Env<'a, F> { // `to` when we're done. if ranges_from.is_empty() { // `from` bundle is empty -- trivial merge. - log::trace!(" -> from bundle{} is empty; trivial merge", from.index()); + trace!(" -> from bundle{} is empty; trivial merge", from.index()); return true; } if ranges_to.is_empty() { // `to` bundle is empty -- just move the list over from // `from` and set `bundle` up-link on all ranges. - log::trace!(" -> to bundle{} is empty; trivial merge", to.index()); + trace!(" -> to bundle{} is empty; trivial merge", to.index()); let list = std::mem::replace(&mut self.bundles[from.index()].ranges, smallvec![]); for entry in &list { self.ranges[entry.index.index()].bundle = to; @@ -149,7 +149,7 @@ impl<'a, F: Function> Env<'a, F> { return true; } - log::trace!( + trace!( "merging: ranges_from = {:?} ranges_to = {:?}", ranges_from, ranges_to @@ -170,12 +170,12 @@ impl<'a, F: Function> Env<'a, F> { .sort_unstable_by_key(|entry| entry.range.from); if self.annotations_enabled { - log::trace!("merging: merged = {:?}", self.bundles[to.index()].ranges); + trace!("merging: merged = {:?}", self.bundles[to.index()].ranges); let mut last_range = None; for i in 0..self.bundles[to.index()].ranges.len() { let entry = self.bundles[to.index()].ranges[i]; if last_range.is_some() { - assert!(last_range.unwrap() < entry.range); + debug_assert!(last_range.unwrap() < entry.range); } last_range = Some(entry.range); @@ -192,7 +192,7 @@ impl<'a, F: Function> Env<'a, F> { ); } - log::trace!( + trace!( " -> merged result for bundle{}: range{}", to.index(), entry.index.index(), @@ -225,7 +225,7 @@ impl<'a, F: Function> Env<'a, F> { pub fn merge_vreg_bundles(&mut self) { // Create a bundle for every vreg, initially. - log::trace!("merge_vreg_bundles: creating vreg bundles"); + trace!("merge_vreg_bundles: creating vreg bundles"); for vreg in 0..self.vregs.len() { let vreg = VRegIndex::new(vreg); if self.vregs[vreg.index()].ranges.is_empty() { @@ -251,9 +251,9 @@ impl<'a, F: Function> Env<'a, F> { let bundle = self.create_bundle(); self.bundles[bundle.index()].ranges = self.vregs[vreg.index()].ranges.clone(); - log::trace!("vreg v{} gets bundle{}", vreg.index(), bundle.index()); + trace!("vreg v{} gets bundle{}", vreg.index(), bundle.index()); for entry in &self.bundles[bundle.index()].ranges { - log::trace!( + trace!( " -> with LR range{}: {:?}", entry.index.index(), entry.range @@ -314,17 +314,17 @@ impl<'a, F: Function> Env<'a, F> { continue; } - log::trace!( + trace!( "trying to merge reused-input def: src {} to dst {}", src_vreg, dst_vreg ); let src_bundle = self.ranges[self.vregs[src_vreg.vreg()].ranges[0].index.index()].bundle; - assert!(src_bundle.is_valid()); + debug_assert!(src_bundle.is_valid()); let dest_bundle = self.ranges[self.vregs[dst_vreg.vreg()].ranges[0].index.index()].bundle; - assert!(dest_bundle.is_valid()); + debug_assert!(dest_bundle.is_valid()); self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle); } } @@ -332,18 +332,20 @@ impl<'a, F: Function> Env<'a, F> { // Attempt to merge blockparams with their inputs. for i in 0..self.blockparam_outs.len() { - let (from_vreg, _, _, to_vreg) = self.blockparam_outs[i]; - log::trace!( + let BlockparamOut { + from_vreg, to_vreg, .. + } = self.blockparam_outs[i]; + trace!( "trying to merge blockparam v{} with input v{}", to_vreg.index(), from_vreg.index() ); let to_bundle = self.ranges[self.vregs[to_vreg.index()].ranges[0].index.index()].bundle; - assert!(to_bundle.is_valid()); + debug_assert!(to_bundle.is_valid()); let from_bundle = self.ranges[self.vregs[from_vreg.index()].ranges[0].index.index()].bundle; - assert!(from_bundle.is_valid()); - log::trace!( + debug_assert!(from_bundle.is_valid()); + trace!( " -> from bundle{} to bundle{}", from_bundle.index(), to_bundle.index() @@ -354,10 +356,10 @@ impl<'a, F: Function> Env<'a, F> { // Attempt to merge move srcs/dsts. for i in 0..self.prog_move_merges.len() { let (src, dst) = self.prog_move_merges[i]; - log::trace!("trying to merge move src LR {:?} to dst LR {:?}", src, dst); + trace!("trying to merge move src LR {:?} to dst LR {:?}", src, dst); let src = self.resolve_merged_lr(src); let dst = self.resolve_merged_lr(dst); - log::trace!( + trace!( "resolved LR-construction merging chains: move-merge is now src LR {:?} to dst LR {:?}", src, dst @@ -384,16 +386,16 @@ impl<'a, F: Function> Env<'a, F> { } let src_bundle = self.ranges[src.index()].bundle; - assert!(src_bundle.is_valid()); + debug_assert!(src_bundle.is_valid()); let dest_bundle = self.ranges[dst.index()].bundle; - assert!(dest_bundle.is_valid()); + debug_assert!(dest_bundle.is_valid()); self.stats.prog_move_merge_attempt += 1; if self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle) { self.stats.prog_move_merge_success += 1; } } - log::trace!("done merging bundles"); + trace!("done merging bundles"); } pub fn resolve_merged_lr(&self, mut lr: LiveRangeIndex) -> LiveRangeIndex { @@ -417,14 +419,14 @@ impl<'a, F: Function> Env<'a, F> { pub fn queue_bundles(&mut self) { for bundle in 0..self.bundles.len() { - log::trace!("enqueueing bundle{}", bundle); + trace!("enqueueing bundle{}", bundle); if self.bundles[bundle].ranges.is_empty() { - log::trace!(" -> no ranges; skipping"); + trace!(" -> no ranges; skipping"); continue; } let bundle = LiveBundleIndex::new(bundle); let prio = self.compute_bundle_prio(bundle); - log::trace!(" -> prio {}", prio); + trace!(" -> prio {}", prio); self.bundles[bundle.index()].prio = prio; self.recompute_bundle_properties(bundle); self.allocation_queue diff --git a/src/ion/mod.rs b/src/ion/mod.rs index 5903f58..5cd8945 100644 --- a/src/ion/mod.rs +++ b/src/ion/mod.rs @@ -133,7 +133,7 @@ pub fn run( edits: env .edits .into_iter() - .map(|(pos, _, edit)| (ProgPoint::from_index(pos), edit)) + .map(|(pos_prio, edit)| (pos_prio.pos, edit)) .collect(), allocs: env.allocs, inst_alloc_offsets: env.inst_alloc_offsets, diff --git a/src/ion/moves.rs b/src/ion/moves.rs index 9f913c4..0002cb5 100644 --- a/src/ion/moves.rs +++ b/src/ion/moves.rs @@ -17,10 +17,11 @@ use super::{ VRegIndex, SLOT_NONE, }; +use crate::ion::data_structures::{BlockparamIn, BlockparamOut, PosWithPrio}; use crate::moves::ParallelMoves; use crate::{ Allocation, Block, Edit, Function, Inst, InstPosition, OperandConstraint, OperandKind, - OperandPos, ProgPoint, RegClass, VReg, + OperandPos, PReg, ProgPoint, RegClass, VReg, }; use smallvec::{smallvec, SmallVec}; use std::fmt::Debug; @@ -53,7 +54,7 @@ impl<'a, F: Function> Env<'a, F> { to_alloc: Allocation, to_vreg: Option, ) { - log::trace!( + trace!( "insert_move: pos {:?} prio {:?} from_alloc {:?} to_alloc {:?}", pos, prio, @@ -62,13 +63,15 @@ impl<'a, F: Function> Env<'a, F> { ); match (from_alloc.as_reg(), to_alloc.as_reg()) { (Some(from), Some(to)) => { - assert_eq!(from.class(), to.class()); + debug_assert_eq!(from.class(), to.class()); } _ => {} } self.inserted_moves.push(InsertedMove { - pos, - prio, + pos_prio: PosWithPrio { + pos, + prio: prio as u32, + }, from_alloc, to_alloc, to_vreg, @@ -86,16 +89,16 @@ impl<'a, F: Function> Env<'a, F> { } pub fn get_alloc_for_range(&self, range: LiveRangeIndex) -> Allocation { - log::trace!("get_alloc_for_range: {:?}", range); + trace!("get_alloc_for_range: {:?}", range); let bundle = self.ranges[range.index()].bundle; - log::trace!(" -> bundle: {:?}", bundle); + trace!(" -> bundle: {:?}", bundle); let bundledata = &self.bundles[bundle.index()]; - log::trace!(" -> allocation {:?}", bundledata.allocation); + trace!(" -> allocation {:?}", bundledata.allocation); if bundledata.allocation != Allocation::none() { bundledata.allocation } else { - log::trace!(" -> spillset {:?}", bundledata.spillset); - log::trace!( + trace!(" -> spillset {:?}", bundledata.spillset); + trace!( " -> spill slot {:?}", self.spillsets[bundledata.spillset.index()].slot ); @@ -104,9 +107,9 @@ impl<'a, F: Function> Env<'a, F> { } pub fn apply_allocations_and_insert_moves(&mut self) { - log::trace!("apply_allocations_and_insert_moves"); - log::trace!("blockparam_ins: {:?}", self.blockparam_ins); - log::trace!("blockparam_outs: {:?}", self.blockparam_outs); + trace!("apply_allocations_and_insert_moves"); + trace!("blockparam_ins: {:?}", self.blockparam_ins); + trace!("blockparam_outs: {:?}", self.blockparam_outs); // Now that all splits are done, we can pay the cost once to // sort VReg range lists and update with the final ranges. @@ -148,9 +151,9 @@ impl<'a, F: Function> Env<'a, F> { to_vreg: VRegIndex, kind: HalfMoveKind, ) -> u64 { - assert!(from_block.index() < 1 << 21); - assert!(to_block.index() < 1 << 21); - assert!(to_vreg.index() < 1 << 21); + debug_assert!(from_block.index() < 1 << 21); + debug_assert!(to_block.index() < 1 << 21); + debug_assert!(to_vreg.index() < 1 << 21); ((from_block.index() as u64) << 43) | ((to_block.index() as u64) << 22) | ((to_vreg.index() as u64) << 1) @@ -202,7 +205,7 @@ impl<'a, F: Function> Env<'a, F> { .map(|preg| Allocation::reg(preg)) .unwrap_or_else(|| self.get_alloc_for_range(entry.index)); let range = entry.range; - log::trace!( + trace!( "apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?} (pinned {:?})", vreg, entry.index, @@ -269,7 +272,7 @@ impl<'a, F: Function> Env<'a, F> { && !self.is_start_of_block(range.from) && !first_is_def { - log::trace!( + trace!( "prev LR {} abuts LR {} in same block; moving {} -> {} for v{}", prev.index(), entry.index.index(), @@ -277,7 +280,7 @@ impl<'a, F: Function> Env<'a, F> { alloc, vreg.index() ); - assert_eq!(range.from.pos(), InstPosition::Before); + debug_assert_eq!(range.from.pos(), InstPosition::Before); self.insert_move( range.from, InsertMovePrio::Regular, @@ -303,9 +306,9 @@ impl<'a, F: Function> Env<'a, F> { if range.to < self.cfginfo.block_exit[block.index()].next() { break; } - log::trace!("examining block with end in range: block{}", block.index()); + trace!("examining block with end in range: block{}", block.index()); for &succ in self.func.block_succs(block) { - log::trace!( + trace!( " -> has succ block {} with entry {:?}", succ.index(), self.cfginfo.block_entry[succ.index()] @@ -313,9 +316,9 @@ impl<'a, F: Function> Env<'a, F> { if range.contains_point(self.cfginfo.block_entry[succ.index()]) { continue; } - log::trace!(" -> out of this range, requires half-move if live"); + trace!(" -> out of this range, requires half-move if live"); if self.is_live_in(succ, vreg) { - log::trace!(" -> live at input to succ, adding halfmove"); + trace!(" -> live at input to succ, adding halfmove"); half_moves.push(HalfMove { key: half_move_key(block, succ, vreg, HalfMoveKind::Source), alloc, @@ -326,20 +329,24 @@ impl<'a, F: Function> Env<'a, F> { // Scan forward in `blockparam_outs`, adding all // half-moves for outgoing values to blockparams // in succs. - log::trace!( + trace!( "scanning blockparam_outs for v{} block{}: blockparam_out_idx = {}", vreg.index(), block.index(), blockparam_out_idx, ); while blockparam_out_idx < self.blockparam_outs.len() { - let (from_vreg, from_block, to_block, to_vreg) = - self.blockparam_outs[blockparam_out_idx]; + let BlockparamOut { + from_vreg, + from_block, + to_block, + to_vreg, + } = self.blockparam_outs[blockparam_out_idx]; if (from_vreg, from_block) > (vreg, block) { break; } if (from_vreg, from_block) == (vreg, block) { - log::trace!( + trace!( " -> found: from v{} block{} to v{} block{}", from_vreg.index(), from_block.index(), @@ -391,15 +398,18 @@ impl<'a, F: Function> Env<'a, F> { } // Add half-moves for blockparam inputs. - log::trace!( + trace!( "scanning blockparam_ins at vreg {} block {}: blockparam_in_idx = {}", vreg.index(), block.index(), blockparam_in_idx ); while blockparam_in_idx < self.blockparam_ins.len() { - let (to_vreg, to_block, from_block) = - self.blockparam_ins[blockparam_in_idx]; + let BlockparamIn { + from_block, + to_block, + to_vreg, + } = self.blockparam_ins[blockparam_in_idx]; if (to_vreg, to_block) > (vreg, block) { break; } @@ -413,7 +423,7 @@ impl<'a, F: Function> Env<'a, F> { ), alloc, }); - log::trace!( + trace!( "match: blockparam_in: v{} in block{} from block{} into {}", to_vreg.index(), to_block.index(), @@ -444,7 +454,7 @@ impl<'a, F: Function> Env<'a, F> { continue; } - log::trace!( + trace!( "scanning preds at vreg {} block {} for ends outside the range", vreg.index(), block.index() @@ -453,7 +463,7 @@ impl<'a, F: Function> Env<'a, F> { // Now find any preds whose ends are not in the // same range, and insert appropriate moves. for &pred in self.func.block_preds(block) { - log::trace!( + trace!( "pred block {} has exit {:?}", pred.index(), self.cfginfo.block_exit[pred.index()] @@ -461,7 +471,7 @@ impl<'a, F: Function> Env<'a, F> { if range.contains_point(self.cfginfo.block_exit[pred.index()]) { continue; } - log::trace!(" -> requires half-move"); + trace!(" -> requires half-move"); half_moves.push(HalfMove { key: half_move_key(pred, block, vreg, HalfMoveKind::Dest), alloc, @@ -471,26 +481,30 @@ impl<'a, F: Function> Env<'a, F> { block = block.next(); } - // If this is a blockparam vreg and the start of block - // is in this range, add to blockparam_allocs. - let (blockparam_block, blockparam_idx) = - self.cfginfo.vreg_def_blockparam[vreg.index()]; - if blockparam_block.is_valid() - && range.contains_point(self.cfginfo.block_entry[blockparam_block.index()]) + #[cfg(feature = "checker")] { - self.blockparam_allocs.push(( - blockparam_block, - blockparam_idx, - vreg, - alloc, - )); + // If this is a blockparam vreg and the start of block + // is in this range, add to blockparam_allocs. + let (blockparam_block, blockparam_idx) = + self.cfginfo.vreg_def_blockparam[vreg.index()]; + if blockparam_block.is_valid() + && range + .contains_point(self.cfginfo.block_entry[blockparam_block.index()]) + { + self.blockparam_allocs.push(( + blockparam_block, + blockparam_idx, + vreg, + alloc, + )); + } } } // Scan over def/uses and apply allocations. for use_idx in 0..self.ranges[entry.index.index()].uses.len() { let usedata = self.ranges[entry.index.index()].uses[use_idx]; - log::trace!("applying to use: {:?}", usedata); + trace!("applying to use: {:?}", usedata); debug_assert!(range.contains_point(usedata.pos)); let inst = usedata.pos.inst(); let slot = usedata.slot; @@ -519,7 +533,7 @@ impl<'a, F: Function> Env<'a, F> { // covers only prev inst's After; so includes move // srcs to (exclusive) inst. let move_src_end = (vreg, range.to.inst()); - log::trace!( + trace!( "vreg {:?} range {:?}: looking for program-move sources from {:?} to {:?}", vreg, range, @@ -529,13 +543,13 @@ impl<'a, F: Function> Env<'a, F> { while prog_move_src_idx < self.prog_move_srcs.len() && self.prog_move_srcs[prog_move_src_idx].0 < move_src_start { - log::trace!(" -> skipping idx {}", prog_move_src_idx); + trace!(" -> skipping idx {}", prog_move_src_idx); prog_move_src_idx += 1; } while prog_move_src_idx < self.prog_move_srcs.len() && self.prog_move_srcs[prog_move_src_idx].0 < move_src_end { - log::trace!( + trace!( " -> setting idx {} ({:?}) to alloc {:?}", prog_move_src_idx, self.prog_move_srcs[prog_move_src_idx].0, @@ -562,7 +576,7 @@ impl<'a, F: Function> Env<'a, F> { } else { (vreg, range.to.inst().next()) }; - log::trace!( + trace!( "vreg {:?} range {:?}: looking for program-move dests from {:?} to {:?}", vreg, range, @@ -572,13 +586,13 @@ impl<'a, F: Function> Env<'a, F> { while prog_move_dst_idx < self.prog_move_dsts.len() && self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_start { - log::trace!(" -> skipping idx {}", prog_move_dst_idx); + trace!(" -> skipping idx {}", prog_move_dst_idx); prog_move_dst_idx += 1; } while prog_move_dst_idx < self.prog_move_dsts.len() && self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_end { - log::trace!( + trace!( " -> setting idx {} ({:?}) to alloc {:?}", prog_move_dst_idx, self.prog_move_dsts[prog_move_dst_idx].0, @@ -596,7 +610,7 @@ impl<'a, F: Function> Env<'a, F> { // from-vreg) tuple, find the from-alloc and all the // to-allocs, and insert moves on the block edge. half_moves.sort_unstable_by_key(|h| h.key); - log::trace!("halfmoves: {:?}", half_moves); + trace!("halfmoves: {:?}", half_moves); self.stats.halfmoves_count = half_moves.len(); let mut i = 0; @@ -619,7 +633,7 @@ impl<'a, F: Function> Env<'a, F> { } let last_dest = i; - log::trace!( + trace!( "halfmove match: src {:?} dests {:?}", src, &half_moves[first_dest..last_dest] @@ -697,8 +711,8 @@ impl<'a, F: Function> Env<'a, F> { // Handle multi-fixed-reg constraints by copying. for fixup in std::mem::replace(&mut self.multi_fixed_reg_fixups, vec![]) { let from_alloc = self.get_alloc(fixup.pos.inst(), fixup.from_slot as usize); - let to_alloc = Allocation::reg(self.pregs[fixup.to_preg.index()].reg); - log::trace!( + let to_alloc = Allocation::reg(PReg::from_index(fixup.to_preg.index())); + trace!( "multi-fixed-move constraint at {:?} from {} to {} for v{}", fixup.pos, from_alloc, @@ -715,7 +729,7 @@ impl<'a, F: Function> Env<'a, F> { self.set_alloc( fixup.pos.inst(), fixup.to_slot as usize, - Allocation::reg(self.pregs[fixup.to_preg.index()].reg), + Allocation::reg(PReg::from_index(fixup.to_preg.index())), ); } @@ -773,7 +787,7 @@ impl<'a, F: Function> Env<'a, F> { input_reused.push(input_idx); let input_alloc = self.get_alloc(inst, input_idx); let output_alloc = self.get_alloc(inst, output_idx); - log::trace!( + trace!( "reuse-input inst {:?}: output {} has alloc {:?}, input {} has alloc {:?}", inst, output_idx, @@ -816,20 +830,20 @@ impl<'a, F: Function> Env<'a, F> { .sort_unstable_by_key(|((_, inst), _)| inst.prev()); let prog_move_srcs = std::mem::replace(&mut self.prog_move_srcs, vec![]); let prog_move_dsts = std::mem::replace(&mut self.prog_move_dsts, vec![]); - assert_eq!(prog_move_srcs.len(), prog_move_dsts.len()); + debug_assert_eq!(prog_move_srcs.len(), prog_move_dsts.len()); for (&((_, from_inst), from_alloc), &((to_vreg, to_inst), to_alloc)) in prog_move_srcs.iter().zip(prog_move_dsts.iter()) { - log::trace!( + trace!( "program move at inst {:?}: alloc {:?} -> {:?} (v{})", from_inst, from_alloc, to_alloc, to_vreg.index(), ); - assert!(from_alloc.is_some()); - assert!(to_alloc.is_some()); - assert_eq!(from_inst, to_inst.prev()); + debug_assert!(from_alloc.is_some()); + debug_assert!(to_alloc.is_some()); + debug_assert_eq!(from_inst, to_inst.prev()); // N.B.: these moves happen with the *same* priority as // LR-to-LR moves, because they work just like them: they // connect a use at one progpoint (move-After) with a def @@ -850,7 +864,7 @@ impl<'a, F: Function> Env<'a, F> { // resolve (see cases below). let mut i = 0; self.inserted_moves - .sort_unstable_by_key(|m| (m.pos.to_index(), m.prio)); + .sort_unstable_by_key(|m| m.pos_prio.key()); // Redundant-move elimination state tracker. let mut redundant_moves = RedundantMoveEliminator::default(); @@ -907,18 +921,14 @@ impl<'a, F: Function> Env<'a, F> { while i < self.inserted_moves.len() { let start = i; - let pos = self.inserted_moves[i].pos; - let prio = self.inserted_moves[i].prio; - while i < self.inserted_moves.len() - && self.inserted_moves[i].pos == pos - && self.inserted_moves[i].prio == prio - { + let pos_prio = self.inserted_moves[i].pos_prio; + while i < self.inserted_moves.len() && self.inserted_moves[i].pos_prio == pos_prio { i += 1; } let moves = &self.inserted_moves[start..i]; - redundant_move_process_side_effects(self, &mut redundant_moves, last_pos, pos); - last_pos = pos; + redundant_move_process_side_effects(self, &mut redundant_moves, last_pos, pos_prio.pos); + last_pos = pos_prio.pos; // Gather all the moves with Int class and Float class // separately. These cannot interact, so it is safe to @@ -929,13 +939,15 @@ impl<'a, F: Function> Env<'a, F> { // regs, but this seems simpler.) let mut int_moves: SmallVec<[InsertedMove; 8]> = smallvec![]; let mut float_moves: SmallVec<[InsertedMove; 8]> = smallvec![]; + #[cfg(feature = "checker")] let mut self_moves: SmallVec<[InsertedMove; 8]> = smallvec![]; for m in moves { if m.from_alloc.is_reg() && m.to_alloc.is_reg() { - assert_eq!(m.from_alloc.class(), m.to_alloc.class()); + debug_assert_eq!(m.from_alloc.class(), m.to_alloc.class()); } if m.from_alloc == m.to_alloc { + #[cfg(feature = "checker")] if m.to_vreg.is_some() { self_moves.push(m.clone()); } @@ -959,10 +971,14 @@ impl<'a, F: Function> Env<'a, F> { // that can be done one at a time. let scratch = self.env.scratch_by_class[regclass as u8 as usize]; let mut parallel_moves = ParallelMoves::new(Allocation::reg(scratch)); - log::trace!("parallel moves at pos {:?} prio {:?}", pos, prio); + trace!( + "parallel moves at pos {:?} prio {:?}", + pos_prio.pos, + pos_prio.prio + ); for m in moves { if (m.from_alloc != m.to_alloc) || m.to_vreg.is_some() { - log::trace!(" {} -> {}", m.from_alloc, m.to_alloc,); + trace!(" {} -> {}", m.from_alloc, m.to_alloc,); parallel_moves.add(m.from_alloc, m.to_alloc, m.to_vreg); } } @@ -993,7 +1009,7 @@ impl<'a, F: Function> Env<'a, F> { let mut scratch_used_yet = false; for (src, dst, to_vreg) in resolved { - log::trace!(" resolved: {} -> {} ({:?})", src, dst, to_vreg); + trace!(" resolved: {} -> {} ({:?})", src, dst, to_vreg); let action = redundant_moves.process_move(src, dst, to_vreg); if !action.elide { if dst == Allocation::reg(scratch) { @@ -1001,135 +1017,117 @@ impl<'a, F: Function> Env<'a, F> { } if self.allocation_is_stack(src) && self.allocation_is_stack(dst) { if !scratch_used_yet { - self.add_edit( - pos, - prio, - Edit::Move { - from: src, - to: Allocation::reg(scratch), - to_vreg, - }, + self.add_move_edit( + pos_prio, + src, + Allocation::reg(scratch), + to_vreg, ); - self.add_edit( - pos, - prio, - Edit::Move { - from: Allocation::reg(scratch), - to: dst, - to_vreg, - }, + self.add_move_edit( + pos_prio, + Allocation::reg(scratch), + dst, + to_vreg, ); } else { - assert!(extra_slot.is_some()); - self.add_edit( - pos, - prio, - Edit::Move { - from: Allocation::reg(scratch), - to: extra_slot.unwrap(), - to_vreg: None, - }, + debug_assert!(extra_slot.is_some()); + self.add_move_edit( + pos_prio, + Allocation::reg(scratch), + extra_slot.unwrap(), + None, ); - self.add_edit( - pos, - prio, - Edit::Move { - from: src, - to: Allocation::reg(scratch), - to_vreg, - }, + self.add_move_edit( + pos_prio, + src, + Allocation::reg(scratch), + to_vreg, ); - self.add_edit( - pos, - prio, - Edit::Move { - from: Allocation::reg(scratch), - to: dst, - to_vreg, - }, + self.add_move_edit( + pos_prio, + Allocation::reg(scratch), + dst, + to_vreg, ); - self.add_edit( - pos, - prio, - Edit::Move { - from: extra_slot.unwrap(), - to: Allocation::reg(scratch), - to_vreg: None, - }, + self.add_move_edit( + pos_prio, + extra_slot.unwrap(), + Allocation::reg(scratch), + None, ); } } else { - self.add_edit( - pos, - prio, - Edit::Move { - from: src, - to: dst, - to_vreg, - }, - ); + self.add_move_edit(pos_prio, src, dst, to_vreg); } } else { - log::trace!(" -> redundant move elided"); + trace!(" -> redundant move elided"); } + #[cfg(feature = "checker")] if let Some((alloc, vreg)) = action.def_alloc { - log::trace!( + trace!( " -> converted to DefAlloc: alloc {} vreg {}", alloc, vreg ); - self.add_edit(pos, prio, Edit::DefAlloc { alloc, vreg }); + self.edits.push((pos_prio, Edit::DefAlloc { alloc, vreg })); } } } + #[cfg(feature = "checker")] for m in &self_moves { - log::trace!( + trace!( "self move at pos {:?} prio {:?}: {} -> {} to_vreg {:?}", - pos, - prio, + pos_prio.pos, + pos_prio.prio, m.from_alloc, m.to_alloc, m.to_vreg ); let action = redundant_moves.process_move(m.from_alloc, m.to_alloc, m.to_vreg); - assert!(action.elide); + debug_assert!(action.elide); if let Some((alloc, vreg)) = action.def_alloc { - log::trace!(" -> DefAlloc: alloc {} vreg {}", alloc, vreg); - self.add_edit(pos, prio, Edit::DefAlloc { alloc, vreg }); + trace!(" -> DefAlloc: alloc {} vreg {}", alloc, vreg); + self.edits.push((pos_prio, Edit::DefAlloc { alloc, vreg })); } } } - // Add edits to describe blockparam locations too. This is - // required by the checker. This comes after any edge-moves. - self.blockparam_allocs - .sort_unstable_by_key(|&(block, idx, _, _)| (block, idx)); - self.stats.blockparam_allocs_count = self.blockparam_allocs.len(); - let mut i = 0; - while i < self.blockparam_allocs.len() { - let start = i; - let block = self.blockparam_allocs[i].0; - while i < self.blockparam_allocs.len() && self.blockparam_allocs[i].0 == block { - i += 1; - } - let params = &self.blockparam_allocs[start..i]; - let vregs = params - .iter() - .map(|(_, _, vreg_idx, _)| self.vreg_regs[vreg_idx.index()]) - .collect::>(); - let allocs = params - .iter() - .map(|(_, _, _, alloc)| *alloc) - .collect::>(); - assert_eq!(vregs.len(), self.func.block_params(block).len()); - assert_eq!(allocs.len(), self.func.block_params(block).len()); - for (vreg, alloc) in vregs.into_iter().zip(allocs.into_iter()) { - self.add_edit( - self.cfginfo.block_entry[block.index()], - InsertMovePrio::BlockParam, - Edit::DefAlloc { alloc, vreg }, - ); + #[cfg(feature = "checker")] + { + // Add edits to describe blockparam locations too. This is + // required by the checker. This comes after any edge-moves. + use crate::ion::data_structures::u64_key; + self.blockparam_allocs + .sort_unstable_by_key(|&(block, idx, _, _)| u64_key(block.raw_u32(), idx)); + self.stats.blockparam_allocs_count = self.blockparam_allocs.len(); + let mut i = 0; + while i < self.blockparam_allocs.len() { + let start = i; + let block = self.blockparam_allocs[i].0; + while i < self.blockparam_allocs.len() && self.blockparam_allocs[i].0 == block { + i += 1; + } + let params = &self.blockparam_allocs[start..i]; + let vregs = params + .iter() + .map(|(_, _, vreg_idx, _)| self.vreg_regs[vreg_idx.index()]) + .collect::>(); + let allocs = params + .iter() + .map(|(_, _, _, alloc)| *alloc) + .collect::>(); + debug_assert_eq!(vregs.len(), self.func.block_params(block).len()); + debug_assert_eq!(allocs.len(), self.func.block_params(block).len()); + for (vreg, alloc) in vregs.into_iter().zip(allocs.into_iter()) { + self.edits.push(( + PosWithPrio { + pos: self.cfginfo.block_entry[block.index()], + prio: InsertMovePrio::BlockParam as u32, + }, + Edit::DefAlloc { alloc, vreg }, + )); + } } } @@ -1137,38 +1135,49 @@ impl<'a, F: Function> Env<'a, F> { // be a stable sort! We have to keep the order produced by the // parallel-move resolver for all moves within a single sort // key. - self.edits.sort_by_key(|&(pos, prio, _)| (pos, prio)); + self.edits.sort_by_key(|&(pos_prio, _)| pos_prio.key()); self.stats.edits_count = self.edits.len(); // Add debug annotations. if self.annotations_enabled { for i in 0..self.edits.len() { - let &(pos, _, ref edit) = &self.edits[i]; + let &(pos_prio, ref edit) = &self.edits[i]; match edit { - &Edit::Move { from, to, to_vreg } => { - self.annotate( - ProgPoint::from_index(pos), - format!("move {} -> {} ({:?})", from, to, to_vreg), - ); + &Edit::Move { from, to } => { + self.annotate(pos_prio.pos, format!("move {} -> {})", from, to)); } &Edit::DefAlloc { alloc, vreg } => { let s = format!("defalloc {:?} := {:?}", alloc, vreg); - self.annotate(ProgPoint::from_index(pos), s); + self.annotate(pos_prio.pos, s); } } } } } - pub fn add_edit(&mut self, pos: ProgPoint, prio: InsertMovePrio, edit: Edit) { - match &edit { - &Edit::Move { from, to, to_vreg } if from == to && to_vreg.is_none() => return, - &Edit::Move { from, to, .. } if from.is_reg() && to.is_reg() => { - assert_eq!(from.as_reg().unwrap().class(), to.as_reg().unwrap().class()); + pub fn add_move_edit( + &mut self, + pos_prio: PosWithPrio, + from: Allocation, + to: Allocation, + _to_vreg: Option, + ) { + if from != to { + if from.is_reg() && to.is_reg() { + debug_assert_eq!(from.as_reg().unwrap().class(), to.as_reg().unwrap().class()); } - _ => {} + self.edits.push((pos_prio, Edit::Move { from, to })); } - self.edits.push((pos.to_index(), prio, edit)); + #[cfg(feature = "checker")] + if let Some(to_vreg) = _to_vreg { + self.edits.push(( + pos_prio, + Edit::DefAlloc { + alloc: to, + vreg: to_vreg, + }, + )); + } } } diff --git a/src/ion/process.rs b/src/ion/process.rs index c45bc5b..48c0381 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -62,7 +62,7 @@ impl<'a, F: Function> Env<'a, F> { // `AllocRegResult::ConflictHighCost`. max_allowable_cost: Option, ) -> AllocRegResult { - log::trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg); + trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg); let mut conflicts = smallvec![]; let mut conflict_set = FxHashSet::default(); let mut max_conflict_weight = 0; @@ -89,7 +89,7 @@ impl<'a, F: Function> Env<'a, F> { .btree .range(from_key..) .peekable(); - log::trace!( + trace!( "alloc map for {:?} in range {:?}..: {:?}", reg, from_key, @@ -98,19 +98,19 @@ impl<'a, F: Function> Env<'a, F> { let mut first_conflict: Option = None; 'ranges: for entry in bundle_ranges { - log::trace!(" -> range LR {:?}: {:?}", entry.index, entry.range); + trace!(" -> range LR {:?}: {:?}", entry.index, entry.range); let key = LiveRangeKey::from_range(&entry.range); let mut skips = 0; 'alloc: loop { - log::trace!(" -> PReg range {:?}", preg_range_iter.peek()); + trace!(" -> PReg range {:?}", preg_range_iter.peek()); // Advance our BTree traversal until it is >= this bundle // range (i.e., skip PReg allocations in the BTree that // are completely before this bundle range). if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key { - log::trace!( + trace!( "Skipping PReg range {:?}", preg_range_iter.peek().unwrap().0 ); @@ -135,13 +135,13 @@ impl<'a, F: Function> Env<'a, F> { // If there are no more PReg allocations, we're done! if preg_range_iter.peek().is_none() { - log::trace!(" -> no more PReg allocations; so no conflict possible!"); + trace!(" -> no more PReg allocations; so no conflict possible!"); break 'ranges; } // If the current PReg range is beyond this range, there is no conflict; continue. if *preg_range_iter.peek().unwrap().0 > key { - log::trace!( + trace!( " -> next PReg allocation is at {:?}; moving to next VReg range", preg_range_iter.peek().unwrap().0 ); @@ -150,16 +150,16 @@ impl<'a, F: Function> Env<'a, F> { // Otherwise, there is a conflict. let preg_key = *preg_range_iter.peek().unwrap().0; - assert_eq!(preg_key, key); // Assert that this range overlaps. + debug_assert_eq!(preg_key, key); // Assert that this range overlaps. let preg_range = preg_range_iter.next().unwrap().1; - log::trace!(" -> btree contains range {:?} that overlaps", preg_range); + trace!(" -> btree contains range {:?} that overlaps", preg_range); if preg_range.is_valid() { - log::trace!(" -> from vreg {:?}", self.ranges[preg_range.index()].vreg); + trace!(" -> from vreg {:?}", self.ranges[preg_range.index()].vreg); // range from an allocated bundle: find the bundle and add to // conflicts list. let conflict_bundle = self.ranges[preg_range.index()].bundle; - log::trace!(" -> conflict bundle {:?}", conflict_bundle); + trace!(" -> conflict bundle {:?}", conflict_bundle); if !conflict_set.contains(&conflict_bundle) { conflicts.push(conflict_bundle); conflict_set.insert(conflict_bundle); @@ -170,7 +170,7 @@ impl<'a, F: Function> Env<'a, F> { if max_allowable_cost.is_some() && max_conflict_weight > max_allowable_cost.unwrap() { - log::trace!(" -> reached high cost, retrying early"); + trace!(" -> reached high cost, retrying early"); return AllocRegResult::ConflictHighCost; } } @@ -182,7 +182,7 @@ impl<'a, F: Function> Env<'a, F> { ))); } } else { - log::trace!(" -> conflict with fixed reservation"); + trace!(" -> conflict with fixed reservation"); // range from a direct use of the PReg (due to clobber). return AllocRegResult::ConflictWithFixed( max_conflict_weight, @@ -197,8 +197,8 @@ impl<'a, F: Function> Env<'a, F> { } // We can allocate! Add our ranges to the preg's BTree. - let preg = self.pregs[reg.index()].reg; - log::trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg); + let preg = PReg::from_index(reg.index()); + trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg); self.bundles[bundle.index()].allocation = Allocation::reg(preg); for entry in &self.bundles[bundle.index()].ranges { self.pregs[reg.index()] @@ -211,7 +211,7 @@ impl<'a, F: Function> Env<'a, F> { } pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) { - log::trace!( + trace!( "evicting bundle {:?}: alloc {:?}", bundle, self.bundles[bundle.index()].allocation @@ -219,7 +219,7 @@ impl<'a, F: Function> Env<'a, F> { let preg = match self.bundles[bundle.index()].allocation.as_reg() { Some(preg) => preg, None => { - log::trace!( + trace!( " -> has no allocation! {:?}", self.bundles[bundle.index()].allocation ); @@ -229,14 +229,14 @@ impl<'a, F: Function> Env<'a, F> { let preg_idx = PRegIndex::new(preg.index()); self.bundles[bundle.index()].allocation = Allocation::none(); for entry in &self.bundles[bundle.index()].ranges { - log::trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx); + trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx); self.pregs[preg_idx.index()] .allocations .btree .remove(&LiveRangeKey::from_range(&entry.range)); } let prio = self.bundles[bundle.index()].prio; - log::trace!(" -> prio {}; back into queue", prio); + trace!(" -> prio {}; back into queue", prio); self.allocation_queue .insert(bundle, prio as usize, PReg::invalid()); } @@ -246,22 +246,22 @@ impl<'a, F: Function> Env<'a, F> { } pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &LiveBundleVec) -> u32 { - log::trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles); + trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles); let m = bundles .iter() .map(|&b| { let w = self.bundles[b.index()].cached_spill_weight(); - log::trace!("bundle{}: {}", b.index(), w); + trace!("bundle{}: {}", b.index(), w); w }) .max() .unwrap_or(0); - log::trace!(" -> max: {}", m); + trace!(" -> max: {}", m); m } pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) { - log::trace!("recompute bundle properties: bundle {:?}", bundle); + trace!("recompute bundle properties: bundle {:?}", bundle); let minimal; let mut fixed = false; @@ -273,18 +273,18 @@ impl<'a, F: Function> Env<'a, F> { self.bundles[bundle.index()].prio = self.compute_bundle_prio(bundle); if first_range_data.vreg.is_invalid() { - log::trace!(" -> no vreg; minimal and fixed"); + trace!(" -> no vreg; minimal and fixed"); minimal = true; fixed = true; } else { for u in &first_range_data.uses { - log::trace!(" -> use: {:?}", u); + trace!(" -> use: {:?}", u); if let OperandConstraint::FixedReg(_) = u.operand.constraint() { - log::trace!(" -> fixed use at {:?}: {:?}", u.pos, u.operand); + trace!(" -> fixed use at {:?}: {:?}", u.pos, u.operand); fixed = true; } if let OperandConstraint::Stack = u.operand.constraint() { - log::trace!(" -> stack use at {:?}: {:?}", u.pos, u.operand); + trace!(" -> stack use at {:?}: {:?}", u.pos, u.operand); stack = true; } if stack && fixed { @@ -295,7 +295,7 @@ impl<'a, F: Function> Env<'a, F> { // that it could cover just one ProgPoint, // i.e. X.Before..X.After, or two ProgPoints, // i.e. X.Before..X+1.Before. - log::trace!(" -> first range has range {:?}", first_range_data.range); + trace!(" -> first range has range {:?}", first_range_data.range); let bundle_start = self.bundles[bundle.index()] .ranges .first() @@ -304,22 +304,22 @@ impl<'a, F: Function> Env<'a, F> { .from; let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to; minimal = bundle_start.inst() == bundle_end.prev().inst(); - log::trace!(" -> minimal: {}", minimal); + trace!(" -> minimal: {}", minimal); } let spill_weight = if minimal { if fixed { - log::trace!(" -> fixed and minimal"); + trace!(" -> fixed and minimal"); MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT } else { - log::trace!(" -> non-fixed and minimal"); + trace!(" -> non-fixed and minimal"); MINIMAL_BUNDLE_SPILL_WEIGHT } } else { let mut total = SpillWeight::zero(); for entry in &self.bundles[bundle.index()].ranges { let range_data = &self.ranges[entry.index.index()]; - log::trace!( + trace!( " -> uses spill weight: +{:?}", range_data.uses_spill_weight() ); @@ -328,7 +328,7 @@ impl<'a, F: Function> Env<'a, F> { if self.bundles[bundle.index()].prio > 0 { let final_weight = (total.to_f32() as u32) / self.bundles[bundle.index()].prio; - log::trace!( + trace!( " -> dividing by prio {}; final weight {}", self.bundles[bundle.index()].prio, final_weight @@ -356,7 +356,7 @@ impl<'a, F: Function> Env<'a, F> { let mut w = SpillWeight::zero(); for u in &rangedata.uses { w = w + SpillWeight::from_bits(u.weight); - log::trace!("range{}: use {:?}", range.index(), u); + trace!("range{}: use {:?}", range.index(), u); } rangedata.set_uses_spill_weight(w); if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def { @@ -397,7 +397,7 @@ impl<'a, F: Function> Env<'a, F> { reg_hint: PReg, ) { self.stats.splits += 1; - log::trace!( + trace!( "split bundle {:?} at {:?} and requeue with reg hint (for first part) {:?}", bundle, split_at, @@ -410,7 +410,7 @@ impl<'a, F: Function> Env<'a, F> { let spillset = self.bundles[bundle.index()].spillset; - assert!(!self.bundles[bundle.index()].ranges.is_empty()); + debug_assert!(!self.bundles[bundle.index()].ranges.is_empty()); // Split point *at* start is OK; this means we peel off // exactly one use to create a minimal bundle. let bundle_start = self.bundles[bundle.index()] @@ -419,9 +419,9 @@ impl<'a, F: Function> Env<'a, F> { .unwrap() .range .from; - assert!(split_at >= bundle_start); + debug_assert!(split_at >= bundle_start); let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to; - assert!(split_at < bundle_end); + debug_assert!(split_at < bundle_end); // Is the split point *at* the start? If so, peel off the // first use: set the split point just after it, or just @@ -435,7 +435,7 @@ impl<'a, F: Function> Env<'a, F> { break 'outer; } } - log::trace!(" -> first use loc is {:?}", first_use); + trace!(" -> first use loc is {:?}", first_use); split_at = match first_use { Some(pos) => { if pos.inst() == bundle_start.inst() { @@ -455,7 +455,7 @@ impl<'a, F: Function> Env<'a, F> { .next(), ), }; - log::trace!( + trace!( "split point is at bundle start; advancing to {:?}", split_at ); @@ -471,13 +471,13 @@ impl<'a, F: Function> Env<'a, F> { } } - assert!(split_at > bundle_start && split_at < bundle_end); + debug_assert!(split_at > bundle_start && split_at < bundle_end); // We need to find which LRs fall on each side of the split, // which LR we need to split down the middle, then update the // current bundle, create a new one, and (re)-queue both. - log::trace!(" -> LRs: {:?}", self.bundles[bundle.index()].ranges); + trace!(" -> LRs: {:?}", self.bundles[bundle.index()].ranges); let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle @@ -492,11 +492,11 @@ impl<'a, F: Function> Env<'a, F> { } } - log::trace!( + trace!( " -> last LR in old bundle: LR {:?}", self.bundles[bundle.index()].ranges[last_lr_in_old_bundle_idx] ); - log::trace!( + trace!( " -> first LR in new bundle: LR {:?}", self.bundles[bundle.index()].ranges[first_lr_in_new_bundle_idx] ); @@ -516,14 +516,14 @@ impl<'a, F: Function> Env<'a, F> { // down the middle, replace it with a new LR and chop off the // end of the same LR in the original list. if split_at > new_lr_list[0].range.from { - assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx); + debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx); let orig_lr = new_lr_list[0].index; let new_lr = self.create_liverange(CodeRange { from: split_at, to: new_lr_list[0].range.to, }); self.ranges[new_lr.index()].vreg = self.ranges[orig_lr.index()].vreg; - log::trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr); + trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr); let first_use = self.ranges[orig_lr.index()] .uses .iter() @@ -560,7 +560,7 @@ impl<'a, F: Function> Env<'a, F> { } let new_bundle = self.create_bundle(); - log::trace!(" -> creating new bundle {:?}", new_bundle); + trace!(" -> creating new bundle {:?}", new_bundle); self.bundles[new_bundle.index()].spillset = spillset; for entry in &new_lr_list { self.ranges[entry.index.index()].bundle = new_bundle; @@ -582,7 +582,7 @@ impl<'a, F: Function> Env<'a, F> { let spill = self .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) .unwrap(); - log::trace!( + trace!( " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", bundle, entry.index, @@ -627,13 +627,13 @@ impl<'a, F: Function> Env<'a, F> { range, index: empty_lr, }); - log::trace!( + trace!( " -> bundle {:?} range {:?}: last use implies split point {:?}", bundle, entry.index, split ); - log::trace!( + trace!( " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", spill, empty_lr @@ -652,7 +652,7 @@ impl<'a, F: Function> Env<'a, F> { let spill = self .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) .unwrap(); - log::trace!( + trace!( " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", new_bundle, entry.index, @@ -697,13 +697,13 @@ impl<'a, F: Function> Env<'a, F> { range, index: empty_lr, }); - log::trace!( + trace!( " -> bundle {:?} range {:?}: first use implies split point {:?}", bundle, entry.index, first_use, ); - log::trace!( + trace!( " -> moving leading empty region to new spill bundle {:?} with new LR {:?}", spill, empty_lr @@ -741,7 +741,7 @@ impl<'a, F: Function> Env<'a, F> { if self.pregs[hint_reg.index()].is_stack { hint_reg = PReg::invalid(); } - log::trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,); + trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,); let req = match self.compute_requirement(bundle) { Ok(req) => req, @@ -749,7 +749,7 @@ impl<'a, F: Function> Env<'a, F> { // We have to split right away. We'll find a point to // split that would allow at least the first half of the // split to be conflict-free. - assert!( + debug_assert!( !self.minimal_bundle(bundle), "Minimal bundle with conflict!" ); @@ -786,7 +786,7 @@ impl<'a, F: Function> Env<'a, F> { let mut attempts = 0; loop { attempts += 1; - log::trace!("attempt {}, req {:?}", attempts, req); + trace!("attempt {}, req {:?}", attempts, req); debug_assert!(attempts < 100 * self.func.num_insts()); let fixed_preg = match req { @@ -836,7 +836,7 @@ impl<'a, F: Function> Env<'a, F> { ) { self.stats.process_bundle_reg_probes_any += 1; let preg_idx = PRegIndex::new(preg.index()); - log::trace!("trying preg {:?}", preg_idx); + trace!("trying preg {:?}", preg_idx); let scan_limit_cost = match ( lowest_cost_evict_conflict_cost, @@ -848,13 +848,13 @@ impl<'a, F: Function> Env<'a, F> { match self.try_to_allocate_bundle_to_reg(bundle, preg_idx, scan_limit_cost) { AllocRegResult::Allocated(alloc) => { self.stats.process_bundle_reg_success_any += 1; - log::trace!(" -> allocated to any {:?}", preg_idx); + trace!(" -> allocated to any {:?}", preg_idx); self.spillsets[self.bundles[bundle.index()].spillset.index()].reg_hint = alloc.as_reg().unwrap(); return Ok(()); } AllocRegResult::Conflict(bundles, first_conflict_point) => { - log::trace!( + trace!( " -> conflict with bundles {:?}, first conflict at {:?}", bundles, first_conflict_point @@ -887,7 +887,7 @@ impl<'a, F: Function> Env<'a, F> { } } AllocRegResult::ConflictWithFixed(max_cost, point) => { - log::trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); + trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); let loop_depth = self.cfginfo.approx_loop_depth [self.cfginfo.insn_block[point.inst().index()].index()]; @@ -919,12 +919,12 @@ impl<'a, F: Function> Env<'a, F> { // any with current bundle assignments. Hence, we will need // to either split or attempt to evict some bundles. - log::trace!( + trace!( " -> lowest cost evict: set {:?}, cost {:?}", lowest_cost_evict_conflict_set, lowest_cost_evict_conflict_cost, ); - log::trace!( + trace!( " -> lowest cost split: cost {:?}, point {:?}, reg {:?}", lowest_cost_split_conflict_cost, lowest_cost_split_conflict_point, @@ -932,13 +932,13 @@ impl<'a, F: Function> Env<'a, F> { ); // If we reach here, we *must* have an option either to split or evict. - assert!( + debug_assert!( lowest_cost_split_conflict_cost.is_some() || lowest_cost_evict_conflict_cost.is_some() ); let our_spill_weight = self.bundle_spill_weight(bundle); - log::trace!(" -> our spill weight: {}", our_spill_weight); + trace!(" -> our spill weight: {}", our_spill_weight); // We detect the "too-many-live-registers" case here and // return an error cleanly, rather than panicking, because @@ -953,7 +953,7 @@ impl<'a, F: Function> Env<'a, F> { if let Requirement::Register = req { // Check if this is a too-many-live-registers situation. let range = self.bundles[bundle.index()].ranges[0].range; - log::trace!("checking for too many live regs"); + trace!("checking for too many live regs"); let mut min_bundles_assigned = 0; let mut fixed_assigned = 0; let mut total_regs = 0; @@ -961,7 +961,7 @@ impl<'a, F: Function> Env<'a, F> { .iter() .chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter()) { - log::trace!(" -> PR {:?}", preg); + trace!(" -> PR {:?}", preg); let start = LiveRangeKey::from_range(&CodeRange { from: range.from.prev(), to: range.from.prev(), @@ -976,19 +976,19 @@ impl<'a, F: Function> Env<'a, F> { } if lr.is_valid() { if self.minimal_bundle(self.ranges[lr.index()].bundle) { - log::trace!(" -> min bundle {:?}", lr); + trace!(" -> min bundle {:?}", lr); min_bundles_assigned += 1; } else { - log::trace!(" -> non-min bundle {:?}", lr); + trace!(" -> non-min bundle {:?}", lr); } } else { - log::trace!(" -> fixed bundle"); + trace!(" -> fixed bundle"); fixed_assigned += 1; } } total_regs += 1; } - log::trace!( + trace!( " -> total {}, fixed {}, min {}", total_regs, fixed_assigned, @@ -1020,7 +1020,7 @@ impl<'a, F: Function> Env<'a, F> { || lowest_cost_evict_conflict_cost.is_none() || our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap()) { - log::trace!( + trace!( " -> deciding to split: our spill weight is {}", self.bundle_spill_weight(bundle) ); @@ -1053,7 +1053,7 @@ impl<'a, F: Function> Env<'a, F> { // Evict all bundles in `conflicting bundles` and try again. self.stats.evict_bundle_event += 1; for &bundle in &lowest_cost_evict_conflict_set.unwrap() { - log::trace!(" -> evicting {:?}", bundle); + trace!(" -> evicting {:?}", bundle); self.evict_bundle(bundle); self.stats.evict_bundle_count += 1; } diff --git a/src/ion/redundant_moves.rs b/src/ion/redundant_moves.rs index 44f15d7..5a0f192 100644 --- a/src/ion/redundant_moves.rs +++ b/src/ion/redundant_moves.rs @@ -18,6 +18,7 @@ pub struct RedundantMoveEliminator { #[derive(Copy, Clone, Debug)] pub struct RedundantMoveAction { pub elide: bool, + #[cfg(feature = "checker")] pub def_alloc: Option<(Allocation, VReg)>, } @@ -40,13 +41,13 @@ impl RedundantMoveEliminator { .map(|&p| p) .unwrap_or(RedundantMoveState::None); - log::trace!( + trace!( " -> redundant move tracker: from {} to {} to_vreg {:?}", from, to, to_vreg ); - log::trace!( + trace!( " -> from_state {:?} to_state {:?}", from_state, to_state @@ -58,6 +59,7 @@ impl RedundantMoveEliminator { .insert(to, RedundantMoveState::Orig(to_vreg.unwrap())); return RedundantMoveAction { elide: true, + #[cfg(feature = "checker")] def_alloc: Some((to, to_vreg.unwrap())), }; } @@ -67,29 +69,29 @@ impl RedundantMoveEliminator { RedundantMoveState::Orig(r) => Some(r), _ => None, }; - log::trace!(" -> src_vreg {:?}", src_vreg); + trace!(" -> src_vreg {:?}", src_vreg); let dst_vreg = to_vreg.or(src_vreg); - log::trace!(" -> dst_vreg {:?}", dst_vreg); + trace!(" -> dst_vreg {:?}", dst_vreg); let existing_dst_vreg = match to_state { RedundantMoveState::Copy(_, opt_r) => opt_r, RedundantMoveState::Orig(r) => Some(r), _ => None, }; - log::trace!(" -> existing_dst_vreg {:?}", existing_dst_vreg); + trace!(" -> existing_dst_vreg {:?}", existing_dst_vreg); let elide = match (from_state, to_state) { (_, RedundantMoveState::Copy(orig_alloc, _)) if orig_alloc == from => true, (RedundantMoveState::Copy(new_alloc, _), _) if new_alloc == to => true, _ => false, }; - log::trace!(" -> elide {}", elide); + trace!(" -> elide {}", elide); let def_alloc = if dst_vreg != existing_dst_vreg && dst_vreg.is_some() { Some((to, dst_vreg.unwrap())) } else { None }; - log::trace!(" -> def_alloc {:?}", def_alloc); + trace!(" -> def_alloc {:?}", def_alloc); // Invalidate all existing copies of `to` if `to` actually changed value. if !elide { @@ -100,7 +102,7 @@ impl RedundantMoveEliminator { if from.is_reg() || to.is_reg() { self.allocs .insert(to, RedundantMoveState::Copy(from, dst_vreg)); - log::trace!( + trace!( " -> create mapping {} -> {:?}", to, RedundantMoveState::Copy(from, dst_vreg) @@ -111,20 +113,24 @@ impl RedundantMoveEliminator { .push(to); } - RedundantMoveAction { elide, def_alloc } + RedundantMoveAction { + elide, + #[cfg(feature = "checker")] + def_alloc, + } } pub fn clear(&mut self) { - log::trace!(" redundant move eliminator cleared"); + trace!(" redundant move eliminator cleared"); self.allocs.clear(); self.reverse_allocs.clear(); } pub fn clear_alloc(&mut self, alloc: Allocation) { - log::trace!(" redundant move eliminator: clear {:?}", alloc); + trace!(" redundant move eliminator: clear {:?}", alloc); if let Some(ref mut existing_copies) = self.reverse_allocs.get_mut(&alloc) { for to_inval in existing_copies.iter() { - log::trace!(" -> clear existing copy: {:?}", to_inval); + trace!(" -> clear existing copy: {:?}", to_inval); if let Some(val) = self.allocs.get_mut(to_inval) { match val { RedundantMoveState::Copy(_, Some(vreg)) => { diff --git a/src/ion/requirement.rs b/src/ion/requirement.rs index c14cc38..1d830b6 100644 --- a/src/ion/requirement.rs +++ b/src/ion/requirement.rs @@ -71,21 +71,21 @@ impl<'a, F: Function> Env<'a, F> { bundle: LiveBundleIndex, ) -> Result { let mut req = Requirement::Any; - log::trace!("compute_requirement: {:?}", bundle); + trace!("compute_requirement: {:?}", bundle); let ranges = &self.bundles[bundle.index()].ranges; for entry in ranges { - log::trace!(" -> LR {:?}", entry.index); + trace!(" -> LR {:?}", entry.index); for u in &self.ranges[entry.index.index()].uses { - log::trace!(" -> use {:?}", u); + trace!(" -> use {:?}", u); let r = self.requirement_from_operand(u.operand); req = req.merge(r).map_err(|_| { - log::trace!(" -> conflict"); + trace!(" -> conflict"); RequirementConflictAt(u.pos) })?; - log::trace!(" -> req {:?}", req); + trace!(" -> req {:?}", req); } } - log::trace!(" -> final: {:?}", req); + trace!(" -> final: {:?}", req); Ok(req) } diff --git a/src/ion/spill.rs b/src/ion/spill.rs index 18607a7..2600cdb 100644 --- a/src/ion/spill.rs +++ b/src/ion/spill.rs @@ -20,7 +20,7 @@ use crate::{Allocation, Function, SpillSlot}; impl<'a, F: Function> Env<'a, F> { pub fn try_allocating_regs_for_spilled_bundles(&mut self) { - log::trace!("allocating regs for spilled bundles"); + trace!("allocating regs for spilled bundles"); for i in 0..self.spilled_bundles.len() { let bundle = self.spilled_bundles[i]; // don't borrow self @@ -38,7 +38,7 @@ impl<'a, F: Function> Env<'a, F> { for preg in RegTraversalIter::new(self.env, class, hint, PReg::invalid(), bundle.index(), None) { - log::trace!("trying bundle {:?} to preg {:?}", bundle, preg); + trace!("trying bundle {:?} to preg {:?}", bundle, preg); let preg_idx = PRegIndex::new(preg.index()); if let AllocRegResult::Allocated(_) = self.try_to_allocate_bundle_to_reg(bundle, preg_idx, None) @@ -49,7 +49,7 @@ impl<'a, F: Function> Env<'a, F> { } } if !success { - log::trace!( + trace!( "spilling bundle {:?}: marking spillset {:?} as required", bundle, self.bundles[bundle.index()].spillset @@ -87,14 +87,14 @@ impl<'a, F: Function> Env<'a, F> { for i in 0..self.spillsets[spillset.index()].vregs.len() { // don't borrow self let vreg = self.spillsets[spillset.index()].vregs[i]; - log::trace!( + trace!( "spillslot {:?} alloc'ed to spillset {:?}: vreg {:?}", spillslot, spillset, vreg, ); for entry in &self.vregs[vreg.index()].ranges { - log::trace!( + trace!( "spillslot {:?} getting range {:?} from LR {:?} from vreg {:?}", spillslot, entry.range, @@ -111,7 +111,7 @@ impl<'a, F: Function> Env<'a, F> { pub fn allocate_spillslots(&mut self) { for spillset in 0..self.spillsets.len() { - log::trace!("allocate spillslot: {}", spillset); + trace!("allocate spillslot: {}", spillset); let spillset = SpillSetIndex::new(spillset); if !self.spillsets[spillset.index()].required { continue; @@ -196,7 +196,7 @@ impl<'a, F: Function> Env<'a, F> { self.spillslots[i].alloc = self.allocate_spillslot(self.spillslots[i].class); } - log::trace!("spillslot allocator done"); + trace!("spillslot allocator done"); } pub fn allocate_spillslot(&mut self, class: RegClass) -> Allocation { diff --git a/src/ion/stackmap.rs b/src/ion/stackmap.rs index 3bae749..517de15 100644 --- a/src/ion/stackmap.rs +++ b/src/ion/stackmap.rs @@ -13,7 +13,7 @@ //! Stackmap computation. use super::{Env, ProgPoint, VRegIndex}; -use crate::Function; +use crate::{ion::data_structures::u64_key, Function}; impl<'a, F: Function> Env<'a, F> { pub fn compute_stackmaps(&mut self) { @@ -30,10 +30,10 @@ impl<'a, F: Function> Env<'a, F> { // safepoints; and for each safepoint in the current range, // emit the allocation into the `safepoint_slots` list. - log::trace!("safepoints_per_vreg = {:?}", self.safepoints_per_vreg); + trace!("safepoints_per_vreg = {:?}", self.safepoints_per_vreg); for vreg in self.func.reftype_vregs() { - log::trace!("generating safepoint info for vreg {}", vreg); + trace!("generating safepoint info for vreg {}", vreg); let vreg = VRegIndex::new(vreg.vreg()); let mut safepoints: Vec = self .safepoints_per_vreg @@ -43,19 +43,19 @@ impl<'a, F: Function> Env<'a, F> { .map(|&inst| ProgPoint::before(inst)) .collect(); safepoints.sort_unstable(); - log::trace!(" -> live over safepoints: {:?}", safepoints); + trace!(" -> live over safepoints: {:?}", safepoints); let mut safepoint_idx = 0; for entry in &self.vregs[vreg.index()].ranges { let range = entry.range; let alloc = self.get_alloc_for_range(entry.index); - log::trace!(" -> range {:?}: alloc {}", range, alloc); + trace!(" -> range {:?}: alloc {}", range, alloc); while safepoint_idx < safepoints.len() && safepoints[safepoint_idx] < range.to { if safepoints[safepoint_idx] < range.from { safepoint_idx += 1; continue; } - log::trace!(" -> covers safepoint {:?}", safepoints[safepoint_idx]); + trace!(" -> covers safepoint {:?}", safepoints[safepoint_idx]); self.safepoint_slots .push((safepoints[safepoint_idx], alloc)); @@ -64,7 +64,8 @@ impl<'a, F: Function> Env<'a, F> { } } - self.safepoint_slots.sort_unstable(); - log::trace!("final safepoint slots info: {:?}", self.safepoint_slots); + self.safepoint_slots + .sort_unstable_by_key(|(progpoint, slot)| u64_key(progpoint.to_index(), slot.bits())); + trace!("final safepoint slots info: {:?}", self.safepoint_slots); } } diff --git a/src/lib.rs b/src/lib.rs index 9639b0f..800dd4f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,16 @@ #![allow(dead_code)] +// Even when trace logging is disabled, the trace macro has a significant +// performance cost so we disable it in release builds. +macro_rules! trace { + ($($tt:tt)*) => { + if cfg!(feature = "trace-log") { + ::log::trace!($($tt)*); + } + }; +} + pub(crate) mod cfg; pub(crate) mod domtree; pub mod indexset; @@ -234,7 +244,7 @@ impl SpillSlot { /// Create a new SpillSlot of a given class. #[inline(always)] pub fn new(slot: usize, class: RegClass) -> Self { - assert!(slot < (1 << 24)); + debug_assert!(slot < (1 << 24)); SpillSlot { bits: (slot as u32) | (class as u8 as u32) << 24, } @@ -412,11 +422,11 @@ impl Operand { OperandConstraint::Reg => 1, OperandConstraint::Stack => 2, OperandConstraint::FixedReg(preg) => { - assert_eq!(preg.class(), vreg.class()); + debug_assert_eq!(preg.class(), vreg.class()); 0b1000000 | preg.hw_enc() as u32 } OperandConstraint::Reuse(which) => { - assert!(which <= 31); + debug_assert!(which <= 31); 0b0100000 | which as u32 } }; @@ -696,7 +706,7 @@ impl Allocation { /// Construct a new Allocation. #[inline(always)] pub(crate) fn new(kind: AllocationKind, index: usize) -> Self { - assert!(index < (1 << 28)); + debug_assert!(index < (1 << 28)); Self { bits: ((kind as u8 as u32) << 29) | (index as u32), } @@ -1105,20 +1115,16 @@ pub enum Edit { /// register or a stack slot (spillslot). However, stack-to-stack /// moves will never be generated. /// - /// `to_vreg`, if defined, is useful as metadata: it indicates - /// that the moved value is a def of a new vreg. - /// /// `Move` edits will be generated even if src and dst allocation /// are the same if the vreg changes; this allows proper metadata /// tracking even when moves are elided. - Move { - from: Allocation, - to: Allocation, - to_vreg: Option, - }, + Move { from: Allocation, to: Allocation }, /// Define a particular Allocation to contain a particular VReg. Useful /// for the checker. + /// + /// `DefAlloc` edits are only emitted when the `"checker"` Cargo feature is + /// enabled. DefAlloc { alloc: Allocation, vreg: VReg }, } diff --git a/src/moves.rs b/src/moves.rs index 0bb388e..3828f1b 100644 --- a/src/moves.rs +++ b/src/moves.rs @@ -3,7 +3,7 @@ * exception. See `LICENSE` for details. */ -use crate::Allocation; +use crate::{ion::data_structures::u64_key, Allocation}; use smallvec::{smallvec, SmallVec}; pub type MoveVec = SmallVec<[(Allocation, Allocation, T); 16]>; @@ -53,7 +53,8 @@ impl ParallelMoves { // Sort moves by source so that we can efficiently test for // presence. - self.parallel_moves.sort_by_key(|&(src, dst, _)| (src, dst)); + self.parallel_moves + .sort_by_key(|&(src, dst, _)| u64_key(src.bits(), dst.bits())); // Do any dests overlap sources? If not, we can also just // return the list. @@ -183,7 +184,7 @@ impl ParallelMoves { scratch_src = Some(src); src = self.scratch; } else { - assert_eq!(last_dst.unwrap(), src); + debug_assert_eq!(last_dst.unwrap(), src); } ret.push((src, dst, dst_t));