T0b1/regalloc2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #25 from Amanieu/perf

Browse Source 
Performance improvements
This commit is contained in:
Chris Fallin
2022-01-11 14:35:10 -08:00
committed by GitHub
parent a27f93f01e 6b1a5e8b1b
commit 56a8f844a8
20 changed files with 594 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Cargo.toml
View File

@@ -1,7 +1,10 @@
[package]
name = "regalloc2"
version = "0.0.1"
authors = ["Chris Fallin <chris@cfallin.org>", "Mozilla SpiderMonkey Developers"]
authors = [
"Chris Fallin <chris@cfallin.org>",
"Mozilla SpiderMonkey Developers",
]
edition = "2018"
license = "Apache-2.0 WITH LLVM-exception"
description = "Backtracking register allocator inspired from IonMonkey"
@@ -17,9 +20,20 @@ slice-group-by = "0.3.0"
# Keep this in sync with libfuzzer_sys's crate version:
arbitrary = { version = "^0.4.6", optional = true }
# When testing regalloc2 by itself, enable debug assertions and overflow checks
[profile.release]
debug = true
debug-assertions = true
overflow-checks = true
[features]
default = []
fuzzing = ["arbitrary"]
# Enables generation of DefAlloc edits for the checker.
checker = []
# Enables detailed logging which can be somewhat expensive.
trace-log = []
# Exposes the internal API for fuzzing.
fuzzing = ["arbitrary", "checker", "trace-log"]

6
fuzz/Cargo.toml
View File

@@ -49,3 +49,9 @@ name = "ion_checker"
path = "fuzz_targets/ion_checker.rs"
test = false
doc = false
# Enable debug assertions and overflow checks when fuzzing
[profile.release]
debug = true
debug-assertions = true
overflow-checks = true

50
src/checker.rs
View File

@@ -191,7 +191,7 @@ impl CheckerValue {
CheckerValue::Reg(r1, ref1)
}
_ => {
log::trace!("{:?} and {:?} meet to Conflicted", self, other);
trace!("{:?} and {:?} meet to Conflicted", self, other);
CheckerValue::Conflicted
}
}
@@ -322,7 +322,7 @@ impl CheckerState {
.get(alloc)
.cloned()
.unwrap_or(Default::default());
log::trace!(
trace!(
"checker: checkinst {:?}: op {:?}, alloc {:?}, checker value {:?}",
checkinst,
op,
@@ -339,7 +339,7 @@ impl CheckerState {
.get(&alloc)
.cloned()
.unwrap_or(Default::default());
log::trace!(
trace!(
"checker: checkinst {:?}: safepoint slot {}, checker value {:?}",
checkinst,
alloc,
@@ -372,7 +372,7 @@ impl CheckerState {
.get(&from)
.cloned()
.unwrap_or(Default::default());
log::trace!(
trace!(
"checker: checkinst {:?} updating: move {:?} -> {:?} val {:?}",
checkinst,
from,
@@ -534,7 +534,7 @@ impl<'a, F: Function> Checker<'a, F> {
/// Build the list of checker instructions based on the given func
/// and allocation results.
pub fn prepare(&mut self, out: &Output) {
log::trace!("checker: out = {:?}", out);
trace!("checker: out = {:?}", out);
// Preprocess safepoint stack-maps into per-inst vecs.
let mut safepoint_slots: HashMap<Inst, Vec<Allocation>> = HashMap::new();
for &(progpoint, slot) in &out.safepoint_slots {
@@ -550,7 +550,7 @@ impl<'a, F: Function> Checker<'a, F> {
for inst_or_edit in out.block_insts_and_edits(self.f, block) {
match inst_or_edit {
InstOrEdit::Inst(inst) => {
assert!(last_inst.is_none() || inst > last_inst.unwrap());
debug_assert!(last_inst.is_none() || inst > last_inst.unwrap());
last_inst = Some(inst);
self.handle_inst(block, inst, &mut safepoint_slots, out);
}
@@ -591,25 +591,19 @@ impl<'a, F: Function> Checker<'a, F> {
allocs,
clobbers,
};
log::trace!("checker: adding inst {:?}", checkinst);
trace!("checker: adding inst {:?}", checkinst);
self.bb_insts.get_mut(&block).unwrap().push(checkinst);
}
}
fn handle_edit(&mut self, block: Block, edit: &Edit) {
log::trace!("checker: adding edit {:?}", edit);
trace!("checker: adding edit {:?}", edit);
match edit {
&Edit::Move { from, to, to_vreg } => {
&Edit::Move { from, to } => {
self.bb_insts
.get_mut(&block)
.unwrap()
.push(CheckerInst::Move { into: to, from });
if let Some(vreg) = to_vreg {
self.bb_insts
.get_mut(&block)
.unwrap()
.push(CheckerInst::DefAlloc { alloc: to, vreg });
}
}
&Edit::DefAlloc { alloc, vreg } => {
self.bb_insts
@@ -634,10 +628,10 @@ impl<'a, F: Function> Checker<'a, F> {
let block = queue.pop_front().unwrap();
queue_set.remove(&block);
let mut state = self.bb_in.get(&block).cloned().unwrap();
log::trace!("analyze: block {} has state {:?}", block.index(), state);
trace!("analyze: block {} has state {:?}", block.index(), state);
for inst in self.bb_insts.get(&block).unwrap() {
state.update(inst, self);
log::trace!("analyze: inst {:?} -> state {:?}", inst, state);
trace!("analyze: inst {:?} -> state {:?}", inst, state);
}
for &succ in self.f.block_succs(block) {
@@ -646,7 +640,7 @@ impl<'a, F: Function> Checker<'a, F> {
new_state.meet_with(cur_succ_in);
let changed = &new_state != cur_succ_in;
if changed {
log::trace!(
trace!(
"analyze: block {} state changed from {:?} to {:?}; pushing onto queue",
succ.index(),
cur_succ_in,
@@ -671,12 +665,12 @@ impl<'a, F: Function> Checker<'a, F> {
let mut state = input.clone();
for inst in self.bb_insts.get(block).unwrap() {
if let Err(e) = state.check(InstPosition::Before, inst) {
log::trace!("Checker error: {:?}", e);
trace!("Checker error: {:?}", e);
errors.push(e);
}
state.update(inst, self);
if let Err(e) = state.check(InstPosition::After, inst) {
log::trace!("Checker error: {:?}", e);
trace!("Checker error: {:?}", e);
errors.push(e);
}
}
@@ -695,20 +689,20 @@ impl<'a, F: Function> Checker<'a, F> {
self.analyze();
let result = self.find_errors();
log::trace!("=== CHECKER RESULT ===");
trace!("=== CHECKER RESULT ===");
fn print_state(state: &CheckerState) {
let mut s = vec![];
for (alloc, state) in &state.allocations {
s.push(format!("{} := {}", alloc, state));
}
log::trace!(" {{ {} }}", s.join(", "))
trace!(" {{ {} }}", s.join(", "))
}
for vreg in self.f.reftype_vregs() {
log::trace!(" REF: {}", vreg);
trace!(" REF: {}", vreg);
}
for bb in 0..self.f.num_blocks() {
let bb = Block::new(bb);
log::trace!("block{}:", bb.index());
trace!("block{}:", bb.index());
let insts = self.bb_insts.get(&bb).unwrap();
let mut state = self.bb_in.get(&bb).unwrap().clone();
print_state(&state);
@@ -720,7 +714,7 @@ impl<'a, F: Function> Checker<'a, F> {
ref allocs,
ref clobbers,
} => {
log::trace!(
trace!(
" inst{}: {:?} ({:?}) clobbers:{:?}",
inst.index(),
operands,
@@ -729,17 +723,17 @@ impl<'a, F: Function> Checker<'a, F> {
);
}
&CheckerInst::Move { from, into } => {
log::trace!(" {} -> {}", from, into);
trace!(" {} -> {}", from, into);
}
&CheckerInst::DefAlloc { alloc, vreg } => {
log::trace!(" defalloc: {}:{}", vreg, alloc);
trace!(" defalloc: {}:{}", vreg, alloc);
}
&CheckerInst::Safepoint { ref allocs, .. } => {
let mut slotargs = vec![];
for &slot in allocs {
slotargs.push(format!("{}", slot));
}
log::trace!(" safepoint: {}", slotargs.join(", "));
trace!(" safepoint: {}", slotargs.join(", "));
}
}
state.update(inst, &self);

2
src/domtree.rs
View File

@@ -33,7 +33,7 @@ fn merge_sets(
node2 = idom[node2.index()];
}
}
assert!(node1 == node2);
debug_assert!(node1 == node2);
node1
}

6
src/fuzzing/func.rs
View File

@@ -79,7 +79,7 @@ impl Function for Func {
}
fn entry_block(&self) -> Block {
assert!(self.blocks.len() > 0);
debug_assert!(self.blocks.len() > 0);
Block::new(0)
}
@@ -237,7 +237,7 @@ fn choose_dominating_block(
allow_self: bool,
u: &mut Unstructured,
) -> ArbitraryResult<Block> {
assert!(block.is_valid());
debug_assert!(block.is_valid());
let orig_block = block;
loop {
if (allow_self || block != orig_block) && bool::arbitrary(u)? {
@@ -445,7 +445,7 @@ impl Func {
if operands.len() > 1 && opts.reused_inputs && bool::arbitrary(u)? {
// Make the def a reused input.
let op = operands[0];
assert_eq!(op.kind(), OperandKind::Def);
debug_assert_eq!(op.kind(), OperandKind::Def);
let reused = u.int_in_range(1..=(operands.len() - 1))?;
operands[0] = Operand::new(
op.vreg(),

34
src/index.rs
View File

@@ -10,7 +10,7 @@ macro_rules! define_index {
}
#[inline(always)]
pub fn index(self) -> usize {
assert!(self.is_valid());
debug_assert!(self.is_valid());
self.0 as usize
}
#[inline(always)]
@@ -27,12 +27,12 @@ macro_rules! define_index {
}
#[inline(always)]
pub fn next(self) -> $ix {
assert!(self.is_valid());
debug_assert!(self.is_valid());
Self(self.0 + 1)
}
#[inline(always)]
pub fn prev(self) -> $ix {
assert!(self.is_valid());
debug_assert!(self.is_valid());
Self(self.0 - 1)
}
@@ -62,19 +62,19 @@ pub struct InstRange(Inst, Inst, bool);
impl InstRange {
#[inline(always)]
pub fn forward(from: Inst, to: Inst) -> Self {
assert!(from.index() <= to.index());
debug_assert!(from.index() <= to.index());
InstRange(from, to, true)
}
#[inline(always)]
pub fn backward(from: Inst, to: Inst) -> Self {
assert!(from.index() >= to.index());
debug_assert!(from.index() >= to.index());
InstRange(to, from, false)
}
#[inline(always)]
pub fn first(self) -> Inst {
assert!(self.len() > 0);
debug_assert!(self.len() > 0);
if self.is_forward() {
self.0
} else {
@@ -84,7 +84,7 @@ impl InstRange {
#[inline(always)]
pub fn last(self) -> Inst {
assert!(self.len() > 0);
debug_assert!(self.len() > 0);
if self.is_forward() {
self.1.prev()
} else {
@@ -94,7 +94,7 @@ impl InstRange {
#[inline(always)]
pub fn rest(self) -> InstRange {
assert!(self.len() > 0);
debug_assert!(self.len() > 0);
if self.is_forward() {
InstRange::forward(self.0.next(), self.1)
} else {
@@ -147,13 +147,13 @@ mod test {
#[test]
fn test_inst_range() {
let range = InstRange::forward(Inst::new(0), Inst::new(0));
assert_eq!(range.len(), 0);
debug_assert_eq!(range.len(), 0);
let range = InstRange::forward(Inst::new(0), Inst::new(5));
assert_eq!(range.first().index(), 0);
assert_eq!(range.last().index(), 4);
assert_eq!(range.len(), 5);
assert_eq!(
debug_assert_eq!(range.first().index(), 0);
debug_assert_eq!(range.last().index(), 4);
debug_assert_eq!(range.len(), 5);
debug_assert_eq!(
range.iter().collect::<Vec<_>>(),
vec![
Inst::new(0),
@@ -164,10 +164,10 @@ mod test {
]
);
let range = range.rev();
assert_eq!(range.first().index(), 4);
assert_eq!(range.last().index(), 0);
assert_eq!(range.len(), 5);
assert_eq!(
debug_assert_eq!(range.first().index(), 4);
debug_assert_eq!(range.last().index(), 0);
debug_assert_eq!(range.len(), 5);
debug_assert_eq!(
range.iter().collect::<Vec<_>>(),
vec![
Inst::new(4),

10
src/indexset.rs
View File

@@ -93,7 +93,7 @@ impl AdaptiveMap {
};
if needs_expand {
assert!(small_mode_idx.is_none());
debug_assert!(small_mode_idx.is_none());
self.expand();
}
@@ -111,7 +111,7 @@ impl AdaptiveMap {
}
// Otherwise, the key must not be present; add a new
// entry.
assert!(*len < SMALL_ELEMS as u32);
debug_assert!(*len < SMALL_ELEMS as u32);
let idx = *len;
*len += 1;
keys[idx as usize] = key;
@@ -344,11 +344,11 @@ mod test {
let mut checksum = 0;
for bit in vec.iter() {
assert!(bit % 17 == 0);
debug_assert!(bit % 17 == 0);
checksum += bit;
}
assert_eq!(sum, checksum);
debug_assert_eq!(sum, checksum);
}
#[test]
@@ -362,6 +362,6 @@ mod test {
// should still be in small mode.
vec.set(64 * 5, false);
vec.set(64 * 100, true);
assert!(vec.is_small());
debug_assert!(vec.is_small());
}
}

97
src/ion/data_structures.rs
View File

@@ -264,7 +264,6 @@ pub struct VRegData {
#[derive(Clone, Debug)]
pub struct PRegData {
pub reg: PReg,
pub allocations: LiveRangeSet,
pub is_stack: bool,
}
@@ -278,6 +277,56 @@ pub struct MultiFixedRegFixup {
pub vreg: VRegIndex,
}
/// The field order is significant: these are sorted so that a
/// scan over vregs, then blocks in each range, can scan in
/// order through this (sorted) list and add allocs to the
/// half-move list.
///
/// The fields in this struct are reversed in sort order so that the entire
/// struct can be treated as a u128 for sorting purposes.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(C)]
pub struct BlockparamOut {
pub to_vreg: VRegIndex,
pub to_block: Block,
pub from_block: Block,
pub from_vreg: VRegIndex,
}
impl BlockparamOut {
#[inline(always)]
pub fn key(&self) -> u128 {
u128_key(
self.from_vreg.raw_u32(),
self.from_block.raw_u32(),
self.to_block.raw_u32(),
self.to_vreg.raw_u32(),
)
}
}
/// As above for `BlockparamIn`, field order is significant.
///
/// The fields in this struct are reversed in sort order so that the entire
/// struct can be treated as a u128 for sorting purposes.
#[derive(Clone, Debug)]
#[repr(C)]
pub struct BlockparamIn {
pub from_block: Block,
pub to_block: Block,
pub to_vreg: VRegIndex,
}
impl BlockparamIn {
#[inline(always)]
pub fn key(&self) -> u128 {
u128_key(
self.to_vreg.raw_u32(),
self.to_block.raw_u32(),
self.from_block.raw_u32(),
0,
)
}
}
#[derive(Clone, Debug)]
pub struct Env<'a, F: Function> {
pub func: &'a F,
@@ -285,16 +334,8 @@ pub struct Env<'a, F: Function> {
pub cfginfo: CFGInfo,
pub liveins: Vec<IndexSet>,
pub liveouts: Vec<IndexSet>,
/// Blockparam outputs: from-vreg, (end of) from-block, (start of)
/// to-block, to-vreg. The field order is significant: these are sorted so
/// that a scan over vregs, then blocks in each range, can scan in
/// order through this (sorted) list and add allocs to the
/// half-move list.
pub blockparam_outs: Vec<(VRegIndex, Block, Block, VRegIndex)>,
/// Blockparam inputs: to-vreg, (start of) to-block, (end of)
/// from-block. As above for `blockparam_outs`, field order is
/// significant.
pub blockparam_ins: Vec<(VRegIndex, Block, Block)>,
pub blockparam_outs: Vec<BlockparamOut>,
pub blockparam_ins: Vec<BlockparamIn>,
/// Blockparam allocs: block, idx, vreg, alloc. Info to describe
/// blockparam locations at block entry, for metadata purposes
/// (e.g. for the checker).
@@ -344,7 +385,7 @@ pub struct Env<'a, F: Function> {
pub inserted_moves: Vec<InsertedMove>,
// Output:
pub edits: Vec<(u32, InsertMovePrio, Edit)>,
pub edits: Vec<(PosWithPrio, Edit)>,
pub allocs: Vec<Allocation>,
pub inst_alloc_offsets: Vec<u32>,
pub num_spillslots: u32,
@@ -488,8 +529,7 @@ impl LiveRangeSet {
#[derive(Clone, Debug)]
pub struct InsertedMove {
pub pos: ProgPoint,
pub prio: InsertMovePrio,
pub pos_prio: PosWithPrio,
pub from_alloc: Allocation,
pub to_alloc: Allocation,
pub to_vreg: Option<VReg>,
@@ -506,6 +546,21 @@ pub enum InsertMovePrio {
OutEdgeMoves,
}
/// The fields in this struct are reversed in sort order so that the entire
/// struct can be treated as a u64 for sorting purposes.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(C)]
pub struct PosWithPrio {
pub prio: u32,
pub pos: ProgPoint,
}
impl PosWithPrio {
pub fn key(self) -> u64 {
u64_key(self.pos.to_index(), self.prio)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct Stats {
pub livein_blocks: usize,
@@ -544,3 +599,17 @@ pub struct Stats {
pub halfmoves_count: usize,
pub edits_count: usize,
}
// Helper function for generating sorting keys. The order of arguments is from
// the most significant field to the least significant one.
//
// These work best when the fields are stored in reverse order in memory so that
// they can be loaded with a single u64 load on little-endian machines.
#[inline(always)]
pub fn u64_key(b: u32, a: u32) -> u64 {
a as u64 | (b as u64) << 32
}
#[inline(always)]
pub fn u128_key(d: u32, c: u32, b: u32, a: u32) -> u128 {
a as u128 | (b as u128) << 32 | (c as u128) << 64 | (d as u128) << 96
}

18
src/ion/dump.rs
View File

@@ -5,16 +5,16 @@ use crate::{Block, Function, ProgPoint};
impl<'a, F: Function> Env<'a, F> {
pub fn dump_state(&self) {
log::trace!("Bundles:");
trace!("Bundles:");
for (i, b) in self.bundles.iter().enumerate() {
log::trace!(
trace!(
"bundle{}: spillset={:?} alloc={:?}",
i,
b.spillset,
b.allocation
);
for entry in &b.ranges {
log::trace!(
trace!(
" * range {:?} -- {:?}: range{}",
entry.range.from,
entry.range.to,
@@ -22,11 +22,11 @@ impl<'a, F: Function> Env<'a, F> {
);
}
}
log::trace!("VRegs:");
trace!("VRegs:");
for (i, v) in self.vregs.iter().enumerate() {
log::trace!("vreg{}:", i);
trace!("vreg{}:", i);
for entry in &v.ranges {
log::trace!(
trace!(
" * range {:?} -- {:?}: range{}",
entry.range.from,
entry.range.to,
@@ -34,9 +34,9 @@ impl<'a, F: Function> Env<'a, F> {
);
}
}
log::trace!("Ranges:");
trace!("Ranges:");
for (i, r) in self.ranges.iter().enumerate() {
log::trace!(
trace!(
"range{}: range={:?} vreg={:?} bundle={:?} weight={:?}",
i,
r.range,
@@ -45,7 +45,7 @@ impl<'a, F: Function> Env<'a, F> {
r.uses_spill_weight(),
);
for u in &r.uses {
log::trace!(" * use at {:?} (slot {}): {:?}", u.pos, u.slot, u.operand);
trace!(" * use at {:?} (slot {}): {:?}", u.pos, u.slot, u.operand);
}
}
}

137
src/ion/liveranges.rs
View File

@@ -18,7 +18,7 @@ use super::{
SpillSetIndex, Use, VRegData, VRegIndex, SLOT_NONE,
};
use crate::indexset::IndexSet;
use crate::ion::data_structures::MultiFixedRegFixup;
use crate::ion::data_structures::{BlockparamIn, BlockparamOut, MultiFixedRegFixup};
use crate::{
Allocation, Block, Function, Inst, InstPosition, Operand, OperandConstraint, OperandKind,
OperandPos, PReg, ProgPoint, RegAllocError, VReg,
@@ -102,17 +102,10 @@ impl<'a, F: Function> Env<'a, F> {
self.pregs.resize(
PReg::NUM_INDEX,
PRegData {
reg: PReg::invalid(),
allocations: LiveRangeSet::new(),
is_stack: false,
},
);
for i in 0..=PReg::MAX {
let preg_int = PReg::new(i, RegClass::Int);
self.pregs[preg_int.index()].reg = preg_int;
let preg_float = PReg::new(i, RegClass::Float);
self.pregs[preg_float.index()].reg = preg_float;
}
for &preg in &self.env.fixed_stack_slots {
self.pregs[preg.index()].is_stack = true;
}
@@ -186,7 +179,7 @@ impl<'a, F: Function> Env<'a, F> {
///
/// Returns the liverange that contains the given range.
pub fn add_liverange_to_vreg(&mut self, vreg: VRegIndex, range: CodeRange) -> LiveRangeIndex {
log::trace!("add_liverange_to_vreg: vreg {:?} range {:?}", vreg, range);
trace!("add_liverange_to_vreg: vreg {:?} range {:?}", vreg, range);
// Invariant: as we are building liveness information, we
// *always* process instructions bottom-to-top, and as a
@@ -199,7 +192,7 @@ impl<'a, F: Function> Env<'a, F> {
// array, then reverse them at the end of
// `compute_liveness()`.
assert!(
debug_assert!(
self.vregs[vreg.index()].ranges.is_empty()
|| range.to
<= self.ranges[self.vregs[vreg.index()]
@@ -235,7 +228,7 @@ impl<'a, F: Function> Env<'a, F> {
// Is contiguous with previously-added range; just extend
// its range and return it.
let lr = self.vregs[vreg.index()].ranges.last().unwrap().index;
assert!(range.to == self.ranges[lr.index()].range.from);
debug_assert!(range.to == self.ranges[lr.index()].range.from);
self.ranges[lr.index()].range.from = range.from;
lr
}
@@ -253,7 +246,7 @@ impl<'a, F: Function> Env<'a, F> {
);
u.weight = weight.to_bits();
log::trace!(
trace!(
"insert use {:?} into lr {:?} with weight {:?}",
u,
into,
@@ -269,7 +262,7 @@ impl<'a, F: Function> Env<'a, F> {
// Update stats.
let range_weight = self.ranges[into.index()].uses_spill_weight() + weight;
self.ranges[into.index()].set_uses_spill_weight(range_weight);
log::trace!(
trace!(
" -> now range has weight {:?}",
self.ranges[into.index()].uses_spill_weight(),
);
@@ -289,7 +282,7 @@ impl<'a, F: Function> Env<'a, F> {
}
pub fn add_liverange_to_preg(&mut self, range: CodeRange, reg: PReg) {
log::trace!("adding liverange to preg: {:?} to {}", range, reg);
trace!("adding liverange to preg: {:?} to {}", range, reg);
let preg_idx = PRegIndex::new(reg.index());
self.pregs[preg_idx.index()]
.allocations
@@ -323,12 +316,12 @@ impl<'a, F: Function> Env<'a, F> {
workqueue_set.remove(&block);
let insns = self.func.block_insns(block);
log::trace!("computing liveins for block{}", block.index());
trace!("computing liveins for block{}", block.index());
self.stats.livein_iterations += 1;
let mut live = self.liveouts[block.index()].clone();
log::trace!(" -> initial liveout set: {:?}", live);
trace!(" -> initial liveout set: {:?}", live);
// Include outgoing blockparams in the initial live set.
if self.func.is_branch(insns.last()) {
@@ -349,7 +342,7 @@ impl<'a, F: Function> Env<'a, F> {
for op in self.func.inst_operands(inst) {
if op.pos() == *pos {
let was_live = live.get(op.vreg().vreg());
log::trace!("op {:?} was_live = {}", op, was_live);
trace!("op {:?} was_live = {}", op, was_live);
match op.kind() {
OperandKind::Use | OperandKind::Mod => {
live.set(op.vreg().vreg(), true);
@@ -375,7 +368,7 @@ impl<'a, F: Function> Env<'a, F> {
}
}
log::trace!("computed liveins at block{}: {:?}", block.index(), live);
trace!("computed liveins at block{}: {:?}", block.index(), live);
self.liveins[block.index()] = live;
}
@@ -387,7 +380,7 @@ impl<'a, F: Function> Env<'a, F> {
.next()
.is_some()
{
log::trace!(
trace!(
"non-empty liveins to entry block: {:?}",
self.liveins[self.func.entry_block().index()]
);
@@ -437,8 +430,12 @@ impl<'a, F: Function> Env<'a, F> {
{
let blockparam_out = VRegIndex::new(blockparam_out.vreg());
let blockparam_in = VRegIndex::new(blockparam_in.vreg());
self.blockparam_outs
.push((blockparam_out, block, succ, blockparam_in));
self.blockparam_outs.push(BlockparamOut {
to_vreg: blockparam_in,
to_block: succ,
from_block: block,
from_vreg: blockparam_out,
});
// Include outgoing blockparams in the initial live set.
live.set(blockparam_out.index(), true);
@@ -452,7 +449,7 @@ impl<'a, F: Function> Env<'a, F> {
from: self.cfginfo.block_entry[block.index()],
to: self.cfginfo.block_exit[block.index()].next(),
};
log::trace!(
trace!(
"vreg {:?} live at end of block --> create range {:?}",
VRegIndex::new(vreg),
range
@@ -502,13 +499,13 @@ impl<'a, F: Function> Env<'a, F> {
// We can completely skip the move if it is
// trivial (vreg to same vreg).
if src.vreg() != dst.vreg() {
log::trace!(" -> move inst{}: src {} -> dst {}", inst.index(), src, dst);
trace!(" -> move inst{}: src {} -> dst {}", inst.index(), src, dst);
assert_eq!(src.class(), dst.class());
assert_eq!(src.kind(), OperandKind::Use);
assert_eq!(src.pos(), OperandPos::Early);
assert_eq!(dst.kind(), OperandKind::Def);
assert_eq!(dst.pos(), OperandPos::Late);
debug_assert_eq!(src.class(), dst.class());
debug_assert_eq!(src.kind(), OperandKind::Use);
debug_assert_eq!(src.pos(), OperandPos::Early);
debug_assert_eq!(dst.kind(), OperandKind::Def);
debug_assert_eq!(dst.pos(), OperandPos::Late);
// If both src and dest are pinned, emit the
// move right here, right now.
@@ -564,9 +561,7 @@ impl<'a, F: Function> Env<'a, F> {
else if self.vregs[src.vreg().vreg()].is_pinned
|| self.vregs[dst.vreg().vreg()].is_pinned
{
log::trace!(
" -> exactly one of src/dst is pinned; converting to ghost use"
);
trace!(" -> exactly one of src/dst is pinned; converting to ghost use");
let (preg, vreg, pinned_vreg, kind, pos, progpoint) =
if self.vregs[src.vreg().vreg()].is_pinned {
// Source is pinned: this is a def on the dst with a pinned preg.
@@ -592,7 +587,7 @@ impl<'a, F: Function> Env<'a, F> {
let constraint = OperandConstraint::FixedReg(preg);
let operand = Operand::new(vreg, constraint, kind, pos);
log::trace!(
trace!(
concat!(
" -> preg {:?} vreg {:?} kind {:?} ",
"pos {:?} progpoint {:?} constraint {:?} operand {:?}"
@@ -619,9 +614,9 @@ impl<'a, F: Function> Env<'a, F> {
VRegIndex::new(vreg.vreg()),
CodeRange { from, to },
);
log::trace!(" -> dead; created LR");
trace!(" -> dead; created LR");
}
log::trace!(" -> LR {:?}", lr);
trace!(" -> LR {:?}", lr);
self.insert_use_into_liverange(
lr,
@@ -655,7 +650,7 @@ impl<'a, F: Function> Env<'a, F> {
// (this is the last use), start it
// before.
if kind == OperandKind::Def {
log::trace!(" -> src on pinned vreg {:?}", pinned_vreg);
trace!(" -> src on pinned vreg {:?}", pinned_vreg);
// The *other* vreg is a def, so the pinned-vreg
// mention is a use. If already live,
// end the existing LR just *after*
@@ -669,7 +664,7 @@ impl<'a, F: Function> Env<'a, F> {
if live.get(pinned_vreg.vreg()) {
let pinned_lr = vreg_ranges[pinned_vreg.vreg()];
let orig_start = self.ranges[pinned_lr.index()].range.from;
log::trace!(
trace!(
" -> live with LR {:?}; truncating to start at {:?}",
pinned_lr,
progpoint.next()
@@ -683,7 +678,7 @@ impl<'a, F: Function> Env<'a, F> {
},
);
vreg_ranges[pinned_vreg.vreg()] = new_lr;
log::trace!(" -> created LR {:?} with remaining range from {:?} to {:?}", new_lr, orig_start, progpoint);
trace!(" -> created LR {:?} with remaining range from {:?} to {:?}", new_lr, orig_start, progpoint);
// Add an edit right now to indicate that at
// this program point, the given
@@ -717,10 +712,7 @@ impl<'a, F: Function> Env<'a, F> {
);
vreg_ranges[pinned_vreg.vreg()] = new_lr;
live.set(pinned_vreg.vreg(), true);
log::trace!(
" -> was not live; created new LR {:?}",
new_lr
);
trace!(" -> was not live; created new LR {:?}", new_lr);
}
// Add an edit right now to indicate that at
@@ -737,7 +729,7 @@ impl<'a, F: Function> Env<'a, F> {
);
}
} else {
log::trace!(" -> dst on pinned vreg {:?}", pinned_vreg);
trace!(" -> dst on pinned vreg {:?}", pinned_vreg);
// The *other* vreg is a use, so the pinned-vreg
// mention is a def. Truncate its LR
// just *after* the `progpoint`
@@ -745,7 +737,7 @@ impl<'a, F: Function> Env<'a, F> {
if live.get(pinned_vreg.vreg()) {
let pinned_lr = vreg_ranges[pinned_vreg.vreg()];
self.ranges[pinned_lr.index()].range.from = progpoint.next();
log::trace!(
trace!(
" -> was live with LR {:?}; truncated start to {:?}",
pinned_lr,
progpoint.next()
@@ -845,14 +837,14 @@ impl<'a, F: Function> Env<'a, F> {
VRegIndex::new(dst.vreg().vreg()),
CodeRange { from, to },
);
log::trace!(" -> invalid LR for def; created {:?}", dst_lr);
trace!(" -> invalid LR for def; created {:?}", dst_lr);
}
log::trace!(" -> has existing LR {:?}", dst_lr);
trace!(" -> has existing LR {:?}", dst_lr);
// Trim the LR to start here.
if self.ranges[dst_lr.index()].range.from
== self.cfginfo.block_entry[block.index()]
{
log::trace!(" -> started at block start; trimming to {:?}", pos);
trace!(" -> started at block start; trimming to {:?}", pos);
self.ranges[dst_lr.index()].range.from = pos;
}
self.ranges[dst_lr.index()].set_flag(LiveRangeFlag::StartsAtDef);
@@ -879,7 +871,7 @@ impl<'a, F: Function> Env<'a, F> {
vreg_ranges[src.vreg().vreg()]
};
log::trace!(" -> src LR {:?}", src_lr);
trace!(" -> src LR {:?}", src_lr);
// Add to live-set.
let src_is_dead_after_move = !live.get(src.vreg().vreg());
@@ -932,7 +924,7 @@ impl<'a, F: Function> Env<'a, F> {
continue;
}
log::trace!(
trace!(
"processing inst{} operand at {:?}: {:?}",
inst.index(),
pos,
@@ -941,14 +933,14 @@ impl<'a, F: Function> Env<'a, F> {
match operand.kind() {
OperandKind::Def | OperandKind::Mod => {
log::trace!("Def of {} at {:?}", operand.vreg(), pos);
trace!("Def of {} at {:?}", operand.vreg(), pos);
// Fill in vreg's actual data.
self.vreg_regs[operand.vreg().vreg()] = operand.vreg();
// Get or create the LiveRange.
let mut lr = vreg_ranges[operand.vreg().vreg()];
log::trace!(" -> has existing LR {:?}", lr);
trace!(" -> has existing LR {:?}", lr);
// If there was no liverange (dead def), create a trivial one.
if !live.get(operand.vreg().vreg()) {
let from = match operand.kind() {
@@ -965,7 +957,7 @@ impl<'a, F: Function> Env<'a, F> {
VRegIndex::new(operand.vreg().vreg()),
CodeRange { from, to },
);
log::trace!(" -> invalid; created {:?}", lr);
trace!(" -> invalid; created {:?}", lr);
vreg_ranges[operand.vreg().vreg()] = lr;
live.set(operand.vreg().vreg(), true);
}
@@ -982,10 +974,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.ranges[lr.index()].range.from
== self.cfginfo.block_entry[block.index()]
{
log::trace!(
" -> started at block start; trimming to {:?}",
pos
);
trace!(" -> started at block start; trimming to {:?}", pos);
self.ranges[lr.index()].range.from = pos;
}
@@ -1012,9 +1001,9 @@ impl<'a, F: Function> Env<'a, F> {
);
vreg_ranges[operand.vreg().vreg()] = lr;
}
assert!(lr.is_valid());
debug_assert!(lr.is_valid());
log::trace!("Use of {:?} at {:?} -> {:?}", operand, pos, lr,);
trace!("Use of {:?} at {:?} -> {:?}", operand, pos, lr,);
self.insert_use_into_liverange(lr, Use::new(operand, pos, i as u8));
@@ -1026,11 +1015,11 @@ impl<'a, F: Function> Env<'a, F> {
}
if self.func.requires_refs_on_stack(inst) {
log::trace!("inst{} is safepoint", inst.index());
trace!("inst{} is safepoint", inst.index());
self.safepoints.push(inst);
for vreg in live.iter() {
if let Some(safepoints) = self.safepoints_per_vreg.get_mut(&vreg) {
log::trace!("vreg v{} live at safepoint inst{}", vreg, inst.index());
trace!("vreg v{} live at safepoint inst{}", vreg, inst.index());
safepoints.insert(inst);
}
}
@@ -1057,7 +1046,11 @@ impl<'a, F: Function> Env<'a, F> {
// add `blockparam_ins` entries.
let vreg_idx = VRegIndex::new(vreg.vreg());
for &pred in self.func.block_preds(block) {
self.blockparam_ins.push((vreg_idx, block, pred));
self.blockparam_ins.push(BlockparamIn {
to_vreg: vreg_idx,
to_block: block,
from_block: pred,
});
}
}
}
@@ -1082,7 +1075,7 @@ impl<'a, F: Function> Env<'a, F> {
// need to update with the final range here.
entry.range = self.ranges[entry.index.index()].range;
// Assert in-order and non-overlapping.
assert!(last.is_none() || last.unwrap() <= entry.range.from);
debug_assert!(last.is_none() || last.unwrap() <= entry.range.from);
last = Some(entry.range.to);
}
}
@@ -1123,7 +1116,7 @@ impl<'a, F: Function> Env<'a, F> {
OperandPos::Early,
);
log::trace!(
trace!(
"Safepoint-induced stack use of {:?} at {:?} -> {:?}",
operand,
pos,
@@ -1148,13 +1141,13 @@ impl<'a, F: Function> Env<'a, F> {
}
}
self.blockparam_ins.sort_unstable();
self.blockparam_outs.sort_unstable();
self.blockparam_ins.sort_unstable_by_key(|x| x.key());
self.blockparam_outs.sort_unstable_by_key(|x| x.key());
self.prog_move_srcs.sort_unstable_by_key(|(pos, _)| *pos);
self.prog_move_dsts.sort_unstable_by_key(|(pos, _)| *pos);
log::trace!("prog_move_srcs = {:?}", self.prog_move_srcs);
log::trace!("prog_move_dsts = {:?}", self.prog_move_dsts);
trace!("prog_move_srcs = {:?}", self.prog_move_srcs);
trace!("prog_move_dsts = {:?}", self.prog_move_dsts);
self.stats.initial_liverange_count = self.ranges.len();
self.stats.blockparam_ins_count = self.blockparam_ins.len();
@@ -1176,7 +1169,7 @@ impl<'a, F: Function> Env<'a, F> {
for range_idx in 0..self.vregs[vreg].ranges.len() {
let entry = self.vregs[vreg].ranges[range_idx];
let range = entry.index;
log::trace!(
trace!(
"multi-fixed-reg cleanup: vreg {:?} range {:?}",
VRegIndex::new(vreg),
range,
@@ -1245,7 +1238,7 @@ impl<'a, F: Function> Env<'a, F> {
if let OperandConstraint::FixedReg(preg) = u.operand.constraint() {
let vreg_idx = VRegIndex::new(u.operand.vreg().vreg());
let preg_idx = PRegIndex::new(preg.index());
log::trace!(
trace!(
"at pos {:?}, vreg {:?} has fixed constraint to preg {:?}",
u.pos,
vreg_idx,
@@ -1264,7 +1257,7 @@ impl<'a, F: Function> Env<'a, F> {
continue;
}
log::trace!(" -> duplicate; switching to constraint Any");
trace!(" -> duplicate; switching to constraint Any");
self.multi_fixed_reg_fixups.push(MultiFixedRegFixup {
pos: u.pos,
from_slot: source_slot,
@@ -1278,11 +1271,7 @@ impl<'a, F: Function> Env<'a, F> {
u.operand.kind(),
u.operand.pos(),
);
log::trace!(
" -> extra clobber {} at inst{}",
preg,
u.pos.inst().index()
);
trace!(" -> extra clobber {} at inst{}", preg, u.pos.inst().index());
extra_clobbers.push((preg, u.pos.inst()));
}
}

72
src/ion/merge.rs
View File

@@ -16,7 +16,7 @@ use super::{
Env, LiveBundleIndex, LiveRangeIndex, LiveRangeKey, SpillSet, SpillSetIndex, SpillSlotIndex,
VRegIndex,
};
use crate::{Function, Inst, OperandConstraint, PReg};
use crate::{ion::data_structures::BlockparamOut, Function, Inst, OperandConstraint, PReg};
use smallvec::smallvec;
impl<'a, F: Function> Env<'a, F> {
@@ -25,7 +25,7 @@ impl<'a, F: Function> Env<'a, F> {
// Merge bundle into self -- trivial merge.
return true;
}
log::trace!(
trace!(
"merging from bundle{} to bundle{}",
from.index(),
to.index()
@@ -35,7 +35,7 @@ impl<'a, F: Function> Env<'a, F> {
let from_rc = self.spillsets[self.bundles[from.index()].spillset.index()].class;
let to_rc = self.spillsets[self.bundles[to.index()].spillset.index()].class;
if from_rc != to_rc {
log::trace!(" -> mismatching reg classes");
trace!(" -> mismatching reg classes");
return false;
}
@@ -43,7 +43,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.bundles[from.index()].allocation.is_some()
|| self.bundles[to.index()].allocation.is_some()
{
log::trace!("one of the bundles is already assigned (pinned)");
trace!("one of the bundles is already assigned (pinned)");
return false;
}
@@ -52,11 +52,11 @@ impl<'a, F: Function> Env<'a, F> {
// Sanity check: both bundles should contain only ranges with appropriate VReg classes.
for entry in &self.bundles[from.index()].ranges {
let vreg = self.ranges[entry.index.index()].vreg;
assert_eq!(from_rc, self.vreg_regs[vreg.index()].class());
debug_assert_eq!(from_rc, self.vreg_regs[vreg.index()].class());
}
for entry in &self.bundles[to.index()].ranges {
let vreg = self.ranges[entry.index.index()].vreg;
assert_eq!(to_rc, self.vreg_regs[vreg.index()].class());
debug_assert_eq!(to_rc, self.vreg_regs[vreg.index()].class());
}
}
@@ -70,7 +70,7 @@ impl<'a, F: Function> Env<'a, F> {
while idx_from < ranges_from.len() && idx_to < ranges_to.len() {
range_count += 1;
if range_count > 200 {
log::trace!(
trace!(
"reached merge complexity (range_count = {}); exiting",
range_count
);
@@ -84,7 +84,7 @@ impl<'a, F: Function> Env<'a, F> {
idx_from += 1;
} else {
// Overlap -- cannot merge.
log::trace!(
trace!(
" -> overlap between {:?} and {:?}, exiting",
ranges_from[idx_from].index,
ranges_to[idx_to].index
@@ -100,12 +100,12 @@ impl<'a, F: Function> Env<'a, F> {
|| self.bundles[to.index()].cached_fixed()
{
if self.merge_bundle_requirements(from, to).is_err() {
log::trace!(" -> conflicting requirements; aborting merge");
trace!(" -> conflicting requirements; aborting merge");
return false;
}
}
log::trace!(" -> committing to merge");
trace!(" -> committing to merge");
// If we reach here, then the bundles do not overlap -- merge
// them! We do this with a merge-sort-like scan over both
@@ -113,13 +113,13 @@ impl<'a, F: Function> Env<'a, F> {
// `to` when we're done.
if ranges_from.is_empty() {
// `from` bundle is empty -- trivial merge.
log::trace!(" -> from bundle{} is empty; trivial merge", from.index());
trace!(" -> from bundle{} is empty; trivial merge", from.index());
return true;
}
if ranges_to.is_empty() {
// `to` bundle is empty -- just move the list over from
// `from` and set `bundle` up-link on all ranges.
log::trace!(" -> to bundle{} is empty; trivial merge", to.index());
trace!(" -> to bundle{} is empty; trivial merge", to.index());
let list = std::mem::replace(&mut self.bundles[from.index()].ranges, smallvec![]);
for entry in &list {
self.ranges[entry.index.index()].bundle = to;
@@ -149,7 +149,7 @@ impl<'a, F: Function> Env<'a, F> {
return true;
}
log::trace!(
trace!(
"merging: ranges_from = {:?} ranges_to = {:?}",
ranges_from,
ranges_to
@@ -170,12 +170,12 @@ impl<'a, F: Function> Env<'a, F> {
.sort_unstable_by_key(|entry| entry.range.from);
if self.annotations_enabled {
log::trace!("merging: merged = {:?}", self.bundles[to.index()].ranges);
trace!("merging: merged = {:?}", self.bundles[to.index()].ranges);
let mut last_range = None;
for i in 0..self.bundles[to.index()].ranges.len() {
let entry = self.bundles[to.index()].ranges[i];
if last_range.is_some() {
assert!(last_range.unwrap() < entry.range);
debug_assert!(last_range.unwrap() < entry.range);
}
last_range = Some(entry.range);
@@ -192,7 +192,7 @@ impl<'a, F: Function> Env<'a, F> {
);
}
log::trace!(
trace!(
" -> merged result for bundle{}: range{}",
to.index(),
entry.index.index(),
@@ -225,7 +225,7 @@ impl<'a, F: Function> Env<'a, F> {
pub fn merge_vreg_bundles(&mut self) {
// Create a bundle for every vreg, initially.
log::trace!("merge_vreg_bundles: creating vreg bundles");
trace!("merge_vreg_bundles: creating vreg bundles");
for vreg in 0..self.vregs.len() {
let vreg = VRegIndex::new(vreg);
if self.vregs[vreg.index()].ranges.is_empty() {
@@ -251,9 +251,9 @@ impl<'a, F: Function> Env<'a, F> {
let bundle = self.create_bundle();
self.bundles[bundle.index()].ranges = self.vregs[vreg.index()].ranges.clone();
log::trace!("vreg v{} gets bundle{}", vreg.index(), bundle.index());
trace!("vreg v{} gets bundle{}", vreg.index(), bundle.index());
for entry in &self.bundles[bundle.index()].ranges {
log::trace!(
trace!(
" -> with LR range{}: {:?}",
entry.index.index(),
entry.range
@@ -314,17 +314,17 @@ impl<'a, F: Function> Env<'a, F> {
continue;
}
log::trace!(
trace!(
"trying to merge reused-input def: src {} to dst {}",
src_vreg,
dst_vreg
);
let src_bundle =
self.ranges[self.vregs[src_vreg.vreg()].ranges[0].index.index()].bundle;
assert!(src_bundle.is_valid());
debug_assert!(src_bundle.is_valid());
let dest_bundle =
self.ranges[self.vregs[dst_vreg.vreg()].ranges[0].index.index()].bundle;
assert!(dest_bundle.is_valid());
debug_assert!(dest_bundle.is_valid());
self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle);
}
}
@@ -332,18 +332,20 @@ impl<'a, F: Function> Env<'a, F> {
// Attempt to merge blockparams with their inputs.
for i in 0..self.blockparam_outs.len() {
let (from_vreg, _, _, to_vreg) = self.blockparam_outs[i];
log::trace!(
let BlockparamOut {
from_vreg, to_vreg, ..
} = self.blockparam_outs[i];
trace!(
"trying to merge blockparam v{} with input v{}",
to_vreg.index(),
from_vreg.index()
);
let to_bundle = self.ranges[self.vregs[to_vreg.index()].ranges[0].index.index()].bundle;
assert!(to_bundle.is_valid());
debug_assert!(to_bundle.is_valid());
let from_bundle =
self.ranges[self.vregs[from_vreg.index()].ranges[0].index.index()].bundle;
assert!(from_bundle.is_valid());
log::trace!(
debug_assert!(from_bundle.is_valid());
trace!(
" -> from bundle{} to bundle{}",
from_bundle.index(),
to_bundle.index()
@@ -354,10 +356,10 @@ impl<'a, F: Function> Env<'a, F> {
// Attempt to merge move srcs/dsts.
for i in 0..self.prog_move_merges.len() {
let (src, dst) = self.prog_move_merges[i];
log::trace!("trying to merge move src LR {:?} to dst LR {:?}", src, dst);
trace!("trying to merge move src LR {:?} to dst LR {:?}", src, dst);
let src = self.resolve_merged_lr(src);
let dst = self.resolve_merged_lr(dst);
log::trace!(
trace!(
"resolved LR-construction merging chains: move-merge is now src LR {:?} to dst LR {:?}",
src,
dst
@@ -384,16 +386,16 @@ impl<'a, F: Function> Env<'a, F> {
}
let src_bundle = self.ranges[src.index()].bundle;
assert!(src_bundle.is_valid());
debug_assert!(src_bundle.is_valid());
let dest_bundle = self.ranges[dst.index()].bundle;
assert!(dest_bundle.is_valid());
debug_assert!(dest_bundle.is_valid());
self.stats.prog_move_merge_attempt += 1;
if self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle) {
self.stats.prog_move_merge_success += 1;
}
}
log::trace!("done merging bundles");
trace!("done merging bundles");
}
pub fn resolve_merged_lr(&self, mut lr: LiveRangeIndex) -> LiveRangeIndex {
@@ -417,14 +419,14 @@ impl<'a, F: Function> Env<'a, F> {
pub fn queue_bundles(&mut self) {
for bundle in 0..self.bundles.len() {
log::trace!("enqueueing bundle{}", bundle);
trace!("enqueueing bundle{}", bundle);
if self.bundles[bundle].ranges.is_empty() {
log::trace!(" -> no ranges; skipping");
trace!(" -> no ranges; skipping");
continue;
}
let bundle = LiveBundleIndex::new(bundle);
let prio = self.compute_bundle_prio(bundle);
log::trace!(" -> prio {}", prio);
trace!(" -> prio {}", prio);
self.bundles[bundle.index()].prio = prio;
self.recompute_bundle_properties(bundle);
self.allocation_queue

2
src/ion/mod.rs
View File

@@ -133,7 +133,7 @@ pub fn run<F: Function>(
edits: env
.edits
.into_iter()
.map(|(pos, _, edit)| (ProgPoint::from_index(pos), edit))
.map(|(pos_prio, edit)| (pos_prio.pos, edit))
.collect(),
allocs: env.allocs,
inst_alloc_offsets: env.inst_alloc_offsets,

385
src/ion/moves.rs
View File

@@ -17,10 +17,11 @@ use super::{
VRegIndex, SLOT_NONE,
};
use crate::ion::data_structures::{BlockparamIn, BlockparamOut, PosWithPrio};
use crate::moves::ParallelMoves;
use crate::{
Allocation, Block, Edit, Function, Inst, InstPosition, OperandConstraint, OperandKind,
OperandPos, ProgPoint, RegClass, VReg,
OperandPos, PReg, ProgPoint, RegClass, VReg,
};
use smallvec::{smallvec, SmallVec};
use std::fmt::Debug;
@@ -53,7 +54,7 @@ impl<'a, F: Function> Env<'a, F> {
to_alloc: Allocation,
to_vreg: Option<VReg>,
) {
log::trace!(
trace!(
"insert_move: pos {:?} prio {:?} from_alloc {:?} to_alloc {:?}",
pos,
prio,
@@ -62,13 +63,15 @@ impl<'a, F: Function> Env<'a, F> {
);
match (from_alloc.as_reg(), to_alloc.as_reg()) {
(Some(from), Some(to)) => {
assert_eq!(from.class(), to.class());
debug_assert_eq!(from.class(), to.class());
}
_ => {}
}
self.inserted_moves.push(InsertedMove {
pos,
prio,
pos_prio: PosWithPrio {
pos,
prio: prio as u32,
},
from_alloc,
to_alloc,
to_vreg,
@@ -86,16 +89,16 @@ impl<'a, F: Function> Env<'a, F> {
}
pub fn get_alloc_for_range(&self, range: LiveRangeIndex) -> Allocation {
log::trace!("get_alloc_for_range: {:?}", range);
trace!("get_alloc_for_range: {:?}", range);
let bundle = self.ranges[range.index()].bundle;
log::trace!(" -> bundle: {:?}", bundle);
trace!(" -> bundle: {:?}", bundle);
let bundledata = &self.bundles[bundle.index()];
log::trace!(" -> allocation {:?}", bundledata.allocation);
trace!(" -> allocation {:?}", bundledata.allocation);
if bundledata.allocation != Allocation::none() {
bundledata.allocation
} else {
log::trace!(" -> spillset {:?}", bundledata.spillset);
log::trace!(
trace!(" -> spillset {:?}", bundledata.spillset);
trace!(
" -> spill slot {:?}",
self.spillsets[bundledata.spillset.index()].slot
);
@@ -104,9 +107,9 @@ impl<'a, F: Function> Env<'a, F> {
}
pub fn apply_allocations_and_insert_moves(&mut self) {
log::trace!("apply_allocations_and_insert_moves");
log::trace!("blockparam_ins: {:?}", self.blockparam_ins);
log::trace!("blockparam_outs: {:?}", self.blockparam_outs);
trace!("apply_allocations_and_insert_moves");
trace!("blockparam_ins: {:?}", self.blockparam_ins);
trace!("blockparam_outs: {:?}", self.blockparam_outs);
// Now that all splits are done, we can pay the cost once to
// sort VReg range lists and update with the final ranges.
@@ -148,9 +151,9 @@ impl<'a, F: Function> Env<'a, F> {
to_vreg: VRegIndex,
kind: HalfMoveKind,
) -> u64 {
assert!(from_block.index() < 1 << 21);
assert!(to_block.index() < 1 << 21);
assert!(to_vreg.index() < 1 << 21);
debug_assert!(from_block.index() < 1 << 21);
debug_assert!(to_block.index() < 1 << 21);
debug_assert!(to_vreg.index() < 1 << 21);
((from_block.index() as u64) << 43)
| ((to_block.index() as u64) << 22)
| ((to_vreg.index() as u64) << 1)
@@ -202,7 +205,7 @@ impl<'a, F: Function> Env<'a, F> {
.map(|preg| Allocation::reg(preg))
.unwrap_or_else(|| self.get_alloc_for_range(entry.index));
let range = entry.range;
log::trace!(
trace!(
"apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?} (pinned {:?})",
vreg,
entry.index,
@@ -269,7 +272,7 @@ impl<'a, F: Function> Env<'a, F> {
&& !self.is_start_of_block(range.from)
&& !first_is_def
{
log::trace!(
trace!(
"prev LR {} abuts LR {} in same block; moving {} -> {} for v{}",
prev.index(),
entry.index.index(),
@@ -277,7 +280,7 @@ impl<'a, F: Function> Env<'a, F> {
alloc,
vreg.index()
);
assert_eq!(range.from.pos(), InstPosition::Before);
debug_assert_eq!(range.from.pos(), InstPosition::Before);
self.insert_move(
range.from,
InsertMovePrio::Regular,
@@ -303,9 +306,9 @@ impl<'a, F: Function> Env<'a, F> {
if range.to < self.cfginfo.block_exit[block.index()].next() {
break;
}
log::trace!("examining block with end in range: block{}", block.index());
trace!("examining block with end in range: block{}", block.index());
for &succ in self.func.block_succs(block) {
log::trace!(
trace!(
" -> has succ block {} with entry {:?}",
succ.index(),
self.cfginfo.block_entry[succ.index()]
@@ -313,9 +316,9 @@ impl<'a, F: Function> Env<'a, F> {
if range.contains_point(self.cfginfo.block_entry[succ.index()]) {
continue;
}
log::trace!(" -> out of this range, requires half-move if live");
trace!(" -> out of this range, requires half-move if live");
if self.is_live_in(succ, vreg) {
log::trace!(" -> live at input to succ, adding halfmove");
trace!(" -> live at input to succ, adding halfmove");
half_moves.push(HalfMove {
key: half_move_key(block, succ, vreg, HalfMoveKind::Source),
alloc,
@@ -326,20 +329,24 @@ impl<'a, F: Function> Env<'a, F> {
// Scan forward in `blockparam_outs`, adding all
// half-moves for outgoing values to blockparams
// in succs.
log::trace!(
trace!(
"scanning blockparam_outs for v{} block{}: blockparam_out_idx = {}",
vreg.index(),
block.index(),
blockparam_out_idx,
);
while blockparam_out_idx < self.blockparam_outs.len() {
let (from_vreg, from_block, to_block, to_vreg) =
self.blockparam_outs[blockparam_out_idx];
let BlockparamOut {
from_vreg,
from_block,
to_block,
to_vreg,
} = self.blockparam_outs[blockparam_out_idx];
if (from_vreg, from_block) > (vreg, block) {
break;
}
if (from_vreg, from_block) == (vreg, block) {
log::trace!(
trace!(
" -> found: from v{} block{} to v{} block{}",
from_vreg.index(),
from_block.index(),
@@ -391,15 +398,18 @@ impl<'a, F: Function> Env<'a, F> {
}
// Add half-moves for blockparam inputs.
log::trace!(
trace!(
"scanning blockparam_ins at vreg {} block {}: blockparam_in_idx = {}",
vreg.index(),
block.index(),
blockparam_in_idx
);
while blockparam_in_idx < self.blockparam_ins.len() {
let (to_vreg, to_block, from_block) =
self.blockparam_ins[blockparam_in_idx];
let BlockparamIn {
from_block,
to_block,
to_vreg,
} = self.blockparam_ins[blockparam_in_idx];
if (to_vreg, to_block) > (vreg, block) {
break;
}
@@ -413,7 +423,7 @@ impl<'a, F: Function> Env<'a, F> {
),
alloc,
});
log::trace!(
trace!(
"match: blockparam_in: v{} in block{} from block{} into {}",
to_vreg.index(),
to_block.index(),
@@ -444,7 +454,7 @@ impl<'a, F: Function> Env<'a, F> {
continue;
}
log::trace!(
trace!(
"scanning preds at vreg {} block {} for ends outside the range",
vreg.index(),
block.index()
@@ -453,7 +463,7 @@ impl<'a, F: Function> Env<'a, F> {
// Now find any preds whose ends are not in the
// same range, and insert appropriate moves.
for &pred in self.func.block_preds(block) {
log::trace!(
trace!(
"pred block {} has exit {:?}",
pred.index(),
self.cfginfo.block_exit[pred.index()]
@@ -461,7 +471,7 @@ impl<'a, F: Function> Env<'a, F> {
if range.contains_point(self.cfginfo.block_exit[pred.index()]) {
continue;
}
log::trace!(" -> requires half-move");
trace!(" -> requires half-move");
half_moves.push(HalfMove {
key: half_move_key(pred, block, vreg, HalfMoveKind::Dest),
alloc,
@@ -471,26 +481,30 @@ impl<'a, F: Function> Env<'a, F> {
block = block.next();
}
// If this is a blockparam vreg and the start of block
// is in this range, add to blockparam_allocs.
let (blockparam_block, blockparam_idx) =
self.cfginfo.vreg_def_blockparam[vreg.index()];
if blockparam_block.is_valid()
&& range.contains_point(self.cfginfo.block_entry[blockparam_block.index()])
#[cfg(feature = "checker")]
{
self.blockparam_allocs.push((
blockparam_block,
blockparam_idx,
vreg,
alloc,
));
// If this is a blockparam vreg and the start of block
// is in this range, add to blockparam_allocs.
let (blockparam_block, blockparam_idx) =
self.cfginfo.vreg_def_blockparam[vreg.index()];
if blockparam_block.is_valid()
&& range
.contains_point(self.cfginfo.block_entry[blockparam_block.index()])
{
self.blockparam_allocs.push((
blockparam_block,
blockparam_idx,
vreg,
alloc,
));
}
}
}
// Scan over def/uses and apply allocations.
for use_idx in 0..self.ranges[entry.index.index()].uses.len() {
let usedata = self.ranges[entry.index.index()].uses[use_idx];
log::trace!("applying to use: {:?}", usedata);
trace!("applying to use: {:?}", usedata);
debug_assert!(range.contains_point(usedata.pos));
let inst = usedata.pos.inst();
let slot = usedata.slot;
@@ -519,7 +533,7 @@ impl<'a, F: Function> Env<'a, F> {
// covers only prev inst's After; so includes move
// srcs to (exclusive) inst.
let move_src_end = (vreg, range.to.inst());
log::trace!(
trace!(
"vreg {:?} range {:?}: looking for program-move sources from {:?} to {:?}",
vreg,
range,
@@ -529,13 +543,13 @@ impl<'a, F: Function> Env<'a, F> {
while prog_move_src_idx < self.prog_move_srcs.len()
&& self.prog_move_srcs[prog_move_src_idx].0 < move_src_start
{
log::trace!(" -> skipping idx {}", prog_move_src_idx);
trace!(" -> skipping idx {}", prog_move_src_idx);
prog_move_src_idx += 1;
}
while prog_move_src_idx < self.prog_move_srcs.len()
&& self.prog_move_srcs[prog_move_src_idx].0 < move_src_end
{
log::trace!(
trace!(
" -> setting idx {} ({:?}) to alloc {:?}",
prog_move_src_idx,
self.prog_move_srcs[prog_move_src_idx].0,
@@ -562,7 +576,7 @@ impl<'a, F: Function> Env<'a, F> {
} else {
(vreg, range.to.inst().next())
};
log::trace!(
trace!(
"vreg {:?} range {:?}: looking for program-move dests from {:?} to {:?}",
vreg,
range,
@@ -572,13 +586,13 @@ impl<'a, F: Function> Env<'a, F> {
while prog_move_dst_idx < self.prog_move_dsts.len()
&& self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_start
{
log::trace!(" -> skipping idx {}", prog_move_dst_idx);
trace!(" -> skipping idx {}", prog_move_dst_idx);
prog_move_dst_idx += 1;
}
while prog_move_dst_idx < self.prog_move_dsts.len()
&& self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_end
{
log::trace!(
trace!(
" -> setting idx {} ({:?}) to alloc {:?}",
prog_move_dst_idx,
self.prog_move_dsts[prog_move_dst_idx].0,
@@ -596,7 +610,7 @@ impl<'a, F: Function> Env<'a, F> {
// from-vreg) tuple, find the from-alloc and all the
// to-allocs, and insert moves on the block edge.
half_moves.sort_unstable_by_key(|h| h.key);
log::trace!("halfmoves: {:?}", half_moves);
trace!("halfmoves: {:?}", half_moves);
self.stats.halfmoves_count = half_moves.len();
let mut i = 0;
@@ -619,7 +633,7 @@ impl<'a, F: Function> Env<'a, F> {
}
let last_dest = i;
log::trace!(
trace!(
"halfmove match: src {:?} dests {:?}",
src,
&half_moves[first_dest..last_dest]
@@ -697,8 +711,8 @@ impl<'a, F: Function> Env<'a, F> {
// Handle multi-fixed-reg constraints by copying.
for fixup in std::mem::replace(&mut self.multi_fixed_reg_fixups, vec![]) {
let from_alloc = self.get_alloc(fixup.pos.inst(), fixup.from_slot as usize);
let to_alloc = Allocation::reg(self.pregs[fixup.to_preg.index()].reg);
log::trace!(
let to_alloc = Allocation::reg(PReg::from_index(fixup.to_preg.index()));
trace!(
"multi-fixed-move constraint at {:?} from {} to {} for v{}",
fixup.pos,
from_alloc,
@@ -715,7 +729,7 @@ impl<'a, F: Function> Env<'a, F> {
self.set_alloc(
fixup.pos.inst(),
fixup.to_slot as usize,
Allocation::reg(self.pregs[fixup.to_preg.index()].reg),
Allocation::reg(PReg::from_index(fixup.to_preg.index())),
);
}
@@ -773,7 +787,7 @@ impl<'a, F: Function> Env<'a, F> {
input_reused.push(input_idx);
let input_alloc = self.get_alloc(inst, input_idx);
let output_alloc = self.get_alloc(inst, output_idx);
log::trace!(
trace!(
"reuse-input inst {:?}: output {} has alloc {:?}, input {} has alloc {:?}",
inst,
output_idx,
@@ -816,20 +830,20 @@ impl<'a, F: Function> Env<'a, F> {
.sort_unstable_by_key(|((_, inst), _)| inst.prev());
let prog_move_srcs = std::mem::replace(&mut self.prog_move_srcs, vec![]);
let prog_move_dsts = std::mem::replace(&mut self.prog_move_dsts, vec![]);
assert_eq!(prog_move_srcs.len(), prog_move_dsts.len());
debug_assert_eq!(prog_move_srcs.len(), prog_move_dsts.len());
for (&((_, from_inst), from_alloc), &((to_vreg, to_inst), to_alloc)) in
prog_move_srcs.iter().zip(prog_move_dsts.iter())
{
log::trace!(
trace!(
"program move at inst {:?}: alloc {:?} -> {:?} (v{})",
from_inst,
from_alloc,
to_alloc,
to_vreg.index(),
);
assert!(from_alloc.is_some());
assert!(to_alloc.is_some());
assert_eq!(from_inst, to_inst.prev());
debug_assert!(from_alloc.is_some());
debug_assert!(to_alloc.is_some());
debug_assert_eq!(from_inst, to_inst.prev());
// N.B.: these moves happen with the *same* priority as
// LR-to-LR moves, because they work just like them: they
// connect a use at one progpoint (move-After) with a def
@@ -850,7 +864,7 @@ impl<'a, F: Function> Env<'a, F> {
// resolve (see cases below).
let mut i = 0;
self.inserted_moves
.sort_unstable_by_key(|m| (m.pos.to_index(), m.prio));
.sort_unstable_by_key(|m| m.pos_prio.key());
// Redundant-move elimination state tracker.
let mut redundant_moves = RedundantMoveEliminator::default();
@@ -907,18 +921,14 @@ impl<'a, F: Function> Env<'a, F> {
while i < self.inserted_moves.len() {
let start = i;
let pos = self.inserted_moves[i].pos;
let prio = self.inserted_moves[i].prio;
while i < self.inserted_moves.len()
&& self.inserted_moves[i].pos == pos
&& self.inserted_moves[i].prio == prio
{
let pos_prio = self.inserted_moves[i].pos_prio;
while i < self.inserted_moves.len() && self.inserted_moves[i].pos_prio == pos_prio {
i += 1;
}
let moves = &self.inserted_moves[start..i];
redundant_move_process_side_effects(self, &mut redundant_moves, last_pos, pos);
last_pos = pos;
redundant_move_process_side_effects(self, &mut redundant_moves, last_pos, pos_prio.pos);
last_pos = pos_prio.pos;
// Gather all the moves with Int class and Float class
// separately. These cannot interact, so it is safe to
@@ -929,13 +939,15 @@ impl<'a, F: Function> Env<'a, F> {
// regs, but this seems simpler.)
let mut int_moves: SmallVec<[InsertedMove; 8]> = smallvec![];
let mut float_moves: SmallVec<[InsertedMove; 8]> = smallvec![];
#[cfg(feature = "checker")]
let mut self_moves: SmallVec<[InsertedMove; 8]> = smallvec![];
for m in moves {
if m.from_alloc.is_reg() && m.to_alloc.is_reg() {
assert_eq!(m.from_alloc.class(), m.to_alloc.class());
debug_assert_eq!(m.from_alloc.class(), m.to_alloc.class());
}
if m.from_alloc == m.to_alloc {
#[cfg(feature = "checker")]
if m.to_vreg.is_some() {
self_moves.push(m.clone());
}
@@ -959,10 +971,14 @@ impl<'a, F: Function> Env<'a, F> {
// that can be done one at a time.
let scratch = self.env.scratch_by_class[regclass as u8 as usize];
let mut parallel_moves = ParallelMoves::new(Allocation::reg(scratch));
log::trace!("parallel moves at pos {:?} prio {:?}", pos, prio);
trace!(
"parallel moves at pos {:?} prio {:?}",
pos_prio.pos,
pos_prio.prio
);
for m in moves {
if (m.from_alloc != m.to_alloc) || m.to_vreg.is_some() {
log::trace!(" {} -> {}", m.from_alloc, m.to_alloc,);
trace!(" {} -> {}", m.from_alloc, m.to_alloc,);
parallel_moves.add(m.from_alloc, m.to_alloc, m.to_vreg);
}
}
@@ -993,7 +1009,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut scratch_used_yet = false;
for (src, dst, to_vreg) in resolved {
log::trace!(" resolved: {} -> {} ({:?})", src, dst, to_vreg);
trace!(" resolved: {} -> {} ({:?})", src, dst, to_vreg);
let action = redundant_moves.process_move(src, dst, to_vreg);
if !action.elide {
if dst == Allocation::reg(scratch) {
@@ -1001,135 +1017,117 @@ impl<'a, F: Function> Env<'a, F> {
}
if self.allocation_is_stack(src) && self.allocation_is_stack(dst) {
if !scratch_used_yet {
self.add_edit(
pos,
prio,
Edit::Move {
from: src,
to: Allocation::reg(scratch),
to_vreg,
},
self.add_move_edit(
pos_prio,
src,
Allocation::reg(scratch),
to_vreg,
);
self.add_edit(
pos,
prio,
Edit::Move {
from: Allocation::reg(scratch),
to: dst,
to_vreg,
},
self.add_move_edit(
pos_prio,
Allocation::reg(scratch),
dst,
to_vreg,
);
} else {
assert!(extra_slot.is_some());
self.add_edit(
pos,
prio,
Edit::Move {
from: Allocation::reg(scratch),
to: extra_slot.unwrap(),
to_vreg: None,
},
debug_assert!(extra_slot.is_some());
self.add_move_edit(
pos_prio,
Allocation::reg(scratch),
extra_slot.unwrap(),
None,
);
self.add_edit(
pos,
prio,
Edit::Move {
from: src,
to: Allocation::reg(scratch),
to_vreg,
},
self.add_move_edit(
pos_prio,
src,
Allocation::reg(scratch),
to_vreg,
);
self.add_edit(
pos,
prio,
Edit::Move {
from: Allocation::reg(scratch),
to: dst,
to_vreg,
},
self.add_move_edit(
pos_prio,
Allocation::reg(scratch),
dst,
to_vreg,
);
self.add_edit(
pos,
prio,
Edit::Move {
from: extra_slot.unwrap(),
to: Allocation::reg(scratch),
to_vreg: None,
},
self.add_move_edit(
pos_prio,
extra_slot.unwrap(),
Allocation::reg(scratch),
None,
);
}
} else {
self.add_edit(
pos,
prio,
Edit::Move {
from: src,
to: dst,
to_vreg,
},
);
self.add_move_edit(pos_prio, src, dst, to_vreg);
}
} else {
log::trace!(" -> redundant move elided");
trace!(" -> redundant move elided");
}
#[cfg(feature = "checker")]
if let Some((alloc, vreg)) = action.def_alloc {
log::trace!(
trace!(
" -> converted to DefAlloc: alloc {} vreg {}",
alloc,
vreg
);
self.add_edit(pos, prio, Edit::DefAlloc { alloc, vreg });
self.edits.push((pos_prio, Edit::DefAlloc { alloc, vreg }));
}
}
}
#[cfg(feature = "checker")]
for m in &self_moves {
log::trace!(
trace!(
"self move at pos {:?} prio {:?}: {} -> {} to_vreg {:?}",
pos,
prio,
pos_prio.pos,
pos_prio.prio,
m.from_alloc,
m.to_alloc,
m.to_vreg
);
let action = redundant_moves.process_move(m.from_alloc, m.to_alloc, m.to_vreg);
assert!(action.elide);
debug_assert!(action.elide);
if let Some((alloc, vreg)) = action.def_alloc {
log::trace!(" -> DefAlloc: alloc {} vreg {}", alloc, vreg);
self.add_edit(pos, prio, Edit::DefAlloc { alloc, vreg });
trace!(" -> DefAlloc: alloc {} vreg {}", alloc, vreg);
self.edits.push((pos_prio, Edit::DefAlloc { alloc, vreg }));
}
}
}
// Add edits to describe blockparam locations too. This is
// required by the checker. This comes after any edge-moves.
self.blockparam_allocs
.sort_unstable_by_key(|&(block, idx, _, _)| (block, idx));
self.stats.blockparam_allocs_count = self.blockparam_allocs.len();
let mut i = 0;
while i < self.blockparam_allocs.len() {
let start = i;
let block = self.blockparam_allocs[i].0;
while i < self.blockparam_allocs.len() && self.blockparam_allocs[i].0 == block {
i += 1;
}
let params = &self.blockparam_allocs[start..i];
let vregs = params
.iter()
.map(|(_, _, vreg_idx, _)| self.vreg_regs[vreg_idx.index()])
.collect::<Vec<_>>();
let allocs = params
.iter()
.map(|(_, _, _, alloc)| *alloc)
.collect::<Vec<_>>();
assert_eq!(vregs.len(), self.func.block_params(block).len());
assert_eq!(allocs.len(), self.func.block_params(block).len());
for (vreg, alloc) in vregs.into_iter().zip(allocs.into_iter()) {
self.add_edit(
self.cfginfo.block_entry[block.index()],
InsertMovePrio::BlockParam,
Edit::DefAlloc { alloc, vreg },
);
#[cfg(feature = "checker")]
{
// Add edits to describe blockparam locations too. This is
// required by the checker. This comes after any edge-moves.
use crate::ion::data_structures::u64_key;
self.blockparam_allocs
.sort_unstable_by_key(|&(block, idx, _, _)| u64_key(block.raw_u32(), idx));
self.stats.blockparam_allocs_count = self.blockparam_allocs.len();
let mut i = 0;
while i < self.blockparam_allocs.len() {
let start = i;
let block = self.blockparam_allocs[i].0;
while i < self.blockparam_allocs.len() && self.blockparam_allocs[i].0 == block {
i += 1;
}
let params = &self.blockparam_allocs[start..i];
let vregs = params
.iter()
.map(|(_, _, vreg_idx, _)| self.vreg_regs[vreg_idx.index()])
.collect::<Vec<_>>();
let allocs = params
.iter()
.map(|(_, _, _, alloc)| *alloc)
.collect::<Vec<_>>();
debug_assert_eq!(vregs.len(), self.func.block_params(block).len());
debug_assert_eq!(allocs.len(), self.func.block_params(block).len());
for (vreg, alloc) in vregs.into_iter().zip(allocs.into_iter()) {
self.edits.push((
PosWithPrio {
pos: self.cfginfo.block_entry[block.index()],
prio: InsertMovePrio::BlockParam as u32,
},
Edit::DefAlloc { alloc, vreg },
));
}
}
}
@@ -1137,38 +1135,49 @@ impl<'a, F: Function> Env<'a, F> {
// be a stable sort! We have to keep the order produced by the
// parallel-move resolver for all moves within a single sort
// key.
self.edits.sort_by_key(|&(pos, prio, _)| (pos, prio));
self.edits.sort_by_key(|&(pos_prio, _)| pos_prio.key());
self.stats.edits_count = self.edits.len();
// Add debug annotations.
if self.annotations_enabled {
for i in 0..self.edits.len() {
let &(pos, _, ref edit) = &self.edits[i];
let &(pos_prio, ref edit) = &self.edits[i];
match edit {
&Edit::Move { from, to, to_vreg } => {
self.annotate(
ProgPoint::from_index(pos),
format!("move {} -> {} ({:?})", from, to, to_vreg),
);
&Edit::Move { from, to } => {
self.annotate(pos_prio.pos, format!("move {} -> {})", from, to));
}
&Edit::DefAlloc { alloc, vreg } => {
let s = format!("defalloc {:?} := {:?}", alloc, vreg);
self.annotate(ProgPoint::from_index(pos), s);
self.annotate(pos_prio.pos, s);
}
}
}
}
}
pub fn add_edit(&mut self, pos: ProgPoint, prio: InsertMovePrio, edit: Edit) {
match &edit {
&Edit::Move { from, to, to_vreg } if from == to && to_vreg.is_none() => return,
&Edit::Move { from, to, .. } if from.is_reg() && to.is_reg() => {
assert_eq!(from.as_reg().unwrap().class(), to.as_reg().unwrap().class());
pub fn add_move_edit(
&mut self,
pos_prio: PosWithPrio,
from: Allocation,
to: Allocation,
_to_vreg: Option<VReg>,
) {
if from != to {
if from.is_reg() && to.is_reg() {
debug_assert_eq!(from.as_reg().unwrap().class(), to.as_reg().unwrap().class());
}
_ => {}
self.edits.push((pos_prio, Edit::Move { from, to }));
}
self.edits.push((pos.to_index(), prio, edit));
#[cfg(feature = "checker")]
if let Some(to_vreg) = _to_vreg {
self.edits.push((
pos_prio,
Edit::DefAlloc {
alloc: to,
vreg: to_vreg,
},
));
}
}
}

144
src/ion/process.rs
View File

@@ -62,7 +62,7 @@ impl<'a, F: Function> Env<'a, F> {
// `AllocRegResult::ConflictHighCost`.
max_allowable_cost: Option<u32>,
) -> AllocRegResult {
log::trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg);
trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg);
let mut conflicts = smallvec![];
let mut conflict_set = FxHashSet::default();
let mut max_conflict_weight = 0;
@@ -89,7 +89,7 @@ impl<'a, F: Function> Env<'a, F> {
.btree
.range(from_key..)
.peekable();
log::trace!(
trace!(
"alloc map for {:?} in range {:?}..: {:?}",
reg,
from_key,
@@ -98,19 +98,19 @@ impl<'a, F: Function> Env<'a, F> {
let mut first_conflict: Option<ProgPoint> = None;
'ranges: for entry in bundle_ranges {
log::trace!(" -> range LR {:?}: {:?}", entry.index, entry.range);
trace!(" -> range LR {:?}: {:?}", entry.index, entry.range);
let key = LiveRangeKey::from_range(&entry.range);
let mut skips = 0;
'alloc: loop {
log::trace!(" -> PReg range {:?}", preg_range_iter.peek());
trace!(" -> PReg range {:?}", preg_range_iter.peek());
// Advance our BTree traversal until it is >= this bundle
// range (i.e., skip PReg allocations in the BTree that
// are completely before this bundle range).
if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key {
log::trace!(
trace!(
"Skipping PReg range {:?}",
preg_range_iter.peek().unwrap().0
);
@@ -135,13 +135,13 @@ impl<'a, F: Function> Env<'a, F> {
// If there are no more PReg allocations, we're done!
if preg_range_iter.peek().is_none() {
log::trace!(" -> no more PReg allocations; so no conflict possible!");
trace!(" -> no more PReg allocations; so no conflict possible!");
break 'ranges;
}
// If the current PReg range is beyond this range, there is no conflict; continue.
if *preg_range_iter.peek().unwrap().0 > key {
log::trace!(
trace!(
" -> next PReg allocation is at {:?}; moving to next VReg range",
preg_range_iter.peek().unwrap().0
);
@@ -150,16 +150,16 @@ impl<'a, F: Function> Env<'a, F> {
// Otherwise, there is a conflict.
let preg_key = *preg_range_iter.peek().unwrap().0;
assert_eq!(preg_key, key); // Assert that this range overlaps.
debug_assert_eq!(preg_key, key); // Assert that this range overlaps.
let preg_range = preg_range_iter.next().unwrap().1;
log::trace!(" -> btree contains range {:?} that overlaps", preg_range);
trace!(" -> btree contains range {:?} that overlaps", preg_range);
if preg_range.is_valid() {
log::trace!(" -> from vreg {:?}", self.ranges[preg_range.index()].vreg);
trace!(" -> from vreg {:?}", self.ranges[preg_range.index()].vreg);
// range from an allocated bundle: find the bundle and add to
// conflicts list.
let conflict_bundle = self.ranges[preg_range.index()].bundle;
log::trace!(" -> conflict bundle {:?}", conflict_bundle);
trace!(" -> conflict bundle {:?}", conflict_bundle);
if !conflict_set.contains(&conflict_bundle) {
conflicts.push(conflict_bundle);
conflict_set.insert(conflict_bundle);
@@ -170,7 +170,7 @@ impl<'a, F: Function> Env<'a, F> {
if max_allowable_cost.is_some()
&& max_conflict_weight > max_allowable_cost.unwrap()
{
log::trace!(" -> reached high cost, retrying early");
trace!(" -> reached high cost, retrying early");
return AllocRegResult::ConflictHighCost;
}
}
@@ -182,7 +182,7 @@ impl<'a, F: Function> Env<'a, F> {
)));
}
} else {
log::trace!(" -> conflict with fixed reservation");
trace!(" -> conflict with fixed reservation");
// range from a direct use of the PReg (due to clobber).
return AllocRegResult::ConflictWithFixed(
max_conflict_weight,
@@ -197,8 +197,8 @@ impl<'a, F: Function> Env<'a, F> {
}
// We can allocate! Add our ranges to the preg's BTree.
let preg = self.pregs[reg.index()].reg;
log::trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg);
let preg = PReg::from_index(reg.index());
trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg);
self.bundles[bundle.index()].allocation = Allocation::reg(preg);
for entry in &self.bundles[bundle.index()].ranges {
self.pregs[reg.index()]
@@ -211,7 +211,7 @@ impl<'a, F: Function> Env<'a, F> {
}
pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) {
log::trace!(
trace!(
"evicting bundle {:?}: alloc {:?}",
bundle,
self.bundles[bundle.index()].allocation
@@ -219,7 +219,7 @@ impl<'a, F: Function> Env<'a, F> {
let preg = match self.bundles[bundle.index()].allocation.as_reg() {
Some(preg) => preg,
None => {
log::trace!(
trace!(
" -> has no allocation! {:?}",
self.bundles[bundle.index()].allocation
);
@@ -229,14 +229,14 @@ impl<'a, F: Function> Env<'a, F> {
let preg_idx = PRegIndex::new(preg.index());
self.bundles[bundle.index()].allocation = Allocation::none();
for entry in &self.bundles[bundle.index()].ranges {
log::trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx);
trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx);
self.pregs[preg_idx.index()]
.allocations
.btree
.remove(&LiveRangeKey::from_range(&entry.range));
}
let prio = self.bundles[bundle.index()].prio;
log::trace!(" -> prio {}; back into queue", prio);
trace!(" -> prio {}; back into queue", prio);
self.allocation_queue
.insert(bundle, prio as usize, PReg::invalid());
}
@@ -246,22 +246,22 @@ impl<'a, F: Function> Env<'a, F> {
}
pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &LiveBundleVec) -> u32 {
log::trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles);
trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles);
let m = bundles
.iter()
.map(|&b| {
let w = self.bundles[b.index()].cached_spill_weight();
log::trace!("bundle{}: {}", b.index(), w);
trace!("bundle{}: {}", b.index(), w);
w
})
.max()
.unwrap_or(0);
log::trace!(" -> max: {}", m);
trace!(" -> max: {}", m);
m
}
pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) {
log::trace!("recompute bundle properties: bundle {:?}", bundle);
trace!("recompute bundle properties: bundle {:?}", bundle);
let minimal;
let mut fixed = false;
@@ -273,18 +273,18 @@ impl<'a, F: Function> Env<'a, F> {
self.bundles[bundle.index()].prio = self.compute_bundle_prio(bundle);
if first_range_data.vreg.is_invalid() {
log::trace!(" -> no vreg; minimal and fixed");
trace!(" -> no vreg; minimal and fixed");
minimal = true;
fixed = true;
} else {
for u in &first_range_data.uses {
log::trace!(" -> use: {:?}", u);
trace!(" -> use: {:?}", u);
if let OperandConstraint::FixedReg(_) = u.operand.constraint() {
log::trace!(" -> fixed use at {:?}: {:?}", u.pos, u.operand);
trace!(" -> fixed use at {:?}: {:?}", u.pos, u.operand);
fixed = true;
}
if let OperandConstraint::Stack = u.operand.constraint() {
log::trace!(" -> stack use at {:?}: {:?}", u.pos, u.operand);
trace!(" -> stack use at {:?}: {:?}", u.pos, u.operand);
stack = true;
}
if stack && fixed {
@@ -295,7 +295,7 @@ impl<'a, F: Function> Env<'a, F> {
// that it could cover just one ProgPoint,
// i.e. X.Before..X.After, or two ProgPoints,
// i.e. X.Before..X+1.Before.
log::trace!(" -> first range has range {:?}", first_range_data.range);
trace!(" -> first range has range {:?}", first_range_data.range);
let bundle_start = self.bundles[bundle.index()]
.ranges
.first()
@@ -304,22 +304,22 @@ impl<'a, F: Function> Env<'a, F> {
.from;
let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to;
minimal = bundle_start.inst() == bundle_end.prev().inst();
log::trace!(" -> minimal: {}", minimal);
trace!(" -> minimal: {}", minimal);
}
let spill_weight = if minimal {
if fixed {
log::trace!(" -> fixed and minimal");
trace!(" -> fixed and minimal");
MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT
} else {
log::trace!(" -> non-fixed and minimal");
trace!(" -> non-fixed and minimal");
MINIMAL_BUNDLE_SPILL_WEIGHT
}
} else {
let mut total = SpillWeight::zero();
for entry in &self.bundles[bundle.index()].ranges {
let range_data = &self.ranges[entry.index.index()];
log::trace!(
trace!(
" -> uses spill weight: +{:?}",
range_data.uses_spill_weight()
);
@@ -328,7 +328,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.bundles[bundle.index()].prio > 0 {
let final_weight = (total.to_f32() as u32) / self.bundles[bundle.index()].prio;
log::trace!(
trace!(
" -> dividing by prio {}; final weight {}",
self.bundles[bundle.index()].prio,
final_weight
@@ -356,7 +356,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut w = SpillWeight::zero();
for u in &rangedata.uses {
w = w + SpillWeight::from_bits(u.weight);
log::trace!("range{}: use {:?}", range.index(), u);
trace!("range{}: use {:?}", range.index(), u);
}
rangedata.set_uses_spill_weight(w);
if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def {
@@ -397,7 +397,7 @@ impl<'a, F: Function> Env<'a, F> {
reg_hint: PReg,
) {
self.stats.splits += 1;
log::trace!(
trace!(
"split bundle {:?} at {:?} and requeue with reg hint (for first part) {:?}",
bundle,
split_at,
@@ -410,7 +410,7 @@ impl<'a, F: Function> Env<'a, F> {
let spillset = self.bundles[bundle.index()].spillset;
assert!(!self.bundles[bundle.index()].ranges.is_empty());
debug_assert!(!self.bundles[bundle.index()].ranges.is_empty());
// Split point *at* start is OK; this means we peel off
// exactly one use to create a minimal bundle.
let bundle_start = self.bundles[bundle.index()]
@@ -419,9 +419,9 @@ impl<'a, F: Function> Env<'a, F> {
.unwrap()
.range
.from;
assert!(split_at >= bundle_start);
debug_assert!(split_at >= bundle_start);
let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to;
assert!(split_at < bundle_end);
debug_assert!(split_at < bundle_end);
// Is the split point *at* the start? If so, peel off the
// first use: set the split point just after it, or just
@@ -435,7 +435,7 @@ impl<'a, F: Function> Env<'a, F> {
break 'outer;
}
}
log::trace!(" -> first use loc is {:?}", first_use);
trace!(" -> first use loc is {:?}", first_use);
split_at = match first_use {
Some(pos) => {
if pos.inst() == bundle_start.inst() {
@@ -455,7 +455,7 @@ impl<'a, F: Function> Env<'a, F> {
.next(),
),
};
log::trace!(
trace!(
"split point is at bundle start; advancing to {:?}",
split_at
);
@@ -471,13 +471,13 @@ impl<'a, F: Function> Env<'a, F> {
}
}
assert!(split_at > bundle_start && split_at < bundle_end);
debug_assert!(split_at > bundle_start && split_at < bundle_end);
// We need to find which LRs fall on each side of the split,
// which LR we need to split down the middle, then update the
// current bundle, create a new one, and (re)-queue both.
log::trace!(" -> LRs: {:?}", self.bundles[bundle.index()].ranges);
trace!(" -> LRs: {:?}", self.bundles[bundle.index()].ranges);
let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle
let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle
@@ -492,11 +492,11 @@ impl<'a, F: Function> Env<'a, F> {
}
}
log::trace!(
trace!(
" -> last LR in old bundle: LR {:?}",
self.bundles[bundle.index()].ranges[last_lr_in_old_bundle_idx]
);
log::trace!(
trace!(
" -> first LR in new bundle: LR {:?}",
self.bundles[bundle.index()].ranges[first_lr_in_new_bundle_idx]
);
@@ -516,14 +516,14 @@ impl<'a, F: Function> Env<'a, F> {
// down the middle, replace it with a new LR and chop off the
// end of the same LR in the original list.
if split_at > new_lr_list[0].range.from {
assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx);
debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx);
let orig_lr = new_lr_list[0].index;
let new_lr = self.create_liverange(CodeRange {
from: split_at,
to: new_lr_list[0].range.to,
});
self.ranges[new_lr.index()].vreg = self.ranges[orig_lr.index()].vreg;
log::trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr);
trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr);
let first_use = self.ranges[orig_lr.index()]
.uses
.iter()
@@ -560,7 +560,7 @@ impl<'a, F: Function> Env<'a, F> {
}
let new_bundle = self.create_bundle();
log::trace!(" -> creating new bundle {:?}", new_bundle);
trace!(" -> creating new bundle {:?}", new_bundle);
self.bundles[new_bundle.index()].spillset = spillset;
for entry in &new_lr_list {
self.ranges[entry.index.index()].bundle = new_bundle;
@@ -582,7 +582,7 @@ impl<'a, F: Function> Env<'a, F> {
let spill = self
.get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
.unwrap();
log::trace!(
trace!(
" -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
bundle,
entry.index,
@@ -627,13 +627,13 @@ impl<'a, F: Function> Env<'a, F> {
range,
index: empty_lr,
});
log::trace!(
trace!(
" -> bundle {:?} range {:?}: last use implies split point {:?}",
bundle,
entry.index,
split
);
log::trace!(
trace!(
" -> moving trailing empty region to new spill bundle {:?} with new LR {:?}",
spill,
empty_lr
@@ -652,7 +652,7 @@ impl<'a, F: Function> Env<'a, F> {
let spill = self
.get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true)
.unwrap();
log::trace!(
trace!(
" -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
new_bundle,
entry.index,
@@ -697,13 +697,13 @@ impl<'a, F: Function> Env<'a, F> {
range,
index: empty_lr,
});
log::trace!(
trace!(
" -> bundle {:?} range {:?}: first use implies split point {:?}",
bundle,
entry.index,
first_use,
);
log::trace!(
trace!(
" -> moving leading empty region to new spill bundle {:?} with new LR {:?}",
spill,
empty_lr
@@ -741,7 +741,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.pregs[hint_reg.index()].is_stack {
hint_reg = PReg::invalid();
}
log::trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,);
trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,);
let req = match self.compute_requirement(bundle) {
Ok(req) => req,
@@ -749,7 +749,7 @@ impl<'a, F: Function> Env<'a, F> {
// We have to split right away. We'll find a point to
// split that would allow at least the first half of the
// split to be conflict-free.
assert!(
debug_assert!(
!self.minimal_bundle(bundle),
"Minimal bundle with conflict!"
);
@@ -786,7 +786,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut attempts = 0;
loop {
attempts += 1;
log::trace!("attempt {}, req {:?}", attempts, req);
trace!("attempt {}, req {:?}", attempts, req);
debug_assert!(attempts < 100 * self.func.num_insts());
let fixed_preg = match req {
@@ -836,7 +836,7 @@ impl<'a, F: Function> Env<'a, F> {
) {
self.stats.process_bundle_reg_probes_any += 1;
let preg_idx = PRegIndex::new(preg.index());
log::trace!("trying preg {:?}", preg_idx);
trace!("trying preg {:?}", preg_idx);
let scan_limit_cost = match (
lowest_cost_evict_conflict_cost,
@@ -848,13 +848,13 @@ impl<'a, F: Function> Env<'a, F> {
match self.try_to_allocate_bundle_to_reg(bundle, preg_idx, scan_limit_cost) {
AllocRegResult::Allocated(alloc) => {
self.stats.process_bundle_reg_success_any += 1;
log::trace!(" -> allocated to any {:?}", preg_idx);
trace!(" -> allocated to any {:?}", preg_idx);
self.spillsets[self.bundles[bundle.index()].spillset.index()].reg_hint =
alloc.as_reg().unwrap();
return Ok(());
}
AllocRegResult::Conflict(bundles, first_conflict_point) => {
log::trace!(
trace!(
" -> conflict with bundles {:?}, first conflict at {:?}",
bundles,
first_conflict_point
@@ -887,7 +887,7 @@ impl<'a, F: Function> Env<'a, F> {
}
}
AllocRegResult::ConflictWithFixed(max_cost, point) => {
log::trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
let loop_depth = self.cfginfo.approx_loop_depth
[self.cfginfo.insn_block[point.inst().index()].index()];
@@ -919,12 +919,12 @@ impl<'a, F: Function> Env<'a, F> {
// any with current bundle assignments. Hence, we will need
// to either split or attempt to evict some bundles.
log::trace!(
trace!(
" -> lowest cost evict: set {:?}, cost {:?}",
lowest_cost_evict_conflict_set,
lowest_cost_evict_conflict_cost,
);
log::trace!(
trace!(
" -> lowest cost split: cost {:?}, point {:?}, reg {:?}",
lowest_cost_split_conflict_cost,
lowest_cost_split_conflict_point,
@@ -932,13 +932,13 @@ impl<'a, F: Function> Env<'a, F> {
);
// If we reach here, we *must* have an option either to split or evict.
assert!(
debug_assert!(
lowest_cost_split_conflict_cost.is_some()
|| lowest_cost_evict_conflict_cost.is_some()
);
let our_spill_weight = self.bundle_spill_weight(bundle);
log::trace!(" -> our spill weight: {}", our_spill_weight);
trace!(" -> our spill weight: {}", our_spill_weight);
// We detect the "too-many-live-registers" case here and
// return an error cleanly, rather than panicking, because
@@ -953,7 +953,7 @@ impl<'a, F: Function> Env<'a, F> {
if let Requirement::Register = req {
// Check if this is a too-many-live-registers situation.
let range = self.bundles[bundle.index()].ranges[0].range;
log::trace!("checking for too many live regs");
trace!("checking for too many live regs");
let mut min_bundles_assigned = 0;
let mut fixed_assigned = 0;
let mut total_regs = 0;
@@ -961,7 +961,7 @@ impl<'a, F: Function> Env<'a, F> {
.iter()
.chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter())
{
log::trace!(" -> PR {:?}", preg);
trace!(" -> PR {:?}", preg);
let start = LiveRangeKey::from_range(&CodeRange {
from: range.from.prev(),
to: range.from.prev(),
@@ -976,19 +976,19 @@ impl<'a, F: Function> Env<'a, F> {
}
if lr.is_valid() {
if self.minimal_bundle(self.ranges[lr.index()].bundle) {
log::trace!(" -> min bundle {:?}", lr);
trace!(" -> min bundle {:?}", lr);
min_bundles_assigned += 1;
} else {
log::trace!(" -> non-min bundle {:?}", lr);
trace!(" -> non-min bundle {:?}", lr);
}
} else {
log::trace!(" -> fixed bundle");
trace!(" -> fixed bundle");
fixed_assigned += 1;
}
}
total_regs += 1;
}
log::trace!(
trace!(
" -> total {}, fixed {}, min {}",
total_regs,
fixed_assigned,
@@ -1020,7 +1020,7 @@ impl<'a, F: Function> Env<'a, F> {
|| lowest_cost_evict_conflict_cost.is_none()
|| our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap())
{
log::trace!(
trace!(
" -> deciding to split: our spill weight is {}",
self.bundle_spill_weight(bundle)
);
@@ -1053,7 +1053,7 @@ impl<'a, F: Function> Env<'a, F> {
// Evict all bundles in `conflicting bundles` and try again.
self.stats.evict_bundle_event += 1;
for &bundle in &lowest_cost_evict_conflict_set.unwrap() {
log::trace!(" -> evicting {:?}", bundle);
trace!(" -> evicting {:?}", bundle);
self.evict_bundle(bundle);
self.stats.evict_bundle_count += 1;
}

30
src/ion/redundant_moves.rs
View File

@@ -18,6 +18,7 @@ pub struct RedundantMoveEliminator {
#[derive(Copy, Clone, Debug)]
pub struct RedundantMoveAction {
pub elide: bool,
#[cfg(feature = "checker")]
pub def_alloc: Option<(Allocation, VReg)>,
}
@@ -40,13 +41,13 @@ impl RedundantMoveEliminator {
.map(|&p| p)
.unwrap_or(RedundantMoveState::None);
log::trace!(
trace!(
" -> redundant move tracker: from {} to {} to_vreg {:?}",
from,
to,
to_vreg
);
log::trace!(
trace!(
" -> from_state {:?} to_state {:?}",
from_state,
to_state
@@ -58,6 +59,7 @@ impl RedundantMoveEliminator {
.insert(to, RedundantMoveState::Orig(to_vreg.unwrap()));
return RedundantMoveAction {
elide: true,
#[cfg(feature = "checker")]
def_alloc: Some((to, to_vreg.unwrap())),
};
}
@@ -67,29 +69,29 @@ impl RedundantMoveEliminator {
RedundantMoveState::Orig(r) => Some(r),
_ => None,
};
log::trace!(" -> src_vreg {:?}", src_vreg);
trace!(" -> src_vreg {:?}", src_vreg);
let dst_vreg = to_vreg.or(src_vreg);
log::trace!(" -> dst_vreg {:?}", dst_vreg);
trace!(" -> dst_vreg {:?}", dst_vreg);
let existing_dst_vreg = match to_state {
RedundantMoveState::Copy(_, opt_r) => opt_r,
RedundantMoveState::Orig(r) => Some(r),
_ => None,
};
log::trace!(" -> existing_dst_vreg {:?}", existing_dst_vreg);
trace!(" -> existing_dst_vreg {:?}", existing_dst_vreg);
let elide = match (from_state, to_state) {
(_, RedundantMoveState::Copy(orig_alloc, _)) if orig_alloc == from => true,
(RedundantMoveState::Copy(new_alloc, _), _) if new_alloc == to => true,
_ => false,
};
log::trace!(" -> elide {}", elide);
trace!(" -> elide {}", elide);
let def_alloc = if dst_vreg != existing_dst_vreg && dst_vreg.is_some() {
Some((to, dst_vreg.unwrap()))
} else {
None
};
log::trace!(" -> def_alloc {:?}", def_alloc);
trace!(" -> def_alloc {:?}", def_alloc);
// Invalidate all existing copies of `to` if `to` actually changed value.
if !elide {
@@ -100,7 +102,7 @@ impl RedundantMoveEliminator {
if from.is_reg() || to.is_reg() {
self.allocs
.insert(to, RedundantMoveState::Copy(from, dst_vreg));
log::trace!(
trace!(
" -> create mapping {} -> {:?}",
to,
RedundantMoveState::Copy(from, dst_vreg)
@@ -111,20 +113,24 @@ impl RedundantMoveEliminator {
.push(to);
}
RedundantMoveAction { elide, def_alloc }
RedundantMoveAction {
elide,
#[cfg(feature = "checker")]
def_alloc,
}
}
pub fn clear(&mut self) {
log::trace!(" redundant move eliminator cleared");
trace!(" redundant move eliminator cleared");
self.allocs.clear();
self.reverse_allocs.clear();
}
pub fn clear_alloc(&mut self, alloc: Allocation) {
log::trace!(" redundant move eliminator: clear {:?}", alloc);
trace!(" redundant move eliminator: clear {:?}", alloc);
if let Some(ref mut existing_copies) = self.reverse_allocs.get_mut(&alloc) {
for to_inval in existing_copies.iter() {
log::trace!(" -> clear existing copy: {:?}", to_inval);
trace!(" -> clear existing copy: {:?}", to_inval);
if let Some(val) = self.allocs.get_mut(to_inval) {
match val {
RedundantMoveState::Copy(_, Some(vreg)) => {

12
src/ion/requirement.rs
View File

@@ -71,21 +71,21 @@ impl<'a, F: Function> Env<'a, F> {
bundle: LiveBundleIndex,
) -> Result<Requirement, RequirementConflictAt> {
let mut req = Requirement::Any;
log::trace!("compute_requirement: {:?}", bundle);
trace!("compute_requirement: {:?}", bundle);
let ranges = &self.bundles[bundle.index()].ranges;
for entry in ranges {
log::trace!(" -> LR {:?}", entry.index);
trace!(" -> LR {:?}", entry.index);
for u in &self.ranges[entry.index.index()].uses {
log::trace!(" -> use {:?}", u);
trace!(" -> use {:?}", u);
let r = self.requirement_from_operand(u.operand);
req = req.merge(r).map_err(|_| {
log::trace!(" -> conflict");
trace!(" -> conflict");
RequirementConflictAt(u.pos)
})?;
log::trace!(" -> req {:?}", req);
trace!(" -> req {:?}", req);
}
}
log::trace!(" -> final: {:?}", req);
trace!(" -> final: {:?}", req);
Ok(req)
}

14
src/ion/spill.rs
View File

@@ -20,7 +20,7 @@ use crate::{Allocation, Function, SpillSlot};
impl<'a, F: Function> Env<'a, F> {
pub fn try_allocating_regs_for_spilled_bundles(&mut self) {
log::trace!("allocating regs for spilled bundles");
trace!("allocating regs for spilled bundles");
for i in 0..self.spilled_bundles.len() {
let bundle = self.spilled_bundles[i]; // don't borrow self
@@ -38,7 +38,7 @@ impl<'a, F: Function> Env<'a, F> {
for preg in
RegTraversalIter::new(self.env, class, hint, PReg::invalid(), bundle.index(), None)
{
log::trace!("trying bundle {:?} to preg {:?}", bundle, preg);
trace!("trying bundle {:?} to preg {:?}", bundle, preg);
let preg_idx = PRegIndex::new(preg.index());
if let AllocRegResult::Allocated(_) =
self.try_to_allocate_bundle_to_reg(bundle, preg_idx, None)
@@ -49,7 +49,7 @@ impl<'a, F: Function> Env<'a, F> {
}
}
if !success {
log::trace!(
trace!(
"spilling bundle {:?}: marking spillset {:?} as required",
bundle,
self.bundles[bundle.index()].spillset
@@ -87,14 +87,14 @@ impl<'a, F: Function> Env<'a, F> {
for i in 0..self.spillsets[spillset.index()].vregs.len() {
// don't borrow self
let vreg = self.spillsets[spillset.index()].vregs[i];
log::trace!(
trace!(
"spillslot {:?} alloc'ed to spillset {:?}: vreg {:?}",
spillslot,
spillset,
vreg,
);
for entry in &self.vregs[vreg.index()].ranges {
log::trace!(
trace!(
"spillslot {:?} getting range {:?} from LR {:?} from vreg {:?}",
spillslot,
entry.range,
@@ -111,7 +111,7 @@ impl<'a, F: Function> Env<'a, F> {
pub fn allocate_spillslots(&mut self) {
for spillset in 0..self.spillsets.len() {
log::trace!("allocate spillslot: {}", spillset);
trace!("allocate spillslot: {}", spillset);
let spillset = SpillSetIndex::new(spillset);
if !self.spillsets[spillset.index()].required {
continue;
@@ -196,7 +196,7 @@ impl<'a, F: Function> Env<'a, F> {
self.spillslots[i].alloc = self.allocate_spillslot(self.spillslots[i].class);
}
log::trace!("spillslot allocator done");
trace!("spillslot allocator done");
}
pub fn allocate_spillslot(&mut self, class: RegClass) -> Allocation {

17
src/ion/stackmap.rs
View File

@@ -13,7 +13,7 @@
//! Stackmap computation.
use super::{Env, ProgPoint, VRegIndex};
use crate::Function;
use crate::{ion::data_structures::u64_key, Function};
impl<'a, F: Function> Env<'a, F> {
pub fn compute_stackmaps(&mut self) {
@@ -30,10 +30,10 @@ impl<'a, F: Function> Env<'a, F> {
// safepoints; and for each safepoint in the current range,
// emit the allocation into the `safepoint_slots` list.
log::trace!("safepoints_per_vreg = {:?}", self.safepoints_per_vreg);
trace!("safepoints_per_vreg = {:?}", self.safepoints_per_vreg);
for vreg in self.func.reftype_vregs() {
log::trace!("generating safepoint info for vreg {}", vreg);
trace!("generating safepoint info for vreg {}", vreg);
let vreg = VRegIndex::new(vreg.vreg());
let mut safepoints: Vec<ProgPoint> = self
.safepoints_per_vreg
@@ -43,19 +43,19 @@ impl<'a, F: Function> Env<'a, F> {
.map(|&inst| ProgPoint::before(inst))
.collect();
safepoints.sort_unstable();
log::trace!(" -> live over safepoints: {:?}", safepoints);
trace!(" -> live over safepoints: {:?}", safepoints);
let mut safepoint_idx = 0;
for entry in &self.vregs[vreg.index()].ranges {
let range = entry.range;
let alloc = self.get_alloc_for_range(entry.index);
log::trace!(" -> range {:?}: alloc {}", range, alloc);
trace!(" -> range {:?}: alloc {}", range, alloc);
while safepoint_idx < safepoints.len() && safepoints[safepoint_idx] < range.to {
if safepoints[safepoint_idx] < range.from {
safepoint_idx += 1;
continue;
}
log::trace!(" -> covers safepoint {:?}", safepoints[safepoint_idx]);
trace!(" -> covers safepoint {:?}", safepoints[safepoint_idx]);
self.safepoint_slots
.push((safepoints[safepoint_idx], alloc));
@@ -64,7 +64,8 @@ impl<'a, F: Function> Env<'a, F> {
}
}
self.safepoint_slots.sort_unstable();
log::trace!("final safepoint slots info: {:?}", self.safepoint_slots);
self.safepoint_slots
.sort_unstable_by_key(|(progpoint, slot)| u64_key(progpoint.to_index(), slot.bits()));
trace!("final safepoint slots info: {:?}", self.safepoint_slots);
}
}

30
src/lib.rs
View File

@@ -12,6 +12,16 @@
#![allow(dead_code)]
// Even when trace logging is disabled, the trace macro has a significant
// performance cost so we disable it in release builds.
macro_rules! trace {
($($tt:tt)*) => {
if cfg!(feature = "trace-log") {
::log::trace!($($tt)*);
}
};
}
pub(crate) mod cfg;
pub(crate) mod domtree;
pub mod indexset;
@@ -234,7 +244,7 @@ impl SpillSlot {
/// Create a new SpillSlot of a given class.
#[inline(always)]
pub fn new(slot: usize, class: RegClass) -> Self {
assert!(slot < (1 << 24));
debug_assert!(slot < (1 << 24));
SpillSlot {
bits: (slot as u32) | (class as u8 as u32) << 24,
}
@@ -412,11 +422,11 @@ impl Operand {
OperandConstraint::Reg => 1,
OperandConstraint::Stack => 2,
OperandConstraint::FixedReg(preg) => {
assert_eq!(preg.class(), vreg.class());
debug_assert_eq!(preg.class(), vreg.class());
0b1000000 | preg.hw_enc() as u32
}
OperandConstraint::Reuse(which) => {
assert!(which <= 31);
debug_assert!(which <= 31);
0b0100000 | which as u32
}
};
@@ -696,7 +706,7 @@ impl Allocation {
/// Construct a new Allocation.
#[inline(always)]
pub(crate) fn new(kind: AllocationKind, index: usize) -> Self {
assert!(index < (1 << 28));
debug_assert!(index < (1 << 28));
Self {
bits: ((kind as u8 as u32) << 29) | (index as u32),
}
@@ -1105,20 +1115,16 @@ pub enum Edit {
/// register or a stack slot (spillslot). However, stack-to-stack
/// moves will never be generated.
///
/// `to_vreg`, if defined, is useful as metadata: it indicates
/// that the moved value is a def of a new vreg.
///
/// `Move` edits will be generated even if src and dst allocation
/// are the same if the vreg changes; this allows proper metadata
/// tracking even when moves are elided.
Move {
from: Allocation,
to: Allocation,
to_vreg: Option<VReg>,
},
Move { from: Allocation, to: Allocation },
/// Define a particular Allocation to contain a particular VReg. Useful
/// for the checker.
///
/// `DefAlloc` edits are only emitted when the `"checker"` Cargo feature is
/// enabled.
DefAlloc { alloc: Allocation, vreg: VReg },
}

7
src/moves.rs
View File

@@ -3,7 +3,7 @@
* exception. See `LICENSE` for details.
*/
use crate::Allocation;
use crate::{ion::data_structures::u64_key, Allocation};
use smallvec::{smallvec, SmallVec};
pub type MoveVec<T> = SmallVec<[(Allocation, Allocation, T); 16]>;
@@ -53,7 +53,8 @@ impl<T: Clone + Copy + Default> ParallelMoves<T> {
// Sort moves by source so that we can efficiently test for
// presence.
self.parallel_moves.sort_by_key(|&(src, dst, _)| (src, dst));
self.parallel_moves
.sort_by_key(|&(src, dst, _)| u64_key(src.bits(), dst.bits()));
// Do any dests overlap sources? If not, we can also just
// return the list.
@@ -183,7 +184,7 @@ impl<T: Clone + Copy + Default> ParallelMoves<T> {
scratch_src = Some(src);
src = self.scratch;
} else {
assert_eq!(last_dst.unwrap(), src);
debug_assert_eq!(last_dst.unwrap(), src);
}
ret.push((src, dst, dst_t));