T0b1/regalloc2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #25 from Amanieu/perf

Browse Source 
Performance improvements
This commit is contained in:
Chris Fallin
2022-01-11 14:35:10 -08:00
committed by GitHub
parent a27f93f01e 6b1a5e8b1b
commit 56a8f844a8
20 changed files with 594 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Cargo.toml
View File

@@ -1,7 +1,10 @@
[package] [package]
name = "regalloc2" name = "regalloc2"
version = "0.0.1" version = "0.0.1"
authors = ["Chris Fallin <chris@cfallin.org>", "Mozilla SpiderMonkey Developers"] authors = [
"Chris Fallin <chris@cfallin.org>",
"Mozilla SpiderMonkey Developers",
]
edition = "2018" edition = "2018"
license = "Apache-2.0 WITH LLVM-exception" license = "Apache-2.0 WITH LLVM-exception"
description = "Backtracking register allocator inspired from IonMonkey" description = "Backtracking register allocator inspired from IonMonkey"
@@ -17,9 +20,20 @@ slice-group-by = "0.3.0"
# Keep this in sync with libfuzzer_sys's crate version: # Keep this in sync with libfuzzer_sys's crate version:
arbitrary = { version = "^0.4.6", optional = true } arbitrary = { version = "^0.4.6", optional = true }
# When testing regalloc2 by itself, enable debug assertions and overflow checks
[profile.release] [profile.release]
debug = true debug = true
debug-assertions = true
overflow-checks = true
[features] [features]
default = [] default = []
fuzzing = ["arbitrary"]
# Enables generation of DefAlloc edits for the checker.
checker = []
# Enables detailed logging which can be somewhat expensive.
trace-log = []
# Exposes the internal API for fuzzing.
fuzzing = ["arbitrary", "checker", "trace-log"]

6
fuzz/Cargo.toml
View File

@@ -49,3 +49,9 @@ name = "ion_checker"
path = "fuzz_targets/ion_checker.rs" path = "fuzz_targets/ion_checker.rs"
test = false test = false
doc = false doc = false
# Enable debug assertions and overflow checks when fuzzing
[profile.release]
debug = true
debug-assertions = true
overflow-checks = true

50
src/checker.rs
View File

@@ -191,7 +191,7 @@ impl CheckerValue {
CheckerValue::Reg(r1, ref1) CheckerValue::Reg(r1, ref1)
} }
_ => { _ => {
log::trace!("{:?} and {:?} meet to Conflicted", self, other); trace!("{:?} and {:?} meet to Conflicted", self, other);
CheckerValue::Conflicted CheckerValue::Conflicted
} }
} }
@@ -322,7 +322,7 @@ impl CheckerState {
.get(alloc) .get(alloc)
.cloned() .cloned()
.unwrap_or(Default::default()); .unwrap_or(Default::default());
log::trace!( trace!(
"checker: checkinst {:?}: op {:?}, alloc {:?}, checker value {:?}", "checker: checkinst {:?}: op {:?}, alloc {:?}, checker value {:?}",
checkinst, checkinst,
op, op,
@@ -339,7 +339,7 @@ impl CheckerState {
.get(&alloc) .get(&alloc)
.cloned() .cloned()
.unwrap_or(Default::default()); .unwrap_or(Default::default());
log::trace!( trace!(
"checker: checkinst {:?}: safepoint slot {}, checker value {:?}", "checker: checkinst {:?}: safepoint slot {}, checker value {:?}",
checkinst, checkinst,
alloc, alloc,
@@ -372,7 +372,7 @@ impl CheckerState {
.get(&from) .get(&from)
.cloned() .cloned()
.unwrap_or(Default::default()); .unwrap_or(Default::default());
log::trace!( trace!(
"checker: checkinst {:?} updating: move {:?} -> {:?} val {:?}", "checker: checkinst {:?} updating: move {:?} -> {:?} val {:?}",
checkinst, checkinst,
from, from,
@@ -534,7 +534,7 @@ impl<'a, F: Function> Checker<'a, F> {
/// Build the list of checker instructions based on the given func /// Build the list of checker instructions based on the given func
/// and allocation results. /// and allocation results.
pub fn prepare(&mut self, out: &Output) { pub fn prepare(&mut self, out: &Output) {
log::trace!("checker: out = {:?}", out); trace!("checker: out = {:?}", out);
// Preprocess safepoint stack-maps into per-inst vecs. // Preprocess safepoint stack-maps into per-inst vecs.
let mut safepoint_slots: HashMap<Inst, Vec<Allocation>> = HashMap::new(); let mut safepoint_slots: HashMap<Inst, Vec<Allocation>> = HashMap::new();
for &(progpoint, slot) in &out.safepoint_slots { for &(progpoint, slot) in &out.safepoint_slots {
@@ -550,7 +550,7 @@ impl<'a, F: Function> Checker<'a, F> {
for inst_or_edit in out.block_insts_and_edits(self.f, block) { for inst_or_edit in out.block_insts_and_edits(self.f, block) {
match inst_or_edit { match inst_or_edit {
InstOrEdit::Inst(inst) => { InstOrEdit::Inst(inst) => {
assert!(last_inst.is_none() || inst > last_inst.unwrap()); debug_assert!(last_inst.is_none() || inst > last_inst.unwrap());
last_inst = Some(inst); last_inst = Some(inst);
self.handle_inst(block, inst, &mut safepoint_slots, out); self.handle_inst(block, inst, &mut safepoint_slots, out);
} }
@@ -591,25 +591,19 @@ impl<'a, F: Function> Checker<'a, F> {
allocs, allocs,
clobbers, clobbers,
}; };
log::trace!("checker: adding inst {:?}", checkinst); trace!("checker: adding inst {:?}", checkinst);
self.bb_insts.get_mut(&block).unwrap().push(checkinst); self.bb_insts.get_mut(&block).unwrap().push(checkinst);
} }
} }
fn handle_edit(&mut self, block: Block, edit: &Edit) { fn handle_edit(&mut self, block: Block, edit: &Edit) {
log::trace!("checker: adding edit {:?}", edit); trace!("checker: adding edit {:?}", edit);
match edit { match edit {
&Edit::Move { from, to, to_vreg } => { &Edit::Move { from, to } => {
self.bb_insts self.bb_insts
.get_mut(&block) .get_mut(&block)
.unwrap() .unwrap()
.push(CheckerInst::Move { into: to, from }); .push(CheckerInst::Move { into: to, from });
if let Some(vreg) = to_vreg {
self.bb_insts
.get_mut(&block)
.unwrap()
.push(CheckerInst::DefAlloc { alloc: to, vreg });
}
} }
&Edit::DefAlloc { alloc, vreg } => { &Edit::DefAlloc { alloc, vreg } => {
self.bb_insts self.bb_insts
@@ -634,10 +628,10 @@ impl<'a, F: Function> Checker<'a, F> {
let block = queue.pop_front().unwrap(); let block = queue.pop_front().unwrap();
queue_set.remove(&block); queue_set.remove(&block);
let mut state = self.bb_in.get(&block).cloned().unwrap(); let mut state = self.bb_in.get(&block).cloned().unwrap();
log::trace!("analyze: block {} has state {:?}", block.index(), state); trace!("analyze: block {} has state {:?}", block.index(), state);
for inst in self.bb_insts.get(&block).unwrap() { for inst in self.bb_insts.get(&block).unwrap() {
state.update(inst, self); state.update(inst, self);
log::trace!("analyze: inst {:?} -> state {:?}", inst, state); trace!("analyze: inst {:?} -> state {:?}", inst, state);
} }
for &succ in self.f.block_succs(block) { for &succ in self.f.block_succs(block) {
@@ -646,7 +640,7 @@ impl<'a, F: Function> Checker<'a, F> {
new_state.meet_with(cur_succ_in); new_state.meet_with(cur_succ_in);
let changed = &new_state != cur_succ_in; let changed = &new_state != cur_succ_in;
if changed { if changed {
log::trace!( trace!(
"analyze: block {} state changed from {:?} to {:?}; pushing onto queue", "analyze: block {} state changed from {:?} to {:?}; pushing onto queue",
succ.index(), succ.index(),
cur_succ_in, cur_succ_in,
@@ -671,12 +665,12 @@ impl<'a, F: Function> Checker<'a, F> {
let mut state = input.clone(); let mut state = input.clone();
for inst in self.bb_insts.get(block).unwrap() { for inst in self.bb_insts.get(block).unwrap() {
if let Err(e) = state.check(InstPosition::Before, inst) { if let Err(e) = state.check(InstPosition::Before, inst) {
log::trace!("Checker error: {:?}", e); trace!("Checker error: {:?}", e);
errors.push(e); errors.push(e);
} }
state.update(inst, self); state.update(inst, self);
if let Err(e) = state.check(InstPosition::After, inst) { if let Err(e) = state.check(InstPosition::After, inst) {
log::trace!("Checker error: {:?}", e); trace!("Checker error: {:?}", e);
errors.push(e); errors.push(e);
} }
} }
@@ -695,20 +689,20 @@ impl<'a, F: Function> Checker<'a, F> {
self.analyze(); self.analyze();
let result = self.find_errors(); let result = self.find_errors();
log::trace!("=== CHECKER RESULT ==="); trace!("=== CHECKER RESULT ===");
fn print_state(state: &CheckerState) { fn print_state(state: &CheckerState) {
let mut s = vec![]; let mut s = vec![];
for (alloc, state) in &state.allocations { for (alloc, state) in &state.allocations {
s.push(format!("{} := {}", alloc, state)); s.push(format!("{} := {}", alloc, state));
} }
log::trace!(" {{ {} }}", s.join(", ")) trace!(" {{ {} }}", s.join(", "))
} }
for vreg in self.f.reftype_vregs() { for vreg in self.f.reftype_vregs() {
log::trace!(" REF: {}", vreg); trace!(" REF: {}", vreg);
} }
for bb in 0..self.f.num_blocks() { for bb in 0..self.f.num_blocks() {
let bb = Block::new(bb); let bb = Block::new(bb);
log::trace!("block{}:", bb.index()); trace!("block{}:", bb.index());
let insts = self.bb_insts.get(&bb).unwrap(); let insts = self.bb_insts.get(&bb).unwrap();
let mut state = self.bb_in.get(&bb).unwrap().clone(); let mut state = self.bb_in.get(&bb).unwrap().clone();
print_state(&state); print_state(&state);
@@ -720,7 +714,7 @@ impl<'a, F: Function> Checker<'a, F> {
ref allocs, ref allocs,
ref clobbers, ref clobbers,
} => { } => {
log::trace!( trace!(
" inst{}: {:?} ({:?}) clobbers:{:?}", " inst{}: {:?} ({:?}) clobbers:{:?}",
inst.index(), inst.index(),
operands, operands,
@@ -729,17 +723,17 @@ impl<'a, F: Function> Checker<'a, F> {
); );
} }
&CheckerInst::Move { from, into } => { &CheckerInst::Move { from, into } => {
log::trace!(" {} -> {}", from, into); trace!(" {} -> {}", from, into);
} }
&CheckerInst::DefAlloc { alloc, vreg } => { &CheckerInst::DefAlloc { alloc, vreg } => {
log::trace!(" defalloc: {}:{}", vreg, alloc); trace!(" defalloc: {}:{}", vreg, alloc);
} }
&CheckerInst::Safepoint { ref allocs, .. } => { &CheckerInst::Safepoint { ref allocs, .. } => {
let mut slotargs = vec![]; let mut slotargs = vec![];
for &slot in allocs { for &slot in allocs {
slotargs.push(format!("{}", slot)); slotargs.push(format!("{}", slot));
} }
log::trace!(" safepoint: {}", slotargs.join(", ")); trace!(" safepoint: {}", slotargs.join(", "));
} }
} }
state.update(inst, &self); state.update(inst, &self);

2
src/domtree.rs
View File

@@ -33,7 +33,7 @@ fn merge_sets(
node2 = idom[node2.index()]; node2 = idom[node2.index()];
} }
} }
assert!(node1 == node2); debug_assert!(node1 == node2);
node1 node1
} }

6
src/fuzzing/func.rs
View File

@@ -79,7 +79,7 @@ impl Function for Func {
} }
fn entry_block(&self) -> Block { fn entry_block(&self) -> Block {
assert!(self.blocks.len() > 0); debug_assert!(self.blocks.len() > 0);
Block::new(0) Block::new(0)
} }
@@ -237,7 +237,7 @@ fn choose_dominating_block(
allow_self: bool, allow_self: bool,
u: &mut Unstructured, u: &mut Unstructured,
) -> ArbitraryResult<Block> { ) -> ArbitraryResult<Block> {
assert!(block.is_valid()); debug_assert!(block.is_valid());
let orig_block = block; let orig_block = block;
loop { loop {
if (allow_self || block != orig_block) && bool::arbitrary(u)? { if (allow_self || block != orig_block) && bool::arbitrary(u)? {
@@ -445,7 +445,7 @@ impl Func {
if operands.len() > 1 && opts.reused_inputs && bool::arbitrary(u)? { if operands.len() > 1 && opts.reused_inputs && bool::arbitrary(u)? {
// Make the def a reused input. // Make the def a reused input.
let op = operands[0]; let op = operands[0];
assert_eq!(op.kind(), OperandKind::Def); debug_assert_eq!(op.kind(), OperandKind::Def);
let reused = u.int_in_range(1..=(operands.len() - 1))?; let reused = u.int_in_range(1..=(operands.len() - 1))?;
operands[0] = Operand::new( operands[0] = Operand::new(
op.vreg(), op.vreg(),

34
src/index.rs
View File

@@ -10,7 +10,7 @@ macro_rules! define_index {
} }
#[inline(always)] #[inline(always)]
pub fn index(self) -> usize { pub fn index(self) -> usize {
assert!(self.is_valid()); debug_assert!(self.is_valid());
self.0 as usize self.0 as usize
} }
#[inline(always)] #[inline(always)]
@@ -27,12 +27,12 @@ macro_rules! define_index {
} }
#[inline(always)] #[inline(always)]
pub fn next(self) -> $ix { pub fn next(self) -> $ix {
assert!(self.is_valid()); debug_assert!(self.is_valid());
Self(self.0 + 1) Self(self.0 + 1)
} }
#[inline(always)] #[inline(always)]
pub fn prev(self) -> $ix { pub fn prev(self) -> $ix {
assert!(self.is_valid()); debug_assert!(self.is_valid());
Self(self.0 - 1) Self(self.0 - 1)
} }
@@ -62,19 +62,19 @@ pub struct InstRange(Inst, Inst, bool);
impl InstRange { impl InstRange {
#[inline(always)] #[inline(always)]
pub fn forward(from: Inst, to: Inst) -> Self { pub fn forward(from: Inst, to: Inst) -> Self {
assert!(from.index() <= to.index()); debug_assert!(from.index() <= to.index());
InstRange(from, to, true) InstRange(from, to, true)
} }
#[inline(always)] #[inline(always)]
pub fn backward(from: Inst, to: Inst) -> Self { pub fn backward(from: Inst, to: Inst) -> Self {
assert!(from.index() >= to.index()); debug_assert!(from.index() >= to.index());
InstRange(to, from, false) InstRange(to, from, false)
} }
#[inline(always)] #[inline(always)]
pub fn first(self) -> Inst { pub fn first(self) -> Inst {
assert!(self.len() > 0); debug_assert!(self.len() > 0);
if self.is_forward() { if self.is_forward() {
self.0 self.0
} else { } else {
@@ -84,7 +84,7 @@ impl InstRange {
#[inline(always)] #[inline(always)]
pub fn last(self) -> Inst { pub fn last(self) -> Inst {
assert!(self.len() > 0); debug_assert!(self.len() > 0);
if self.is_forward() { if self.is_forward() {
self.1.prev() self.1.prev()
} else { } else {
@@ -94,7 +94,7 @@ impl InstRange {
#[inline(always)] #[inline(always)]
pub fn rest(self) -> InstRange { pub fn rest(self) -> InstRange {
assert!(self.len() > 0); debug_assert!(self.len() > 0);
if self.is_forward() { if self.is_forward() {
InstRange::forward(self.0.next(), self.1) InstRange::forward(self.0.next(), self.1)
} else { } else {
@@ -147,13 +147,13 @@ mod test {
#[test] #[test]
fn test_inst_range() { fn test_inst_range() {
let range = InstRange::forward(Inst::new(0), Inst::new(0)); let range = InstRange::forward(Inst::new(0), Inst::new(0));
assert_eq!(range.len(), 0); debug_assert_eq!(range.len(), 0);
let range = InstRange::forward(Inst::new(0), Inst::new(5)); let range = InstRange::forward(Inst::new(0), Inst::new(5));
assert_eq!(range.first().index(), 0); debug_assert_eq!(range.first().index(), 0);
assert_eq!(range.last().index(), 4); debug_assert_eq!(range.last().index(), 4);
assert_eq!(range.len(), 5); debug_assert_eq!(range.len(), 5);
assert_eq!( debug_assert_eq!(
range.iter().collect::<Vec<_>>(), range.iter().collect::<Vec<_>>(),
vec![ vec![
Inst::new(0), Inst::new(0),
@@ -164,10 +164,10 @@ mod test {
] ]
); );
let range = range.rev(); let range = range.rev();
assert_eq!(range.first().index(), 4); debug_assert_eq!(range.first().index(), 4);
assert_eq!(range.last().index(), 0); debug_assert_eq!(range.last().index(), 0);
assert_eq!(range.len(), 5); debug_assert_eq!(range.len(), 5);
assert_eq!( debug_assert_eq!(
range.iter().collect::<Vec<_>>(), range.iter().collect::<Vec<_>>(),
vec![ vec![
Inst::new(4), Inst::new(4),

10
src/indexset.rs
View File

@@ -93,7 +93,7 @@ impl AdaptiveMap {
}; };
if needs_expand { if needs_expand {
assert!(small_mode_idx.is_none()); debug_assert!(small_mode_idx.is_none());
self.expand(); self.expand();
} }
@@ -111,7 +111,7 @@ impl AdaptiveMap {
} }
// Otherwise, the key must not be present; add a new // Otherwise, the key must not be present; add a new
// entry. // entry.
assert!(*len < SMALL_ELEMS as u32); debug_assert!(*len < SMALL_ELEMS as u32);
let idx = *len; let idx = *len;
*len += 1; *len += 1;
keys[idx as usize] = key; keys[idx as usize] = key;
@@ -344,11 +344,11 @@ mod test {
let mut checksum = 0; let mut checksum = 0;
for bit in vec.iter() { for bit in vec.iter() {
assert!(bit % 17 == 0); debug_assert!(bit % 17 == 0);
checksum += bit; checksum += bit;
} }
assert_eq!(sum, checksum); debug_assert_eq!(sum, checksum);
} }
#[test] #[test]
@@ -362,6 +362,6 @@ mod test {
// should still be in small mode. // should still be in small mode.
vec.set(64 * 5, false); vec.set(64 * 5, false);
vec.set(64 * 100, true); vec.set(64 * 100, true);
assert!(vec.is_small()); debug_assert!(vec.is_small());
} }
} }

97
src/ion/data_structures.rs
View File

@@ -264,7 +264,6 @@ pub struct VRegData {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct PRegData { pub struct PRegData {
pub reg: PReg,
pub allocations: LiveRangeSet, pub allocations: LiveRangeSet,
pub is_stack: bool, pub is_stack: bool,
} }
@@ -278,6 +277,56 @@ pub struct MultiFixedRegFixup {
pub vreg: VRegIndex, pub vreg: VRegIndex,
} }
/// The field order is significant: these are sorted so that a
/// scan over vregs, then blocks in each range, can scan in
/// order through this (sorted) list and add allocs to the
/// half-move list.
///
/// The fields in this struct are reversed in sort order so that the entire
/// struct can be treated as a u128 for sorting purposes.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(C)]
pub struct BlockparamOut {
pub to_vreg: VRegIndex,
pub to_block: Block,
pub from_block: Block,
pub from_vreg: VRegIndex,
}
impl BlockparamOut {
#[inline(always)]
pub fn key(&self) -> u128 {
u128_key(
self.from_vreg.raw_u32(),
self.from_block.raw_u32(),
self.to_block.raw_u32(),
self.to_vreg.raw_u32(),
)
}
}
/// As above for `BlockparamIn`, field order is significant.
///
/// The fields in this struct are reversed in sort order so that the entire
/// struct can be treated as a u128 for sorting purposes.
#[derive(Clone, Debug)]
#[repr(C)]
pub struct BlockparamIn {
pub from_block: Block,
pub to_block: Block,
pub to_vreg: VRegIndex,
}
impl BlockparamIn {
#[inline(always)]
pub fn key(&self) -> u128 {
u128_key(
self.to_vreg.raw_u32(),
self.to_block.raw_u32(),
self.from_block.raw_u32(),
0,
)
}
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct Env<'a, F: Function> { pub struct Env<'a, F: Function> {
pub func: &'a F, pub func: &'a F,
@@ -285,16 +334,8 @@ pub struct Env<'a, F: Function> {
pub cfginfo: CFGInfo, pub cfginfo: CFGInfo,
pub liveins: Vec<IndexSet>, pub liveins: Vec<IndexSet>,
pub liveouts: Vec<IndexSet>, pub liveouts: Vec<IndexSet>,
/// Blockparam outputs: from-vreg, (end of) from-block, (start of) pub blockparam_outs: Vec<BlockparamOut>,
/// to-block, to-vreg. The field order is significant: these are sorted so pub blockparam_ins: Vec<BlockparamIn>,
/// that a scan over vregs, then blocks in each range, can scan in
/// order through this (sorted) list and add allocs to the
/// half-move list.
pub blockparam_outs: Vec<(VRegIndex, Block, Block, VRegIndex)>,
/// Blockparam inputs: to-vreg, (start of) to-block, (end of)
/// from-block. As above for `blockparam_outs`, field order is
/// significant.
pub blockparam_ins: Vec<(VRegIndex, Block, Block)>,
/// Blockparam allocs: block, idx, vreg, alloc. Info to describe /// Blockparam allocs: block, idx, vreg, alloc. Info to describe
/// blockparam locations at block entry, for metadata purposes /// blockparam locations at block entry, for metadata purposes
/// (e.g. for the checker). /// (e.g. for the checker).
@@ -344,7 +385,7 @@ pub struct Env<'a, F: Function> {
pub inserted_moves: Vec<InsertedMove>, pub inserted_moves: Vec<InsertedMove>,
// Output: // Output:
pub edits: Vec<(u32, InsertMovePrio, Edit)>, pub edits: Vec<(PosWithPrio, Edit)>,
pub allocs: Vec<Allocation>, pub allocs: Vec<Allocation>,
pub inst_alloc_offsets: Vec<u32>, pub inst_alloc_offsets: Vec<u32>,
pub num_spillslots: u32, pub num_spillslots: u32,
@@ -488,8 +529,7 @@ impl LiveRangeSet {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct InsertedMove { pub struct InsertedMove {
pub pos: ProgPoint, pub pos_prio: PosWithPrio,
pub prio: InsertMovePrio,
pub from_alloc: Allocation, pub from_alloc: Allocation,
pub to_alloc: Allocation, pub to_alloc: Allocation,
pub to_vreg: Option<VReg>, pub to_vreg: Option<VReg>,
@@ -506,6 +546,21 @@ pub enum InsertMovePrio {
OutEdgeMoves, OutEdgeMoves,
} }
/// The fields in this struct are reversed in sort order so that the entire
/// struct can be treated as a u64 for sorting purposes.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(C)]
pub struct PosWithPrio {
pub prio: u32,
pub pos: ProgPoint,
}
impl PosWithPrio {
pub fn key(self) -> u64 {
u64_key(self.pos.to_index(), self.prio)
}
}
#[derive(Clone, Copy, Debug, Default)] #[derive(Clone, Copy, Debug, Default)]
pub struct Stats { pub struct Stats {
pub livein_blocks: usize, pub livein_blocks: usize,
@@ -544,3 +599,17 @@ pub struct Stats {
pub halfmoves_count: usize, pub halfmoves_count: usize,
pub edits_count: usize, pub edits_count: usize,
} }
// Helper function for generating sorting keys. The order of arguments is from
// the most significant field to the least significant one.
//
// These work best when the fields are stored in reverse order in memory so that
// they can be loaded with a single u64 load on little-endian machines.
#[inline(always)]
pub fn u64_key(b: u32, a: u32) -> u64 {
a as u64 | (b as u64) << 32
}
#[inline(always)]
pub fn u128_key(d: u32, c: u32, b: u32, a: u32) -> u128 {
a as u128 | (b as u128) << 32 | (c as u128) << 64 | (d as u128) << 96
}

18
src/ion/dump.rs
View File

@@ -5,16 +5,16 @@ use crate::{Block, Function, ProgPoint};
impl<'a, F: Function> Env<'a, F> { impl<'a, F: Function> Env<'a, F> {
pub fn dump_state(&self) { pub fn dump_state(&self) {
log::trace!("Bundles:"); trace!("Bundles:");
for (i, b) in self.bundles.iter().enumerate() { for (i, b) in self.bundles.iter().enumerate() {
log::trace!( trace!(
"bundle{}: spillset={:?} alloc={:?}", "bundle{}: spillset={:?} alloc={:?}",
i, i,
b.spillset, b.spillset,
b.allocation b.allocation
); );
for entry in &b.ranges { for entry in &b.ranges {
log::trace!( trace!(
" * range {:?} -- {:?}: range{}", " * range {:?} -- {:?}: range{}",
entry.range.from, entry.range.from,
entry.range.to, entry.range.to,
@@ -22,11 +22,11 @@ impl<'a, F: Function> Env<'a, F> {
); );
} }
} }
log::trace!("VRegs:"); trace!("VRegs:");
for (i, v) in self.vregs.iter().enumerate() { for (i, v) in self.vregs.iter().enumerate() {
log::trace!("vreg{}:", i); trace!("vreg{}:", i);
for entry in &v.ranges { for entry in &v.ranges {
log::trace!( trace!(
" * range {:?} -- {:?}: range{}", " * range {:?} -- {:?}: range{}",
entry.range.from, entry.range.from,
entry.range.to, entry.range.to,
@@ -34,9 +34,9 @@ impl<'a, F: Function> Env<'a, F> {
); );
} }
} }
log::trace!("Ranges:"); trace!("Ranges:");
for (i, r) in self.ranges.iter().enumerate() { for (i, r) in self.ranges.iter().enumerate() {
log::trace!( trace!(
"range{}: range={:?} vreg={:?} bundle={:?} weight={:?}", "range{}: range={:?} vreg={:?} bundle={:?} weight={:?}",
i, i,
r.range, r.range,
@@ -45,7 +45,7 @@ impl<'a, F: Function> Env<'a, F> {
r.uses_spill_weight(), r.uses_spill_weight(),
); );
for u in &r.uses { for u in &r.uses {
log::trace!(" * use at {:?} (slot {}): {:?}", u.pos, u.slot, u.operand); trace!(" * use at {:?} (slot {}): {:?}", u.pos, u.slot, u.operand);
} }
} }
} }

137
src/ion/liveranges.rs
View File

@@ -18,7 +18,7 @@ use super::{
SpillSetIndex, Use, VRegData, VRegIndex, SLOT_NONE, SpillSetIndex, Use, VRegData, VRegIndex, SLOT_NONE,
}; };
use crate::indexset::IndexSet; use crate::indexset::IndexSet;
use crate::ion::data_structures::MultiFixedRegFixup; use crate::ion::data_structures::{BlockparamIn, BlockparamOut, MultiFixedRegFixup};
use crate::{ use crate::{
Allocation, Block, Function, Inst, InstPosition, Operand, OperandConstraint, OperandKind, Allocation, Block, Function, Inst, InstPosition, Operand, OperandConstraint, OperandKind,
OperandPos, PReg, ProgPoint, RegAllocError, VReg, OperandPos, PReg, ProgPoint, RegAllocError, VReg,
@@ -102,17 +102,10 @@ impl<'a, F: Function> Env<'a, F> {
self.pregs.resize( self.pregs.resize(
PReg::NUM_INDEX, PReg::NUM_INDEX,
PRegData { PRegData {
reg: PReg::invalid(),
allocations: LiveRangeSet::new(), allocations: LiveRangeSet::new(),
is_stack: false, is_stack: false,
}, },
); );
for i in 0..=PReg::MAX {
let preg_int = PReg::new(i, RegClass::Int);
self.pregs[preg_int.index()].reg = preg_int;
let preg_float = PReg::new(i, RegClass::Float);
self.pregs[preg_float.index()].reg = preg_float;
}
for &preg in &self.env.fixed_stack_slots { for &preg in &self.env.fixed_stack_slots {
self.pregs[preg.index()].is_stack = true; self.pregs[preg.index()].is_stack = true;
} }
@@ -186,7 +179,7 @@ impl<'a, F: Function> Env<'a, F> {
/// ///
/// Returns the liverange that contains the given range. /// Returns the liverange that contains the given range.
pub fn add_liverange_to_vreg(&mut self, vreg: VRegIndex, range: CodeRange) -> LiveRangeIndex { pub fn add_liverange_to_vreg(&mut self, vreg: VRegIndex, range: CodeRange) -> LiveRangeIndex {
log::trace!("add_liverange_to_vreg: vreg {:?} range {:?}", vreg, range); trace!("add_liverange_to_vreg: vreg {:?} range {:?}", vreg, range);
// Invariant: as we are building liveness information, we // Invariant: as we are building liveness information, we
// *always* process instructions bottom-to-top, and as a // *always* process instructions bottom-to-top, and as a
@@ -199,7 +192,7 @@ impl<'a, F: Function> Env<'a, F> {
// array, then reverse them at the end of // array, then reverse them at the end of
// `compute_liveness()`. // `compute_liveness()`.
assert!( debug_assert!(
self.vregs[vreg.index()].ranges.is_empty() self.vregs[vreg.index()].ranges.is_empty()
|| range.to || range.to
<= self.ranges[self.vregs[vreg.index()] <= self.ranges[self.vregs[vreg.index()]
@@ -235,7 +228,7 @@ impl<'a, F: Function> Env<'a, F> {
// Is contiguous with previously-added range; just extend // Is contiguous with previously-added range; just extend
// its range and return it. // its range and return it.
let lr = self.vregs[vreg.index()].ranges.last().unwrap().index; let lr = self.vregs[vreg.index()].ranges.last().unwrap().index;
assert!(range.to == self.ranges[lr.index()].range.from); debug_assert!(range.to == self.ranges[lr.index()].range.from);
self.ranges[lr.index()].range.from = range.from; self.ranges[lr.index()].range.from = range.from;
lr lr
} }
@@ -253,7 +246,7 @@ impl<'a, F: Function> Env<'a, F> {
); );
u.weight = weight.to_bits(); u.weight = weight.to_bits();
log::trace!( trace!(
"insert use {:?} into lr {:?} with weight {:?}", "insert use {:?} into lr {:?} with weight {:?}",
u, u,
into, into,
@@ -269,7 +262,7 @@ impl<'a, F: Function> Env<'a, F> {
// Update stats. // Update stats.
let range_weight = self.ranges[into.index()].uses_spill_weight() + weight; let range_weight = self.ranges[into.index()].uses_spill_weight() + weight;
self.ranges[into.index()].set_uses_spill_weight(range_weight); self.ranges[into.index()].set_uses_spill_weight(range_weight);
log::trace!( trace!(
" -> now range has weight {:?}", " -> now range has weight {:?}",
self.ranges[into.index()].uses_spill_weight(), self.ranges[into.index()].uses_spill_weight(),
); );
@@ -289,7 +282,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
pub fn add_liverange_to_preg(&mut self, range: CodeRange, reg: PReg) { pub fn add_liverange_to_preg(&mut self, range: CodeRange, reg: PReg) {
log::trace!("adding liverange to preg: {:?} to {}", range, reg); trace!("adding liverange to preg: {:?} to {}", range, reg);
let preg_idx = PRegIndex::new(reg.index()); let preg_idx = PRegIndex::new(reg.index());
self.pregs[preg_idx.index()] self.pregs[preg_idx.index()]
.allocations .allocations
@@ -323,12 +316,12 @@ impl<'a, F: Function> Env<'a, F> {
workqueue_set.remove(&block); workqueue_set.remove(&block);
let insns = self.func.block_insns(block); let insns = self.func.block_insns(block);
log::trace!("computing liveins for block{}", block.index()); trace!("computing liveins for block{}", block.index());
self.stats.livein_iterations += 1; self.stats.livein_iterations += 1;
let mut live = self.liveouts[block.index()].clone(); let mut live = self.liveouts[block.index()].clone();
log::trace!(" -> initial liveout set: {:?}", live); trace!(" -> initial liveout set: {:?}", live);
// Include outgoing blockparams in the initial live set. // Include outgoing blockparams in the initial live set.
if self.func.is_branch(insns.last()) { if self.func.is_branch(insns.last()) {
@@ -349,7 +342,7 @@ impl<'a, F: Function> Env<'a, F> {
for op in self.func.inst_operands(inst) { for op in self.func.inst_operands(inst) {
if op.pos() == *pos { if op.pos() == *pos {
let was_live = live.get(op.vreg().vreg()); let was_live = live.get(op.vreg().vreg());
log::trace!("op {:?} was_live = {}", op, was_live); trace!("op {:?} was_live = {}", op, was_live);
match op.kind() { match op.kind() {
OperandKind::Use | OperandKind::Mod => { OperandKind::Use | OperandKind::Mod => {
live.set(op.vreg().vreg(), true); live.set(op.vreg().vreg(), true);
@@ -375,7 +368,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
} }
log::trace!("computed liveins at block{}: {:?}", block.index(), live); trace!("computed liveins at block{}: {:?}", block.index(), live);
self.liveins[block.index()] = live; self.liveins[block.index()] = live;
} }
@@ -387,7 +380,7 @@ impl<'a, F: Function> Env<'a, F> {
.next() .next()
.is_some() .is_some()
{ {
log::trace!( trace!(
"non-empty liveins to entry block: {:?}", "non-empty liveins to entry block: {:?}",
self.liveins[self.func.entry_block().index()] self.liveins[self.func.entry_block().index()]
); );
@@ -437,8 +430,12 @@ impl<'a, F: Function> Env<'a, F> {
{ {
let blockparam_out = VRegIndex::new(blockparam_out.vreg()); let blockparam_out = VRegIndex::new(blockparam_out.vreg());
let blockparam_in = VRegIndex::new(blockparam_in.vreg()); let blockparam_in = VRegIndex::new(blockparam_in.vreg());
self.blockparam_outs self.blockparam_outs.push(BlockparamOut {
.push((blockparam_out, block, succ, blockparam_in)); to_vreg: blockparam_in,
to_block: succ,
from_block: block,
from_vreg: blockparam_out,
});
// Include outgoing blockparams in the initial live set. // Include outgoing blockparams in the initial live set.
live.set(blockparam_out.index(), true); live.set(blockparam_out.index(), true);
@@ -452,7 +449,7 @@ impl<'a, F: Function> Env<'a, F> {
from: self.cfginfo.block_entry[block.index()], from: self.cfginfo.block_entry[block.index()],
to: self.cfginfo.block_exit[block.index()].next(), to: self.cfginfo.block_exit[block.index()].next(),
}; };
log::trace!( trace!(
"vreg {:?} live at end of block --> create range {:?}", "vreg {:?} live at end of block --> create range {:?}",
VRegIndex::new(vreg), VRegIndex::new(vreg),
range range
@@ -502,13 +499,13 @@ impl<'a, F: Function> Env<'a, F> {
// We can completely skip the move if it is // We can completely skip the move if it is
// trivial (vreg to same vreg). // trivial (vreg to same vreg).
if src.vreg() != dst.vreg() { if src.vreg() != dst.vreg() {
log::trace!(" -> move inst{}: src {} -> dst {}", inst.index(), src, dst); trace!(" -> move inst{}: src {} -> dst {}", inst.index(), src, dst);
assert_eq!(src.class(), dst.class()); debug_assert_eq!(src.class(), dst.class());
assert_eq!(src.kind(), OperandKind::Use); debug_assert_eq!(src.kind(), OperandKind::Use);
assert_eq!(src.pos(), OperandPos::Early); debug_assert_eq!(src.pos(), OperandPos::Early);
assert_eq!(dst.kind(), OperandKind::Def); debug_assert_eq!(dst.kind(), OperandKind::Def);
assert_eq!(dst.pos(), OperandPos::Late); debug_assert_eq!(dst.pos(), OperandPos::Late);
// If both src and dest are pinned, emit the // If both src and dest are pinned, emit the
// move right here, right now. // move right here, right now.
@@ -564,9 +561,7 @@ impl<'a, F: Function> Env<'a, F> {
else if self.vregs[src.vreg().vreg()].is_pinned else if self.vregs[src.vreg().vreg()].is_pinned
|| self.vregs[dst.vreg().vreg()].is_pinned || self.vregs[dst.vreg().vreg()].is_pinned
{ {
log::trace!( trace!(" -> exactly one of src/dst is pinned; converting to ghost use");
" -> exactly one of src/dst is pinned; converting to ghost use"
);
let (preg, vreg, pinned_vreg, kind, pos, progpoint) = let (preg, vreg, pinned_vreg, kind, pos, progpoint) =
if self.vregs[src.vreg().vreg()].is_pinned { if self.vregs[src.vreg().vreg()].is_pinned {
// Source is pinned: this is a def on the dst with a pinned preg. // Source is pinned: this is a def on the dst with a pinned preg.
@@ -592,7 +587,7 @@ impl<'a, F: Function> Env<'a, F> {
let constraint = OperandConstraint::FixedReg(preg); let constraint = OperandConstraint::FixedReg(preg);
let operand = Operand::new(vreg, constraint, kind, pos); let operand = Operand::new(vreg, constraint, kind, pos);
log::trace!( trace!(
concat!( concat!(
" -> preg {:?} vreg {:?} kind {:?} ", " -> preg {:?} vreg {:?} kind {:?} ",
"pos {:?} progpoint {:?} constraint {:?} operand {:?}" "pos {:?} progpoint {:?} constraint {:?} operand {:?}"
@@ -619,9 +614,9 @@ impl<'a, F: Function> Env<'a, F> {
VRegIndex::new(vreg.vreg()), VRegIndex::new(vreg.vreg()),
CodeRange { from, to }, CodeRange { from, to },
); );
log::trace!(" -> dead; created LR"); trace!(" -> dead; created LR");
} }
log::trace!(" -> LR {:?}", lr); trace!(" -> LR {:?}", lr);
self.insert_use_into_liverange( self.insert_use_into_liverange(
lr, lr,
@@ -655,7 +650,7 @@ impl<'a, F: Function> Env<'a, F> {
// (this is the last use), start it // (this is the last use), start it
// before. // before.
if kind == OperandKind::Def { if kind == OperandKind::Def {
log::trace!(" -> src on pinned vreg {:?}", pinned_vreg); trace!(" -> src on pinned vreg {:?}", pinned_vreg);
// The *other* vreg is a def, so the pinned-vreg // The *other* vreg is a def, so the pinned-vreg
// mention is a use. If already live, // mention is a use. If already live,
// end the existing LR just *after* // end the existing LR just *after*
@@ -669,7 +664,7 @@ impl<'a, F: Function> Env<'a, F> {
if live.get(pinned_vreg.vreg()) { if live.get(pinned_vreg.vreg()) {
let pinned_lr = vreg_ranges[pinned_vreg.vreg()]; let pinned_lr = vreg_ranges[pinned_vreg.vreg()];
let orig_start = self.ranges[pinned_lr.index()].range.from; let orig_start = self.ranges[pinned_lr.index()].range.from;
log::trace!( trace!(
" -> live with LR {:?}; truncating to start at {:?}", " -> live with LR {:?}; truncating to start at {:?}",
pinned_lr, pinned_lr,
progpoint.next() progpoint.next()
@@ -683,7 +678,7 @@ impl<'a, F: Function> Env<'a, F> {
}, },
); );
vreg_ranges[pinned_vreg.vreg()] = new_lr; vreg_ranges[pinned_vreg.vreg()] = new_lr;
log::trace!(" -> created LR {:?} with remaining range from {:?} to {:?}", new_lr, orig_start, progpoint); trace!(" -> created LR {:?} with remaining range from {:?} to {:?}", new_lr, orig_start, progpoint);
// Add an edit right now to indicate that at // Add an edit right now to indicate that at
// this program point, the given // this program point, the given
@@ -717,10 +712,7 @@ impl<'a, F: Function> Env<'a, F> {
); );
vreg_ranges[pinned_vreg.vreg()] = new_lr; vreg_ranges[pinned_vreg.vreg()] = new_lr;
live.set(pinned_vreg.vreg(), true); live.set(pinned_vreg.vreg(), true);
log::trace!( trace!(" -> was not live; created new LR {:?}", new_lr);
" -> was not live; created new LR {:?}",
new_lr
);
} }
// Add an edit right now to indicate that at // Add an edit right now to indicate that at
@@ -737,7 +729,7 @@ impl<'a, F: Function> Env<'a, F> {
); );
} }
} else { } else {
log::trace!(" -> dst on pinned vreg {:?}", pinned_vreg); trace!(" -> dst on pinned vreg {:?}", pinned_vreg);
// The *other* vreg is a use, so the pinned-vreg // The *other* vreg is a use, so the pinned-vreg
// mention is a def. Truncate its LR // mention is a def. Truncate its LR
// just *after* the `progpoint` // just *after* the `progpoint`
@@ -745,7 +737,7 @@ impl<'a, F: Function> Env<'a, F> {
if live.get(pinned_vreg.vreg()) { if live.get(pinned_vreg.vreg()) {
let pinned_lr = vreg_ranges[pinned_vreg.vreg()]; let pinned_lr = vreg_ranges[pinned_vreg.vreg()];
self.ranges[pinned_lr.index()].range.from = progpoint.next(); self.ranges[pinned_lr.index()].range.from = progpoint.next();
log::trace!( trace!(
" -> was live with LR {:?}; truncated start to {:?}", " -> was live with LR {:?}; truncated start to {:?}",
pinned_lr, pinned_lr,
progpoint.next() progpoint.next()
@@ -845,14 +837,14 @@ impl<'a, F: Function> Env<'a, F> {
VRegIndex::new(dst.vreg().vreg()), VRegIndex::new(dst.vreg().vreg()),
CodeRange { from, to }, CodeRange { from, to },
); );
log::trace!(" -> invalid LR for def; created {:?}", dst_lr); trace!(" -> invalid LR for def; created {:?}", dst_lr);
} }
log::trace!(" -> has existing LR {:?}", dst_lr); trace!(" -> has existing LR {:?}", dst_lr);
// Trim the LR to start here. // Trim the LR to start here.
if self.ranges[dst_lr.index()].range.from if self.ranges[dst_lr.index()].range.from
== self.cfginfo.block_entry[block.index()] == self.cfginfo.block_entry[block.index()]
{ {
log::trace!(" -> started at block start; trimming to {:?}", pos); trace!(" -> started at block start; trimming to {:?}", pos);
self.ranges[dst_lr.index()].range.from = pos; self.ranges[dst_lr.index()].range.from = pos;
} }
self.ranges[dst_lr.index()].set_flag(LiveRangeFlag::StartsAtDef); self.ranges[dst_lr.index()].set_flag(LiveRangeFlag::StartsAtDef);
@@ -879,7 +871,7 @@ impl<'a, F: Function> Env<'a, F> {
vreg_ranges[src.vreg().vreg()] vreg_ranges[src.vreg().vreg()]
}; };
log::trace!(" -> src LR {:?}", src_lr); trace!(" -> src LR {:?}", src_lr);
// Add to live-set. // Add to live-set.
let src_is_dead_after_move = !live.get(src.vreg().vreg()); let src_is_dead_after_move = !live.get(src.vreg().vreg());
@@ -932,7 +924,7 @@ impl<'a, F: Function> Env<'a, F> {
continue; continue;
} }
log::trace!( trace!(
"processing inst{} operand at {:?}: {:?}", "processing inst{} operand at {:?}: {:?}",
inst.index(), inst.index(),
pos, pos,
@@ -941,14 +933,14 @@ impl<'a, F: Function> Env<'a, F> {
match operand.kind() { match operand.kind() {
OperandKind::Def | OperandKind::Mod => { OperandKind::Def | OperandKind::Mod => {
log::trace!("Def of {} at {:?}", operand.vreg(), pos); trace!("Def of {} at {:?}", operand.vreg(), pos);
// Fill in vreg's actual data. // Fill in vreg's actual data.
self.vreg_regs[operand.vreg().vreg()] = operand.vreg(); self.vreg_regs[operand.vreg().vreg()] = operand.vreg();
// Get or create the LiveRange. // Get or create the LiveRange.
let mut lr = vreg_ranges[operand.vreg().vreg()]; let mut lr = vreg_ranges[operand.vreg().vreg()];
log::trace!(" -> has existing LR {:?}", lr); trace!(" -> has existing LR {:?}", lr);
// If there was no liverange (dead def), create a trivial one. // If there was no liverange (dead def), create a trivial one.
if !live.get(operand.vreg().vreg()) { if !live.get(operand.vreg().vreg()) {
let from = match operand.kind() { let from = match operand.kind() {
@@ -965,7 +957,7 @@ impl<'a, F: Function> Env<'a, F> {
VRegIndex::new(operand.vreg().vreg()), VRegIndex::new(operand.vreg().vreg()),
CodeRange { from, to }, CodeRange { from, to },
); );
log::trace!(" -> invalid; created {:?}", lr); trace!(" -> invalid; created {:?}", lr);
vreg_ranges[operand.vreg().vreg()] = lr; vreg_ranges[operand.vreg().vreg()] = lr;
live.set(operand.vreg().vreg(), true); live.set(operand.vreg().vreg(), true);
} }
@@ -982,10 +974,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.ranges[lr.index()].range.from if self.ranges[lr.index()].range.from
== self.cfginfo.block_entry[block.index()] == self.cfginfo.block_entry[block.index()]
{ {
log::trace!( trace!(" -> started at block start; trimming to {:?}", pos);
" -> started at block start; trimming to {:?}",
pos
);
self.ranges[lr.index()].range.from = pos; self.ranges[lr.index()].range.from = pos;
} }
@@ -1012,9 +1001,9 @@ impl<'a, F: Function> Env<'a, F> {
); );
vreg_ranges[operand.vreg().vreg()] = lr; vreg_ranges[operand.vreg().vreg()] = lr;
} }
assert!(lr.is_valid()); debug_assert!(lr.is_valid());
log::trace!("Use of {:?} at {:?} -> {:?}", operand, pos, lr,); trace!("Use of {:?} at {:?} -> {:?}", operand, pos, lr,);
self.insert_use_into_liverange(lr, Use::new(operand, pos, i as u8)); self.insert_use_into_liverange(lr, Use::new(operand, pos, i as u8));
@@ -1026,11 +1015,11 @@ impl<'a, F: Function> Env<'a, F> {
} }
if self.func.requires_refs_on_stack(inst) { if self.func.requires_refs_on_stack(inst) {
log::trace!("inst{} is safepoint", inst.index()); trace!("inst{} is safepoint", inst.index());
self.safepoints.push(inst); self.safepoints.push(inst);
for vreg in live.iter() { for vreg in live.iter() {
if let Some(safepoints) = self.safepoints_per_vreg.get_mut(&vreg) { if let Some(safepoints) = self.safepoints_per_vreg.get_mut(&vreg) {
log::trace!("vreg v{} live at safepoint inst{}", vreg, inst.index()); trace!("vreg v{} live at safepoint inst{}", vreg, inst.index());
safepoints.insert(inst); safepoints.insert(inst);
} }
} }
@@ -1057,7 +1046,11 @@ impl<'a, F: Function> Env<'a, F> {
// add `blockparam_ins` entries. // add `blockparam_ins` entries.
let vreg_idx = VRegIndex::new(vreg.vreg()); let vreg_idx = VRegIndex::new(vreg.vreg());
for &pred in self.func.block_preds(block) { for &pred in self.func.block_preds(block) {
self.blockparam_ins.push((vreg_idx, block, pred)); self.blockparam_ins.push(BlockparamIn {
to_vreg: vreg_idx,
to_block: block,
from_block: pred,
});
} }
} }
} }
@@ -1082,7 +1075,7 @@ impl<'a, F: Function> Env<'a, F> {
// need to update with the final range here. // need to update with the final range here.
entry.range = self.ranges[entry.index.index()].range; entry.range = self.ranges[entry.index.index()].range;
// Assert in-order and non-overlapping. // Assert in-order and non-overlapping.
assert!(last.is_none() || last.unwrap() <= entry.range.from); debug_assert!(last.is_none() || last.unwrap() <= entry.range.from);
last = Some(entry.range.to); last = Some(entry.range.to);
} }
} }
@@ -1123,7 +1116,7 @@ impl<'a, F: Function> Env<'a, F> {
OperandPos::Early, OperandPos::Early,
); );
log::trace!( trace!(
"Safepoint-induced stack use of {:?} at {:?} -> {:?}", "Safepoint-induced stack use of {:?} at {:?} -> {:?}",
operand, operand,
pos, pos,
@@ -1148,13 +1141,13 @@ impl<'a, F: Function> Env<'a, F> {
} }
} }
self.blockparam_ins.sort_unstable(); self.blockparam_ins.sort_unstable_by_key(|x| x.key());
self.blockparam_outs.sort_unstable(); self.blockparam_outs.sort_unstable_by_key(|x| x.key());
self.prog_move_srcs.sort_unstable_by_key(|(pos, _)| *pos); self.prog_move_srcs.sort_unstable_by_key(|(pos, _)| *pos);
self.prog_move_dsts.sort_unstable_by_key(|(pos, _)| *pos); self.prog_move_dsts.sort_unstable_by_key(|(pos, _)| *pos);
log::trace!("prog_move_srcs = {:?}", self.prog_move_srcs); trace!("prog_move_srcs = {:?}", self.prog_move_srcs);
log::trace!("prog_move_dsts = {:?}", self.prog_move_dsts); trace!("prog_move_dsts = {:?}", self.prog_move_dsts);
self.stats.initial_liverange_count = self.ranges.len(); self.stats.initial_liverange_count = self.ranges.len();
self.stats.blockparam_ins_count = self.blockparam_ins.len(); self.stats.blockparam_ins_count = self.blockparam_ins.len();
@@ -1176,7 +1169,7 @@ impl<'a, F: Function> Env<'a, F> {
for range_idx in 0..self.vregs[vreg].ranges.len() { for range_idx in 0..self.vregs[vreg].ranges.len() {
let entry = self.vregs[vreg].ranges[range_idx]; let entry = self.vregs[vreg].ranges[range_idx];
let range = entry.index; let range = entry.index;
log::trace!( trace!(
"multi-fixed-reg cleanup: vreg {:?} range {:?}", "multi-fixed-reg cleanup: vreg {:?} range {:?}",
VRegIndex::new(vreg), VRegIndex::new(vreg),
range, range,
@@ -1245,7 +1238,7 @@ impl<'a, F: Function> Env<'a, F> {
if let OperandConstraint::FixedReg(preg) = u.operand.constraint() { if let OperandConstraint::FixedReg(preg) = u.operand.constraint() {
let vreg_idx = VRegIndex::new(u.operand.vreg().vreg()); let vreg_idx = VRegIndex::new(u.operand.vreg().vreg());
let preg_idx = PRegIndex::new(preg.index()); let preg_idx = PRegIndex::new(preg.index());
log::trace!( trace!(
"at pos {:?}, vreg {:?} has fixed constraint to preg {:?}", "at pos {:?}, vreg {:?} has fixed constraint to preg {:?}",
u.pos, u.pos,
vreg_idx, vreg_idx,
@@ -1264,7 +1257,7 @@ impl<'a, F: Function> Env<'a, F> {
continue; continue;
} }
log::trace!(" -> duplicate; switching to constraint Any"); trace!(" -> duplicate; switching to constraint Any");
self.multi_fixed_reg_fixups.push(MultiFixedRegFixup { self.multi_fixed_reg_fixups.push(MultiFixedRegFixup {
pos: u.pos, pos: u.pos,
from_slot: source_slot, from_slot: source_slot,
@@ -1278,11 +1271,7 @@ impl<'a, F: Function> Env<'a, F> {
u.operand.kind(), u.operand.kind(),
u.operand.pos(), u.operand.pos(),
); );
log::trace!( trace!(" -> extra clobber {} at inst{}", preg, u.pos.inst().index());
" -> extra clobber {} at inst{}",
preg,
u.pos.inst().index()
);
extra_clobbers.push((preg, u.pos.inst())); extra_clobbers.push((preg, u.pos.inst()));
} }
} }

72
src/ion/merge.rs
View File

@@ -16,7 +16,7 @@ use super::{
Env, LiveBundleIndex, LiveRangeIndex, LiveRangeKey, SpillSet, SpillSetIndex, SpillSlotIndex, Env, LiveBundleIndex, LiveRangeIndex, LiveRangeKey, SpillSet, SpillSetIndex, SpillSlotIndex,
VRegIndex, VRegIndex,
}; };
use crate::{Function, Inst, OperandConstraint, PReg}; use crate::{ion::data_structures::BlockparamOut, Function, Inst, OperandConstraint, PReg};
use smallvec::smallvec; use smallvec::smallvec;
impl<'a, F: Function> Env<'a, F> { impl<'a, F: Function> Env<'a, F> {
@@ -25,7 +25,7 @@ impl<'a, F: Function> Env<'a, F> {
// Merge bundle into self -- trivial merge. // Merge bundle into self -- trivial merge.
return true; return true;
} }
log::trace!( trace!(
"merging from bundle{} to bundle{}", "merging from bundle{} to bundle{}",
from.index(), from.index(),
to.index() to.index()
@@ -35,7 +35,7 @@ impl<'a, F: Function> Env<'a, F> {
let from_rc = self.spillsets[self.bundles[from.index()].spillset.index()].class; let from_rc = self.spillsets[self.bundles[from.index()].spillset.index()].class;
let to_rc = self.spillsets[self.bundles[to.index()].spillset.index()].class; let to_rc = self.spillsets[self.bundles[to.index()].spillset.index()].class;
if from_rc != to_rc { if from_rc != to_rc {
log::trace!(" -> mismatching reg classes"); trace!(" -> mismatching reg classes");
return false; return false;
} }
@@ -43,7 +43,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.bundles[from.index()].allocation.is_some() if self.bundles[from.index()].allocation.is_some()
|| self.bundles[to.index()].allocation.is_some() || self.bundles[to.index()].allocation.is_some()
{ {
log::trace!("one of the bundles is already assigned (pinned)"); trace!("one of the bundles is already assigned (pinned)");
return false; return false;
} }
@@ -52,11 +52,11 @@ impl<'a, F: Function> Env<'a, F> {
// Sanity check: both bundles should contain only ranges with appropriate VReg classes. // Sanity check: both bundles should contain only ranges with appropriate VReg classes.
for entry in &self.bundles[from.index()].ranges { for entry in &self.bundles[from.index()].ranges {
let vreg = self.ranges[entry.index.index()].vreg; let vreg = self.ranges[entry.index.index()].vreg;
assert_eq!(from_rc, self.vreg_regs[vreg.index()].class()); debug_assert_eq!(from_rc, self.vreg_regs[vreg.index()].class());
} }
for entry in &self.bundles[to.index()].ranges { for entry in &self.bundles[to.index()].ranges {
let vreg = self.ranges[entry.index.index()].vreg; let vreg = self.ranges[entry.index.index()].vreg;
assert_eq!(to_rc, self.vreg_regs[vreg.index()].class()); debug_assert_eq!(to_rc, self.vreg_regs[vreg.index()].class());
} }
} }
@@ -70,7 +70,7 @@ impl<'a, F: Function> Env<'a, F> {
while idx_from < ranges_from.len() && idx_to < ranges_to.len() { while idx_from < ranges_from.len() && idx_to < ranges_to.len() {
range_count += 1; range_count += 1;
if range_count > 200 { if range_count > 200 {
log::trace!( trace!(
"reached merge complexity (range_count = {}); exiting", "reached merge complexity (range_count = {}); exiting",
range_count range_count
); );
@@ -84,7 +84,7 @@ impl<'a, F: Function> Env<'a, F> {
idx_from += 1; idx_from += 1;
} else { } else {
// Overlap -- cannot merge. // Overlap -- cannot merge.
log::trace!( trace!(
" -> overlap between {:?} and {:?}, exiting", " -> overlap between {:?} and {:?}, exiting",
ranges_from[idx_from].index, ranges_from[idx_from].index,
ranges_to[idx_to].index ranges_to[idx_to].index
@@ -100,12 +100,12 @@ impl<'a, F: Function> Env<'a, F> {
|| self.bundles[to.index()].cached_fixed() || self.bundles[to.index()].cached_fixed()
{ {
if self.merge_bundle_requirements(from, to).is_err() { if self.merge_bundle_requirements(from, to).is_err() {
log::trace!(" -> conflicting requirements; aborting merge"); trace!(" -> conflicting requirements; aborting merge");
return false; return false;
} }
} }
log::trace!(" -> committing to merge"); trace!(" -> committing to merge");
// If we reach here, then the bundles do not overlap -- merge // If we reach here, then the bundles do not overlap -- merge
// them! We do this with a merge-sort-like scan over both // them! We do this with a merge-sort-like scan over both
@@ -113,13 +113,13 @@ impl<'a, F: Function> Env<'a, F> {
// `to` when we're done. // `to` when we're done.
if ranges_from.is_empty() { if ranges_from.is_empty() {
// `from` bundle is empty -- trivial merge. // `from` bundle is empty -- trivial merge.
log::trace!(" -> from bundle{} is empty; trivial merge", from.index()); trace!(" -> from bundle{} is empty; trivial merge", from.index());
return true; return true;
} }
if ranges_to.is_empty() { if ranges_to.is_empty() {
// `to` bundle is empty -- just move the list over from // `to` bundle is empty -- just move the list over from
// `from` and set `bundle` up-link on all ranges. // `from` and set `bundle` up-link on all ranges.
log::trace!(" -> to bundle{} is empty; trivial merge", to.index()); trace!(" -> to bundle{} is empty; trivial merge", to.index());
let list = std::mem::replace(&mut self.bundles[from.index()].ranges, smallvec![]); let list = std::mem::replace(&mut self.bundles[from.index()].ranges, smallvec![]);
for entry in &list { for entry in &list {
self.ranges[entry.index.index()].bundle = to; self.ranges[entry.index.index()].bundle = to;
@@ -149,7 +149,7 @@ impl<'a, F: Function> Env<'a, F> {
return true; return true;
} }
log::trace!( trace!(
"merging: ranges_from = {:?} ranges_to = {:?}", "merging: ranges_from = {:?} ranges_to = {:?}",
ranges_from, ranges_from,
ranges_to ranges_to
@@ -170,12 +170,12 @@ impl<'a, F: Function> Env<'a, F> {
.sort_unstable_by_key(|entry| entry.range.from); .sort_unstable_by_key(|entry| entry.range.from);
if self.annotations_enabled { if self.annotations_enabled {
log::trace!("merging: merged = {:?}", self.bundles[to.index()].ranges); trace!("merging: merged = {:?}", self.bundles[to.index()].ranges);
let mut last_range = None; let mut last_range = None;
for i in 0..self.bundles[to.index()].ranges.len() { for i in 0..self.bundles[to.index()].ranges.len() {
let entry = self.bundles[to.index()].ranges[i]; let entry = self.bundles[to.index()].ranges[i];
if last_range.is_some() { if last_range.is_some() {
assert!(last_range.unwrap() < entry.range); debug_assert!(last_range.unwrap() < entry.range);
} }
last_range = Some(entry.range); last_range = Some(entry.range);
@@ -192,7 +192,7 @@ impl<'a, F: Function> Env<'a, F> {
); );
} }
log::trace!( trace!(
" -> merged result for bundle{}: range{}", " -> merged result for bundle{}: range{}",
to.index(), to.index(),
entry.index.index(), entry.index.index(),
@@ -225,7 +225,7 @@ impl<'a, F: Function> Env<'a, F> {
pub fn merge_vreg_bundles(&mut self) { pub fn merge_vreg_bundles(&mut self) {
// Create a bundle for every vreg, initially. // Create a bundle for every vreg, initially.
log::trace!("merge_vreg_bundles: creating vreg bundles"); trace!("merge_vreg_bundles: creating vreg bundles");
for vreg in 0..self.vregs.len() { for vreg in 0..self.vregs.len() {
let vreg = VRegIndex::new(vreg); let vreg = VRegIndex::new(vreg);
if self.vregs[vreg.index()].ranges.is_empty() { if self.vregs[vreg.index()].ranges.is_empty() {
@@ -251,9 +251,9 @@ impl<'a, F: Function> Env<'a, F> {
let bundle = self.create_bundle(); let bundle = self.create_bundle();
self.bundles[bundle.index()].ranges = self.vregs[vreg.index()].ranges.clone(); self.bundles[bundle.index()].ranges = self.vregs[vreg.index()].ranges.clone();
log::trace!("vreg v{} gets bundle{}", vreg.index(), bundle.index()); trace!("vreg v{} gets bundle{}", vreg.index(), bundle.index());
for entry in &self.bundles[bundle.index()].ranges { for entry in &self.bundles[bundle.index()].ranges {
log::trace!( trace!(
" -> with LR range{}: {:?}", " -> with LR range{}: {:?}",
entry.index.index(), entry.index.index(),
entry.range entry.range
@@ -314,17 +314,17 @@ impl<'a, F: Function> Env<'a, F> {
continue; continue;
} }
log::trace!( trace!(
"trying to merge reused-input def: src {} to dst {}", "trying to merge reused-input def: src {} to dst {}",
src_vreg, src_vreg,
dst_vreg dst_vreg
); );
let src_bundle = let src_bundle =
self.ranges[self.vregs[src_vreg.vreg()].ranges[0].index.index()].bundle; self.ranges[self.vregs[src_vreg.vreg()].ranges[0].index.index()].bundle;
assert!(src_bundle.is_valid()); debug_assert!(src_bundle.is_valid());
let dest_bundle = let dest_bundle =
self.ranges[self.vregs[dst_vreg.vreg()].ranges[0].index.index()].bundle; self.ranges[self.vregs[dst_vreg.vreg()].ranges[0].index.index()].bundle;
assert!(dest_bundle.is_valid()); debug_assert!(dest_bundle.is_valid());
self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle); self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle);
} }
} }
@@ -332,18 +332,20 @@ impl<'a, F: Function> Env<'a, F> {
// Attempt to merge blockparams with their inputs. // Attempt to merge blockparams with their inputs.
for i in 0..self.blockparam_outs.len() { for i in 0..self.blockparam_outs.len() {
let (from_vreg, _, _, to_vreg) = self.blockparam_outs[i]; let BlockparamOut {
log::trace!( from_vreg, to_vreg, ..
} = self.blockparam_outs[i];
trace!(
"trying to merge blockparam v{} with input v{}", "trying to merge blockparam v{} with input v{}",
to_vreg.index(), to_vreg.index(),
from_vreg.index() from_vreg.index()
); );
let to_bundle = self.ranges[self.vregs[to_vreg.index()].ranges[0].index.index()].bundle; let to_bundle = self.ranges[self.vregs[to_vreg.index()].ranges[0].index.index()].bundle;
assert!(to_bundle.is_valid()); debug_assert!(to_bundle.is_valid());
let from_bundle = let from_bundle =
self.ranges[self.vregs[from_vreg.index()].ranges[0].index.index()].bundle; self.ranges[self.vregs[from_vreg.index()].ranges[0].index.index()].bundle;
assert!(from_bundle.is_valid()); debug_assert!(from_bundle.is_valid());
log::trace!( trace!(
" -> from bundle{} to bundle{}", " -> from bundle{} to bundle{}",
from_bundle.index(), from_bundle.index(),
to_bundle.index() to_bundle.index()
@@ -354,10 +356,10 @@ impl<'a, F: Function> Env<'a, F> {
// Attempt to merge move srcs/dsts. // Attempt to merge move srcs/dsts.
for i in 0..self.prog_move_merges.len() { for i in 0..self.prog_move_merges.len() {
let (src, dst) = self.prog_move_merges[i]; let (src, dst) = self.prog_move_merges[i];
log::trace!("trying to merge move src LR {:?} to dst LR {:?}", src, dst); trace!("trying to merge move src LR {:?} to dst LR {:?}", src, dst);
let src = self.resolve_merged_lr(src); let src = self.resolve_merged_lr(src);
let dst = self.resolve_merged_lr(dst); let dst = self.resolve_merged_lr(dst);
log::trace!( trace!(
"resolved LR-construction merging chains: move-merge is now src LR {:?} to dst LR {:?}", "resolved LR-construction merging chains: move-merge is now src LR {:?} to dst LR {:?}",
src, src,
dst dst
@@ -384,16 +386,16 @@ impl<'a, F: Function> Env<'a, F> {
} }
let src_bundle = self.ranges[src.index()].bundle; let src_bundle = self.ranges[src.index()].bundle;
assert!(src_bundle.is_valid()); debug_assert!(src_bundle.is_valid());
let dest_bundle = self.ranges[dst.index()].bundle; let dest_bundle = self.ranges[dst.index()].bundle;
assert!(dest_bundle.is_valid()); debug_assert!(dest_bundle.is_valid());
self.stats.prog_move_merge_attempt += 1; self.stats.prog_move_merge_attempt += 1;
if self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle) { if self.merge_bundles(/* from */ dest_bundle, /* to */ src_bundle) {
self.stats.prog_move_merge_success += 1; self.stats.prog_move_merge_success += 1;
} }
} }
log::trace!("done merging bundles"); trace!("done merging bundles");
} }
pub fn resolve_merged_lr(&self, mut lr: LiveRangeIndex) -> LiveRangeIndex { pub fn resolve_merged_lr(&self, mut lr: LiveRangeIndex) -> LiveRangeIndex {
@@ -417,14 +419,14 @@ impl<'a, F: Function> Env<'a, F> {
pub fn queue_bundles(&mut self) { pub fn queue_bundles(&mut self) {
for bundle in 0..self.bundles.len() { for bundle in 0..self.bundles.len() {
log::trace!("enqueueing bundle{}", bundle); trace!("enqueueing bundle{}", bundle);
if self.bundles[bundle].ranges.is_empty() { if self.bundles[bundle].ranges.is_empty() {
log::trace!(" -> no ranges; skipping"); trace!(" -> no ranges; skipping");
continue; continue;
} }
let bundle = LiveBundleIndex::new(bundle); let bundle = LiveBundleIndex::new(bundle);
let prio = self.compute_bundle_prio(bundle); let prio = self.compute_bundle_prio(bundle);
log::trace!(" -> prio {}", prio); trace!(" -> prio {}", prio);
self.bundles[bundle.index()].prio = prio; self.bundles[bundle.index()].prio = prio;
self.recompute_bundle_properties(bundle); self.recompute_bundle_properties(bundle);
self.allocation_queue self.allocation_queue

2
src/ion/mod.rs
View File

@@ -133,7 +133,7 @@ pub fn run<F: Function>(
edits: env edits: env
.edits .edits
.into_iter() .into_iter()
.map(|(pos, _, edit)| (ProgPoint::from_index(pos), edit)) .map(|(pos_prio, edit)| (pos_prio.pos, edit))
.collect(), .collect(),
allocs: env.allocs, allocs: env.allocs,
inst_alloc_offsets: env.inst_alloc_offsets, inst_alloc_offsets: env.inst_alloc_offsets,

385
src/ion/moves.rs
View File

@@ -17,10 +17,11 @@ use super::{
VRegIndex, SLOT_NONE, VRegIndex, SLOT_NONE,
}; };
use crate::ion::data_structures::{BlockparamIn, BlockparamOut, PosWithPrio};
use crate::moves::ParallelMoves; use crate::moves::ParallelMoves;
use crate::{ use crate::{
Allocation, Block, Edit, Function, Inst, InstPosition, OperandConstraint, OperandKind, Allocation, Block, Edit, Function, Inst, InstPosition, OperandConstraint, OperandKind,
OperandPos, ProgPoint, RegClass, VReg, OperandPos, PReg, ProgPoint, RegClass, VReg,
}; };
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use std::fmt::Debug; use std::fmt::Debug;
@@ -53,7 +54,7 @@ impl<'a, F: Function> Env<'a, F> {
to_alloc: Allocation, to_alloc: Allocation,
to_vreg: Option<VReg>, to_vreg: Option<VReg>,
) { ) {
log::trace!( trace!(
"insert_move: pos {:?} prio {:?} from_alloc {:?} to_alloc {:?}", "insert_move: pos {:?} prio {:?} from_alloc {:?} to_alloc {:?}",
pos, pos,
prio, prio,
@@ -62,13 +63,15 @@ impl<'a, F: Function> Env<'a, F> {
); );
match (from_alloc.as_reg(), to_alloc.as_reg()) { match (from_alloc.as_reg(), to_alloc.as_reg()) {
(Some(from), Some(to)) => { (Some(from), Some(to)) => {
assert_eq!(from.class(), to.class()); debug_assert_eq!(from.class(), to.class());
} }
_ => {} _ => {}
} }
self.inserted_moves.push(InsertedMove { self.inserted_moves.push(InsertedMove {
pos, pos_prio: PosWithPrio {
prio, pos,
prio: prio as u32,
},
from_alloc, from_alloc,
to_alloc, to_alloc,
to_vreg, to_vreg,
@@ -86,16 +89,16 @@ impl<'a, F: Function> Env<'a, F> {
} }
pub fn get_alloc_for_range(&self, range: LiveRangeIndex) -> Allocation { pub fn get_alloc_for_range(&self, range: LiveRangeIndex) -> Allocation {
log::trace!("get_alloc_for_range: {:?}", range); trace!("get_alloc_for_range: {:?}", range);
let bundle = self.ranges[range.index()].bundle; let bundle = self.ranges[range.index()].bundle;
log::trace!(" -> bundle: {:?}", bundle); trace!(" -> bundle: {:?}", bundle);
let bundledata = &self.bundles[bundle.index()]; let bundledata = &self.bundles[bundle.index()];
log::trace!(" -> allocation {:?}", bundledata.allocation); trace!(" -> allocation {:?}", bundledata.allocation);
if bundledata.allocation != Allocation::none() { if bundledata.allocation != Allocation::none() {
bundledata.allocation bundledata.allocation
} else { } else {
log::trace!(" -> spillset {:?}", bundledata.spillset); trace!(" -> spillset {:?}", bundledata.spillset);
log::trace!( trace!(
" -> spill slot {:?}", " -> spill slot {:?}",
self.spillsets[bundledata.spillset.index()].slot self.spillsets[bundledata.spillset.index()].slot
); );
@@ -104,9 +107,9 @@ impl<'a, F: Function> Env<'a, F> {
} }
pub fn apply_allocations_and_insert_moves(&mut self) { pub fn apply_allocations_and_insert_moves(&mut self) {
log::trace!("apply_allocations_and_insert_moves"); trace!("apply_allocations_and_insert_moves");
log::trace!("blockparam_ins: {:?}", self.blockparam_ins); trace!("blockparam_ins: {:?}", self.blockparam_ins);
log::trace!("blockparam_outs: {:?}", self.blockparam_outs); trace!("blockparam_outs: {:?}", self.blockparam_outs);
// Now that all splits are done, we can pay the cost once to // Now that all splits are done, we can pay the cost once to
// sort VReg range lists and update with the final ranges. // sort VReg range lists and update with the final ranges.
@@ -148,9 +151,9 @@ impl<'a, F: Function> Env<'a, F> {
to_vreg: VRegIndex, to_vreg: VRegIndex,
kind: HalfMoveKind, kind: HalfMoveKind,
) -> u64 { ) -> u64 {
assert!(from_block.index() < 1 << 21); debug_assert!(from_block.index() < 1 << 21);
assert!(to_block.index() < 1 << 21); debug_assert!(to_block.index() < 1 << 21);
assert!(to_vreg.index() < 1 << 21); debug_assert!(to_vreg.index() < 1 << 21);
((from_block.index() as u64) << 43) ((from_block.index() as u64) << 43)
| ((to_block.index() as u64) << 22) | ((to_block.index() as u64) << 22)
| ((to_vreg.index() as u64) << 1) | ((to_vreg.index() as u64) << 1)
@@ -202,7 +205,7 @@ impl<'a, F: Function> Env<'a, F> {
.map(|preg| Allocation::reg(preg)) .map(|preg| Allocation::reg(preg))
.unwrap_or_else(|| self.get_alloc_for_range(entry.index)); .unwrap_or_else(|| self.get_alloc_for_range(entry.index));
let range = entry.range; let range = entry.range;
log::trace!( trace!(
"apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?} (pinned {:?})", "apply_allocations: vreg {:?} LR {:?} with range {:?} has alloc {:?} (pinned {:?})",
vreg, vreg,
entry.index, entry.index,
@@ -269,7 +272,7 @@ impl<'a, F: Function> Env<'a, F> {
&& !self.is_start_of_block(range.from) && !self.is_start_of_block(range.from)
&& !first_is_def && !first_is_def
{ {
log::trace!( trace!(
"prev LR {} abuts LR {} in same block; moving {} -> {} for v{}", "prev LR {} abuts LR {} in same block; moving {} -> {} for v{}",
prev.index(), prev.index(),
entry.index.index(), entry.index.index(),
@@ -277,7 +280,7 @@ impl<'a, F: Function> Env<'a, F> {
alloc, alloc,
vreg.index() vreg.index()
); );
assert_eq!(range.from.pos(), InstPosition::Before); debug_assert_eq!(range.from.pos(), InstPosition::Before);
self.insert_move( self.insert_move(
range.from, range.from,
InsertMovePrio::Regular, InsertMovePrio::Regular,
@@ -303,9 +306,9 @@ impl<'a, F: Function> Env<'a, F> {
if range.to < self.cfginfo.block_exit[block.index()].next() { if range.to < self.cfginfo.block_exit[block.index()].next() {
break; break;
} }
log::trace!("examining block with end in range: block{}", block.index()); trace!("examining block with end in range: block{}", block.index());
for &succ in self.func.block_succs(block) { for &succ in self.func.block_succs(block) {
log::trace!( trace!(
" -> has succ block {} with entry {:?}", " -> has succ block {} with entry {:?}",
succ.index(), succ.index(),
self.cfginfo.block_entry[succ.index()] self.cfginfo.block_entry[succ.index()]
@@ -313,9 +316,9 @@ impl<'a, F: Function> Env<'a, F> {
if range.contains_point(self.cfginfo.block_entry[succ.index()]) { if range.contains_point(self.cfginfo.block_entry[succ.index()]) {
continue; continue;
} }
log::trace!(" -> out of this range, requires half-move if live"); trace!(" -> out of this range, requires half-move if live");
if self.is_live_in(succ, vreg) { if self.is_live_in(succ, vreg) {
log::trace!(" -> live at input to succ, adding halfmove"); trace!(" -> live at input to succ, adding halfmove");
half_moves.push(HalfMove { half_moves.push(HalfMove {
key: half_move_key(block, succ, vreg, HalfMoveKind::Source), key: half_move_key(block, succ, vreg, HalfMoveKind::Source),
alloc, alloc,
@@ -326,20 +329,24 @@ impl<'a, F: Function> Env<'a, F> {
// Scan forward in `blockparam_outs`, adding all // Scan forward in `blockparam_outs`, adding all
// half-moves for outgoing values to blockparams // half-moves for outgoing values to blockparams
// in succs. // in succs.
log::trace!( trace!(
"scanning blockparam_outs for v{} block{}: blockparam_out_idx = {}", "scanning blockparam_outs for v{} block{}: blockparam_out_idx = {}",
vreg.index(), vreg.index(),
block.index(), block.index(),
blockparam_out_idx, blockparam_out_idx,
); );
while blockparam_out_idx < self.blockparam_outs.len() { while blockparam_out_idx < self.blockparam_outs.len() {
let (from_vreg, from_block, to_block, to_vreg) = let BlockparamOut {
self.blockparam_outs[blockparam_out_idx]; from_vreg,
from_block,
to_block,
to_vreg,
} = self.blockparam_outs[blockparam_out_idx];
if (from_vreg, from_block) > (vreg, block) { if (from_vreg, from_block) > (vreg, block) {
break; break;
} }
if (from_vreg, from_block) == (vreg, block) { if (from_vreg, from_block) == (vreg, block) {
log::trace!( trace!(
" -> found: from v{} block{} to v{} block{}", " -> found: from v{} block{} to v{} block{}",
from_vreg.index(), from_vreg.index(),
from_block.index(), from_block.index(),
@@ -391,15 +398,18 @@ impl<'a, F: Function> Env<'a, F> {
} }
// Add half-moves for blockparam inputs. // Add half-moves for blockparam inputs.
log::trace!( trace!(
"scanning blockparam_ins at vreg {} block {}: blockparam_in_idx = {}", "scanning blockparam_ins at vreg {} block {}: blockparam_in_idx = {}",
vreg.index(), vreg.index(),
block.index(), block.index(),
blockparam_in_idx blockparam_in_idx
); );
while blockparam_in_idx < self.blockparam_ins.len() { while blockparam_in_idx < self.blockparam_ins.len() {
let (to_vreg, to_block, from_block) = let BlockparamIn {
self.blockparam_ins[blockparam_in_idx]; from_block,
to_block,
to_vreg,
} = self.blockparam_ins[blockparam_in_idx];
if (to_vreg, to_block) > (vreg, block) { if (to_vreg, to_block) > (vreg, block) {
break; break;
} }
@@ -413,7 +423,7 @@ impl<'a, F: Function> Env<'a, F> {
), ),
alloc, alloc,
}); });
log::trace!( trace!(
"match: blockparam_in: v{} in block{} from block{} into {}", "match: blockparam_in: v{} in block{} from block{} into {}",
to_vreg.index(), to_vreg.index(),
to_block.index(), to_block.index(),
@@ -444,7 +454,7 @@ impl<'a, F: Function> Env<'a, F> {
continue; continue;
} }
log::trace!( trace!(
"scanning preds at vreg {} block {} for ends outside the range", "scanning preds at vreg {} block {} for ends outside the range",
vreg.index(), vreg.index(),
block.index() block.index()
@@ -453,7 +463,7 @@ impl<'a, F: Function> Env<'a, F> {
// Now find any preds whose ends are not in the // Now find any preds whose ends are not in the
// same range, and insert appropriate moves. // same range, and insert appropriate moves.
for &pred in self.func.block_preds(block) { for &pred in self.func.block_preds(block) {
log::trace!( trace!(
"pred block {} has exit {:?}", "pred block {} has exit {:?}",
pred.index(), pred.index(),
self.cfginfo.block_exit[pred.index()] self.cfginfo.block_exit[pred.index()]
@@ -461,7 +471,7 @@ impl<'a, F: Function> Env<'a, F> {
if range.contains_point(self.cfginfo.block_exit[pred.index()]) { if range.contains_point(self.cfginfo.block_exit[pred.index()]) {
continue; continue;
} }
log::trace!(" -> requires half-move"); trace!(" -> requires half-move");
half_moves.push(HalfMove { half_moves.push(HalfMove {
key: half_move_key(pred, block, vreg, HalfMoveKind::Dest), key: half_move_key(pred, block, vreg, HalfMoveKind::Dest),
alloc, alloc,
@@ -471,26 +481,30 @@ impl<'a, F: Function> Env<'a, F> {
block = block.next(); block = block.next();
} }
// If this is a blockparam vreg and the start of block #[cfg(feature = "checker")]
// is in this range, add to blockparam_allocs.
let (blockparam_block, blockparam_idx) =
self.cfginfo.vreg_def_blockparam[vreg.index()];
if blockparam_block.is_valid()
&& range.contains_point(self.cfginfo.block_entry[blockparam_block.index()])
{ {
self.blockparam_allocs.push(( // If this is a blockparam vreg and the start of block
blockparam_block, // is in this range, add to blockparam_allocs.
blockparam_idx, let (blockparam_block, blockparam_idx) =
vreg, self.cfginfo.vreg_def_blockparam[vreg.index()];
alloc, if blockparam_block.is_valid()
)); && range
.contains_point(self.cfginfo.block_entry[blockparam_block.index()])
{
self.blockparam_allocs.push((
blockparam_block,
blockparam_idx,
vreg,
alloc,
));
}
} }
} }
// Scan over def/uses and apply allocations. // Scan over def/uses and apply allocations.
for use_idx in 0..self.ranges[entry.index.index()].uses.len() { for use_idx in 0..self.ranges[entry.index.index()].uses.len() {
let usedata = self.ranges[entry.index.index()].uses[use_idx]; let usedata = self.ranges[entry.index.index()].uses[use_idx];
log::trace!("applying to use: {:?}", usedata); trace!("applying to use: {:?}", usedata);
debug_assert!(range.contains_point(usedata.pos)); debug_assert!(range.contains_point(usedata.pos));
let inst = usedata.pos.inst(); let inst = usedata.pos.inst();
let slot = usedata.slot; let slot = usedata.slot;
@@ -519,7 +533,7 @@ impl<'a, F: Function> Env<'a, F> {
// covers only prev inst's After; so includes move // covers only prev inst's After; so includes move
// srcs to (exclusive) inst. // srcs to (exclusive) inst.
let move_src_end = (vreg, range.to.inst()); let move_src_end = (vreg, range.to.inst());
log::trace!( trace!(
"vreg {:?} range {:?}: looking for program-move sources from {:?} to {:?}", "vreg {:?} range {:?}: looking for program-move sources from {:?} to {:?}",
vreg, vreg,
range, range,
@@ -529,13 +543,13 @@ impl<'a, F: Function> Env<'a, F> {
while prog_move_src_idx < self.prog_move_srcs.len() while prog_move_src_idx < self.prog_move_srcs.len()
&& self.prog_move_srcs[prog_move_src_idx].0 < move_src_start && self.prog_move_srcs[prog_move_src_idx].0 < move_src_start
{ {
log::trace!(" -> skipping idx {}", prog_move_src_idx); trace!(" -> skipping idx {}", prog_move_src_idx);
prog_move_src_idx += 1; prog_move_src_idx += 1;
} }
while prog_move_src_idx < self.prog_move_srcs.len() while prog_move_src_idx < self.prog_move_srcs.len()
&& self.prog_move_srcs[prog_move_src_idx].0 < move_src_end && self.prog_move_srcs[prog_move_src_idx].0 < move_src_end
{ {
log::trace!( trace!(
" -> setting idx {} ({:?}) to alloc {:?}", " -> setting idx {} ({:?}) to alloc {:?}",
prog_move_src_idx, prog_move_src_idx,
self.prog_move_srcs[prog_move_src_idx].0, self.prog_move_srcs[prog_move_src_idx].0,
@@ -562,7 +576,7 @@ impl<'a, F: Function> Env<'a, F> {
} else { } else {
(vreg, range.to.inst().next()) (vreg, range.to.inst().next())
}; };
log::trace!( trace!(
"vreg {:?} range {:?}: looking for program-move dests from {:?} to {:?}", "vreg {:?} range {:?}: looking for program-move dests from {:?} to {:?}",
vreg, vreg,
range, range,
@@ -572,13 +586,13 @@ impl<'a, F: Function> Env<'a, F> {
while prog_move_dst_idx < self.prog_move_dsts.len() while prog_move_dst_idx < self.prog_move_dsts.len()
&& self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_start && self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_start
{ {
log::trace!(" -> skipping idx {}", prog_move_dst_idx); trace!(" -> skipping idx {}", prog_move_dst_idx);
prog_move_dst_idx += 1; prog_move_dst_idx += 1;
} }
while prog_move_dst_idx < self.prog_move_dsts.len() while prog_move_dst_idx < self.prog_move_dsts.len()
&& self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_end && self.prog_move_dsts[prog_move_dst_idx].0 < move_dst_end
{ {
log::trace!( trace!(
" -> setting idx {} ({:?}) to alloc {:?}", " -> setting idx {} ({:?}) to alloc {:?}",
prog_move_dst_idx, prog_move_dst_idx,
self.prog_move_dsts[prog_move_dst_idx].0, self.prog_move_dsts[prog_move_dst_idx].0,
@@ -596,7 +610,7 @@ impl<'a, F: Function> Env<'a, F> {
// from-vreg) tuple, find the from-alloc and all the // from-vreg) tuple, find the from-alloc and all the
// to-allocs, and insert moves on the block edge. // to-allocs, and insert moves on the block edge.
half_moves.sort_unstable_by_key(|h| h.key); half_moves.sort_unstable_by_key(|h| h.key);
log::trace!("halfmoves: {:?}", half_moves); trace!("halfmoves: {:?}", half_moves);
self.stats.halfmoves_count = half_moves.len(); self.stats.halfmoves_count = half_moves.len();
let mut i = 0; let mut i = 0;
@@ -619,7 +633,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
let last_dest = i; let last_dest = i;
log::trace!( trace!(
"halfmove match: src {:?} dests {:?}", "halfmove match: src {:?} dests {:?}",
src, src,
&half_moves[first_dest..last_dest] &half_moves[first_dest..last_dest]
@@ -697,8 +711,8 @@ impl<'a, F: Function> Env<'a, F> {
// Handle multi-fixed-reg constraints by copying. // Handle multi-fixed-reg constraints by copying.
for fixup in std::mem::replace(&mut self.multi_fixed_reg_fixups, vec![]) { for fixup in std::mem::replace(&mut self.multi_fixed_reg_fixups, vec![]) {
let from_alloc = self.get_alloc(fixup.pos.inst(), fixup.from_slot as usize); let from_alloc = self.get_alloc(fixup.pos.inst(), fixup.from_slot as usize);
let to_alloc = Allocation::reg(self.pregs[fixup.to_preg.index()].reg); let to_alloc = Allocation::reg(PReg::from_index(fixup.to_preg.index()));
log::trace!( trace!(
"multi-fixed-move constraint at {:?} from {} to {} for v{}", "multi-fixed-move constraint at {:?} from {} to {} for v{}",
fixup.pos, fixup.pos,
from_alloc, from_alloc,
@@ -715,7 +729,7 @@ impl<'a, F: Function> Env<'a, F> {
self.set_alloc( self.set_alloc(
fixup.pos.inst(), fixup.pos.inst(),
fixup.to_slot as usize, fixup.to_slot as usize,
Allocation::reg(self.pregs[fixup.to_preg.index()].reg), Allocation::reg(PReg::from_index(fixup.to_preg.index())),
); );
} }
@@ -773,7 +787,7 @@ impl<'a, F: Function> Env<'a, F> {
input_reused.push(input_idx); input_reused.push(input_idx);
let input_alloc = self.get_alloc(inst, input_idx); let input_alloc = self.get_alloc(inst, input_idx);
let output_alloc = self.get_alloc(inst, output_idx); let output_alloc = self.get_alloc(inst, output_idx);
log::trace!( trace!(
"reuse-input inst {:?}: output {} has alloc {:?}, input {} has alloc {:?}", "reuse-input inst {:?}: output {} has alloc {:?}, input {} has alloc {:?}",
inst, inst,
output_idx, output_idx,
@@ -816,20 +830,20 @@ impl<'a, F: Function> Env<'a, F> {
.sort_unstable_by_key(|((_, inst), _)| inst.prev()); .sort_unstable_by_key(|((_, inst), _)| inst.prev());
let prog_move_srcs = std::mem::replace(&mut self.prog_move_srcs, vec![]); let prog_move_srcs = std::mem::replace(&mut self.prog_move_srcs, vec![]);
let prog_move_dsts = std::mem::replace(&mut self.prog_move_dsts, vec![]); let prog_move_dsts = std::mem::replace(&mut self.prog_move_dsts, vec![]);
assert_eq!(prog_move_srcs.len(), prog_move_dsts.len()); debug_assert_eq!(prog_move_srcs.len(), prog_move_dsts.len());
for (&((_, from_inst), from_alloc), &((to_vreg, to_inst), to_alloc)) in for (&((_, from_inst), from_alloc), &((to_vreg, to_inst), to_alloc)) in
prog_move_srcs.iter().zip(prog_move_dsts.iter()) prog_move_srcs.iter().zip(prog_move_dsts.iter())
{ {
log::trace!( trace!(
"program move at inst {:?}: alloc {:?} -> {:?} (v{})", "program move at inst {:?}: alloc {:?} -> {:?} (v{})",
from_inst, from_inst,
from_alloc, from_alloc,
to_alloc, to_alloc,
to_vreg.index(), to_vreg.index(),
); );
assert!(from_alloc.is_some()); debug_assert!(from_alloc.is_some());
assert!(to_alloc.is_some()); debug_assert!(to_alloc.is_some());
assert_eq!(from_inst, to_inst.prev()); debug_assert_eq!(from_inst, to_inst.prev());
// N.B.: these moves happen with the *same* priority as // N.B.: these moves happen with the *same* priority as
// LR-to-LR moves, because they work just like them: they // LR-to-LR moves, because they work just like them: they
// connect a use at one progpoint (move-After) with a def // connect a use at one progpoint (move-After) with a def
@@ -850,7 +864,7 @@ impl<'a, F: Function> Env<'a, F> {
// resolve (see cases below). // resolve (see cases below).
let mut i = 0; let mut i = 0;
self.inserted_moves self.inserted_moves
.sort_unstable_by_key(|m| (m.pos.to_index(), m.prio)); .sort_unstable_by_key(|m| m.pos_prio.key());
// Redundant-move elimination state tracker. // Redundant-move elimination state tracker.
let mut redundant_moves = RedundantMoveEliminator::default(); let mut redundant_moves = RedundantMoveEliminator::default();
@@ -907,18 +921,14 @@ impl<'a, F: Function> Env<'a, F> {
while i < self.inserted_moves.len() { while i < self.inserted_moves.len() {
let start = i; let start = i;
let pos = self.inserted_moves[i].pos; let pos_prio = self.inserted_moves[i].pos_prio;
let prio = self.inserted_moves[i].prio; while i < self.inserted_moves.len() && self.inserted_moves[i].pos_prio == pos_prio {
while i < self.inserted_moves.len()
&& self.inserted_moves[i].pos == pos
&& self.inserted_moves[i].prio == prio
{
i += 1; i += 1;
} }
let moves = &self.inserted_moves[start..i]; let moves = &self.inserted_moves[start..i];
redundant_move_process_side_effects(self, &mut redundant_moves, last_pos, pos); redundant_move_process_side_effects(self, &mut redundant_moves, last_pos, pos_prio.pos);
last_pos = pos; last_pos = pos_prio.pos;
// Gather all the moves with Int class and Float class // Gather all the moves with Int class and Float class
// separately. These cannot interact, so it is safe to // separately. These cannot interact, so it is safe to
@@ -929,13 +939,15 @@ impl<'a, F: Function> Env<'a, F> {
// regs, but this seems simpler.) // regs, but this seems simpler.)
let mut int_moves: SmallVec<[InsertedMove; 8]> = smallvec![]; let mut int_moves: SmallVec<[InsertedMove; 8]> = smallvec![];
let mut float_moves: SmallVec<[InsertedMove; 8]> = smallvec![]; let mut float_moves: SmallVec<[InsertedMove; 8]> = smallvec![];
#[cfg(feature = "checker")]
let mut self_moves: SmallVec<[InsertedMove; 8]> = smallvec![]; let mut self_moves: SmallVec<[InsertedMove; 8]> = smallvec![];
for m in moves { for m in moves {
if m.from_alloc.is_reg() && m.to_alloc.is_reg() { if m.from_alloc.is_reg() && m.to_alloc.is_reg() {
assert_eq!(m.from_alloc.class(), m.to_alloc.class()); debug_assert_eq!(m.from_alloc.class(), m.to_alloc.class());
} }
if m.from_alloc == m.to_alloc { if m.from_alloc == m.to_alloc {
#[cfg(feature = "checker")]
if m.to_vreg.is_some() { if m.to_vreg.is_some() {
self_moves.push(m.clone()); self_moves.push(m.clone());
} }
@@ -959,10 +971,14 @@ impl<'a, F: Function> Env<'a, F> {
// that can be done one at a time. // that can be done one at a time.
let scratch = self.env.scratch_by_class[regclass as u8 as usize]; let scratch = self.env.scratch_by_class[regclass as u8 as usize];
let mut parallel_moves = ParallelMoves::new(Allocation::reg(scratch)); let mut parallel_moves = ParallelMoves::new(Allocation::reg(scratch));
log::trace!("parallel moves at pos {:?} prio {:?}", pos, prio); trace!(
"parallel moves at pos {:?} prio {:?}",
pos_prio.pos,
pos_prio.prio
);
for m in moves { for m in moves {
if (m.from_alloc != m.to_alloc) || m.to_vreg.is_some() { if (m.from_alloc != m.to_alloc) || m.to_vreg.is_some() {
log::trace!(" {} -> {}", m.from_alloc, m.to_alloc,); trace!(" {} -> {}", m.from_alloc, m.to_alloc,);
parallel_moves.add(m.from_alloc, m.to_alloc, m.to_vreg); parallel_moves.add(m.from_alloc, m.to_alloc, m.to_vreg);
} }
} }
@@ -993,7 +1009,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut scratch_used_yet = false; let mut scratch_used_yet = false;
for (src, dst, to_vreg) in resolved { for (src, dst, to_vreg) in resolved {
log::trace!(" resolved: {} -> {} ({:?})", src, dst, to_vreg); trace!(" resolved: {} -> {} ({:?})", src, dst, to_vreg);
let action = redundant_moves.process_move(src, dst, to_vreg); let action = redundant_moves.process_move(src, dst, to_vreg);
if !action.elide { if !action.elide {
if dst == Allocation::reg(scratch) { if dst == Allocation::reg(scratch) {
@@ -1001,135 +1017,117 @@ impl<'a, F: Function> Env<'a, F> {
} }
if self.allocation_is_stack(src) && self.allocation_is_stack(dst) { if self.allocation_is_stack(src) && self.allocation_is_stack(dst) {
if !scratch_used_yet { if !scratch_used_yet {
self.add_edit( self.add_move_edit(
pos, pos_prio,
prio, src,
Edit::Move { Allocation::reg(scratch),
from: src, to_vreg,
to: Allocation::reg(scratch),
to_vreg,
},
); );
self.add_edit( self.add_move_edit(
pos, pos_prio,
prio, Allocation::reg(scratch),
Edit::Move { dst,
from: Allocation::reg(scratch), to_vreg,
to: dst,
to_vreg,
},
); );
} else { } else {
assert!(extra_slot.is_some()); debug_assert!(extra_slot.is_some());
self.add_edit( self.add_move_edit(
pos, pos_prio,
prio, Allocation::reg(scratch),
Edit::Move { extra_slot.unwrap(),
from: Allocation::reg(scratch), None,
to: extra_slot.unwrap(),
to_vreg: None,
},
); );
self.add_edit( self.add_move_edit(
pos, pos_prio,
prio, src,
Edit::Move { Allocation::reg(scratch),
from: src, to_vreg,
to: Allocation::reg(scratch),
to_vreg,
},
); );
self.add_edit( self.add_move_edit(
pos, pos_prio,
prio, Allocation::reg(scratch),
Edit::Move { dst,
from: Allocation::reg(scratch), to_vreg,
to: dst,
to_vreg,
},
); );
self.add_edit( self.add_move_edit(
pos, pos_prio,
prio, extra_slot.unwrap(),
Edit::Move { Allocation::reg(scratch),
from: extra_slot.unwrap(), None,
to: Allocation::reg(scratch),
to_vreg: None,
},
); );
} }
} else { } else {
self.add_edit( self.add_move_edit(pos_prio, src, dst, to_vreg);
pos,
prio,
Edit::Move {
from: src,
to: dst,
to_vreg,
},
);
} }
} else { } else {
log::trace!(" -> redundant move elided"); trace!(" -> redundant move elided");
} }
#[cfg(feature = "checker")]
if let Some((alloc, vreg)) = action.def_alloc { if let Some((alloc, vreg)) = action.def_alloc {
log::trace!( trace!(
" -> converted to DefAlloc: alloc {} vreg {}", " -> converted to DefAlloc: alloc {} vreg {}",
alloc, alloc,
vreg vreg
); );
self.add_edit(pos, prio, Edit::DefAlloc { alloc, vreg }); self.edits.push((pos_prio, Edit::DefAlloc { alloc, vreg }));
} }
} }
} }
#[cfg(feature = "checker")]
for m in &self_moves { for m in &self_moves {
log::trace!( trace!(
"self move at pos {:?} prio {:?}: {} -> {} to_vreg {:?}", "self move at pos {:?} prio {:?}: {} -> {} to_vreg {:?}",
pos, pos_prio.pos,
prio, pos_prio.prio,
m.from_alloc, m.from_alloc,
m.to_alloc, m.to_alloc,
m.to_vreg m.to_vreg
); );
let action = redundant_moves.process_move(m.from_alloc, m.to_alloc, m.to_vreg); let action = redundant_moves.process_move(m.from_alloc, m.to_alloc, m.to_vreg);
assert!(action.elide); debug_assert!(action.elide);
if let Some((alloc, vreg)) = action.def_alloc { if let Some((alloc, vreg)) = action.def_alloc {
log::trace!(" -> DefAlloc: alloc {} vreg {}", alloc, vreg); trace!(" -> DefAlloc: alloc {} vreg {}", alloc, vreg);
self.add_edit(pos, prio, Edit::DefAlloc { alloc, vreg }); self.edits.push((pos_prio, Edit::DefAlloc { alloc, vreg }));
} }
} }
} }
// Add edits to describe blockparam locations too. This is #[cfg(feature = "checker")]
// required by the checker. This comes after any edge-moves. {
self.blockparam_allocs // Add edits to describe blockparam locations too. This is
.sort_unstable_by_key(|&(block, idx, _, _)| (block, idx)); // required by the checker. This comes after any edge-moves.
self.stats.blockparam_allocs_count = self.blockparam_allocs.len(); use crate::ion::data_structures::u64_key;
let mut i = 0; self.blockparam_allocs
while i < self.blockparam_allocs.len() { .sort_unstable_by_key(|&(block, idx, _, _)| u64_key(block.raw_u32(), idx));
let start = i; self.stats.blockparam_allocs_count = self.blockparam_allocs.len();
let block = self.blockparam_allocs[i].0; let mut i = 0;
while i < self.blockparam_allocs.len() && self.blockparam_allocs[i].0 == block { while i < self.blockparam_allocs.len() {
i += 1; let start = i;
} let block = self.blockparam_allocs[i].0;
let params = &self.blockparam_allocs[start..i]; while i < self.blockparam_allocs.len() && self.blockparam_allocs[i].0 == block {
let vregs = params i += 1;
.iter() }
.map(|(_, _, vreg_idx, _)| self.vreg_regs[vreg_idx.index()]) let params = &self.blockparam_allocs[start..i];
.collect::<Vec<_>>(); let vregs = params
let allocs = params .iter()
.iter() .map(|(_, _, vreg_idx, _)| self.vreg_regs[vreg_idx.index()])
.map(|(_, _, _, alloc)| *alloc) .collect::<Vec<_>>();
.collect::<Vec<_>>(); let allocs = params
assert_eq!(vregs.len(), self.func.block_params(block).len()); .iter()
assert_eq!(allocs.len(), self.func.block_params(block).len()); .map(|(_, _, _, alloc)| *alloc)
for (vreg, alloc) in vregs.into_iter().zip(allocs.into_iter()) { .collect::<Vec<_>>();
self.add_edit( debug_assert_eq!(vregs.len(), self.func.block_params(block).len());
self.cfginfo.block_entry[block.index()], debug_assert_eq!(allocs.len(), self.func.block_params(block).len());
InsertMovePrio::BlockParam, for (vreg, alloc) in vregs.into_iter().zip(allocs.into_iter()) {
Edit::DefAlloc { alloc, vreg }, self.edits.push((
); PosWithPrio {
pos: self.cfginfo.block_entry[block.index()],
prio: InsertMovePrio::BlockParam as u32,
},
Edit::DefAlloc { alloc, vreg },
));
}
} }
} }
@@ -1137,38 +1135,49 @@ impl<'a, F: Function> Env<'a, F> {
// be a stable sort! We have to keep the order produced by the // be a stable sort! We have to keep the order produced by the
// parallel-move resolver for all moves within a single sort // parallel-move resolver for all moves within a single sort
// key. // key.
self.edits.sort_by_key(|&(pos, prio, _)| (pos, prio)); self.edits.sort_by_key(|&(pos_prio, _)| pos_prio.key());
self.stats.edits_count = self.edits.len(); self.stats.edits_count = self.edits.len();
// Add debug annotations. // Add debug annotations.
if self.annotations_enabled { if self.annotations_enabled {
for i in 0..self.edits.len() { for i in 0..self.edits.len() {
let &(pos, _, ref edit) = &self.edits[i]; let &(pos_prio, ref edit) = &self.edits[i];
match edit { match edit {
&Edit::Move { from, to, to_vreg } => { &Edit::Move { from, to } => {
self.annotate( self.annotate(pos_prio.pos, format!("move {} -> {})", from, to));
ProgPoint::from_index(pos),
format!("move {} -> {} ({:?})", from, to, to_vreg),
);
} }
&Edit::DefAlloc { alloc, vreg } => { &Edit::DefAlloc { alloc, vreg } => {
let s = format!("defalloc {:?} := {:?}", alloc, vreg); let s = format!("defalloc {:?} := {:?}", alloc, vreg);
self.annotate(ProgPoint::from_index(pos), s); self.annotate(pos_prio.pos, s);
} }
} }
} }
} }
} }
pub fn add_edit(&mut self, pos: ProgPoint, prio: InsertMovePrio, edit: Edit) { pub fn add_move_edit(
match &edit { &mut self,
&Edit::Move { from, to, to_vreg } if from == to && to_vreg.is_none() => return, pos_prio: PosWithPrio,
&Edit::Move { from, to, .. } if from.is_reg() && to.is_reg() => { from: Allocation,
assert_eq!(from.as_reg().unwrap().class(), to.as_reg().unwrap().class()); to: Allocation,
_to_vreg: Option<VReg>,
) {
if from != to {
if from.is_reg() && to.is_reg() {
debug_assert_eq!(from.as_reg().unwrap().class(), to.as_reg().unwrap().class());
} }
_ => {} self.edits.push((pos_prio, Edit::Move { from, to }));
} }
self.edits.push((pos.to_index(), prio, edit)); #[cfg(feature = "checker")]
if let Some(to_vreg) = _to_vreg {
self.edits.push((
pos_prio,
Edit::DefAlloc {
alloc: to,
vreg: to_vreg,
},
));
}
} }
} }

144
src/ion/process.rs
View File

@@ -62,7 +62,7 @@ impl<'a, F: Function> Env<'a, F> {
// `AllocRegResult::ConflictHighCost`. // `AllocRegResult::ConflictHighCost`.
max_allowable_cost: Option<u32>, max_allowable_cost: Option<u32>,
) -> AllocRegResult { ) -> AllocRegResult {
log::trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg); trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg);
let mut conflicts = smallvec![]; let mut conflicts = smallvec![];
let mut conflict_set = FxHashSet::default(); let mut conflict_set = FxHashSet::default();
let mut max_conflict_weight = 0; let mut max_conflict_weight = 0;
@@ -89,7 +89,7 @@ impl<'a, F: Function> Env<'a, F> {
.btree .btree
.range(from_key..) .range(from_key..)
.peekable(); .peekable();
log::trace!( trace!(
"alloc map for {:?} in range {:?}..: {:?}", "alloc map for {:?} in range {:?}..: {:?}",
reg, reg,
from_key, from_key,
@@ -98,19 +98,19 @@ impl<'a, F: Function> Env<'a, F> {
let mut first_conflict: Option<ProgPoint> = None; let mut first_conflict: Option<ProgPoint> = None;
'ranges: for entry in bundle_ranges { 'ranges: for entry in bundle_ranges {
log::trace!(" -> range LR {:?}: {:?}", entry.index, entry.range); trace!(" -> range LR {:?}: {:?}", entry.index, entry.range);
let key = LiveRangeKey::from_range(&entry.range); let key = LiveRangeKey::from_range(&entry.range);
let mut skips = 0; let mut skips = 0;
'alloc: loop { 'alloc: loop {
log::trace!(" -> PReg range {:?}", preg_range_iter.peek()); trace!(" -> PReg range {:?}", preg_range_iter.peek());
// Advance our BTree traversal until it is >= this bundle // Advance our BTree traversal until it is >= this bundle
// range (i.e., skip PReg allocations in the BTree that // range (i.e., skip PReg allocations in the BTree that
// are completely before this bundle range). // are completely before this bundle range).
if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key { if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key {
log::trace!( trace!(
"Skipping PReg range {:?}", "Skipping PReg range {:?}",
preg_range_iter.peek().unwrap().0 preg_range_iter.peek().unwrap().0
); );
@@ -135,13 +135,13 @@ impl<'a, F: Function> Env<'a, F> {
// If there are no more PReg allocations, we're done! // If there are no more PReg allocations, we're done!
if preg_range_iter.peek().is_none() { if preg_range_iter.peek().is_none() {
log::trace!(" -> no more PReg allocations; so no conflict possible!"); trace!(" -> no more PReg allocations; so no conflict possible!");
break 'ranges; break 'ranges;
} }
// If the current PReg range is beyond this range, there is no conflict; continue. // If the current PReg range is beyond this range, there is no conflict; continue.
if *preg_range_iter.peek().unwrap().0 > key { if *preg_range_iter.peek().unwrap().0 > key {
log::trace!( trace!(
" -> next PReg allocation is at {:?}; moving to next VReg range", " -> next PReg allocation is at {:?}; moving to next VReg range",
preg_range_iter.peek().unwrap().0 preg_range_iter.peek().unwrap().0
); );
@@ -150,16 +150,16 @@ impl<'a, F: Function> Env<'a, F> {
// Otherwise, there is a conflict. // Otherwise, there is a conflict.
let preg_key = *preg_range_iter.peek().unwrap().0; let preg_key = *preg_range_iter.peek().unwrap().0;
assert_eq!(preg_key, key); // Assert that this range overlaps. debug_assert_eq!(preg_key, key); // Assert that this range overlaps.
let preg_range = preg_range_iter.next().unwrap().1; let preg_range = preg_range_iter.next().unwrap().1;
log::trace!(" -> btree contains range {:?} that overlaps", preg_range); trace!(" -> btree contains range {:?} that overlaps", preg_range);
if preg_range.is_valid() { if preg_range.is_valid() {
log::trace!(" -> from vreg {:?}", self.ranges[preg_range.index()].vreg); trace!(" -> from vreg {:?}", self.ranges[preg_range.index()].vreg);
// range from an allocated bundle: find the bundle and add to // range from an allocated bundle: find the bundle and add to
// conflicts list. // conflicts list.
let conflict_bundle = self.ranges[preg_range.index()].bundle; let conflict_bundle = self.ranges[preg_range.index()].bundle;
log::trace!(" -> conflict bundle {:?}", conflict_bundle); trace!(" -> conflict bundle {:?}", conflict_bundle);
if !conflict_set.contains(&conflict_bundle) { if !conflict_set.contains(&conflict_bundle) {
conflicts.push(conflict_bundle); conflicts.push(conflict_bundle);
conflict_set.insert(conflict_bundle); conflict_set.insert(conflict_bundle);
@@ -170,7 +170,7 @@ impl<'a, F: Function> Env<'a, F> {
if max_allowable_cost.is_some() if max_allowable_cost.is_some()
&& max_conflict_weight > max_allowable_cost.unwrap() && max_conflict_weight > max_allowable_cost.unwrap()
{ {
log::trace!(" -> reached high cost, retrying early"); trace!(" -> reached high cost, retrying early");
return AllocRegResult::ConflictHighCost; return AllocRegResult::ConflictHighCost;
} }
} }
@@ -182,7 +182,7 @@ impl<'a, F: Function> Env<'a, F> {
))); )));
} }
} else { } else {
log::trace!(" -> conflict with fixed reservation"); trace!(" -> conflict with fixed reservation");
// range from a direct use of the PReg (due to clobber). // range from a direct use of the PReg (due to clobber).
return AllocRegResult::ConflictWithFixed( return AllocRegResult::ConflictWithFixed(
max_conflict_weight, max_conflict_weight,
@@ -197,8 +197,8 @@ impl<'a, F: Function> Env<'a, F> {
} }
// We can allocate! Add our ranges to the preg's BTree. // We can allocate! Add our ranges to the preg's BTree.
let preg = self.pregs[reg.index()].reg; let preg = PReg::from_index(reg.index());
log::trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg); trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg);
self.bundles[bundle.index()].allocation = Allocation::reg(preg); self.bundles[bundle.index()].allocation = Allocation::reg(preg);
for entry in &self.bundles[bundle.index()].ranges { for entry in &self.bundles[bundle.index()].ranges {
self.pregs[reg.index()] self.pregs[reg.index()]
@@ -211,7 +211,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) { pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) {
log::trace!( trace!(
"evicting bundle {:?}: alloc {:?}", "evicting bundle {:?}: alloc {:?}",
bundle, bundle,
self.bundles[bundle.index()].allocation self.bundles[bundle.index()].allocation
@@ -219,7 +219,7 @@ impl<'a, F: Function> Env<'a, F> {
let preg = match self.bundles[bundle.index()].allocation.as_reg() { let preg = match self.bundles[bundle.index()].allocation.as_reg() {
Some(preg) => preg, Some(preg) => preg,
None => { None => {
log::trace!( trace!(
" -> has no allocation! {:?}", " -> has no allocation! {:?}",
self.bundles[bundle.index()].allocation self.bundles[bundle.index()].allocation
); );
@@ -229,14 +229,14 @@ impl<'a, F: Function> Env<'a, F> {
let preg_idx = PRegIndex::new(preg.index()); let preg_idx = PRegIndex::new(preg.index());
self.bundles[bundle.index()].allocation = Allocation::none(); self.bundles[bundle.index()].allocation = Allocation::none();
for entry in &self.bundles[bundle.index()].ranges { for entry in &self.bundles[bundle.index()].ranges {
log::trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx); trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx);
self.pregs[preg_idx.index()] self.pregs[preg_idx.index()]
.allocations .allocations
.btree .btree
.remove(&LiveRangeKey::from_range(&entry.range)); .remove(&LiveRangeKey::from_range(&entry.range));
} }
let prio = self.bundles[bundle.index()].prio; let prio = self.bundles[bundle.index()].prio;
log::trace!(" -> prio {}; back into queue", prio); trace!(" -> prio {}; back into queue", prio);
self.allocation_queue self.allocation_queue
.insert(bundle, prio as usize, PReg::invalid()); .insert(bundle, prio as usize, PReg::invalid());
} }
@@ -246,22 +246,22 @@ impl<'a, F: Function> Env<'a, F> {
} }
pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &LiveBundleVec) -> u32 { pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &LiveBundleVec) -> u32 {
log::trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles); trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles);
let m = bundles let m = bundles
.iter() .iter()
.map(|&b| { .map(|&b| {
let w = self.bundles[b.index()].cached_spill_weight(); let w = self.bundles[b.index()].cached_spill_weight();
log::trace!("bundle{}: {}", b.index(), w); trace!("bundle{}: {}", b.index(), w);
w w
}) })
.max() .max()
.unwrap_or(0); .unwrap_or(0);
log::trace!(" -> max: {}", m); trace!(" -> max: {}", m);
m m
} }
pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) { pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) {
log::trace!("recompute bundle properties: bundle {:?}", bundle); trace!("recompute bundle properties: bundle {:?}", bundle);
let minimal; let minimal;
let mut fixed = false; let mut fixed = false;
@@ -273,18 +273,18 @@ impl<'a, F: Function> Env<'a, F> {
self.bundles[bundle.index()].prio = self.compute_bundle_prio(bundle); self.bundles[bundle.index()].prio = self.compute_bundle_prio(bundle);
if first_range_data.vreg.is_invalid() { if first_range_data.vreg.is_invalid() {
log::trace!(" -> no vreg; minimal and fixed"); trace!(" -> no vreg; minimal and fixed");
minimal = true; minimal = true;
fixed = true; fixed = true;
} else { } else {
for u in &first_range_data.uses { for u in &first_range_data.uses {
log::trace!(" -> use: {:?}", u); trace!(" -> use: {:?}", u);
if let OperandConstraint::FixedReg(_) = u.operand.constraint() { if let OperandConstraint::FixedReg(_) = u.operand.constraint() {
log::trace!(" -> fixed use at {:?}: {:?}", u.pos, u.operand); trace!(" -> fixed use at {:?}: {:?}", u.pos, u.operand);
fixed = true; fixed = true;
} }
if let OperandConstraint::Stack = u.operand.constraint() { if let OperandConstraint::Stack = u.operand.constraint() {
log::trace!(" -> stack use at {:?}: {:?}", u.pos, u.operand); trace!(" -> stack use at {:?}: {:?}", u.pos, u.operand);
stack = true; stack = true;
} }
if stack && fixed { if stack && fixed {
@@ -295,7 +295,7 @@ impl<'a, F: Function> Env<'a, F> {
// that it could cover just one ProgPoint, // that it could cover just one ProgPoint,
// i.e. X.Before..X.After, or two ProgPoints, // i.e. X.Before..X.After, or two ProgPoints,
// i.e. X.Before..X+1.Before. // i.e. X.Before..X+1.Before.
log::trace!(" -> first range has range {:?}", first_range_data.range); trace!(" -> first range has range {:?}", first_range_data.range);
let bundle_start = self.bundles[bundle.index()] let bundle_start = self.bundles[bundle.index()]
.ranges .ranges
.first() .first()
@@ -304,22 +304,22 @@ impl<'a, F: Function> Env<'a, F> {
.from; .from;
let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to; let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to;
minimal = bundle_start.inst() == bundle_end.prev().inst(); minimal = bundle_start.inst() == bundle_end.prev().inst();
log::trace!(" -> minimal: {}", minimal); trace!(" -> minimal: {}", minimal);
} }
let spill_weight = if minimal { let spill_weight = if minimal {
if fixed { if fixed {
log::trace!(" -> fixed and minimal"); trace!(" -> fixed and minimal");
MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT
} else { } else {
log::trace!(" -> non-fixed and minimal"); trace!(" -> non-fixed and minimal");
MINIMAL_BUNDLE_SPILL_WEIGHT MINIMAL_BUNDLE_SPILL_WEIGHT
} }
} else { } else {
let mut total = SpillWeight::zero(); let mut total = SpillWeight::zero();
for entry in &self.bundles[bundle.index()].ranges { for entry in &self.bundles[bundle.index()].ranges {
let range_data = &self.ranges[entry.index.index()]; let range_data = &self.ranges[entry.index.index()];
log::trace!( trace!(
" -> uses spill weight: +{:?}", " -> uses spill weight: +{:?}",
range_data.uses_spill_weight() range_data.uses_spill_weight()
); );
@@ -328,7 +328,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.bundles[bundle.index()].prio > 0 { if self.bundles[bundle.index()].prio > 0 {
let final_weight = (total.to_f32() as u32) / self.bundles[bundle.index()].prio; let final_weight = (total.to_f32() as u32) / self.bundles[bundle.index()].prio;
log::trace!( trace!(
" -> dividing by prio {}; final weight {}", " -> dividing by prio {}; final weight {}",
self.bundles[bundle.index()].prio, self.bundles[bundle.index()].prio,
final_weight final_weight
@@ -356,7 +356,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut w = SpillWeight::zero(); let mut w = SpillWeight::zero();
for u in &rangedata.uses { for u in &rangedata.uses {
w = w + SpillWeight::from_bits(u.weight); w = w + SpillWeight::from_bits(u.weight);
log::trace!("range{}: use {:?}", range.index(), u); trace!("range{}: use {:?}", range.index(), u);
} }
rangedata.set_uses_spill_weight(w); rangedata.set_uses_spill_weight(w);
if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def { if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def {
@@ -397,7 +397,7 @@ impl<'a, F: Function> Env<'a, F> {
reg_hint: PReg, reg_hint: PReg,
) { ) {
self.stats.splits += 1; self.stats.splits += 1;
log::trace!( trace!(
"split bundle {:?} at {:?} and requeue with reg hint (for first part) {:?}", "split bundle {:?} at {:?} and requeue with reg hint (for first part) {:?}",
bundle, bundle,
split_at, split_at,
@@ -410,7 +410,7 @@ impl<'a, F: Function> Env<'a, F> {
let spillset = self.bundles[bundle.index()].spillset; let spillset = self.bundles[bundle.index()].spillset;
assert!(!self.bundles[bundle.index()].ranges.is_empty()); debug_assert!(!self.bundles[bundle.index()].ranges.is_empty());
// Split point *at* start is OK; this means we peel off // Split point *at* start is OK; this means we peel off
// exactly one use to create a minimal bundle. // exactly one use to create a minimal bundle.
let bundle_start = self.bundles[bundle.index()] let bundle_start = self.bundles[bundle.index()]
@@ -419,9 +419,9 @@ impl<'a, F: Function> Env<'a, F> {
.unwrap() .unwrap()
.range .range
.from; .from;
assert!(split_at >= bundle_start); debug_assert!(split_at >= bundle_start);
let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to; let bundle_end = self.bundles[bundle.index()].ranges.last().unwrap().range.to;
assert!(split_at < bundle_end); debug_assert!(split_at < bundle_end);
// Is the split point *at* the start? If so, peel off the // Is the split point *at* the start? If so, peel off the
// first use: set the split point just after it, or just // first use: set the split point just after it, or just
@@ -435,7 +435,7 @@ impl<'a, F: Function> Env<'a, F> {
break 'outer; break 'outer;
} }
} }
log::trace!(" -> first use loc is {:?}", first_use); trace!(" -> first use loc is {:?}", first_use);
split_at = match first_use { split_at = match first_use {
Some(pos) => { Some(pos) => {
if pos.inst() == bundle_start.inst() { if pos.inst() == bundle_start.inst() {
@@ -455,7 +455,7 @@ impl<'a, F: Function> Env<'a, F> {
.next(), .next(),
), ),
}; };
log::trace!( trace!(
"split point is at bundle start; advancing to {:?}", "split point is at bundle start; advancing to {:?}",
split_at split_at
); );
@@ -471,13 +471,13 @@ impl<'a, F: Function> Env<'a, F> {
} }
} }
assert!(split_at > bundle_start && split_at < bundle_end); debug_assert!(split_at > bundle_start && split_at < bundle_end);
// We need to find which LRs fall on each side of the split, // We need to find which LRs fall on each side of the split,
// which LR we need to split down the middle, then update the // which LR we need to split down the middle, then update the
// current bundle, create a new one, and (re)-queue both. // current bundle, create a new one, and (re)-queue both.
log::trace!(" -> LRs: {:?}", self.bundles[bundle.index()].ranges); trace!(" -> LRs: {:?}", self.bundles[bundle.index()].ranges);
let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle
let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle
@@ -492,11 +492,11 @@ impl<'a, F: Function> Env<'a, F> {
} }
} }
log::trace!( trace!(
" -> last LR in old bundle: LR {:?}", " -> last LR in old bundle: LR {:?}",
self.bundles[bundle.index()].ranges[last_lr_in_old_bundle_idx] self.bundles[bundle.index()].ranges[last_lr_in_old_bundle_idx]
); );
log::trace!( trace!(
" -> first LR in new bundle: LR {:?}", " -> first LR in new bundle: LR {:?}",
self.bundles[bundle.index()].ranges[first_lr_in_new_bundle_idx] self.bundles[bundle.index()].ranges[first_lr_in_new_bundle_idx]
); );
@@ -516,14 +516,14 @@ impl<'a, F: Function> Env<'a, F> {
// down the middle, replace it with a new LR and chop off the // down the middle, replace it with a new LR and chop off the
// end of the same LR in the original list. // end of the same LR in the original list.
if split_at > new_lr_list[0].range.from { if split_at > new_lr_list[0].range.from {
assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx); debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx);
let orig_lr = new_lr_list[0].index; let orig_lr = new_lr_list[0].index;
let new_lr = self.create_liverange(CodeRange { let new_lr = self.create_liverange(CodeRange {
from: split_at, from: split_at,
to: new_lr_list[0].range.to, to: new_lr_list[0].range.to,
}); });
self.ranges[new_lr.index()].vreg = self.ranges[orig_lr.index()].vreg; self.ranges[new_lr.index()].vreg = self.ranges[orig_lr.index()].vreg;
log::trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr); trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr);
let first_use = self.ranges[orig_lr.index()] let first_use = self.ranges[orig_lr.index()]
.uses .uses
.iter() .iter()
@@ -560,7 +560,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
let new_bundle = self.create_bundle(); let new_bundle = self.create_bundle();
log::trace!(" -> creating new bundle {:?}", new_bundle); trace!(" -> creating new bundle {:?}", new_bundle);
self.bundles[new_bundle.index()].spillset = spillset; self.bundles[new_bundle.index()].spillset = spillset;
for entry in &new_lr_list { for entry in &new_lr_list {
self.ranges[entry.index.index()].bundle = new_bundle; self.ranges[entry.index.index()].bundle = new_bundle;
@@ -582,7 +582,7 @@ impl<'a, F: Function> Env<'a, F> {
let spill = self let spill = self
.get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
.unwrap(); .unwrap();
log::trace!( trace!(
" -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
bundle, bundle,
entry.index, entry.index,
@@ -627,13 +627,13 @@ impl<'a, F: Function> Env<'a, F> {
range, range,
index: empty_lr, index: empty_lr,
}); });
log::trace!( trace!(
" -> bundle {:?} range {:?}: last use implies split point {:?}", " -> bundle {:?} range {:?}: last use implies split point {:?}",
bundle, bundle,
entry.index, entry.index,
split split
); );
log::trace!( trace!(
" -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}",
spill, spill,
empty_lr empty_lr
@@ -652,7 +652,7 @@ impl<'a, F: Function> Env<'a, F> {
let spill = self let spill = self
.get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true)
.unwrap(); .unwrap();
log::trace!( trace!(
" -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
new_bundle, new_bundle,
entry.index, entry.index,
@@ -697,13 +697,13 @@ impl<'a, F: Function> Env<'a, F> {
range, range,
index: empty_lr, index: empty_lr,
}); });
log::trace!( trace!(
" -> bundle {:?} range {:?}: first use implies split point {:?}", " -> bundle {:?} range {:?}: first use implies split point {:?}",
bundle, bundle,
entry.index, entry.index,
first_use, first_use,
); );
log::trace!( trace!(
" -> moving leading empty region to new spill bundle {:?} with new LR {:?}", " -> moving leading empty region to new spill bundle {:?} with new LR {:?}",
spill, spill,
empty_lr empty_lr
@@ -741,7 +741,7 @@ impl<'a, F: Function> Env<'a, F> {
if self.pregs[hint_reg.index()].is_stack { if self.pregs[hint_reg.index()].is_stack {
hint_reg = PReg::invalid(); hint_reg = PReg::invalid();
} }
log::trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,); trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,);
let req = match self.compute_requirement(bundle) { let req = match self.compute_requirement(bundle) {
Ok(req) => req, Ok(req) => req,
@@ -749,7 +749,7 @@ impl<'a, F: Function> Env<'a, F> {
// We have to split right away. We'll find a point to // We have to split right away. We'll find a point to
// split that would allow at least the first half of the // split that would allow at least the first half of the
// split to be conflict-free. // split to be conflict-free.
assert!( debug_assert!(
!self.minimal_bundle(bundle), !self.minimal_bundle(bundle),
"Minimal bundle with conflict!" "Minimal bundle with conflict!"
); );
@@ -786,7 +786,7 @@ impl<'a, F: Function> Env<'a, F> {
let mut attempts = 0; let mut attempts = 0;
loop { loop {
attempts += 1; attempts += 1;
log::trace!("attempt {}, req {:?}", attempts, req); trace!("attempt {}, req {:?}", attempts, req);
debug_assert!(attempts < 100 * self.func.num_insts()); debug_assert!(attempts < 100 * self.func.num_insts());
let fixed_preg = match req { let fixed_preg = match req {
@@ -836,7 +836,7 @@ impl<'a, F: Function> Env<'a, F> {
) { ) {
self.stats.process_bundle_reg_probes_any += 1; self.stats.process_bundle_reg_probes_any += 1;
let preg_idx = PRegIndex::new(preg.index()); let preg_idx = PRegIndex::new(preg.index());
log::trace!("trying preg {:?}", preg_idx); trace!("trying preg {:?}", preg_idx);
let scan_limit_cost = match ( let scan_limit_cost = match (
lowest_cost_evict_conflict_cost, lowest_cost_evict_conflict_cost,
@@ -848,13 +848,13 @@ impl<'a, F: Function> Env<'a, F> {
match self.try_to_allocate_bundle_to_reg(bundle, preg_idx, scan_limit_cost) { match self.try_to_allocate_bundle_to_reg(bundle, preg_idx, scan_limit_cost) {
AllocRegResult::Allocated(alloc) => { AllocRegResult::Allocated(alloc) => {
self.stats.process_bundle_reg_success_any += 1; self.stats.process_bundle_reg_success_any += 1;
log::trace!(" -> allocated to any {:?}", preg_idx); trace!(" -> allocated to any {:?}", preg_idx);
self.spillsets[self.bundles[bundle.index()].spillset.index()].reg_hint = self.spillsets[self.bundles[bundle.index()].spillset.index()].reg_hint =
alloc.as_reg().unwrap(); alloc.as_reg().unwrap();
return Ok(()); return Ok(());
} }
AllocRegResult::Conflict(bundles, first_conflict_point) => { AllocRegResult::Conflict(bundles, first_conflict_point) => {
log::trace!( trace!(
" -> conflict with bundles {:?}, first conflict at {:?}", " -> conflict with bundles {:?}, first conflict at {:?}",
bundles, bundles,
first_conflict_point first_conflict_point
@@ -887,7 +887,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
} }
AllocRegResult::ConflictWithFixed(max_cost, point) => { AllocRegResult::ConflictWithFixed(max_cost, point) => {
log::trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
let loop_depth = self.cfginfo.approx_loop_depth let loop_depth = self.cfginfo.approx_loop_depth
[self.cfginfo.insn_block[point.inst().index()].index()]; [self.cfginfo.insn_block[point.inst().index()].index()];
@@ -919,12 +919,12 @@ impl<'a, F: Function> Env<'a, F> {
// any with current bundle assignments. Hence, we will need // any with current bundle assignments. Hence, we will need
// to either split or attempt to evict some bundles. // to either split or attempt to evict some bundles.
log::trace!( trace!(
" -> lowest cost evict: set {:?}, cost {:?}", " -> lowest cost evict: set {:?}, cost {:?}",
lowest_cost_evict_conflict_set, lowest_cost_evict_conflict_set,
lowest_cost_evict_conflict_cost, lowest_cost_evict_conflict_cost,
); );
log::trace!( trace!(
" -> lowest cost split: cost {:?}, point {:?}, reg {:?}", " -> lowest cost split: cost {:?}, point {:?}, reg {:?}",
lowest_cost_split_conflict_cost, lowest_cost_split_conflict_cost,
lowest_cost_split_conflict_point, lowest_cost_split_conflict_point,
@@ -932,13 +932,13 @@ impl<'a, F: Function> Env<'a, F> {
); );
// If we reach here, we *must* have an option either to split or evict. // If we reach here, we *must* have an option either to split or evict.
assert!( debug_assert!(
lowest_cost_split_conflict_cost.is_some() lowest_cost_split_conflict_cost.is_some()
|| lowest_cost_evict_conflict_cost.is_some() || lowest_cost_evict_conflict_cost.is_some()
); );
let our_spill_weight = self.bundle_spill_weight(bundle); let our_spill_weight = self.bundle_spill_weight(bundle);
log::trace!(" -> our spill weight: {}", our_spill_weight); trace!(" -> our spill weight: {}", our_spill_weight);
// We detect the "too-many-live-registers" case here and // We detect the "too-many-live-registers" case here and
// return an error cleanly, rather than panicking, because // return an error cleanly, rather than panicking, because
@@ -953,7 +953,7 @@ impl<'a, F: Function> Env<'a, F> {
if let Requirement::Register = req { if let Requirement::Register = req {
// Check if this is a too-many-live-registers situation. // Check if this is a too-many-live-registers situation.
let range = self.bundles[bundle.index()].ranges[0].range; let range = self.bundles[bundle.index()].ranges[0].range;
log::trace!("checking for too many live regs"); trace!("checking for too many live regs");
let mut min_bundles_assigned = 0; let mut min_bundles_assigned = 0;
let mut fixed_assigned = 0; let mut fixed_assigned = 0;
let mut total_regs = 0; let mut total_regs = 0;
@@ -961,7 +961,7 @@ impl<'a, F: Function> Env<'a, F> {
.iter() .iter()
.chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter()) .chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter())
{ {
log::trace!(" -> PR {:?}", preg); trace!(" -> PR {:?}", preg);
let start = LiveRangeKey::from_range(&CodeRange { let start = LiveRangeKey::from_range(&CodeRange {
from: range.from.prev(), from: range.from.prev(),
to: range.from.prev(), to: range.from.prev(),
@@ -976,19 +976,19 @@ impl<'a, F: Function> Env<'a, F> {
} }
if lr.is_valid() { if lr.is_valid() {
if self.minimal_bundle(self.ranges[lr.index()].bundle) { if self.minimal_bundle(self.ranges[lr.index()].bundle) {
log::trace!(" -> min bundle {:?}", lr); trace!(" -> min bundle {:?}", lr);
min_bundles_assigned += 1; min_bundles_assigned += 1;
} else { } else {
log::trace!(" -> non-min bundle {:?}", lr); trace!(" -> non-min bundle {:?}", lr);
} }
} else { } else {
log::trace!(" -> fixed bundle"); trace!(" -> fixed bundle");
fixed_assigned += 1; fixed_assigned += 1;
} }
} }
total_regs += 1; total_regs += 1;
} }
log::trace!( trace!(
" -> total {}, fixed {}, min {}", " -> total {}, fixed {}, min {}",
total_regs, total_regs,
fixed_assigned, fixed_assigned,
@@ -1020,7 +1020,7 @@ impl<'a, F: Function> Env<'a, F> {
|| lowest_cost_evict_conflict_cost.is_none() || lowest_cost_evict_conflict_cost.is_none()
|| our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap()) || our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap())
{ {
log::trace!( trace!(
" -> deciding to split: our spill weight is {}", " -> deciding to split: our spill weight is {}",
self.bundle_spill_weight(bundle) self.bundle_spill_weight(bundle)
); );
@@ -1053,7 +1053,7 @@ impl<'a, F: Function> Env<'a, F> {
// Evict all bundles in `conflicting bundles` and try again. // Evict all bundles in `conflicting bundles` and try again.
self.stats.evict_bundle_event += 1; self.stats.evict_bundle_event += 1;
for &bundle in &lowest_cost_evict_conflict_set.unwrap() { for &bundle in &lowest_cost_evict_conflict_set.unwrap() {
log::trace!(" -> evicting {:?}", bundle); trace!(" -> evicting {:?}", bundle);
self.evict_bundle(bundle); self.evict_bundle(bundle);
self.stats.evict_bundle_count += 1; self.stats.evict_bundle_count += 1;
} }

30
src/ion/redundant_moves.rs
View File

@@ -18,6 +18,7 @@ pub struct RedundantMoveEliminator {
#[derive(Copy, Clone, Debug)] #[derive(Copy, Clone, Debug)]
pub struct RedundantMoveAction { pub struct RedundantMoveAction {
pub elide: bool, pub elide: bool,
#[cfg(feature = "checker")]
pub def_alloc: Option<(Allocation, VReg)>, pub def_alloc: Option<(Allocation, VReg)>,
} }
@@ -40,13 +41,13 @@ impl RedundantMoveEliminator {
.map(|&p| p) .map(|&p| p)
.unwrap_or(RedundantMoveState::None); .unwrap_or(RedundantMoveState::None);
log::trace!( trace!(
" -> redundant move tracker: from {} to {} to_vreg {:?}", " -> redundant move tracker: from {} to {} to_vreg {:?}",
from, from,
to, to,
to_vreg to_vreg
); );
log::trace!( trace!(
" -> from_state {:?} to_state {:?}", " -> from_state {:?} to_state {:?}",
from_state, from_state,
to_state to_state
@@ -58,6 +59,7 @@ impl RedundantMoveEliminator {
.insert(to, RedundantMoveState::Orig(to_vreg.unwrap())); .insert(to, RedundantMoveState::Orig(to_vreg.unwrap()));
return RedundantMoveAction { return RedundantMoveAction {
elide: true, elide: true,
#[cfg(feature = "checker")]
def_alloc: Some((to, to_vreg.unwrap())), def_alloc: Some((to, to_vreg.unwrap())),
}; };
} }
@@ -67,29 +69,29 @@ impl RedundantMoveEliminator {
RedundantMoveState::Orig(r) => Some(r), RedundantMoveState::Orig(r) => Some(r),
_ => None, _ => None,
}; };
log::trace!(" -> src_vreg {:?}", src_vreg); trace!(" -> src_vreg {:?}", src_vreg);
let dst_vreg = to_vreg.or(src_vreg); let dst_vreg = to_vreg.or(src_vreg);
log::trace!(" -> dst_vreg {:?}", dst_vreg); trace!(" -> dst_vreg {:?}", dst_vreg);
let existing_dst_vreg = match to_state { let existing_dst_vreg = match to_state {
RedundantMoveState::Copy(_, opt_r) => opt_r, RedundantMoveState::Copy(_, opt_r) => opt_r,
RedundantMoveState::Orig(r) => Some(r), RedundantMoveState::Orig(r) => Some(r),
_ => None, _ => None,
}; };
log::trace!(" -> existing_dst_vreg {:?}", existing_dst_vreg); trace!(" -> existing_dst_vreg {:?}", existing_dst_vreg);
let elide = match (from_state, to_state) { let elide = match (from_state, to_state) {
(_, RedundantMoveState::Copy(orig_alloc, _)) if orig_alloc == from => true, (_, RedundantMoveState::Copy(orig_alloc, _)) if orig_alloc == from => true,
(RedundantMoveState::Copy(new_alloc, _), _) if new_alloc == to => true, (RedundantMoveState::Copy(new_alloc, _), _) if new_alloc == to => true,
_ => false, _ => false,
}; };
log::trace!(" -> elide {}", elide); trace!(" -> elide {}", elide);
let def_alloc = if dst_vreg != existing_dst_vreg && dst_vreg.is_some() { let def_alloc = if dst_vreg != existing_dst_vreg && dst_vreg.is_some() {
Some((to, dst_vreg.unwrap())) Some((to, dst_vreg.unwrap()))
} else { } else {
None None
}; };
log::trace!(" -> def_alloc {:?}", def_alloc); trace!(" -> def_alloc {:?}", def_alloc);
// Invalidate all existing copies of `to` if `to` actually changed value. // Invalidate all existing copies of `to` if `to` actually changed value.
if !elide { if !elide {
@@ -100,7 +102,7 @@ impl RedundantMoveEliminator {
if from.is_reg() || to.is_reg() { if from.is_reg() || to.is_reg() {
self.allocs self.allocs
.insert(to, RedundantMoveState::Copy(from, dst_vreg)); .insert(to, RedundantMoveState::Copy(from, dst_vreg));
log::trace!( trace!(
" -> create mapping {} -> {:?}", " -> create mapping {} -> {:?}",
to, to,
RedundantMoveState::Copy(from, dst_vreg) RedundantMoveState::Copy(from, dst_vreg)
@@ -111,20 +113,24 @@ impl RedundantMoveEliminator {
.push(to); .push(to);
} }
RedundantMoveAction { elide, def_alloc } RedundantMoveAction {
elide,
#[cfg(feature = "checker")]
def_alloc,
}
} }
pub fn clear(&mut self) { pub fn clear(&mut self) {
log::trace!(" redundant move eliminator cleared"); trace!(" redundant move eliminator cleared");
self.allocs.clear(); self.allocs.clear();
self.reverse_allocs.clear(); self.reverse_allocs.clear();
} }
pub fn clear_alloc(&mut self, alloc: Allocation) { pub fn clear_alloc(&mut self, alloc: Allocation) {
log::trace!(" redundant move eliminator: clear {:?}", alloc); trace!(" redundant move eliminator: clear {:?}", alloc);
if let Some(ref mut existing_copies) = self.reverse_allocs.get_mut(&alloc) { if let Some(ref mut existing_copies) = self.reverse_allocs.get_mut(&alloc) {
for to_inval in existing_copies.iter() { for to_inval in existing_copies.iter() {
log::trace!(" -> clear existing copy: {:?}", to_inval); trace!(" -> clear existing copy: {:?}", to_inval);
if let Some(val) = self.allocs.get_mut(to_inval) { if let Some(val) = self.allocs.get_mut(to_inval) {
match val { match val {
RedundantMoveState::Copy(_, Some(vreg)) => { RedundantMoveState::Copy(_, Some(vreg)) => {

12
src/ion/requirement.rs
View File

@@ -71,21 +71,21 @@ impl<'a, F: Function> Env<'a, F> {
bundle: LiveBundleIndex, bundle: LiveBundleIndex,
) -> Result<Requirement, RequirementConflictAt> { ) -> Result<Requirement, RequirementConflictAt> {
let mut req = Requirement::Any; let mut req = Requirement::Any;
log::trace!("compute_requirement: {:?}", bundle); trace!("compute_requirement: {:?}", bundle);
let ranges = &self.bundles[bundle.index()].ranges; let ranges = &self.bundles[bundle.index()].ranges;
for entry in ranges { for entry in ranges {
log::trace!(" -> LR {:?}", entry.index); trace!(" -> LR {:?}", entry.index);
for u in &self.ranges[entry.index.index()].uses { for u in &self.ranges[entry.index.index()].uses {
log::trace!(" -> use {:?}", u); trace!(" -> use {:?}", u);
let r = self.requirement_from_operand(u.operand); let r = self.requirement_from_operand(u.operand);
req = req.merge(r).map_err(|_| { req = req.merge(r).map_err(|_| {
log::trace!(" -> conflict"); trace!(" -> conflict");
RequirementConflictAt(u.pos) RequirementConflictAt(u.pos)
})?; })?;
log::trace!(" -> req {:?}", req); trace!(" -> req {:?}", req);
} }
} }
log::trace!(" -> final: {:?}", req); trace!(" -> final: {:?}", req);
Ok(req) Ok(req)
} }

14
src/ion/spill.rs
View File

@@ -20,7 +20,7 @@ use crate::{Allocation, Function, SpillSlot};
impl<'a, F: Function> Env<'a, F> { impl<'a, F: Function> Env<'a, F> {
pub fn try_allocating_regs_for_spilled_bundles(&mut self) { pub fn try_allocating_regs_for_spilled_bundles(&mut self) {
log::trace!("allocating regs for spilled bundles"); trace!("allocating regs for spilled bundles");
for i in 0..self.spilled_bundles.len() { for i in 0..self.spilled_bundles.len() {
let bundle = self.spilled_bundles[i]; // don't borrow self let bundle = self.spilled_bundles[i]; // don't borrow self
@@ -38,7 +38,7 @@ impl<'a, F: Function> Env<'a, F> {
for preg in for preg in
RegTraversalIter::new(self.env, class, hint, PReg::invalid(), bundle.index(), None) RegTraversalIter::new(self.env, class, hint, PReg::invalid(), bundle.index(), None)
{ {
log::trace!("trying bundle {:?} to preg {:?}", bundle, preg); trace!("trying bundle {:?} to preg {:?}", bundle, preg);
let preg_idx = PRegIndex::new(preg.index()); let preg_idx = PRegIndex::new(preg.index());
if let AllocRegResult::Allocated(_) = if let AllocRegResult::Allocated(_) =
self.try_to_allocate_bundle_to_reg(bundle, preg_idx, None) self.try_to_allocate_bundle_to_reg(bundle, preg_idx, None)
@@ -49,7 +49,7 @@ impl<'a, F: Function> Env<'a, F> {
} }
} }
if !success { if !success {
log::trace!( trace!(
"spilling bundle {:?}: marking spillset {:?} as required", "spilling bundle {:?}: marking spillset {:?} as required",
bundle, bundle,
self.bundles[bundle.index()].spillset self.bundles[bundle.index()].spillset
@@ -87,14 +87,14 @@ impl<'a, F: Function> Env<'a, F> {
for i in 0..self.spillsets[spillset.index()].vregs.len() { for i in 0..self.spillsets[spillset.index()].vregs.len() {
// don't borrow self // don't borrow self
let vreg = self.spillsets[spillset.index()].vregs[i]; let vreg = self.spillsets[spillset.index()].vregs[i];
log::trace!( trace!(
"spillslot {:?} alloc'ed to spillset {:?}: vreg {:?}", "spillslot {:?} alloc'ed to spillset {:?}: vreg {:?}",
spillslot, spillslot,
spillset, spillset,
vreg, vreg,
); );
for entry in &self.vregs[vreg.index()].ranges { for entry in &self.vregs[vreg.index()].ranges {
log::trace!( trace!(
"spillslot {:?} getting range {:?} from LR {:?} from vreg {:?}", "spillslot {:?} getting range {:?} from LR {:?} from vreg {:?}",
spillslot, spillslot,
entry.range, entry.range,
@@ -111,7 +111,7 @@ impl<'a, F: Function> Env<'a, F> {
pub fn allocate_spillslots(&mut self) { pub fn allocate_spillslots(&mut self) {
for spillset in 0..self.spillsets.len() { for spillset in 0..self.spillsets.len() {
log::trace!("allocate spillslot: {}", spillset); trace!("allocate spillslot: {}", spillset);
let spillset = SpillSetIndex::new(spillset); let spillset = SpillSetIndex::new(spillset);
if !self.spillsets[spillset.index()].required { if !self.spillsets[spillset.index()].required {
continue; continue;
@@ -196,7 +196,7 @@ impl<'a, F: Function> Env<'a, F> {
self.spillslots[i].alloc = self.allocate_spillslot(self.spillslots[i].class); self.spillslots[i].alloc = self.allocate_spillslot(self.spillslots[i].class);
} }
log::trace!("spillslot allocator done"); trace!("spillslot allocator done");
} }
pub fn allocate_spillslot(&mut self, class: RegClass) -> Allocation { pub fn allocate_spillslot(&mut self, class: RegClass) -> Allocation {

17
src/ion/stackmap.rs
View File

@@ -13,7 +13,7 @@
//! Stackmap computation. //! Stackmap computation.
use super::{Env, ProgPoint, VRegIndex}; use super::{Env, ProgPoint, VRegIndex};
use crate::Function; use crate::{ion::data_structures::u64_key, Function};
impl<'a, F: Function> Env<'a, F> { impl<'a, F: Function> Env<'a, F> {
pub fn compute_stackmaps(&mut self) { pub fn compute_stackmaps(&mut self) {
@@ -30,10 +30,10 @@ impl<'a, F: Function> Env<'a, F> {
// safepoints; and for each safepoint in the current range, // safepoints; and for each safepoint in the current range,
// emit the allocation into the `safepoint_slots` list. // emit the allocation into the `safepoint_slots` list.
log::trace!("safepoints_per_vreg = {:?}", self.safepoints_per_vreg); trace!("safepoints_per_vreg = {:?}", self.safepoints_per_vreg);
for vreg in self.func.reftype_vregs() { for vreg in self.func.reftype_vregs() {
log::trace!("generating safepoint info for vreg {}", vreg); trace!("generating safepoint info for vreg {}", vreg);
let vreg = VRegIndex::new(vreg.vreg()); let vreg = VRegIndex::new(vreg.vreg());
let mut safepoints: Vec<ProgPoint> = self let mut safepoints: Vec<ProgPoint> = self
.safepoints_per_vreg .safepoints_per_vreg
@@ -43,19 +43,19 @@ impl<'a, F: Function> Env<'a, F> {
.map(|&inst| ProgPoint::before(inst)) .map(|&inst| ProgPoint::before(inst))
.collect(); .collect();
safepoints.sort_unstable(); safepoints.sort_unstable();
log::trace!(" -> live over safepoints: {:?}", safepoints); trace!(" -> live over safepoints: {:?}", safepoints);
let mut safepoint_idx = 0; let mut safepoint_idx = 0;
for entry in &self.vregs[vreg.index()].ranges { for entry in &self.vregs[vreg.index()].ranges {
let range = entry.range; let range = entry.range;
let alloc = self.get_alloc_for_range(entry.index); let alloc = self.get_alloc_for_range(entry.index);
log::trace!(" -> range {:?}: alloc {}", range, alloc); trace!(" -> range {:?}: alloc {}", range, alloc);
while safepoint_idx < safepoints.len() && safepoints[safepoint_idx] < range.to { while safepoint_idx < safepoints.len() && safepoints[safepoint_idx] < range.to {
if safepoints[safepoint_idx] < range.from { if safepoints[safepoint_idx] < range.from {
safepoint_idx += 1; safepoint_idx += 1;
continue; continue;
} }
log::trace!(" -> covers safepoint {:?}", safepoints[safepoint_idx]); trace!(" -> covers safepoint {:?}", safepoints[safepoint_idx]);
self.safepoint_slots self.safepoint_slots
.push((safepoints[safepoint_idx], alloc)); .push((safepoints[safepoint_idx], alloc));
@@ -64,7 +64,8 @@ impl<'a, F: Function> Env<'a, F> {
} }
} }
self.safepoint_slots.sort_unstable(); self.safepoint_slots
log::trace!("final safepoint slots info: {:?}", self.safepoint_slots); .sort_unstable_by_key(|(progpoint, slot)| u64_key(progpoint.to_index(), slot.bits()));
trace!("final safepoint slots info: {:?}", self.safepoint_slots);
} }
} }

30
src/lib.rs
View File

@@ -12,6 +12,16 @@
#![allow(dead_code)] #![allow(dead_code)]
// Even when trace logging is disabled, the trace macro has a significant
// performance cost so we disable it in release builds.
macro_rules! trace {
($($tt:tt)*) => {
if cfg!(feature = "trace-log") {
::log::trace!($($tt)*);
}
};
}
pub(crate) mod cfg; pub(crate) mod cfg;
pub(crate) mod domtree; pub(crate) mod domtree;
pub mod indexset; pub mod indexset;
@@ -234,7 +244,7 @@ impl SpillSlot {
/// Create a new SpillSlot of a given class. /// Create a new SpillSlot of a given class.
#[inline(always)] #[inline(always)]
pub fn new(slot: usize, class: RegClass) -> Self { pub fn new(slot: usize, class: RegClass) -> Self {
assert!(slot < (1 << 24)); debug_assert!(slot < (1 << 24));
SpillSlot { SpillSlot {
bits: (slot as u32) | (class as u8 as u32) << 24, bits: (slot as u32) | (class as u8 as u32) << 24,
} }
@@ -412,11 +422,11 @@ impl Operand {
OperandConstraint::Reg => 1, OperandConstraint::Reg => 1,
OperandConstraint::Stack => 2, OperandConstraint::Stack => 2,
OperandConstraint::FixedReg(preg) => { OperandConstraint::FixedReg(preg) => {
assert_eq!(preg.class(), vreg.class()); debug_assert_eq!(preg.class(), vreg.class());
0b1000000 | preg.hw_enc() as u32 0b1000000 | preg.hw_enc() as u32
} }
OperandConstraint::Reuse(which) => { OperandConstraint::Reuse(which) => {
assert!(which <= 31); debug_assert!(which <= 31);
0b0100000 | which as u32 0b0100000 | which as u32
} }
}; };
@@ -696,7 +706,7 @@ impl Allocation {
/// Construct a new Allocation. /// Construct a new Allocation.
#[inline(always)] #[inline(always)]
pub(crate) fn new(kind: AllocationKind, index: usize) -> Self { pub(crate) fn new(kind: AllocationKind, index: usize) -> Self {
assert!(index < (1 << 28)); debug_assert!(index < (1 << 28));
Self { Self {
bits: ((kind as u8 as u32) << 29) | (index as u32), bits: ((kind as u8 as u32) << 29) | (index as u32),
} }
@@ -1105,20 +1115,16 @@ pub enum Edit {
/// register or a stack slot (spillslot). However, stack-to-stack /// register or a stack slot (spillslot). However, stack-to-stack
/// moves will never be generated. /// moves will never be generated.
/// ///
/// `to_vreg`, if defined, is useful as metadata: it indicates
/// that the moved value is a def of a new vreg.
///
/// `Move` edits will be generated even if src and dst allocation /// `Move` edits will be generated even if src and dst allocation
/// are the same if the vreg changes; this allows proper metadata /// are the same if the vreg changes; this allows proper metadata
/// tracking even when moves are elided. /// tracking even when moves are elided.
Move { Move { from: Allocation, to: Allocation },
from: Allocation,
to: Allocation,
to_vreg: Option<VReg>,
},
/// Define a particular Allocation to contain a particular VReg. Useful /// Define a particular Allocation to contain a particular VReg. Useful
/// for the checker. /// for the checker.
///
/// `DefAlloc` edits are only emitted when the `"checker"` Cargo feature is
/// enabled.
DefAlloc { alloc: Allocation, vreg: VReg }, DefAlloc { alloc: Allocation, vreg: VReg },
} }

7
src/moves.rs
View File

@@ -3,7 +3,7 @@
* exception. See `LICENSE` for details. * exception. See `LICENSE` for details.
*/ */
use crate::Allocation; use crate::{ion::data_structures::u64_key, Allocation};
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
pub type MoveVec<T> = SmallVec<[(Allocation, Allocation, T); 16]>; pub type MoveVec<T> = SmallVec<[(Allocation, Allocation, T); 16]>;
@@ -53,7 +53,8 @@ impl<T: Clone + Copy + Default> ParallelMoves<T> {
// Sort moves by source so that we can efficiently test for // Sort moves by source so that we can efficiently test for
// presence. // presence.
self.parallel_moves.sort_by_key(|&(src, dst, _)| (src, dst)); self.parallel_moves
.sort_by_key(|&(src, dst, _)| u64_key(src.bits(), dst.bits()));
// Do any dests overlap sources? If not, we can also just // Do any dests overlap sources? If not, we can also just
// return the list. // return the list.
@@ -183,7 +184,7 @@ impl<T: Clone + Copy + Default> ParallelMoves<T> {
scratch_src = Some(src); scratch_src = Some(src);
src = self.scratch; src = self.scratch;
} else { } else {
assert_eq!(last_dst.unwrap(), src); debug_assert_eq!(last_dst.unwrap(), src);
} }
ret.push((src, dst, dst_t)); ret.push((src, dst, dst_t));