From b19fa4857f1a2612e5ae37bcaa62f035456358c8 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 31 Aug 2021 17:31:23 -0700 Subject: [PATCH] Rename operand positions to Early and Late, and make weights f16/f32 values. --- src/checker.rs | 4 +- src/fuzzing/func.rs | 10 ++-- src/ion/data_structures.rs | 12 +++-- src/ion/dump.rs | 2 +- src/ion/liveranges.rs | 104 +++++++++++++++++++++++++++---------- src/ion/moves.rs | 2 +- src/ion/process.rs | 23 ++++---- src/lib.rs | 80 ++++++++++++++-------------- 8 files changed, 145 insertions(+), 92 deletions(-) diff --git a/src/checker.rs b/src/checker.rs index 94660d7..d074101 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -306,8 +306,8 @@ impl CheckerState { // the requirements of the OperandConstraint. for (op, alloc) in operands.iter().zip(allocs.iter()) { let is_here = match (op.pos(), pos) { - (OperandPos::Before, InstPosition::Before) => true, - (OperandPos::After, InstPosition::After) => true, + (OperandPos::Early, InstPosition::Before) => true, + (OperandPos::Late, InstPosition::After) => true, _ => false, }; if !is_here { diff --git a/src/fuzzing/func.rs b/src/fuzzing/func.rs index 0d0eeda..657ed61 100644 --- a/src/fuzzing/func.rs +++ b/src/fuzzing/func.rs @@ -402,9 +402,9 @@ impl Func { while let Some(vreg) = vregs_by_block_to_be_defined[block].pop() { let def_constraint = OperandConstraint::arbitrary(u)?; let def_pos = if bool::arbitrary(u)? { - OperandPos::Before + OperandPos::Early } else { - OperandPos::After + OperandPos::Late }; let mut operands = vec![Operand::new( vreg, @@ -442,7 +442,7 @@ impl Func { vreg, use_constraint, OperandKind::Use, - OperandPos::Before, + OperandPos::Early, )); allocations.push(Allocation::none()); } @@ -456,7 +456,7 @@ impl Func { op.vreg(), OperandConstraint::Reuse(reused), op.kind(), - OperandPos::After, + OperandPos::Late, ); // Make sure reused input is a Reg. let op = operands[reused]; @@ -464,7 +464,7 @@ impl Func { op.vreg(), OperandConstraint::Reg, op.kind(), - OperandPos::Before, + OperandPos::Early, ); } else if opts.fixed_regs && bool::arbitrary(u)? { let mut fixed = vec![]; diff --git a/src/ion/data_structures.rs b/src/ion/data_structures.rs index 33f5684..dbf1d77 100644 --- a/src/ion/data_structures.rs +++ b/src/ion/data_structures.rs @@ -13,6 +13,7 @@ //! Data structures for backtracking allocator. +use super::liveranges::SpillWeight; use crate::cfg::CFGInfo; use crate::index::ContainerComparator; use crate::indexset::IndexSet; @@ -141,14 +142,15 @@ impl LiveRange { self.uses_spill_weight_and_flags |= flag_word; } #[inline(always)] - pub fn uses_spill_weight(&self) -> u32 { - self.uses_spill_weight_and_flags & 0x1fff_ffff + pub fn uses_spill_weight(&self) -> SpillWeight { + let bits = (self.uses_spill_weight_and_flags & 0x1fff_ffff) << 2; + SpillWeight::from_f32(f32::from_bits(bits)) } #[inline(always)] - pub fn set_uses_spill_weight(&mut self, weight: u32) { - assert!(weight < (1 << 29)); + pub fn set_uses_spill_weight(&mut self, weight: SpillWeight) { + let weight_bits = (weight.to_f32().to_bits() >> 2) & 0x1fff_ffff; self.uses_spill_weight_and_flags = - (self.uses_spill_weight_and_flags & 0xe000_0000) | weight; + (self.uses_spill_weight_and_flags & 0xe000_0000) | weight_bits; } } diff --git a/src/ion/dump.rs b/src/ion/dump.rs index b45d90c..0048f80 100644 --- a/src/ion/dump.rs +++ b/src/ion/dump.rs @@ -37,7 +37,7 @@ impl<'a, F: Function> Env<'a, F> { log::trace!("Ranges:"); for (i, r) in self.ranges.iter().enumerate() { log::trace!( - "range{}: range={:?} vreg={:?} bundle={:?} weight={}", + "range{}: range={:?} vreg={:?} bundle={:?} weight={:?}", i, r.range, r.vreg, diff --git a/src/ion/liveranges.rs b/src/ion/liveranges.rs index 9e74ed3..e73c821 100644 --- a/src/ion/liveranges.rs +++ b/src/ion/liveranges.rs @@ -26,26 +26,73 @@ use crate::{ use fxhash::FxHashSet; use smallvec::{smallvec, SmallVec}; use std::collections::{HashSet, VecDeque}; -use std::convert::TryFrom; + +/// A spill weight computed for a certain Use. +#[derive(Clone, Copy, Debug)] +pub struct SpillWeight(f32); #[inline(always)] pub fn spill_weight_from_constraint( constraint: OperandConstraint, loop_depth: usize, is_def: bool, -) -> u32 { +) -> SpillWeight { // A bonus of 1000 for one loop level, 4000 for two loop levels, // 16000 for three loop levels, etc. Avoids exponentiation. - // Bound `loop_depth` at 2 so that `hot_bonus` is at most 16000. - let loop_depth = std::cmp::min(2, loop_depth); - let hot_bonus = 1000 * (1 << (2 * loop_depth)); - let def_bonus = if is_def { 2000 } else { 0 }; - let constraint_bonus = match constraint { - OperandConstraint::Any => 1000, - OperandConstraint::Reg | OperandConstraint::FixedReg(_) => 2000, - _ => 0, + let loop_depth = std::cmp::min(10, loop_depth); + let hot_bonus: f32 = (0..loop_depth).fold(1000.0, |a, _| a * 4.0); + let def_bonus: f32 = if is_def { 2000.0 } else { 0.0 }; + let constraint_bonus: f32 = match constraint { + OperandConstraint::Any => 1000.0, + OperandConstraint::Reg | OperandConstraint::FixedReg(_) => 2000.0, + _ => 0.0, }; - hot_bonus + def_bonus + constraint_bonus + SpillWeight(hot_bonus + def_bonus + constraint_bonus) +} + +impl SpillWeight { + /// Convert a floating-point weight to a u16 that can be compactly + /// stored in a `Use`. We simply take the top 16 bits of the f32; this + /// is equivalent to the bfloat16 format + /// (https://en.wikipedia.org/wiki/Bfloat16_floating-point_format). + pub fn to_bits(self) -> u16 { + (self.0.to_bits() >> 15) as u16 + } + + /// Convert a value that was returned from + /// `SpillWeight::to_bits()` back into a `SpillWeight`. Note that + /// some precision may be lost when round-tripping from a spill + /// weight to packed bits and back. + pub fn from_bits(bits: u16) -> SpillWeight { + let x = f32::from_bits((bits as u32) << 15); + SpillWeight(x) + } + + /// Get a zero spill weight. + pub fn zero() -> SpillWeight { + SpillWeight(0.0) + } + + /// Convert to a raw floating-point value. + pub fn to_f32(self) -> f32 { + self.0 + } + + /// Create a `SpillWeight` from a raw floating-point value. + pub fn from_f32(x: f32) -> SpillWeight { + SpillWeight(x) + } + + pub fn to_int(self) -> u32 { + self.0 as u32 + } +} + +impl std::ops::Add for SpillWeight { + type Output = SpillWeight; + fn add(self, other: SpillWeight) -> Self { + SpillWeight(self.0 + other.0) + } } impl<'a, F: Function> Env<'a, F> { @@ -196,10 +243,10 @@ impl<'a, F: Function> Env<'a, F> { loop_depth, operand.kind() != OperandKind::Use, ); - u.weight = u16::try_from(weight).expect("weight too large for u16 field"); + u.weight = weight.to_bits(); log::trace!( - "insert use {:?} into lr {:?} with weight {}", + "insert use {:?} into lr {:?} with weight {:?}", u, into, weight, @@ -212,9 +259,10 @@ impl<'a, F: Function> Env<'a, F> { self.ranges[into.index()].uses.push(u); // Update stats. - self.ranges[into.index()].uses_spill_weight_and_flags += weight; + let range_weight = self.ranges[into.index()].uses_spill_weight() + weight; + self.ranges[into.index()].set_uses_spill_weight(range_weight); log::trace!( - " -> now range has weight {}", + " -> now range has weight {:?}", self.ranges[into.index()].uses_spill_weight(), ); } @@ -279,7 +327,7 @@ impl<'a, F: Function> Env<'a, F> { live.set(src.vreg().vreg(), true); } - for pos in &[OperandPos::After, OperandPos::Before] { + for pos in &[OperandPos::Late, OperandPos::Early] { for op in self.func.inst_operands(inst) { if op.pos() == *pos { let was_live = live.get(op.vreg().vreg()); @@ -437,9 +485,9 @@ impl<'a, F: Function> Env<'a, F> { assert_eq!(src.class(), dst.class()); assert_eq!(src.kind(), OperandKind::Use); - assert_eq!(src.pos(), OperandPos::Before); + assert_eq!(src.pos(), OperandPos::Early); assert_eq!(dst.kind(), OperandKind::Def); - assert_eq!(dst.pos(), OperandPos::After); + assert_eq!(dst.pos(), OperandPos::Late); // If both src and dest are pinned, emit the // move right here, right now. @@ -506,7 +554,7 @@ impl<'a, F: Function> Env<'a, F> { dst.vreg(), src.vreg(), OperandKind::Def, - OperandPos::After, + OperandPos::Late, ProgPoint::after(inst), ) } else { @@ -516,7 +564,7 @@ impl<'a, F: Function> Env<'a, F> { src.vreg(), dst.vreg(), OperandKind::Use, - OperandPos::Before, + OperandPos::Early, ProgPoint::after(inst), ) }; @@ -720,13 +768,13 @@ impl<'a, F: Function> Env<'a, F> { src.vreg(), src_constraint, OperandKind::Use, - OperandPos::After, + OperandPos::Late, ); let dst = Operand::new( dst.vreg(), dst_constraint, OperandKind::Def, - OperandPos::Before, + OperandPos::Early, ); if self.annotations_enabled { @@ -843,9 +891,9 @@ impl<'a, F: Function> Env<'a, F> { let operand = self.func.inst_operands(inst)[i]; let pos = match (operand.kind(), operand.pos()) { (OperandKind::Mod, _) => ProgPoint::before(inst), - (OperandKind::Def, OperandPos::Before) => ProgPoint::before(inst), - (OperandKind::Def, OperandPos::After) => ProgPoint::after(inst), - (OperandKind::Use, OperandPos::After) => ProgPoint::after(inst), + (OperandKind::Def, OperandPos::Early) => ProgPoint::before(inst), + (OperandKind::Def, OperandPos::Late) => ProgPoint::after(inst), + (OperandKind::Use, OperandPos::Late) => ProgPoint::after(inst), // If this is a branch, extend `pos` to // the end of the block. (Branch uses are // blockparams and need to be live at the @@ -858,12 +906,12 @@ impl<'a, F: Function> Env<'a, F> { // reused input, force `pos` to // `After`. (See note below for why; it's // very subtle!) - (OperandKind::Use, OperandPos::Before) + (OperandKind::Use, OperandPos::Early) if reused_input.is_some() && reused_input.unwrap() != i => { ProgPoint::after(inst) } - (OperandKind::Use, OperandPos::Before) => ProgPoint::before(inst), + (OperandKind::Use, OperandPos::Early) => ProgPoint::before(inst), }; if pos.pos() != cur_pos { @@ -1058,7 +1106,7 @@ impl<'a, F: Function> Env<'a, F> { self.vreg_regs[vreg.index()], OperandConstraint::Stack, OperandKind::Use, - OperandPos::Before, + OperandPos::Early, ); log::trace!( diff --git a/src/ion/moves.rs b/src/ion/moves.rs index d8479e7..6ba259c 100644 --- a/src/ion/moves.rs +++ b/src/ion/moves.rs @@ -759,7 +759,7 @@ impl<'a, F: Function> Env<'a, F> { let operand = self.func.inst_operands(inst)[output_idx]; if let OperandConstraint::Reuse(input_idx) = operand.constraint() { debug_assert!(!input_reused.contains(&input_idx)); - debug_assert_eq!(operand.pos(), OperandPos::After); + debug_assert_eq!(operand.pos(), OperandPos::Late); input_reused.push(input_idx); let input_alloc = self.get_alloc(inst, input_idx); let output_alloc = self.get_alloc(inst, output_idx); diff --git a/src/ion/process.rs b/src/ion/process.rs index 8f37b13..8c10a9b 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -16,7 +16,7 @@ use super::{ spill_weight_from_constraint, CodeRange, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, LiveRangeIndex, LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, - Requirement, UseList, + Requirement, SpillWeight, UseList, }; use crate::{ Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint, @@ -310,23 +310,24 @@ impl<'a, F: Function> Env<'a, F> { 1_000_000 } } else { - let mut total = 0; + let mut total = SpillWeight::zero(); for entry in &self.bundles[bundle.index()].ranges { let range_data = &self.ranges[entry.index.index()]; log::trace!( - " -> uses spill weight: +{}", + " -> uses spill weight: +{:?}", range_data.uses_spill_weight() ); - total += range_data.uses_spill_weight(); + total = total + range_data.uses_spill_weight(); } if self.bundles[bundle.index()].prio > 0 { + let final_weight = (total.to_f32() as u32) / self.bundles[bundle.index()].prio; log::trace!( " -> dividing by prio {}; final weight {}", self.bundles[bundle.index()].prio, - total / self.bundles[bundle.index()].prio + final_weight ); - total / self.bundles[bundle.index()].prio + final_weight } else { 0 } @@ -346,9 +347,9 @@ impl<'a, F: Function> Env<'a, F> { pub fn recompute_range_properties(&mut self, range: LiveRangeIndex) { let rangedata = &mut self.ranges[range.index()]; - let mut w = 0; + let mut w = SpillWeight::zero(); for u in &rangedata.uses { - w += u.weight as u32; + w = w + SpillWeight::from_bits(u.weight); log::trace!("range{}: use {:?}", range.index(), u); } rangedata.set_uses_spill_weight(w); @@ -890,7 +891,8 @@ impl<'a, F: Function> Env<'a, F> { OperandConstraint::Reg, loop_depth as usize, /* is_def = */ true, - ); + ) + .to_int(); if lowest_cost_split_conflict_cost.is_none() || (conflict_cost + move_cost) < lowest_cost_split_conflict_cost.unwrap() @@ -909,7 +911,8 @@ impl<'a, F: Function> Env<'a, F> { OperandConstraint::Reg, loop_depth as usize, /* is_def = */ true, - ); + ) + .to_int(); if lowest_cost_split_conflict_cost.is_none() || (max_cost + move_cost) < lowest_cost_split_conflict_cost.unwrap() diff --git a/src/lib.rs b/src/lib.rs index 379c05f..e5cda3b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -335,26 +335,27 @@ pub enum OperandKind { } /// The "position" of the operand: where it has its read/write -/// effects. These are positions "in" the instruction, and "before" -/// and "after" are relative to the instruction's actual semantics. In -/// other words, the allocator assumes that the instruction (i) -/// performs all reads of "before" operands, (ii) does its work, and -/// (iii) performs all writes of its "after" operands. +/// effects. These are positions "in" the instruction, and "early" and +/// "late" are relative to the instruction's main effect or +/// computation. In other words, the allocator assumes that the +/// instruction (i) performs all reads and writes of "early" operands, +/// (ii) does its work, and (iii) performs all reads and writes of its +/// "late" operands. /// -/// A "write" (def) at "before" or a "read" (use) at "after" may be -/// slightly nonsensical, given the above; but, it is consistent with -/// the notion that the value (even if a result of execution) *could* -/// have been written to the register at "Before", or the value (even -/// if depended upon by the execution) *could* have been read from the -/// regster at "After". In other words, these write-before or -/// use-after operands ensure that the particular allocations are -/// valid for longer than usual and that a register is not reused -/// between the use (normally complete at "Before") and the def -/// (normally starting at "After"). See `Operand` for more. +/// A "write" (def) at "early" or a "read" (use) at "late" may be +/// slightly nonsensical, given the above, if the read is necessary +/// for the computation or the write is a result of it. A way to think +/// of it is that the value (even if a result of execution) *could* +/// have been read or written at the given location without causing +/// any register-usage conflicts. In other words, these write-early or +/// use-late operands ensure that the particular allocations are valid +/// for longer than usual and that a register is not reused between +/// the use (normally complete at "Early") and the def (normally +/// starting at "Late"). See `Operand` for more. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum OperandPos { - Before = 0, - After = 1, + Early = 0, + Late = 1, } /// An `Operand` encodes everything about a mention of a register in @@ -365,21 +366,20 @@ pub enum OperandPos { /// `LAllocation` in Ion). /// /// Generally, regalloc2 considers operands to have their effects at -/// one of two program points that surround an instruction: "Before" -/// or "After". All operands at a given program-point are assigned +/// one of two points that exist in an instruction: "Early" or +/// "Late". All operands at a given program-point are assigned /// non-conflicting locations based on their constraints. Each operand /// has a "kind", one of use/def/mod, corresponding to /// read/write/read-write, respectively. /// -/// Usually, an instruction's inputs will be uses-at-Before and -/// outputs will be defs-at-After, though there are valid use-cases -/// for other combinations too. For example, a single "instruction" -/// seen by the regalloc that lowers into multiple machine -/// instructions and reads some of its inputs after it starts to write -/// outputs must either make those input(s) uses-at-After or those -/// output(s) defs-at-Before so that the conflict (overlap) is -/// properly accounted for. See comments on the constructors below for -/// more. +/// Usually, an instruction's inputs will be "early uses" and outputs +/// will be "late defs", though there are valid use-cases for other +/// combinations too. For example, a single "instruction" seen by the +/// regalloc that lowers into multiple machine instructions and reads +/// some of its inputs after it starts to write outputs must either +/// make those input(s) "late uses" or those output(s) "early defs" so +/// that the conflict (overlap) is properly accounted for. See +/// comments on the constructors below for more. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Operand { /// Bit-pack into 32 bits. @@ -437,7 +437,7 @@ impl Operand { vreg, OperandConstraint::Reg, OperandKind::Use, - OperandPos::Before, + OperandPos::Early, ) } @@ -450,7 +450,7 @@ impl Operand { vreg, OperandConstraint::Reg, OperandKind::Use, - OperandPos::After, + OperandPos::Late, ) } @@ -464,7 +464,7 @@ impl Operand { vreg, OperandConstraint::Reg, OperandKind::Def, - OperandPos::After, + OperandPos::Late, ) } @@ -478,7 +478,7 @@ impl Operand { vreg, OperandConstraint::Reg, OperandKind::Def, - OperandPos::Before, + OperandPos::Early, ) } @@ -496,7 +496,7 @@ impl Operand { vreg, OperandConstraint::Reg, OperandKind::Def, - OperandPos::Before, + OperandPos::Early, ) } @@ -511,7 +511,7 @@ impl Operand { vreg, OperandConstraint::Reuse(idx), OperandKind::Def, - OperandPos::After, + OperandPos::Late, ) } @@ -525,7 +525,7 @@ impl Operand { vreg, OperandConstraint::FixedReg(preg), OperandKind::Use, - OperandPos::Before, + OperandPos::Early, ) } @@ -539,7 +539,7 @@ impl Operand { vreg, OperandConstraint::FixedReg(preg), OperandKind::Def, - OperandPos::After, + OperandPos::Late, ) } @@ -585,8 +585,8 @@ impl Operand { pub fn pos(self) -> OperandPos { let pos_field = (self.bits >> 26) & 1; match pos_field { - 0 => OperandPos::Before, - 1 => OperandPos::After, + 0 => OperandPos::Early, + 1 => OperandPos::Late, _ => unreachable!(), } } @@ -631,8 +631,8 @@ impl std::fmt::Debug for Operand { impl std::fmt::Display for Operand { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match (self.kind(), self.pos()) { - (OperandKind::Def, OperandPos::After) - | (OperandKind::Mod | OperandKind::Use, OperandPos::Before) => { + (OperandKind::Def, OperandPos::Late) + | (OperandKind::Mod | OperandKind::Use, OperandPos::Early) => { write!(f, "{:?}", self.kind())?; } _ => {