T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove ancient register allocation (#3401)

Browse Source
This commit is contained in:
Benjamin Bouvier
2021-09-30 21:27:23 +02:00
committed by GitHub
parent 80336f4535
commit bae4ec6427
66 changed files with 112 additions and 15380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cranelift/codegen/src/verifier/cssa.rs
View File

@@ -4,10 +4,8 @@ use crate::dbg::DisplayList;
use crate::dominator_tree::{DominatorTree, DominatorTreePreorder};
use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
use crate::ir::{ExpandedProgramPoint, Function};
use crate::regalloc::liveness::Liveness;
use crate::regalloc::virtregs::VirtRegs;
use crate::timing;
use crate::verifier::{VerifierErrors, VerifierStepResult};
use crate::verifier::{virtregs::VirtRegs, VerifierErrors, VerifierStepResult};
/// Verify conventional SSA form for `func`.
///
@@ -27,7 +25,6 @@ pub fn verify_cssa(
func: &Function,
cfg: &ControlFlowGraph,
domtree: &DominatorTree,
liveness: &Liveness,
virtregs: &VirtRegs,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
@@ -41,7 +38,6 @@ pub fn verify_cssa(
cfg,
domtree,
virtregs,
liveness,
preorder,
};
verifier.check_virtregs(errors)?;
@@ -54,7 +50,6 @@ struct CssaVerifier<'a> {
cfg: &'a ControlFlowGraph,
domtree: &'a DominatorTree,
virtregs: &'a VirtRegs,
liveness: &'a Liveness,
preorder: DominatorTreePreorder,
}
@@ -70,9 +65,6 @@ impl<'a> CssaVerifier<'a> {
if !self.func.dfg.value_is_attached(val) {
return errors.fatal((val, format!("Detached value in {}", vreg)));
}
if self.liveness.get(val).is_none() {
return errors.fatal((val, format!("Value in {} has no live range", vreg)));
};
// Check topological ordering with the previous values in the virtual register.
let def: ExpandedProgramPoint = self.func.dfg.value_def(val).into();
@@ -120,19 +112,7 @@ impl<'a> CssaVerifier<'a> {
if self.preorder.dominates(prev_block, def_block)
&& self.domtree.dominates(prev_def, def, &self.func.layout)
{
if self.liveness[prev_val].overlaps_def(def, def_block, &self.func.layout) {
return errors.fatal((
val,
format!(
"Value def in {} = {} interferes with {}",
vreg,
DisplayList(values),
prev_val
),
));
} else {
break;
}
break;
}
}
}

24
cranelift/codegen/src/verifier/flags.rs
View File

@@ -4,7 +4,6 @@ use crate::entity::{EntitySet, SecondaryMap};
use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
use crate::ir;
use crate::ir::instructions::BranchInfo;
use crate::isa;
use crate::packed_option::PackedOption;
use crate::timing;
use crate::verifier::{VerifierErrors, VerifierStepResult};
@@ -24,19 +23,12 @@ use crate::verifier::{VerifierErrors, VerifierStepResult};
pub fn verify_flags(
func: &ir::Function,
cfg: &ControlFlowGraph,
isa: Option<&dyn isa::TargetIsa>,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let _tt = timing::verify_flags();
let encinfo = if isa.is_none() || isa.unwrap().get_mach_backend().is_some() {
None
} else {
Some(isa.unwrap().encoding_info())
};
let mut verifier = FlagsVerifier {
func,
cfg,
encinfo,
livein: SecondaryMap::new(),
};
verifier.check(errors)
@@ -45,7 +37,6 @@ pub fn verify_flags(
struct FlagsVerifier<'a> {
func: &'a ir::Function,
cfg: &'a ControlFlowGraph,
encinfo: Option<isa::EncInfo>,
/// The single live-in flags value (if any) for each block.
livein: SecondaryMap<ir::Block, PackedOption<ir::Value>>,
@@ -111,21 +102,6 @@ impl<'a> FlagsVerifier<'a> {
return Err(());
}
}
// Does the instruction have an encoding that clobbers the CPU flags?
if self
.encinfo
.as_ref()
.and_then(|ei| ei.operand_constraints(self.func.encodings[inst]))
.map_or(false, |c| c.clobbers_flags)
&& live_val.is_some()
{
errors.report((
inst,
format!("encoding clobbers live CPU flags in {}", live),
));
return Err(());
}
}
// Now look for live ranges of CPU flags that end here.

235
cranelift/codegen/src/verifier/liveness.rs
View File

@@ -1,235 +0,0 @@
//! Liveness verifier.
use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
use crate::ir::entities::AnyEntity;
use crate::ir::{ExpandedProgramPoint, Function, ProgramPoint, Value};
use crate::isa::TargetIsa;
use crate::regalloc::liveness::Liveness;
use crate::regalloc::liverange::LiveRange;
use crate::timing;
use crate::verifier::{VerifierErrors, VerifierStepResult};
/// Verify liveness information for `func`.
///
/// The provided control flow graph is assumed to be sound.
///
/// - All values in the program must have a live range.
/// - The live range def point must match where the value is defined.
/// - The live range must reach all uses.
/// - When a live range is live-in to a block, it must be live at all the predecessors.
/// - The live range affinity must be compatible with encoding constraints.
///
/// We don't verify that live ranges are minimal. This would require recomputing live ranges for
/// all values.
pub fn verify_liveness(
isa: &dyn TargetIsa,
func: &Function,
cfg: &ControlFlowGraph,
liveness: &Liveness,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let _tt = timing::verify_liveness();
let verifier = LivenessVerifier {
isa,
func,
cfg,
liveness,
};
verifier.check_blocks(errors)?;
verifier.check_insts(errors)?;
Ok(())
}
struct LivenessVerifier<'a> {
isa: &'a dyn TargetIsa,
func: &'a Function,
cfg: &'a ControlFlowGraph,
liveness: &'a Liveness,
}
impl<'a> LivenessVerifier<'a> {
/// Check all block arguments.
fn check_blocks(&self, errors: &mut VerifierErrors) -> VerifierStepResult<()> {
for block in self.func.layout.blocks() {
for &val in self.func.dfg.block_params(block) {
let lr = match self.liveness.get(val) {
Some(lr) => lr,
None => {
return errors
.fatal((block, format!("block arg {} has no live range", val)))
}
};
self.check_lr(block.into(), val, lr, errors)?;
}
}
Ok(())
}
/// Check all instructions.
fn check_insts(&self, errors: &mut VerifierErrors) -> VerifierStepResult<()> {
for block in self.func.layout.blocks() {
for inst in self.func.layout.block_insts(block) {
let encoding = self.func.encodings[inst];
// Check the defs.
for &val in self.func.dfg.inst_results(inst) {
let lr = match self.liveness.get(val) {
Some(lr) => lr,
None => return errors.fatal((inst, format!("{} has no live range", val))),
};
self.check_lr(inst.into(), val, lr, errors)?;
if encoding.is_legal() {
// A legal instruction is not allowed to define ghost values.
if lr.affinity.is_unassigned() {
return errors.fatal((
inst,
format!(
"{} is a ghost value defined by a real [{}] instruction",
val,
self.isa.encoding_info().display(encoding)
),
));
}
} else if !lr.affinity.is_unassigned() {
// A non-encoded instruction can only define ghost values.
return errors.fatal((
inst,
format!(
"{} is a real {} value defined by a ghost instruction",
val,
lr.affinity.display(&self.isa.register_info())
),
));
}
}
// Check the uses.
for &val in self.func.dfg.inst_args(inst) {
let lr = match self.liveness.get(val) {
Some(lr) => lr,
None => return errors.fatal((inst, format!("{} has no live range", val))),
};
debug_assert!(self.func.layout.inst_block(inst).unwrap() == block);
if !lr.reaches_use(inst, block, &self.func.layout) {
return errors.fatal((inst, format!("{} is not live at this use", val)));
}
// A legal instruction is not allowed to depend on ghost values.
if encoding.is_legal() && lr.affinity.is_unassigned() {
return errors.fatal((
inst,
format!(
"{} is a ghost value used by a real [{}] instruction",
val,
self.isa.encoding_info().display(encoding),
),
));
}
}
}
}
Ok(())
}
/// Check the integrity of the live range `lr`.
fn check_lr(
&self,
def: ProgramPoint,
val: Value,
lr: &LiveRange,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let l = &self.func.layout;
let loc: AnyEntity = match def.into() {
ExpandedProgramPoint::Block(e) => e.into(),
ExpandedProgramPoint::Inst(i) => i.into(),
};
if lr.def() != def {
return errors.fatal((
loc,
format!("Wrong live range def ({}) for {}", lr.def(), val),
));
}
if lr.is_dead() {
if !lr.is_local() {
return errors.fatal((loc, format!("Dead live range {} should be local", val)));
} else {
return Ok(());
}
}
let def_block = match def.into() {
ExpandedProgramPoint::Block(e) => e,
ExpandedProgramPoint::Inst(i) => l.inst_block(i).unwrap(),
};
match lr.def_local_end().into() {
ExpandedProgramPoint::Block(e) => {
return errors.fatal((
loc,
format!("Def local range for {} can't end at {}", val, e),
));
}
ExpandedProgramPoint::Inst(i) => {
if self.func.layout.inst_block(i) != Some(def_block) {
return errors
.fatal((loc, format!("Def local end for {} in wrong block", val)));
}
}
}
// Now check the live-in intervals against the CFG.
for (mut block, end) in lr.liveins() {
if !l.is_block_inserted(block) {
return errors.fatal((
loc,
format!("{} livein at {} which is not in the layout", val, block),
));
}
let end_block = match l.inst_block(end) {
Some(e) => e,
None => {
return errors.fatal((
loc,
format!(
"{} livein for {} ends at {} which is not in the layout",
val, block, end
),
));
}
};
// Check all the blocks in the interval independently.
loop {
// If `val` is live-in at `block`, it must be live at all the predecessors.
for BlockPredecessor { inst: pred, block } in self.cfg.pred_iter(block) {
if !lr.reaches_use(pred, block, &self.func.layout) {
return errors.fatal((
pred,
format!(
"{} is live in to {} but not live at predecessor",
val, block
),
));
}
}
if block == end_block {
break;
}
block = match l.next_block(block) {
Some(e) => e,
None => {
return errors.fatal((
loc,
format!("end of {} livein ({}) never reached", val, end_block),
));
}
};
}
}
Ok(())
}
}

399
cranelift/codegen/src/verifier/locations.rs
View File

@@ -1,399 +0,0 @@
//! Verify value locations.
use crate::flowgraph::ControlFlowGraph;
use crate::ir;
use crate::isa;
use crate::regalloc::liveness::Liveness;
use crate::regalloc::RegDiversions;
use crate::timing;
use crate::verifier::{VerifierErrors, VerifierStepResult};
/// Verify value locations for `func`.
///
/// After register allocation, every value must be assigned to a location - either a register or a
/// stack slot. These locations must be compatible with the constraints described by the
/// instruction encoding recipes.
///
/// Values can be temporarily diverted to a different location by using the `regmove`, `regspill`,
/// and `regfill` instructions, but only inside a block.
///
/// If a liveness analysis is provided, it is used to verify that there are no active register
/// diversions across control flow edges.
pub fn verify_locations(
isa: &dyn isa::TargetIsa,
func: &ir::Function,
cfg: &ControlFlowGraph,
liveness: Option<&Liveness>,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let _tt = timing::verify_locations();
let verifier = LocationVerifier {
isa,
func,
reginfo: isa.register_info(),
encinfo: isa.encoding_info(),
cfg,
liveness,
};
verifier.check_constraints(errors)?;
Ok(())
}
struct LocationVerifier<'a> {
isa: &'a dyn isa::TargetIsa,
func: &'a ir::Function,
reginfo: isa::RegInfo,
encinfo: isa::EncInfo,
cfg: &'a ControlFlowGraph,
liveness: Option<&'a Liveness>,
}
impl<'a> LocationVerifier<'a> {
/// Check that the assigned value locations match the operand constraints of their uses.
fn check_constraints(&self, errors: &mut VerifierErrors) -> VerifierStepResult<()> {
let dfg = &self.func.dfg;
let mut divert = RegDiversions::new();
for block in self.func.layout.blocks() {
divert.at_block(&self.func.entry_diversions, block);
let mut is_after_branch = false;
for inst in self.func.layout.block_insts(block) {
let enc = self.func.encodings[inst];
if enc.is_legal() {
self.check_enc_constraints(inst, enc, &divert, errors)?
} else {
self.check_ghost_results(inst, errors)?;
}
if let Some(sig) = dfg.call_signature(inst) {
self.check_call_abi(inst, sig, &divert, errors)?;
}
let opcode = dfg[inst].opcode();
if opcode.is_return() {
self.check_return_abi(inst, &divert, errors)?;
} else if opcode.is_branch() && !divert.is_empty() {
self.check_cfg_edges(inst, &mut divert, is_after_branch, errors)?;
}
self.update_diversions(inst, &mut divert, errors)?;
is_after_branch = opcode.is_branch();
}
}
Ok(())
}
/// Check encoding constraints against the current value locations.
fn check_enc_constraints(
&self,
inst: ir::Inst,
enc: isa::Encoding,
divert: &RegDiversions,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let constraints = self
.encinfo
.operand_constraints(enc)
.expect("check_enc_constraints requires a legal encoding");
if constraints.satisfied(inst, divert, self.func) {
return Ok(());
}
// TODO: We could give a better error message here.
errors.fatal((
inst,
format!(
"{} constraints not satisfied in: {}\n{}",
self.encinfo.display(enc),
self.func.dfg.display_inst(inst, self.isa),
self.func.display(self.isa),
),
))
}
/// Check that the result values produced by a ghost instruction are not assigned a value
/// location.
fn check_ghost_results(
&self,
inst: ir::Inst,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let results = self.func.dfg.inst_results(inst);
for &res in results {
let loc = self.func.locations[res];
if loc.is_assigned() {
return errors.fatal((
inst,
format!(
"ghost result {} value must not have a location ({}).",
res,
loc.display(&self.reginfo)
),
));
}
}
Ok(())
}
/// Check the ABI argument and result locations for a call.
fn check_call_abi(
&self,
inst: ir::Inst,
sig: ir::SigRef,
divert: &RegDiversions,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let sig = &self.func.dfg.signatures[sig];
let varargs = self.func.dfg.inst_variable_args(inst);
let results = self.func.dfg.inst_results(inst);
for (abi, &value) in sig.params.iter().zip(varargs) {
self.check_abi_location(
inst,
value,
abi,
divert.get(value, &self.func.locations),
ir::StackSlotKind::OutgoingArg,
errors,
)?;
}
for (abi, &value) in sig.returns.iter().zip(results) {
self.check_abi_location(
inst,
value,
abi,
self.func.locations[value],
ir::StackSlotKind::OutgoingArg,
errors,
)?;
}
Ok(())
}
/// Check the ABI argument locations for a return.
fn check_return_abi(
&self,
inst: ir::Inst,
divert: &RegDiversions,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let sig = &self.func.signature;
let varargs = self.func.dfg.inst_variable_args(inst);
for (abi, &value) in sig.returns.iter().zip(varargs) {
self.check_abi_location(
inst,
value,
abi,
divert.get(value, &self.func.locations),
ir::StackSlotKind::IncomingArg,
errors,
)?;
}
Ok(())
}
/// Check a single ABI location.
fn check_abi_location(
&self,
inst: ir::Inst,
value: ir::Value,
abi: &ir::AbiParam,
loc: ir::ValueLoc,
want_kind: ir::StackSlotKind,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
match abi.location {
ir::ArgumentLoc::Unassigned => {}
ir::ArgumentLoc::Reg(reg) => {
if loc != ir::ValueLoc::Reg(reg) {
return errors.fatal((
inst,
format!(
"ABI expects {} in {}, got {}",
value,
abi.location.display(&self.reginfo),
loc.display(&self.reginfo),
),
));
}
}
ir::ArgumentLoc::Stack(offset) => {
if let ir::ValueLoc::Stack(ss) = loc {
let slot = &self.func.stack_slots[ss];
if slot.kind != want_kind {
return errors.fatal((
inst,
format!(
"call argument {} should be in a {} slot, but {} is {}",
value, want_kind, ss, slot.kind
),
));
}
if slot.offset.unwrap() != offset {
return errors.fatal((
inst,
format!(
"ABI expects {} at stack offset {}, but {} is at {}",
value,
offset,
ss,
slot.offset.unwrap()
),
));
}
} else {
return errors.fatal((
inst,
format!(
"ABI expects {} at stack offset {}, got {}",
value,
offset,
loc.display(&self.reginfo)
),
));
}
}
}
Ok(())
}
/// Update diversions to reflect the current instruction and check their consistency.
fn update_diversions(
&self,
inst: ir::Inst,
divert: &mut RegDiversions,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
let (arg, src) = match self.func.dfg[inst] {
ir::InstructionData::RegMove { arg, src, .. }
| ir::InstructionData::RegSpill { arg, src, .. } => (arg, ir::ValueLoc::Reg(src)),
ir::InstructionData::RegFill { arg, src, .. } => (arg, ir::ValueLoc::Stack(src)),
_ => return Ok(()),
};
if let Some(d) = divert.diversion(arg) {
if d.to != src {
return errors.fatal((
inst,
format!(
"inconsistent with current diversion to {}",
d.to.display(&self.reginfo)
),
));
}
} else if self.func.locations[arg] != src {
return errors.fatal((
inst,
format!(
"inconsistent with global location {} ({})",
self.func.locations[arg].display(&self.reginfo),
self.func.dfg.display_inst(inst, None)
),
));
}
divert.apply(&self.func.dfg[inst]);
Ok(())
}
/// We have active diversions before a branch. Make sure none of the diverted values are live
/// on the outgoing CFG edges.
fn check_cfg_edges(
&self,
inst: ir::Inst,
divert: &mut RegDiversions,
is_after_branch: bool,
errors: &mut VerifierErrors,
) -> VerifierStepResult<()> {
use crate::ir::instructions::BranchInfo::*;
let dfg = &self.func.dfg;
let branch_kind = dfg.analyze_branch(inst);
// We can only check CFG edges if we have a liveness analysis.
let liveness = match self.liveness {
Some(l) => l,
None => return Ok(()),
};
match branch_kind {
NotABranch => panic!(
"No branch information for {}",
dfg.display_inst(inst, self.isa)
),
SingleDest(block, _) => {
let unique_predecessor = self.cfg.pred_iter(block).count() == 1;
let mut val_to_remove = vec![];
for (&value, d) in divert.iter() {
let lr = &liveness[value];
if is_after_branch && unique_predecessor {
// Forward diversions based on the targeted branch.
if !lr.is_livein(block, &self.func.layout) {
val_to_remove.push(value)
}
} else if lr.is_livein(block, &self.func.layout) {
return errors.fatal((
inst,
format!(
"SingleDest: {} is diverted to {} and live in to {}",
value,
d.to.display(&self.reginfo),
block,
),
));
}
}
if is_after_branch && unique_predecessor {
for val in val_to_remove.into_iter() {
divert.remove(val);
}
debug_assert!(divert.check_block_entry(&self.func.entry_diversions, block));
}
}
Table(jt, block) => {
for (&value, d) in divert.iter() {
let lr = &liveness[value];
if let Some(block) = block {
if lr.is_livein(block, &self.func.layout) {
return errors.fatal((
inst,
format!(
"Table.default: {} is diverted to {} and live in to {}",
value,
d.to.display(&self.reginfo),
block,
),
));
}
}
for block in self.func.jump_tables[jt].iter() {
if lr.is_livein(*block, &self.func.layout) {
return errors.fatal((
inst,
format!(
"Table.case: {} is diverted to {} and live in to {}",
value,
d.to.display(&self.reginfo),
block,
),
));
}
}
}
}
}
Ok(())
}
}

149
cranelift/codegen/src/verifier/mod.rs
View File

@@ -78,16 +78,13 @@ use alloc::collections::BTreeSet;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
use core::cmp::Ordering;
use core::fmt::{self, Display, Formatter, Write};
use core::fmt::{self, Display, Formatter};
pub use self::cssa::verify_cssa;
pub use self::liveness::verify_liveness;
pub use self::locations::verify_locations;
mod cssa;
mod flags;
mod liveness;
mod locations;
mod virtregs;
/// A verifier error.
#[derive(Debug, PartialEq, Eq, Clone)]
@@ -1763,145 +1760,6 @@ impl<'a> Verifier<'a> {
errors.as_result()
}
/// If the verifier has been set up with an ISA, make sure that the recorded encoding for the
/// instruction (if any) matches how the ISA would encode it.
fn verify_encoding(&self, inst: Inst, errors: &mut VerifierErrors) -> VerifierStepResult<()> {
// When the encodings table is empty, we don't require any instructions to be encoded.
//
// Once some instructions are encoded, we require all side-effecting instructions to have a
// legal encoding.
if self.func.encodings.is_empty() {
return Ok(());
}
let isa = match self.isa {
Some(isa) => isa,
None => return Ok(()),
};
let encoding = self.func.encodings[inst];
if encoding.is_legal() {
if self.func.dfg[inst].opcode().is_ghost() {
return errors.nonfatal((
inst,
self.context(inst),
format!(
"Ghost instruction has an encoding: {}",
isa.encoding_info().display(encoding),
),
));
}
let mut encodings = isa
.legal_encodings(
&self.func,
&self.func.dfg[inst],
self.func.dfg.ctrl_typevar(inst),
)
.peekable();
if encodings.peek().is_none() {
return errors.nonfatal((
inst,
self.context(inst),
format!(
"Instruction failed to re-encode {}",
isa.encoding_info().display(encoding),
),
));
}
let has_valid_encoding = encodings.any(|possible_enc| encoding == possible_enc);
if !has_valid_encoding {
let mut possible_encodings = String::new();
let mut multiple_encodings = false;
for enc in isa.legal_encodings(
&self.func,
&self.func.dfg[inst],
self.func.dfg.ctrl_typevar(inst),
) {
if !possible_encodings.is_empty() {
possible_encodings.push_str(", ");
multiple_encodings = true;
}
possible_encodings
.write_fmt(format_args!("{}", isa.encoding_info().display(enc)))
.unwrap();
}
return errors.nonfatal((
inst,
self.context(inst),
format!(
"encoding {} should be {}{}",
isa.encoding_info().display(encoding),
if multiple_encodings { "one of: " } else { "" },
possible_encodings,
),
));
}
return Ok(());
}
// Instruction is not encoded, so it is a ghost instruction.
// Instructions with side effects are not allowed to be ghost instructions.
let opcode = self.func.dfg[inst].opcode();
// The `fallthrough`, `fallthrough_return`, and `safepoint` instructions are not required
// to have an encoding.
if opcode == Opcode::Fallthrough
|| opcode == Opcode::FallthroughReturn
|| opcode == Opcode::Safepoint
{
return Ok(());
}
// Check if this opcode must be encoded.
let mut needs_enc = None;
if opcode.is_branch() {
needs_enc = Some("Branch");
} else if opcode.is_call() {
needs_enc = Some("Call");
} else if opcode.is_return() {
needs_enc = Some("Return");
} else if opcode.can_store() {
needs_enc = Some("Store");
} else if opcode.can_trap() {
needs_enc = Some("Trapping instruction");
} else if opcode.other_side_effects() {
needs_enc = Some("Instruction with side effects");
}
if let Some(text) = needs_enc {
// This instruction needs an encoding, so generate an error.
// Provide the ISA default encoding as a hint.
match self.func.encode(inst, isa) {
Ok(enc) => {
return errors.nonfatal((
inst,
self.context(inst),
format!(
"{} must have an encoding (e.g., {})))",
text,
isa.encoding_info().display(enc),
),
));
}
Err(_) => {
return errors.nonfatal((
inst,
self.context(inst),
format!("{} must have an encoding", text),
))
}
}
}
Ok(())
}
fn immediate_constraints(
&self,
inst: Inst,
@@ -2034,14 +1892,13 @@ impl<'a> Verifier<'a> {
self.instruction_integrity(inst, errors)?;
self.verify_safepoint_unused(inst, errors)?;
self.typecheck(inst, errors)?;
self.verify_encoding(inst, errors)?;
self.immediate_constraints(inst, errors)?;
}
self.encodable_as_bb(block, errors)?;
}
verify_flags(self.func, &self.expected_cfg, self.isa, errors)?;
verify_flags(self.func, &self.expected_cfg, errors)?;
if !errors.is_empty() {
log::warn!(

505
cranelift/codegen/src/verifier/virtregs.rs Normal file
View File

@@ -0,0 +1,505 @@
//! Virtual registers.
//!
//! A virtual register is a set of related SSA values whose live ranges don't interfere. If all the
//! values in a virtual register are assigned to the same location, fewer copies will result in the
//! output.
//!
//! A virtual register is typically built by merging together SSA values that are "phi-related" -
//! that is, one value is passed as a block argument to a branch and the other is the block parameter
//! value itself.
//!
//! If any values in a virtual register are spilled, they will use the same stack slot. This avoids
//! memory-to-memory copies when a spilled value is passed as a block argument.
use crate::dbg::DisplayList;
use crate::dominator_tree::DominatorTreePreorder;
use crate::entity::entity_impl;
use crate::entity::{EntityList, ListPool};
use crate::entity::{Keys, PrimaryMap, SecondaryMap};
use crate::ir::{Function, Value};
use crate::packed_option::PackedOption;
use alloc::vec::Vec;
use core::cmp::Ordering;
use core::fmt;
use core::slice;
use smallvec::SmallVec;
/// A virtual register reference.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct VirtReg(u32);
entity_impl!(VirtReg, "vreg");
type ValueList = EntityList<Value>;
/// Collection of virtual registers.
///
/// Each virtual register is a list of values. Also maintain a map from values to their unique
/// virtual register, if any.
pub struct VirtRegs {
/// Memory pool for the value lists.
pool: ListPool<Value>,
/// The primary table of virtual registers.
vregs: PrimaryMap<VirtReg, ValueList>,
/// Allocated virtual register numbers that are no longer in use.
unused_vregs: Vec<VirtReg>,
/// Each value belongs to at most one virtual register.
value_vregs: SecondaryMap<Value, PackedOption<VirtReg>>,
/// Table used during the union-find phase while `vregs` is empty.
union_find: SecondaryMap<Value, i32>,
/// Values that have been activated in the `union_find` table, but not yet added to any virtual
/// registers by the `finish_union_find()` function.
pending_values: Vec<Value>,
}
impl VirtRegs {
/// Create a new virtual register collection.
pub fn new() -> Self {
Self {
pool: ListPool::new(),
vregs: PrimaryMap::new(),
unused_vregs: Vec::new(),
value_vregs: SecondaryMap::new(),
union_find: SecondaryMap::new(),
pending_values: Vec::new(),
}
}
/// Clear all virtual registers.
pub fn clear(&mut self) {
self.vregs.clear();
self.unused_vregs.clear();
self.value_vregs.clear();
self.pool.clear();
self.union_find.clear();
self.pending_values.clear();
}
/// Get the virtual register containing `value`, if any.
pub fn get(&self, value: Value) -> Option<VirtReg> {
self.value_vregs[value].into()
}
/// Get the list of values in `vreg`.
pub fn values(&self, vreg: VirtReg) -> &[Value] {
self.vregs[vreg].as_slice(&self.pool)
}
/// Get an iterator over all virtual registers.
pub fn all_virtregs(&self) -> Keys<VirtReg> {
self.vregs.keys()
}
/// Get the congruence class of `value`.
///
/// If `value` belongs to a virtual register, the congruence class is the values of the virtual
/// register. Otherwise it is just the value itself.
#[cfg_attr(feature = "cargo-clippy", allow(clippy::trivially_copy_pass_by_ref))]
pub fn congruence_class<'a, 'b>(&'a self, value: &'b Value) -> &'b [Value]
where
'a: 'b,
{
self.get(*value)
.map_or_else(|| slice::from_ref(value), |vr| self.values(vr))
}
/// Check if `a` and `b` belong to the same congruence class.
pub fn same_class(&self, a: Value, b: Value) -> bool {
match (self.get(a), self.get(b)) {
(Some(va), Some(vb)) => va == vb,
_ => a == b,
}
}
/// Sort the values in `vreg` according to the dominator tree pre-order.
///
/// Returns the slice of sorted values which `values(vreg)` will also return from now on.
pub fn sort_values(
&mut self,
vreg: VirtReg,
func: &Function,
preorder: &DominatorTreePreorder,
) -> &[Value] {
let s = self.vregs[vreg].as_mut_slice(&mut self.pool);
s.sort_unstable_by(|&a, &b| preorder.pre_cmp_def(a, b, func));
s
}
/// Insert a single value into a sorted virtual register.
///
/// It is assumed that the virtual register containing `big` is already sorted by
/// `sort_values()`, and that `single` does not already belong to a virtual register.
///
/// If `big` is not part of a virtual register, one will be created.
pub fn insert_single(
&mut self,
big: Value,
single: Value,
func: &Function,
preorder: &DominatorTreePreorder,
) -> VirtReg {
debug_assert_eq!(self.get(single), None, "Expected singleton {}", single);
// Make sure `big` has a vreg.
let vreg = self.get(big).unwrap_or_else(|| {
let vr = self.alloc();
self.vregs[vr].push(big, &mut self.pool);
self.value_vregs[big] = vr.into();
vr
});
// Determine the insertion position for `single`.
let index = match self
.values(vreg)
.binary_search_by(|&v| preorder.pre_cmp_def(v, single, func))
{
Ok(_) => panic!("{} already in {}", single, vreg),
Err(i) => i,
};
self.vregs[vreg].insert(index, single, &mut self.pool);
self.value_vregs[single] = vreg.into();
vreg
}
/// Remove a virtual register.
///
/// The values in `vreg` become singletons, and the virtual register number may be reused in
/// the future.
pub fn remove(&mut self, vreg: VirtReg) {
// Start by reassigning all the values.
for &v in self.vregs[vreg].as_slice(&self.pool) {
let old = self.value_vregs[v].take();
debug_assert_eq!(old, Some(vreg));
}
self.vregs[vreg].clear(&mut self.pool);
self.unused_vregs.push(vreg);
}
/// Allocate a new empty virtual register.
fn alloc(&mut self) -> VirtReg {
self.unused_vregs
.pop()
.unwrap_or_else(|| self.vregs.push(Default::default()))
}
/// Unify `values` into a single virtual register.
///
/// The values in the slice can be singletons or they can belong to a virtual register already.
/// If a value belongs to a virtual register, all of the values in that register must be
/// present.
///
/// The values are assumed to already be in topological order.
pub fn unify(&mut self, values: &[Value]) -> VirtReg {
// Start by clearing all virtual registers involved.
let mut singletons = 0;
let mut cleared = 0;
for &val in values {
match self.get(val) {
None => singletons += 1,
Some(vreg) => {
if !self.vregs[vreg].is_empty() {
cleared += self.vregs[vreg].len(&self.pool);
self.vregs[vreg].clear(&mut self.pool);
self.unused_vregs.push(vreg);
}
}
}
}
debug_assert_eq!(
values.len(),
singletons + cleared,
"Can't unify partial virtual registers"
);
let vreg = self.alloc();
self.vregs[vreg].extend(values.iter().cloned(), &mut self.pool);
for &v in values {
self.value_vregs[v] = vreg.into();
}
vreg
}
}
impl fmt::Display for VirtRegs {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
for vreg in self.all_virtregs() {
write!(f, "\n{} = {}", vreg, DisplayList(self.values(vreg)))?;
}
Ok(())
}
}
/// Expanded version of a union-find table entry.
enum UFEntry {
/// This value is a a set leader. The embedded number is the set's rank.
Rank(u32),
/// This value belongs to the same set as the linked value.
Link(Value),
}
/// The `union_find` table contains `i32` entries that are interpreted as follows:
///
/// x = 0: The value belongs to its own singleton set.
/// x > 0: The value is the leader of a set with rank x.
/// x < 0: The value belongs to the same set as the value numbered !x.
///
/// The rank of a set is an upper bound on the number of links that must be followed from a member
/// of the set to the set leader.
///
/// A singleton set is the same as a set with rank 0. It contains only the leader value.
impl UFEntry {
/// Decode a table entry.
fn decode(x: i32) -> Self {
if x < 0 {
Self::Link(Value::from_u32((!x) as u32))
} else {
Self::Rank(x as u32)
}
}
/// Encode a link entry.
fn encode_link(v: Value) -> i32 {
!(v.as_u32() as i32)
}
}
/// Union-find algorithm for building virtual registers.
///
/// Before values are added to virtual registers, it is possible to use a union-find algorithm to
/// construct virtual registers efficiently. This support implemented here is used as follows:
///
/// 1. Repeatedly call the `union(a, b)` method to request that `a` and `b` are placed in the same
/// virtual register.
/// 2. When done, call `finish_union_find()` to construct the virtual register sets based on the
/// `union()` calls.
///
/// The values that were passed to `union(a, b)` must not belong to any existing virtual registers
/// by the time `finish_union_find()` is called.
///
/// For more information on the algorithm implemented here, see Chapter 21 "Data Structures for
/// Disjoint Sets" of Cormen, Leiserson, Rivest, Stein, "Introduction to algorithms", 3rd Ed.
///
/// The [Wikipedia entry on disjoint-set data
/// structures](https://en.wikipedia.org/wiki/Disjoint-set_data_structure) is also good.
impl VirtRegs {
/// Find the leader value and rank of the set containing `v`.
/// Compress the path if needed.
fn find(&mut self, mut val: Value) -> (Value, u32) {
let mut val_stack = SmallVec::<[Value; 8]>::new();
let found = loop {
match UFEntry::decode(self.union_find[val]) {
UFEntry::Rank(rank) => break (val, rank),
UFEntry::Link(parent) => {
val_stack.push(val);
val = parent;
}
}
};
// Compress the path
while let Some(val) = val_stack.pop() {
self.union_find[val] = UFEntry::encode_link(found.0);
}
found
}
/// Union the two sets containing `a` and `b`.
///
/// This ensures that `a` and `b` will belong to the same virtual register after calling
/// `finish_union_find()`.
pub fn union(&mut self, a: Value, b: Value) {
let (leader_a, rank_a) = self.find(a);
let (leader_b, rank_b) = self.find(b);
if leader_a == leader_b {
return;
}
// The first time we see a value, its rank will be 0. Add it to the list of pending values.
if rank_a == 0 {
debug_assert_eq!(a, leader_a);
self.pending_values.push(a);
}
if rank_b == 0 {
debug_assert_eq!(b, leader_b);
self.pending_values.push(b);
}
// Merge into the set with the greater rank. This preserves the invariant that the rank is
// an upper bound on the number of links to the leader.
match rank_a.cmp(&rank_b) {
Ordering::Less => {
self.union_find[leader_a] = UFEntry::encode_link(leader_b);
}
Ordering::Greater => {
self.union_find[leader_b] = UFEntry::encode_link(leader_a);
}
Ordering::Equal => {
// When the two sets have the same rank, we arbitrarily pick the a-set to preserve.
// We need to increase the rank by one since the elements in the b-set are now one
// link further away from the leader.
self.union_find[leader_a] += 1;
self.union_find[leader_b] = UFEntry::encode_link(leader_a);
}
}
}
/// Compute virtual registers based on previous calls to `union(a, b)`.
///
/// This terminates the union-find algorithm, so the next time `union()` is called, it is for a
/// new independent batch of values.
///
/// The values in each virtual register will be ordered according to when they were first
/// passed to `union()`, but backwards. It is expected that `sort_values()` will be used to
/// create a more sensible value order.
///
/// The new virtual registers will be appended to `new_vregs`, if present.
pub fn finish_union_find(&mut self, mut new_vregs: Option<&mut Vec<VirtReg>>) {
debug_assert_eq!(
self.pending_values.iter().find(|&&v| self.get(v).is_some()),
None,
"Values participating in union-find must not belong to existing virtual registers"
);
while let Some(val) = self.pending_values.pop() {
let (leader, _) = self.find(val);
// Get the vreg for `leader`, or create it.
let vreg = self.get(leader).unwrap_or_else(|| {
// Allocate a vreg for `leader`, but leave it empty.
let vr = self.alloc();
if let Some(ref mut vec) = new_vregs {
vec.push(vr);
}
self.value_vregs[leader] = vr.into();
vr
});
// Push values in `pending_values` order, including when `v == leader`.
self.vregs[vreg].push(val, &mut self.pool);
self.value_vregs[val] = vreg.into();
// Clear the entry in the union-find table. The `find(val)` call may still look at this
// entry in a future iteration, but that it ok. It will return a rank 0 leader that has
// already been assigned to the correct virtual register.
self.union_find[val] = 0;
}
// We do *not* call `union_find.clear()` table here because re-initializing the table for
// sparse use takes time linear in the number of values in the function. Instead we reset
// the entries that are known to be non-zero in the loop above.
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::entity::EntityRef;
use crate::ir::Value;
#[test]
fn empty_union_find() {
let mut vregs = VirtRegs::new();
vregs.finish_union_find(None);
assert_eq!(vregs.all_virtregs().count(), 0);
}
#[test]
fn union_self() {
let mut vregs = VirtRegs::new();
let v1 = Value::new(1);
vregs.union(v1, v1);
vregs.finish_union_find(None);
assert_eq!(vregs.get(v1), None);
assert_eq!(vregs.all_virtregs().count(), 0);
}
#[test]
fn union_pair() {
let mut vregs = VirtRegs::new();
let v1 = Value::new(1);
let v2 = Value::new(2);
vregs.union(v1, v2);
vregs.finish_union_find(None);
assert_eq!(vregs.congruence_class(&v1), &[v2, v1]);
assert_eq!(vregs.congruence_class(&v2), &[v2, v1]);
assert_eq!(vregs.all_virtregs().count(), 1);
}
#[test]
fn union_pair_backwards() {
let mut vregs = VirtRegs::new();
let v1 = Value::new(1);
let v2 = Value::new(2);
vregs.union(v2, v1);
vregs.finish_union_find(None);
assert_eq!(vregs.congruence_class(&v1), &[v1, v2]);
assert_eq!(vregs.congruence_class(&v2), &[v1, v2]);
assert_eq!(vregs.all_virtregs().count(), 1);
}
#[test]
fn union_tree() {
let mut vregs = VirtRegs::new();
let v1 = Value::new(1);
let v2 = Value::new(2);
let v3 = Value::new(3);
let v4 = Value::new(4);
vregs.union(v2, v4);
vregs.union(v3, v1);
// Leaders: v2, v3
vregs.union(v4, v1);
vregs.finish_union_find(None);
assert_eq!(vregs.congruence_class(&v1), &[v1, v3, v4, v2]);
assert_eq!(vregs.congruence_class(&v2), &[v1, v3, v4, v2]);
assert_eq!(vregs.congruence_class(&v3), &[v1, v3, v4, v2]);
assert_eq!(vregs.congruence_class(&v4), &[v1, v3, v4, v2]);
assert_eq!(vregs.all_virtregs().count(), 1);
}
#[test]
fn union_two() {
let mut vregs = VirtRegs::new();
let v1 = Value::new(1);
let v2 = Value::new(2);
let v3 = Value::new(3);
let v4 = Value::new(4);
vregs.union(v2, v4);
vregs.union(v3, v1);
// Leaders: v2, v3
vregs.finish_union_find(None);
assert_eq!(vregs.congruence_class(&v1), &[v1, v3]);
assert_eq!(vregs.congruence_class(&v2), &[v4, v2]);
assert_eq!(vregs.congruence_class(&v3), &[v1, v3]);
assert_eq!(vregs.congruence_class(&v4), &[v4, v2]);
assert_eq!(vregs.all_virtregs().count(), 2);
}
#[test]
fn union_uneven() {
let mut vregs = VirtRegs::new();
let v1 = Value::new(1);
let v2 = Value::new(2);
let v3 = Value::new(3);
let v4 = Value::new(4);
vregs.union(v2, v4); // Rank 0-0
vregs.union(v3, v2); // Rank 0-1
vregs.union(v2, v1); // Rank 1-0
vregs.finish_union_find(None);
assert_eq!(vregs.congruence_class(&v1), &[v1, v3, v4, v2]);
assert_eq!(vregs.congruence_class(&v2), &[v1, v3, v4, v2]);
assert_eq!(vregs.congruence_class(&v3), &[v1, v3, v4, v2]);
assert_eq!(vregs.congruence_class(&v4), &[v1, v3, v4, v2]);
assert_eq!(vregs.all_virtregs().count(), 1);
}
}