T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update rustfmt to 0.9.0.

Browse Source
This commit is contained in:
Dan Gohman
2017-08-31 10:44:59 -07:00
parent 46fb64cbb4
commit 2efdc0ed37
111 changed files with 4692 additions and 3379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/cretonne/src/regalloc/affinity.rs
View File

@@ -91,7 +91,8 @@ impl Affinity {
// If the preferred register class is a subclass of the constraint, there's no need
// to change anything.
if constraint.kind != ConstraintKind::Stack &&
!constraint.regclass.has_subclass(rc) {
!constraint.regclass.has_subclass(rc)
{
// If the register classes don't overlap, `intersect` returns `None`, and we
// just keep our previous affinity.
if let Some(subclass) = constraint.regclass.intersect(reg_info.rc(rc)) {

7
lib/cretonne/src/regalloc/allocatable_set.rs
View File

@@ -86,10 +86,9 @@ impl AllocatableSet {
///
/// This assumes that unused bits are 1.
pub fn interferes_with(&self, other: &AllocatableSet) -> bool {
self.avail
.iter()
.zip(&other.avail)
.any(|(&x, &y)| (x | y) != !0)
self.avail.iter().zip(&other.avail).any(
|(&x, &y)| (x | y) != !0,
)
}
/// Intersect this set of allocatable registers with `other`. This has the effect of removing

234
lib/cretonne/src/regalloc/coalescing.rs
View File

@@ -132,14 +132,15 @@ impl DomForest {
///
/// If the merge succeeds, returns `Ok(())`. The merged sequence can be extracted with
/// `swap()`.
pub fn try_merge(&mut self,
va: &[Value],
vb: &[Value],
dfg: &DataFlowGraph,
layout: &Layout,
domtree: &DominatorTree,
liveness: &Liveness)
-> Result<(), (Value, Value)> {
pub fn try_merge(
&mut self,
va: &[Value],
vb: &[Value],
dfg: &DataFlowGraph,
layout: &Layout,
domtree: &DominatorTree,
liveness: &Liveness,
) -> Result<(), (Value, Value)> {
self.stack.clear();
self.values.clear();
self.values.reserve(va.len() + vb.len());
@@ -154,16 +155,16 @@ impl DomForest {
for node in merged {
if let Some(parent) = self.push_node(node, layout, domtree) {
// Check if `parent` live range contains `node.def`.
let lr = liveness
.get(parent)
.expect("No live range for parent value");
let lr = liveness.get(parent).expect(
"No live range for parent value",
);
if lr.overlaps_def(node.def, layout.pp_ebb(node.def), layout) {
// Interference detected. Get the `(a, b)` order right in the error.
return Err(if node.set == 0 {
(node.value, parent)
} else {
(parent, node.value)
});
(node.value, parent)
} else {
(parent, node.value)
});
}
}
}
@@ -177,8 +178,9 @@ impl DomForest {
/// Given two ordered sequences of nodes, yield an ordered sequence containing all of them.
/// Duplicates are removed.
struct MergedNodes<'a, IA, IB>
where IA: Iterator<Item = Node>,
IB: Iterator<Item = Node>
where
IA: Iterator<Item = Node>,
IB: Iterator<Item = Node>,
{
a: Peekable<IA>,
b: Peekable<IB>,
@@ -187,8 +189,9 @@ struct MergedNodes<'a, IA, IB>
}
impl<'a, IA, IB> Iterator for MergedNodes<'a, IA, IB>
where IA: Iterator<Item = Node>,
IB: Iterator<Item = Node>
where
IA: Iterator<Item = Node>,
IB: Iterator<Item = Node>,
{
type Item = Node;
@@ -198,9 +201,12 @@ impl<'a, IA, IB> Iterator for MergedNodes<'a, IA, IB>
// If the two values are defined at the same point, compare value numbers instead
// this is going to cause an interference conflict unless its actually the same
// value appearing in both streams.
self.domtree
.rpo_cmp(a.def, b.def, self.layout)
.then(Ord::cmp(&a.value, &b.value))
self.domtree.rpo_cmp(a.def, b.def, self.layout).then(
Ord::cmp(
&a.value,
&b.value,
),
)
}
(Some(_), None) => Ordering::Less,
(None, Some(_)) => Ordering::Greater,
@@ -256,13 +262,15 @@ impl Coalescing {
}
/// Convert `func` to conventional SSA form and build virtual registers in the process.
pub fn conventional_ssa(&mut self,
isa: &TargetIsa,
func: &mut Function,
cfg: &ControlFlowGraph,
domtree: &DominatorTree,
liveness: &mut Liveness,
virtregs: &mut VirtRegs) {
pub fn conventional_ssa(
&mut self,
isa: &TargetIsa,
func: &mut Function,
cfg: &ControlFlowGraph,
domtree: &DominatorTree,
liveness: &mut Liveness,
virtregs: &mut VirtRegs,
) {
dbg!("Coalescing for:\n{}", func.display(isa));
let mut context = Context {
isa,
@@ -329,9 +337,11 @@ impl<'a> Context<'a> {
//
// Try to catch infinite splitting loops. The values created by splitting should never
// have irreconcilable interferences.
assert!(!self.split_values.contains(&bad_value),
"{} was already isolated",
bad_value);
assert!(
!self.split_values.contains(&bad_value),
"{} was already isolated",
bad_value
);
let split_len = self.split_values.len();
// The bad value can be both the successor value and a predecessor value at the same
@@ -349,18 +359,22 @@ impl<'a> Context<'a> {
}
// Second loop check.
assert_ne!(split_len,
self.split_values.len(),
"Couldn't isolate {}",
bad_value);
assert_ne!(
split_len,
self.split_values.len(),
"Couldn't isolate {}",
bad_value
);
}
let vreg = self.virtregs.unify(self.values);
dbg!("Coalesced {} arg {} into {} = {}",
ebb,
argnum,
vreg,
DisplayList(self.virtregs.values(vreg)));
dbg!(
"Coalesced {} arg {} into {} = {}",
ebb,
argnum,
vreg,
DisplayList(self.virtregs.values(vreg))
);
}
/// Reset `self.values` to just the set of split values.
@@ -369,21 +383,21 @@ impl<'a> Context<'a> {
self.values.extend_from_slice(self.split_values);
let domtree = &self.domtree;
let func = &self.func;
self.values
.sort_by(|&a, &b| {
domtree.rpo_cmp(func.dfg.value_def(a), func.dfg.value_def(b), &func.layout)
});
self.values.sort_by(|&a, &b| {
domtree.rpo_cmp(func.dfg.value_def(a), func.dfg.value_def(b), &func.layout)
});
}
/// Try coalescing predecessors with `succ_val`.
///
/// Returns a value from a congruence class that needs to be split before starting over, or
/// `None` if everything was successfully coalesced into `self.values`.
fn try_coalesce(&mut self,
argnum: usize,
succ_val: Value,
preds: &[BasicBlock])
-> Option<Value> {
fn try_coalesce(
&mut self,
argnum: usize,
succ_val: Value,
preds: &[BasicBlock],
) -> Option<Value> {
// Initialize the value list with the split values. These are guaranteed to be
// interference free, and anything that interferes with them must be split away.
self.reset_values();
@@ -397,19 +411,22 @@ impl<'a> Context<'a> {
for &(pred_ebb, pred_inst) in preds {
let pred_val = self.func.dfg.inst_variable_args(pred_inst)[argnum];
dbg!("Checking {}: {}: {}",
pred_val,
pred_ebb,
self.func.dfg.display_inst(pred_inst, self.isa));
dbg!(
"Checking {}: {}: {}",
pred_val,
pred_ebb,
self.func.dfg.display_inst(pred_inst, self.isa)
);
// Never coalesce incoming function arguments on the stack. These arguments are
// pre-spilled, and the rest of the virtual register would be forced to spill to the
// `incoming_arg` stack slot too.
if let ValueDef::Arg(def_ebb, def_num) = self.func.dfg.value_def(pred_val) {
if Some(def_ebb) == self.func.layout.entry_block() &&
self.func.signature.argument_types[def_num]
.location
.is_stack() {
self.func.signature.argument_types[def_num]
.location
.is_stack()
{
dbg!("Isolating incoming stack parameter {}", pred_val);
let new_val = self.split_pred(pred_inst, pred_ebb, argnum, pred_val);
assert!(self.add_class(new_val).is_ok());
@@ -424,9 +441,10 @@ impl<'a> Context<'a> {
//
// Check if the `a` live range is fundamentally incompatible with `pred_inst`.
if self.liveness
.get(a)
.expect("No live range for interfering value")
.reaches_use(pred_inst, pred_ebb, &self.func.layout) {
.get(a)
.expect("No live range for interfering value")
.reaches_use(pred_inst, pred_ebb, &self.func.layout)
{
// Splitting at `pred_inst` wouldn't resolve the interference, so we need to
// start over.
return Some(a);
@@ -435,8 +453,10 @@ impl<'a> Context<'a> {
// The local conflict could be avoided by splitting at this predecessor, so try
// that. This split is not necessarily required, but it allows us to make progress.
let new_val = self.split_pred(pred_inst, pred_ebb, argnum, pred_val);
assert!(self.add_class(new_val).is_ok(),
"Splitting didn't resolve conflict.");
assert!(
self.add_class(new_val).is_ok(),
"Splitting didn't resolve conflict."
);
}
}
@@ -447,42 +467,52 @@ impl<'a> Context<'a> {
///
/// Leave `self.values` unchanged on failure.
fn add_class(&mut self, value: Value) -> Result<(), (Value, Value)> {
self.forest
.try_merge(&self.values,
self.virtregs.congruence_class(&value),
&self.func.dfg,
&self.func.layout,
self.domtree,
self.liveness)?;
self.forest.try_merge(
&self.values,
self.virtregs.congruence_class(&value),
&self.func.dfg,
&self.func.layout,
self.domtree,
self.liveness,
)?;
self.forest.swap(&mut self.values);
Ok(())
}
/// Split the congruence class for the `argnum` argument to `pred_inst` by inserting a copy.
fn split_pred(&mut self,
pred_inst: Inst,
pred_ebb: Ebb,
argnum: usize,
pred_val: Value)
-> Value {
fn split_pred(
&mut self,
pred_inst: Inst,
pred_ebb: Ebb,
argnum: usize,
pred_val: Value,
) -> Value {
let mut pos = EncCursor::new(self.func, self.isa).at_inst(pred_inst);
let copy = pos.ins().copy(pred_val);
let inst = pos.built_inst();
dbg!("Inserted {}, before {}: {}",
pos.display_inst(inst),
pred_ebb,
pos.display_inst(pred_inst));
dbg!(
"Inserted {}, before {}: {}",
pos.display_inst(inst),
pred_ebb,
pos.display_inst(pred_inst)
);
// Create a live range for the new value.
let affinity = Affinity::new(&self.encinfo
.operand_constraints(pos.func.encodings[inst])
.expect("Bad copy encoding")
.outs
[0]);
let affinity = Affinity::new(
&self.encinfo
.operand_constraints(pos.func.encodings[inst])
.expect("Bad copy encoding")
.outs
[0],
);
self.liveness.create_dead(copy, inst, affinity);
self.liveness
.extend_locally(copy, pred_ebb, pred_inst, &pos.func.layout);
self.liveness.extend_locally(
copy,
pred_ebb,
pred_inst,
&pos.func.layout,
);
pos.func.dfg.inst_variable_args_mut(pred_inst)[argnum] = copy;
self.split_values.push(copy);
@@ -500,21 +530,29 @@ impl<'a> Context<'a> {
let inst = pos.built_inst();
self.liveness.move_def_locally(succ_val, inst);
dbg!("Inserted {}, following {}({}: {})",
pos.display_inst(inst),
ebb,
new_val,
ty);
dbg!(
"Inserted {}, following {}({}: {})",
pos.display_inst(inst),
ebb,
new_val,
ty
);
// Create a live range for the new value.
let affinity = Affinity::new(&self.encinfo
.operand_constraints(pos.func.encodings[inst])
.expect("Bad copy encoding")
.outs
[0]);
let affinity = Affinity::new(
&self.encinfo
.operand_constraints(pos.func.encodings[inst])
.expect("Bad copy encoding")
.outs
[0],
);
self.liveness.create_dead(new_val, ebb, affinity);
self.liveness
.extend_locally(new_val, ebb, inst, &pos.func.layout);
self.liveness.extend_locally(
new_val,
ebb,
inst,
&pos.func.layout,
);
self.split_values.push(new_val);
new_val

313
lib/cretonne/src/regalloc/coloring.rs
View File

@@ -105,12 +105,14 @@ impl Coloring {
}
/// Run the coloring algorithm over `func`.
pub fn run(&mut self,
isa: &TargetIsa,
func: &mut Function,
domtree: &DominatorTree,
liveness: &mut Liveness,
tracker: &mut LiveValueTracker) {
pub fn run(
&mut self,
isa: &TargetIsa,
func: &mut Function,
domtree: &DominatorTree,
liveness: &mut Liveness,
tracker: &mut LiveValueTracker,
) {
dbg!("Coloring for:\n{}", func.display(isa));
let mut ctx = Context {
isa,
@@ -150,15 +152,17 @@ impl<'a> Context<'a> {
pos.goto_top(ebb);
while let Some(inst) = pos.next_inst() {
if let Some(constraints) = self.encinfo.operand_constraints(func.encodings[inst]) {
self.visit_inst(inst,
constraints,
&mut pos,
&mut func.dfg,
tracker,
&mut regs,
&mut func.locations,
&mut func.encodings,
&func.signature);
self.visit_inst(
inst,
constraints,
&mut pos,
&mut func.dfg,
tracker,
&mut regs,
&mut func.locations,
&mut func.encodings,
&func.signature,
);
} else {
let (_throughs, kills) = tracker.process_ghost(inst);
self.process_ghost_kills(kills, &mut regs, &func.locations);
@@ -170,11 +174,12 @@ impl<'a> Context<'a> {
/// Visit the `ebb` header.
///
/// Initialize the set of live registers and color the arguments to `ebb`.
fn visit_ebb_header(&self,
ebb: Ebb,
func: &mut Function,
tracker: &mut LiveValueTracker)
-> AllocatableSet {
fn visit_ebb_header(
&self,
ebb: Ebb,
func: &mut Function,
tracker: &mut LiveValueTracker,
) -> AllocatableSet {
// Reposition the live value tracker and deal with the EBB arguments.
tracker.ebb_top(ebb, &func.dfg, self.liveness, &func.layout, self.domtree);
@@ -204,10 +209,12 @@ impl<'a> Context<'a> {
.get(value)
.expect("No live range for live-in")
.affinity;
dbg!("Live-in: {}:{} in {}",
value,
affinity.display(&self.reginfo),
func.locations[value].display(&self.reginfo));
dbg!(
"Live-in: {}:{} in {}",
value,
affinity.display(&self.reginfo),
func.locations[value].display(&self.reginfo)
);
if let Affinity::Reg(rci) = affinity {
let rc = self.reginfo.rc(rci);
let loc = func.locations[value];
@@ -230,11 +237,12 @@ impl<'a> Context<'a> {
/// function signature.
///
/// Return the set of remaining allocatable registers after filtering out the dead arguments.
fn color_entry_args(&self,
sig: &Signature,
args: &[LiveValue],
locations: &mut ValueLocations)
-> AllocatableSet {
fn color_entry_args(
&self,
sig: &Signature,
args: &[LiveValue],
locations: &mut ValueLocations,
) -> AllocatableSet {
assert_eq!(sig.argument_types.len(), args.len());
let mut regs = self.usable_regs.clone();
@@ -250,10 +258,12 @@ impl<'a> Context<'a> {
locations[lv.value] = ValueLoc::Reg(reg);
} else {
// This should have been fixed by the reload pass.
panic!("Entry arg {} has {} affinity, but ABI {}",
lv.value,
lv.affinity.display(&self.reginfo),
abi.display(&self.reginfo));
panic!(
"Entry arg {} has {} affinity, but ABI {}",
lv.value,
lv.affinity.display(&self.reginfo),
abi.display(&self.reginfo)
);
}
}
@@ -273,19 +283,23 @@ impl<'a> Context<'a> {
///
/// Update `regs` to reflect the allocated registers after `inst`, including removing any dead
/// or killed values from the set.
fn visit_inst(&mut self,
inst: Inst,
constraints: &RecipeConstraints,
pos: &mut Cursor,
dfg: &mut DataFlowGraph,
tracker: &mut LiveValueTracker,
regs: &mut AllocatableSet,
locations: &mut ValueLocations,
encodings: &mut InstEncodings,
func_signature: &Signature) {
dbg!("Coloring {}\n {}",
dfg.display_inst(inst, self.isa),
regs.display(&self.reginfo));
fn visit_inst(
&mut self,
inst: Inst,
constraints: &RecipeConstraints,
pos: &mut Cursor,
dfg: &mut DataFlowGraph,
tracker: &mut LiveValueTracker,
regs: &mut AllocatableSet,
locations: &mut ValueLocations,
encodings: &mut InstEncodings,
func_signature: &Signature,
) {
dbg!(
"Coloring {}\n {}",
dfg.display_inst(inst, self.isa),
regs.display(&self.reginfo)
);
// EBB whose arguments should be colored to match the current branch instruction's
// arguments.
@@ -310,10 +324,12 @@ impl<'a> Context<'a> {
} else {
// This is a multi-way branch like `br_table`. We only support arguments on
// single-destination branches.
assert_eq!(dfg.inst_variable_args(inst).len(),
0,
"Can't handle EBB arguments: {}",
dfg.display_inst(inst, self.isa));
assert_eq!(
dfg.inst_variable_args(inst).len(),
0,
"Can't handle EBB arguments: {}",
dfg.display_inst(inst, self.isa)
);
self.undivert_regs(|lr| !lr.is_local());
}
}
@@ -329,10 +345,11 @@ impl<'a> Context<'a> {
// Get rid of the killed values.
for lv in kills {
if let Affinity::Reg(rci) = lv.affinity {
self.solver
.add_kill(lv.value,
self.reginfo.rc(rci),
self.divert.reg(lv.value, locations));
self.solver.add_kill(
lv.value,
self.reginfo.rc(rci),
self.divert.reg(lv.value, locations),
);
}
}
@@ -350,9 +367,9 @@ impl<'a> Context<'a> {
// Finally, we've fully programmed the constraint solver.
// We expect a quick solution in most cases.
let mut output_regs = self.solver
.quick_solve()
.unwrap_or_else(|_| self.iterate_solution());
let mut output_regs = self.solver.quick_solve().unwrap_or_else(
|_| self.iterate_solution(),
);
// The solution and/or fixed input constraints may require us to shuffle the set of live
@@ -399,30 +416,42 @@ impl<'a> Context<'a> {
}
/// Program the input-side constraints for `inst` into the constraint solver.
fn program_input_constraints(&mut self,
inst: Inst,
constraints: &[OperandConstraint],
dfg: &DataFlowGraph,
locations: &ValueLocations) {
for (op, &value) in constraints
.iter()
.zip(dfg.inst_args(inst))
.filter(|&(op, _)| op.kind != ConstraintKind::Stack) {
fn program_input_constraints(
&mut self,
inst: Inst,
constraints: &[OperandConstraint],
dfg: &DataFlowGraph,
locations: &ValueLocations,
) {
for (op, &value) in constraints.iter().zip(dfg.inst_args(inst)).filter(
|&(op, _)| {
op.kind != ConstraintKind::Stack
},
)
{
// Reload pass is supposed to ensure that all arguments to register operands are
// already in a register.
let cur_reg = self.divert.reg(value, locations);
match op.kind {
ConstraintKind::FixedReg(regunit) => {
if regunit != cur_reg {
self.solver
.reassign_in(value, op.regclass, cur_reg, regunit);
self.solver.reassign_in(
value,
op.regclass,
cur_reg,
regunit,
);
}
}
ConstraintKind::Reg |
ConstraintKind::Tied(_) => {
if !op.regclass.contains(cur_reg) {
self.solver
.add_var(value, op.regclass, cur_reg, &self.reginfo);
self.solver.add_var(
value,
op.regclass,
cur_reg,
&self.reginfo,
);
}
}
ConstraintKind::Stack => unreachable!(),
@@ -433,18 +462,21 @@ impl<'a> Context<'a> {
/// Program the input-side ABI constraints for `inst` into the constraint solver.
///
/// ABI constraints are the fixed register assignments used for calls and returns.
fn program_input_abi(&mut self,
inst: Inst,
abi_types: &[ArgumentType],
dfg: &DataFlowGraph,
locations: &ValueLocations) {
fn program_input_abi(
&mut self,
inst: Inst,
abi_types: &[ArgumentType],
dfg: &DataFlowGraph,
locations: &ValueLocations,
) {
for (abi, &value) in abi_types.iter().zip(dfg.inst_variable_args(inst)) {
if let ArgumentLoc::Reg(reg) = abi.location {
if let Affinity::Reg(rci) =
self.liveness
.get(value)
.expect("ABI register must have live range")
.affinity {
.affinity
{
let rc = self.reginfo.rc(rci);
let cur_reg = self.divert.reg(value, locations);
self.solver.reassign_in(value, rc, cur_reg, reg);
@@ -464,13 +496,14 @@ impl<'a> Context<'a> {
///
/// Returns true if this is the first time a branch to `dest` is seen, so the `dest` argument
/// values should be colored after `shuffle_inputs`.
fn program_ebb_arguments(&mut self,
inst: Inst,
dest: Ebb,
dfg: &DataFlowGraph,
layout: &Layout,
locations: &ValueLocations)
-> bool {
fn program_ebb_arguments(
&mut self,
inst: Inst,
dest: Ebb,
dfg: &DataFlowGraph,
layout: &Layout,
locations: &ValueLocations,
) -> bool {
// Find diverted registers that are live-in to `dest` and reassign them to their global
// home.
//
@@ -523,11 +556,13 @@ impl<'a> Context<'a> {
/// register state.
///
/// This function is only called when `program_ebb_arguments()` returned `true`.
fn color_ebb_arguments(&mut self,
inst: Inst,
dest: Ebb,
dfg: &DataFlowGraph,
locations: &mut ValueLocations) {
fn color_ebb_arguments(
&mut self,
inst: Inst,
dest: Ebb,
dfg: &DataFlowGraph,
locations: &mut ValueLocations,
) {
let br_args = dfg.inst_variable_args(inst);
let dest_args = dfg.ebb_args(dest);
assert_eq!(br_args.len(), dest_args.len());
@@ -549,20 +584,23 @@ impl<'a> Context<'a> {
/// Find all diverted registers where `pred` returns `true` and undo their diversion so they
/// are reallocated to their global register assignments.
fn undivert_regs<Pred>(&mut self, mut pred: Pred)
where Pred: FnMut(&LiveRange) -> bool
where
Pred: FnMut(&LiveRange) -> bool,
{
for rdiv in self.divert.all() {
let lr = self.liveness
.get(rdiv.value)
.expect("Missing live range for diverted register");
let lr = self.liveness.get(rdiv.value).expect(
"Missing live range for diverted register",
);
if pred(lr) {
if let Affinity::Reg(rci) = lr.affinity {
let rc = self.reginfo.rc(rci);
self.solver.reassign_in(rdiv.value, rc, rdiv.to, rdiv.from);
} else {
panic!("Diverted register {} with {} affinity",
rdiv.value,
lr.affinity.display(&self.reginfo));
panic!(
"Diverted register {} with {} affinity",
rdiv.value,
lr.affinity.display(&self.reginfo)
);
}
}
}
@@ -570,9 +608,7 @@ impl<'a> Context<'a> {
// Find existing live values that conflict with the fixed input register constraints programmed
// into the constraint solver. Convert them to solver variables so they can be diverted.
fn divert_fixed_input_conflicts(&mut self,
live: &[LiveValue],
locations: &mut ValueLocations) {
fn divert_fixed_input_conflicts(&mut self, live: &[LiveValue], locations: &mut ValueLocations) {
for lv in live {
if let Affinity::Reg(rci) = lv.affinity {
let rc = self.reginfo.rc(rci);
@@ -587,11 +623,13 @@ impl<'a> Context<'a> {
/// Program any fixed-register output constraints into the solver. This may also detect
/// conflicts between live-through registers and fixed output registers. These live-through
/// values need to be turned into solver variables so they can be reassigned.
fn program_fixed_outputs(&mut self,
constraints: &[OperandConstraint],
defs: &[LiveValue],
throughs: &[LiveValue],
locations: &mut ValueLocations) {
fn program_fixed_outputs(
&mut self,
constraints: &[OperandConstraint],
defs: &[LiveValue],
throughs: &[LiveValue],
locations: &mut ValueLocations,
) {
for (op, lv) in constraints.iter().zip(defs) {
if let ConstraintKind::FixedReg(reg) = op.kind {
self.add_fixed_output(lv.value, op.regclass, reg, throughs, locations);
@@ -602,11 +640,13 @@ impl<'a> Context<'a> {
/// Program the output-side ABI constraints for `inst` into the constraint solver.
///
/// That means return values for a call instruction.
fn program_output_abi(&mut self,
abi_types: &[ArgumentType],
defs: &[LiveValue],
throughs: &[LiveValue],
locations: &mut ValueLocations) {
fn program_output_abi(
&mut self,
abi_types: &[ArgumentType],
defs: &[LiveValue],
throughs: &[LiveValue],
locations: &mut ValueLocations,
) {
// It's technically possible for a call instruction to have fixed results before the
// variable list of results, but we have no known instances of that.
// Just assume all results are variable return values.
@@ -624,12 +664,14 @@ impl<'a> Context<'a> {
}
/// Add a single fixed output value to the solver.
fn add_fixed_output(&mut self,
value: Value,
rc: RegClass,
reg: RegUnit,
throughs: &[LiveValue],
locations: &mut ValueLocations) {
fn add_fixed_output(
&mut self,
value: Value,
rc: RegClass,
reg: RegUnit,
throughs: &[LiveValue],
locations: &mut ValueLocations,
) {
if !self.solver.add_fixed_output(rc, reg) {
// The fixed output conflicts with some of the live-through registers.
for lv in throughs {
@@ -656,12 +698,14 @@ impl<'a> Context<'a> {
/// Program the output-side constraints for `inst` into the constraint solver.
///
/// It is assumed that all fixed outputs have already been handled.
fn program_output_constraints(&mut self,
inst: Inst,
constraints: &[OperandConstraint],
defs: &[LiveValue],
dfg: &mut DataFlowGraph,
locations: &mut ValueLocations) {
fn program_output_constraints(
&mut self,
inst: Inst,
constraints: &[OperandConstraint],
defs: &[LiveValue],
dfg: &mut DataFlowGraph,
locations: &mut ValueLocations,
) {
for (op, lv) in constraints.iter().zip(defs) {
match op.kind {
ConstraintKind::FixedReg(_) |
@@ -673,8 +717,11 @@ impl<'a> Context<'a> {
// Find the input operand we're tied to.
// The solver doesn't care about the output value.
let arg = dfg.inst_args(inst)[num as usize];
self.solver
.add_tied_input(arg, op.regclass, self.divert.reg(arg, locations));
self.solver.add_tied_input(
arg,
op.regclass,
self.divert.reg(arg, locations),
);
}
}
}
@@ -695,11 +742,13 @@ impl<'a> Context<'a> {
/// before.
///
/// The solver needs to be reminded of the available registers before any moves are inserted.
fn shuffle_inputs(&mut self,
pos: &mut Cursor,
dfg: &mut DataFlowGraph,
regs: &mut AllocatableSet,
encodings: &mut InstEncodings) {
fn shuffle_inputs(
&mut self,
pos: &mut Cursor,
dfg: &mut DataFlowGraph,
regs: &mut AllocatableSet,
encodings: &mut InstEncodings,
) {
self.solver.schedule_moves(regs);
for m in self.solver.moves() {
@@ -729,10 +778,12 @@ impl<'a> Context<'a> {
/// Process kills on a ghost instruction.
/// - Forget diversions.
/// - Free killed registers.
fn process_ghost_kills(&mut self,
kills: &[LiveValue],
regs: &mut AllocatableSet,
locations: &ValueLocations) {
fn process_ghost_kills(
&mut self,
kills: &[LiveValue],
regs: &mut AllocatableSet,
locations: &ValueLocations,
) {
for lv in kills {
if let Affinity::Reg(rci) = lv.affinity {
let rc = self.reginfo.rc(rci);

69
lib/cretonne/src/regalloc/context.rs
View File

@@ -53,12 +53,13 @@ impl Context {
///
/// After register allocation, all values in `func` have been assigned to a register or stack
/// location that is consistent with instruction encoding constraints.
pub fn run(&mut self,
isa: &TargetIsa,
func: &mut Function,
cfg: &ControlFlowGraph,
domtree: &DominatorTree)
-> CtonResult {
pub fn run(
&mut self,
isa: &TargetIsa,
func: &mut Function,
cfg: &ControlFlowGraph,
domtree: &DominatorTree,
) -> CtonResult {
// `Liveness` and `Coloring` are self-clearing.
self.virtregs.clear();
@@ -74,13 +75,14 @@ impl Context {
}
// Pass: Coalesce and create conventional SSA form.
self.coalescing
.conventional_ssa(isa,
func,
cfg,
domtree,
&mut self.liveness,
&mut self.virtregs);
self.coalescing.conventional_ssa(
isa,
func,
cfg,
domtree,
&mut self.liveness,
&mut self.virtregs,
);
if isa.flags().enable_verifier() {
verify_context(func, cfg, domtree, Some(isa))?;
@@ -90,14 +92,15 @@ impl Context {
// Pass: Spilling.
self.spilling
.run(isa,
func,
domtree,
&mut self.liveness,
&self.virtregs,
&mut self.topo,
&mut self.tracker);
self.spilling.run(
isa,
func,
domtree,
&mut self.liveness,
&self.virtregs,
&mut self.topo,
&mut self.tracker,
);
if isa.flags().enable_verifier() {
verify_context(func, cfg, domtree, Some(isa))?;
@@ -106,13 +109,14 @@ impl Context {
}
// Pass: Reload.
self.reload
.run(isa,
func,
domtree,
&mut self.liveness,
&mut self.topo,
&mut self.tracker);
self.reload.run(
isa,
func,
domtree,
&mut self.liveness,
&mut self.topo,
&mut self.tracker,
);
if isa.flags().enable_verifier() {
verify_context(func, cfg, domtree, Some(isa))?;
@@ -121,8 +125,13 @@ impl Context {
}
// Pass: Coloring.
self.coloring
.run(isa, func, domtree, &mut self.liveness, &mut self.tracker);
self.coloring.run(
isa,
func,
domtree,
&mut self.liveness,
&mut self.tracker,
);
if isa.flags().enable_verifier() {
verify_context(func, cfg, domtree, Some(isa))?;

23
lib/cretonne/src/regalloc/diversion.rs
View File

@@ -93,10 +93,11 @@ impl RegDiversions {
///
/// Returns the `to` register of the removed diversion.
pub fn remove(&mut self, value: Value) -> Option<RegUnit> {
self.current
.iter()
.position(|d| d.value == value)
.map(|i| self.current.swap_remove(i).to)
self.current.iter().position(|d| d.value == value).map(
|i| {
self.current.swap_remove(i).to
},
)
}
}
@@ -113,12 +114,14 @@ mod tests {
let v2 = Value::new(2);
divs.regmove(v1, 10, 12);
assert_eq!(divs.diversion(v1),
Some(&Diversion {
value: v1,
from: 10,
to: 12,
}));
assert_eq!(
divs.diversion(v1),
Some(&Diversion {
value: v1,
from: 10,
to: 12,
})
);
assert_eq!(divs.diversion(v2), None);
divs.regmove(v1, 12, 11);

97
lib/cretonne/src/regalloc/live_value_tracker.rs
View File

@@ -74,14 +74,13 @@ impl LiveValueVec {
/// Add a new live value to `values`. Copy some properties from `lr`.
fn push(&mut self, value: Value, endpoint: Inst, lr: &LiveRange) {
self.values
.push(LiveValue {
value,
endpoint,
affinity: lr.affinity,
is_local: lr.is_local(),
is_dead: lr.is_dead(),
});
self.values.push(LiveValue {
value,
endpoint,
affinity: lr.affinity,
is_local: lr.is_local(),
is_dead: lr.is_dead(),
});
}
/// Remove all elements.
@@ -157,13 +156,14 @@ impl LiveValueTracker {
/// from the immediate dominator. The second slice is the set of `ebb` arguments that are live.
///
/// Dead arguments with no uses are included in `args`. Call `drop_dead_args()` to remove them.
pub fn ebb_top(&mut self,
ebb: Ebb,
dfg: &DataFlowGraph,
liveness: &Liveness,
layout: &Layout,
domtree: &DominatorTree)
-> (&[LiveValue], &[LiveValue]) {
pub fn ebb_top(
&mut self,
ebb: Ebb,
dfg: &DataFlowGraph,
liveness: &Liveness,
layout: &Layout,
domtree: &DominatorTree,
) -> (&[LiveValue], &[LiveValue]) {
// Start over, compute the set of live values at the top of the EBB from two sources:
//
// 1. Values that were live before `ebb`'s immediate dominator, filtered for those that are
@@ -179,14 +179,14 @@ impl LiveValueTracker {
// If the immediate dominator exits, we must have a stored list for it. This is a
// requirement to the order EBBs are visited: All dominators must have been processed
// before the current EBB.
let idom_live_list = self.idom_sets
.get(&idom)
.expect("No stored live set for dominator");
let idom_live_list = self.idom_sets.get(&idom).expect(
"No stored live set for dominator",
);
// Get just the values that are live-in to `ebb`.
for &value in idom_live_list.as_slice(&self.idom_pool) {
let lr = liveness
.get(value)
.expect("Immediate dominator value has no live range");
let lr = liveness.get(value).expect(
"Immediate dominator value has no live range",
);
// Check if this value is live-in here.
if let Some(endpoint) = lr.livein_local_end(ebb, layout) {
@@ -198,9 +198,9 @@ impl LiveValueTracker {
// Now add all the live arguments to `ebb`.
let first_arg = self.live.values.len();
for &value in dfg.ebb_args(ebb) {
let lr = liveness
.get(value)
.expect("EBB argument value has no live range");
let lr = liveness.get(value).expect(
"EBB argument value has no live range",
);
assert_eq!(lr.def(), ebb.into());
match lr.def_local_end().into() {
ExpandedProgramPoint::Inst(endpoint) => {
@@ -209,13 +209,18 @@ impl LiveValueTracker {
ExpandedProgramPoint::Ebb(local_ebb) => {
// This is a dead EBB argument which is not even live into the first
// instruction in the EBB.
assert_eq!(local_ebb,
ebb,
"EBB argument live range ends at wrong EBB header");
assert_eq!(
local_ebb,
ebb,
"EBB argument live range ends at wrong EBB header"
);
// Give this value a fake endpoint that is the first instruction in the EBB.
// We expect it to be removed by calling `drop_dead_args()`.
self.live
.push(value, layout.first_inst(ebb).expect("Empty EBB"), lr);
self.live.push(
value,
layout.first_inst(ebb).expect("Empty EBB"),
lr,
);
}
}
}
@@ -241,11 +246,12 @@ impl LiveValueTracker {
///
/// The `drop_dead()` method must be called next to actually remove the dead values from the
/// tracked set after the two returned slices are no longer needed.
pub fn process_inst(&mut self,
inst: Inst,
dfg: &DataFlowGraph,
liveness: &Liveness)
-> (&[LiveValue], &[LiveValue], &[LiveValue]) {
pub fn process_inst(
&mut self,
inst: Inst,
dfg: &DataFlowGraph,
liveness: &Liveness,
) -> (&[LiveValue], &[LiveValue], &[LiveValue]) {
// Save a copy of the live values before any branches or jumps that could be somebody's
// immediate dominator.
match dfg[inst].analyze_branch(&dfg.value_lists) {
@@ -272,9 +278,11 @@ impl LiveValueTracker {
}
}
(&self.live.values[0..first_kill],
&self.live.values[first_kill..first_def],
&self.live.values[first_def..])
(
&self.live.values[0..first_kill],
&self.live.values[first_kill..first_def],
&self.live.values[first_def..],
)
}
/// Prepare to move past a ghost instruction.
@@ -310,7 +318,8 @@ impl LiveValueTracker {
/// Any values where `f` returns true are spilled and will be treated as if their affinity was
/// `Stack`.
pub fn process_spills<F>(&mut self, mut f: F)
where F: FnMut(Value) -> bool
where
F: FnMut(Value) -> bool,
{
for lv in &mut self.live.values {
if f(lv.value) {
@@ -324,12 +333,10 @@ impl LiveValueTracker {
let values = self.live.values.iter().map(|lv| lv.value);
let pool = &mut self.idom_pool;
// If there already is a set saved for `idom`, just keep it.
self.idom_sets
.entry(idom)
.or_insert_with(|| {
let mut list = ValueList::default();
list.extend(values, pool);
list
});
self.idom_sets.entry(idom).or_insert_with(|| {
let mut list = ValueList::default();
list.extend(values, pool);
list
});
}
}

54
lib/cretonne/src/regalloc/liveness.rs
View File

@@ -190,12 +190,13 @@ type LiveRangeSet = SparseMap<Value, LiveRange>;
/// Get a mutable reference to the live range for `value`.
/// Create it if necessary.
fn get_or_create<'a>(lrset: &'a mut LiveRangeSet,
value: Value,
isa: &TargetIsa,
func: &Function,
enc_info: &EncInfo)
-> &'a mut LiveRange {
fn get_or_create<'a>(
lrset: &'a mut LiveRangeSet,
value: Value,
isa: &TargetIsa,
func: &Function,
enc_info: &EncInfo,
) -> &'a mut LiveRange {
// It would be better to use `get_mut()` here, but that leads to borrow checker fighting
// which can probably only be resolved by non-lexical lifetimes.
// https://github.com/rust-lang/rfcs/issues/811
@@ -233,12 +234,14 @@ fn get_or_create<'a>(lrset: &'a mut LiveRangeSet,
}
/// Extend the live range for `value` so it reaches `to` which must live in `ebb`.
fn extend_to_use(lr: &mut LiveRange,
ebb: Ebb,
to: Inst,
worklist: &mut Vec<Ebb>,
func: &Function,
cfg: &ControlFlowGraph) {
fn extend_to_use(
lr: &mut LiveRange,
ebb: Ebb,
to: Inst,
worklist: &mut Vec<Ebb>,
func: &Function,
cfg: &ControlFlowGraph,
) {
// This is our scratch working space, and we'll leave it empty when we return.
assert!(worklist.is_empty());
@@ -309,10 +312,12 @@ impl Liveness {
///
/// This asserts that `value` does not have an existing live range.
pub fn create_dead<PP>(&mut self, value: Value, def: PP, affinity: Affinity)
where PP: Into<ProgramPoint>
where
PP: Into<ProgramPoint>,
{
let old = self.ranges
.insert(LiveRange::new(value, def.into(), affinity));
let old = self.ranges.insert(
LiveRange::new(value, def.into(), affinity),
);
assert!(old.is_none(), "{} already has a live range", value);
}
@@ -320,7 +325,8 @@ impl Liveness {
///
/// The old and new def points must be in the same EBB, and before the end of the live range.
pub fn move_def_locally<PP>(&mut self, value: Value, def: PP)
where PP: Into<ProgramPoint>
where
PP: Into<ProgramPoint>,
{
let mut lr = self.ranges.get_mut(value).expect("Value has no live range");
lr.move_def_locally(def.into());
@@ -331,12 +337,13 @@ impl Liveness {
/// It is assumed the `value` is already live before `user` in `ebb`.
///
/// Returns a mutable reference to the value's affinity in case that also needs to be updated.
pub fn extend_locally(&mut self,
value: Value,
ebb: Ebb,
user: Inst,
layout: &Layout)
-> &mut Affinity {
pub fn extend_locally(
&mut self,
value: Value,
ebb: Ebb,
user: Inst,
layout: &Layout,
) -> &mut Affinity {
debug_assert_eq!(Some(ebb), layout.inst_ebb(user));
let mut lr = self.ranges.get_mut(value).expect("Value has no live range");
let livein = lr.extend_in_ebb(ebb, user, layout);
@@ -401,7 +408,8 @@ impl Liveness {
if let Some(constraint) = operand_constraints.next() {
lr.affinity.merge(constraint, &reg_info);
} else if lr.affinity.is_none() && encoding.is_legal() &&
!func.dfg[inst].opcode().is_branch() {
!func.dfg[inst].opcode().is_branch()
{
// This is a real encoded instruction using a value that doesn't yet have a
// concrete affinity. Most likely a call argument or a return value. Give
// the value a register affinity matching the ABI type.

76
lib/cretonne/src/regalloc/liverange.rs
View File

@@ -224,13 +224,13 @@ impl LiveRange {
self.liveins
.binary_search_by(|intv| order.cmp(intv.begin, ebb))
.or_else(|n| {
// The interval at `n-1` may cover `ebb`.
if n > 0 && order.cmp(self.liveins[n - 1].end, ebb) == Ordering::Greater {
Ok(n - 1)
} else {
Err(n)
}
})
// The interval at `n-1` may cover `ebb`.
if n > 0 && order.cmp(self.liveins[n - 1].end, ebb) == Ordering::Greater {
Ok(n - 1)
} else {
Err(n)
}
})
}
/// Extend the local interval for `ebb` so it reaches `to` which must belong to `ebb`.
@@ -250,11 +250,14 @@ impl LiveRange {
// We're assuming here that `to` never precedes `def_begin` in the same EBB, but we can't
// check it without a method for getting `to`'s EBB.
if order.cmp(ebb, self.def_end) != Ordering::Greater &&
order.cmp(to, self.def_begin) != Ordering::Less {
order.cmp(to, self.def_begin) != Ordering::Less
{
let to_pp = to.into();
assert_ne!(to_pp,
self.def_begin,
"Can't use value in the defining instruction.");
assert_ne!(
to_pp,
self.def_begin,
"Can't use value in the defining instruction."
);
if order.cmp(to, self.def_end) == Ordering::Greater {
self.def_end = to_pp;
}
@@ -288,8 +291,10 @@ impl LiveRange {
let prev = n.checked_sub(1).and_then(|i| self.liveins.get(i));
let next = self.liveins.get(n);
(prev.map_or(false, |prev| order.is_ebb_gap(prev.end, ebb)),
next.map_or(false, |next| order.is_ebb_gap(to, next.begin)))
(
prev.map_or(false, |prev| order.is_ebb_gap(prev.end, ebb)),
next.map_or(false, |next| order.is_ebb_gap(to, next.begin)),
)
};
match (coalesce_prev, coalesce_next) {
@@ -309,12 +314,13 @@ impl LiveRange {
}
// Cannot coalesce; insert new interval
(false, false) => {
self.liveins
.insert(n,
Interval {
begin: ebb,
end: to,
});
self.liveins.insert(
n,
Interval {
begin: ebb,
end: to,
},
);
}
}
@@ -372,9 +378,9 @@ impl LiveRange {
/// answer, but it is also possible that an even later program point is returned. So don't
/// depend on the returned `Inst` to belong to `ebb`.
pub fn livein_local_end<PO: ProgramOrder>(&self, ebb: Ebb, order: &PO) -> Option<Inst> {
self.find_ebb_interval(ebb, order)
.ok()
.map(|n| self.liveins[n].end)
self.find_ebb_interval(ebb, order).ok().map(|n| {
self.liveins[n].end
})
}
/// Get all the live-in intervals.
@@ -384,11 +390,13 @@ impl LiveRange {
/// Check if this live range overlaps a definition in `ebb`.
pub fn overlaps_def<PO>(&self, def: ExpandedProgramPoint, ebb: Ebb, order: &PO) -> bool
where PO: ProgramOrder
where
PO: ProgramOrder,
{
// Check for an overlap with the local range.
if order.cmp(def, self.def_begin) != Ordering::Less &&
order.cmp(def, self.def_end) == Ordering::Less {
order.cmp(def, self.def_end) == Ordering::Less
{
return true;
}
@@ -401,11 +409,13 @@ impl LiveRange {
/// Check if this live range reaches a use at `user` in `ebb`.
pub fn reaches_use<PO>(&self, user: Inst, ebb: Ebb, order: &PO) -> bool
where PO: ProgramOrder
where
PO: ProgramOrder,
{
// Check for an overlap with the local range.
if order.cmp(user, self.def_begin) == Ordering::Greater &&
order.cmp(user, self.def_end) != Ordering::Greater {
order.cmp(user, self.def_end) != Ordering::Greater
{
return true;
}
@@ -418,7 +428,8 @@ impl LiveRange {
/// Check if this live range is killed at `user` in `ebb`.
pub fn killed_at<PO>(&self, user: Inst, ebb: Ebb, order: &PO) -> bool
where PO: ProgramOrder
where
PO: ProgramOrder,
{
self.def_local_end() == user.into() || self.livein_local_end(ebb, order) == Some(user)
}
@@ -447,8 +458,9 @@ mod tests {
impl ProgramOrder for ProgOrder {
fn cmp<A, B>(&self, a: A, b: B) -> Ordering
where A: Into<ExpandedProgramPoint>,
B: Into<ExpandedProgramPoint>
where
A: Into<ExpandedProgramPoint>,
B: Into<ExpandedProgramPoint>,
{
fn idx(pp: ExpandedProgramPoint) -> usize {
match pp {
@@ -505,9 +517,11 @@ mod tests {
assert_eq!(self.cmp(e, li.begin), Ordering::Less);
}
assert!(self.cmp(lr.def_end, li.begin) == Ordering::Less ||
assert!(
self.cmp(lr.def_end, li.begin) == Ordering::Less ||
self.cmp(lr.def_begin, li.end) == Ordering::Greater,
"Interval can't overlap the def EBB");
"Interval can't overlap the def EBB"
);
// Save for next round.
prev_end = Some(li.end);

8
lib/cretonne/src/regalloc/pressure.rs
View File

@@ -103,10 +103,10 @@ impl Pressure {
}
// Compute per-class limits from `usable`.
for (toprc, rc) in p.toprc
.iter_mut()
.take_while(|t| t.num_toprcs > 0)
.zip(reginfo.classes) {
for (toprc, rc) in p.toprc.iter_mut().take_while(|t| t.num_toprcs > 0).zip(
reginfo.classes,
)
{
toprc.limit = usable.iter(rc).len() as u32;
toprc.width = rc.width;
}

126
lib/cretonne/src/regalloc/reload.rs
View File

@@ -54,13 +54,15 @@ impl Reload {
}
/// Run the reload algorithm over `func`.
pub fn run(&mut self,
isa: &TargetIsa,
func: &mut Function,
domtree: &DominatorTree,
liveness: &mut Liveness,
topo: &mut TopoOrder,
tracker: &mut LiveValueTracker) {
pub fn run(
&mut self,
isa: &TargetIsa,
func: &mut Function,
domtree: &DominatorTree,
liveness: &mut Liveness,
topo: &mut TopoOrder,
tracker: &mut LiveValueTracker,
) {
dbg!("Reload for:\n{}", func.display(isa));
let mut ctx = Context {
cur: EncCursor::new(func, isa),
@@ -125,11 +127,13 @@ impl<'a> Context<'a> {
/// Process the EBB parameters. Move to the next instruction in the EBB to be processed
fn visit_ebb_header(&mut self, ebb: Ebb, tracker: &mut LiveValueTracker) {
let (liveins, args) = tracker.ebb_top(ebb,
&self.cur.func.dfg,
self.liveness,
&self.cur.func.layout,
self.domtree);
let (liveins, args) = tracker.ebb_top(
ebb,
&self.cur.func.dfg,
self.liveness,
&self.cur.func.layout,
self.domtree,
);
if self.cur.func.layout.entry_block() == Some(ebb) {
assert_eq!(liveins.len(), 0);
@@ -172,15 +176,17 @@ impl<'a> Context<'a> {
/// Process the instruction pointed to by `pos`, and advance the cursor to the next instruction
/// that needs processing.
fn visit_inst(&mut self,
ebb: Ebb,
inst: Inst,
encoding: Encoding,
tracker: &mut LiveValueTracker) {
fn visit_inst(
&mut self,
ebb: Ebb,
inst: Inst,
encoding: Encoding,
tracker: &mut LiveValueTracker,
) {
// Get the operand constraints for `inst` that we are trying to satisfy.
let constraints = self.encinfo
.operand_constraints(encoding)
.expect("Missing instruction encoding");
let constraints = self.encinfo.operand_constraints(encoding).expect(
"Missing instruction encoding",
);
// Identify reload candidates.
assert!(self.candidates.is_empty());
@@ -195,17 +201,20 @@ impl<'a> Context<'a> {
let reg = self.cur.ins().fill(cand.value);
let fill = self.cur.built_inst();
self.reloads
.insert(ReloadedValue {
stack: cand.value,
reg: reg,
});
self.reloads.insert(ReloadedValue {
stack: cand.value,
reg: reg,
});
// Create a live range for the new reload.
let affinity = Affinity::Reg(cand.regclass.into());
self.liveness.create_dead(reg, fill, affinity);
self.liveness
.extend_locally(reg, ebb, inst, &self.cur.func.layout);
self.liveness.extend_locally(
reg,
ebb,
inst,
&self.cur.func.layout,
);
}
// Rewrite arguments.
@@ -218,8 +227,8 @@ impl<'a> Context<'a> {
// TODO: Reuse reloads for future instructions.
self.reloads.clear();
let (_throughs, _kills, defs) = tracker
.process_inst(inst, &self.cur.func.dfg, self.liveness);
let (_throughs, _kills, defs) =
tracker.process_inst(inst, &self.cur.func.dfg, self.liveness);
// Advance to the next instruction so we can insert any spills after the instruction.
self.cur.next_inst();
@@ -255,11 +264,10 @@ impl<'a> Context<'a> {
for (op, &arg) in constraints.ins.iter().zip(args) {
if op.kind != ConstraintKind::Stack {
if self.liveness[arg].affinity.is_stack() {
self.candidates
.push(ReloadCandidate {
value: arg,
regclass: op.regclass,
})
self.candidates.push(ReloadCandidate {
value: arg,
regclass: op.regclass,
})
}
}
}
@@ -272,17 +280,21 @@ impl<'a> Context<'a> {
// Handle ABI arguments.
if let Some(sig) = self.cur.func.dfg.call_signature(inst) {
handle_abi_args(self.candidates,
&self.cur.func.dfg.signatures[sig].argument_types,
var_args,
self.cur.isa,
self.liveness);
handle_abi_args(
self.candidates,
&self.cur.func.dfg.signatures[sig].argument_types,
var_args,
self.cur.isa,
self.liveness,
);
} else if self.cur.func.dfg[inst].opcode().is_return() {
handle_abi_args(self.candidates,
&self.cur.func.signature.return_types,
var_args,
self.cur.isa,
self.liveness);
handle_abi_args(
self.candidates,
&self.cur.func.signature.return_types,
var_args,
self.cur.isa,
self.liveness,
);
}
}
@@ -297,27 +309,33 @@ impl<'a> Context<'a> {
// Update live ranges.
self.liveness.move_def_locally(stack, inst);
self.liveness
.extend_locally(reg, ebb, inst, &self.cur.func.layout);
self.liveness.extend_locally(
reg,
ebb,
inst,
&self.cur.func.layout,
);
}
}
/// Find reload candidates in the instruction's ABI variable arguments. This handles both
/// return values and call arguments.
fn handle_abi_args(candidates: &mut Vec<ReloadCandidate>,
abi_types: &[ArgumentType],
var_args: &[Value],
isa: &TargetIsa,
liveness: &Liveness) {
fn handle_abi_args(
candidates: &mut Vec<ReloadCandidate>,
abi_types: &[ArgumentType],
var_args: &[Value],
isa: &TargetIsa,
liveness: &Liveness,
) {
assert_eq!(abi_types.len(), var_args.len());
for (abi, &arg) in abi_types.iter().zip(var_args) {
if abi.location.is_reg() {
let lv = liveness.get(arg).expect("Missing live range for ABI arg");
if lv.affinity.is_stack() {
candidates.push(ReloadCandidate {
value: arg,
regclass: isa.regclass_for_abi_type(abi.value_type),
});
value: arg,
regclass: isa.regclass_for_abi_type(abi.value_type),
});
}
}
}

186
lib/cretonne/src/regalloc/solver.rs
View File

@@ -231,12 +231,14 @@ impl SparseMapValue<Value> for Assignment {
impl fmt::Display for Assignment {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f,
"{}:{}(%{} -> %{})",
self.value,
self.rc,
self.from,
self.to)
write!(
f,
"{}:{}(%{} -> %{})",
self.value,
self.rc,
self.from,
self.to
)
}
}
@@ -244,7 +246,7 @@ impl fmt::Display for Assignment {
impl PartialEq for Assignment {
fn eq(&self, other: &Assignment) -> bool {
self.value == other.value && self.from == other.from && self.to == other.to &&
self.rc.index == other.rc.index
self.rc.index == other.rc.index
}
}
@@ -363,22 +365,23 @@ impl Solver {
dbg!("-> converting variable {} to a fixed constraint", v);
// The spiller is responsible for ensuring that all constraints on the uses of a
// value are compatible.
assert!(v.constraint.contains(to),
"Incompatible constraints for {}",
value);
assert!(
v.constraint.contains(to),
"Incompatible constraints for {}",
value
);
} else {
panic!("Invalid from register for fixed {} constraint", value);
}
}
self.regs_in.free(rc, from);
self.regs_out.take(rc, to);
self.assignments
.insert(Assignment {
value,
rc,
from,
to,
});
self.assignments.insert(Assignment {
value,
rc,
from,
to,
});
}
/// Add a variable representing an input side value with an existing register assignment.
@@ -388,18 +391,22 @@ impl Solver {
///
/// It is assumed initially that the value is also live on the output side of the instruction.
/// This can be changed by calling to `add_kill()`.
pub fn add_var(&mut self,
value: Value,
constraint: RegClass,
from: RegUnit,
reginfo: &RegInfo) {
pub fn add_var(
&mut self,
value: Value,
constraint: RegClass,
from: RegUnit,
reginfo: &RegInfo,
) {
// Check for existing entries for this value.
if self.regs_in.is_avail(constraint, from) {
dbg!("add_var({}:{}, from={}/%{}) for existing entry",
value,
constraint,
reginfo.display_regunit(from),
from);
dbg!(
"add_var({}:{}, from={}/%{}) for existing entry",
value,
constraint,
reginfo.display_regunit(from),
from
);
// There could be an existing variable entry.
if let Some(v) = self.vars.iter_mut().find(|v| v.value == value) {
@@ -419,9 +426,11 @@ impl Solver {
// No variable, then it must be a fixed reassignment.
if let Some(a) = self.assignments.get(value) {
dbg!("-> already fixed assignment {}", a);
assert!(constraint.contains(a.to),
"Incompatible constraints for {}",
value);
assert!(
constraint.contains(a.to),
"Incompatible constraints for {}",
value
);
return;
}
@@ -430,12 +439,14 @@ impl Solver {
}
let new_var = Variable::new_live(value, constraint, from);
dbg!("add_var({}:{}, from={}/%{}) new entry: {}",
value,
constraint,
reginfo.display_regunit(from),
from,
new_var);
dbg!(
"add_var({}:{}, from={}/%{}) new entry: {}",
value,
constraint,
reginfo.display_regunit(from),
from,
new_var
);
self.regs_in.free(constraint, from);
if self.inputs_done {
@@ -623,23 +634,20 @@ impl Solver {
// Collect moves from the chosen solution for all non-define variables.
for v in &self.vars {
if let Some(from) = v.from {
self.moves
.push(Assignment {
value: v.value,
from,
to: v.solution,
rc: v.constraint,
});
self.moves.push(Assignment {
value: v.value,
from,
to: v.solution,
rc: v.constraint,
});
}
}
// Convert all of the fixed register assignments into moves, but omit the ones that are
// already in the right register.
self.moves
.extend(self.assignments
.values()
.cloned()
.filter(|v| v.from != v.to));
self.moves.extend(self.assignments.values().cloned().filter(
|v| v.from != v.to,
));
dbg!("collect_moves: {}", DisplayList(self.moves.as_slice()));
}
@@ -661,9 +669,10 @@ impl Solver {
let mut i = 0;
while i < self.moves.len() {
// Find the first move that can be executed now.
if let Some(j) = self.moves[i..]
.iter()
.position(|m| avail.is_avail(m.rc, m.to)) {
if let Some(j) = self.moves[i..].iter().position(
|m| avail.is_avail(m.rc, m.to),
)
{
// This move can be executed now.
self.moves.swap(i, i + j);
let m = &self.moves[i];
@@ -709,17 +718,16 @@ impl Solver {
// Append a fixup move so we end up in the right place. This move will be scheduled
// later. That's ok because it is the single remaining move of `m.value` after the
// next iteration.
self.moves
.push(Assignment {
value: m.value,
rc: m.rc,
from: reg,
to: m.to,
});
// TODO: What if allocating an extra register is not enough to break a cycle? This
// can happen when there are registers of different widths in a cycle. For ARM, we
// may have to move two S-registers out of the way before we can resolve a cycle
// involving a D-register.
self.moves.push(Assignment {
value: m.value,
rc: m.rc,
from: reg,
to: m.to,
});
// TODO: What if allocating an extra register is not enough to break a cycle? This
// can happen when there are registers of different widths in a cycle. For ARM, we
// may have to move two S-registers out of the way before we can resolve a cycle
// involving a D-register.
} else {
panic!("Not enough registers in {} to schedule moves", m.rc);
}
@@ -738,9 +746,11 @@ impl Solver {
impl fmt::Display for Solver {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
writeln!(f, "Solver {{ inputs_done: {},", self.inputs_done)?;
writeln!(f,
" assignments: {}",
DisplayList(self.assignments.as_slice()))?;
writeln!(
f,
" assignments: {}",
DisplayList(self.assignments.as_slice())
)?;
writeln!(f, " vars: {}", DisplayList(self.vars.as_slice()))?;
writeln!(f, " moves: {}", DisplayList(self.moves.as_slice()))?;
writeln!(f, "}}")
@@ -817,8 +827,10 @@ mod tests {
solver.inputs_done();
assert!(solver.quick_solve().is_ok());
assert_eq!(solver.schedule_moves(&regs), 0);
assert_eq!(solver.moves(),
&[mov(v11, gpr, r1, r2), mov(v10, gpr, r0, r1)]);
assert_eq!(
solver.moves(),
&[mov(v11, gpr, r1, r2), mov(v10, gpr, r0, r1)]
);
// Swap r0 and r1 in three moves using r2 as a scratch.
solver.reset(&regs);
@@ -827,10 +839,14 @@ mod tests {
solver.inputs_done();
assert!(solver.quick_solve().is_ok());
assert_eq!(solver.schedule_moves(&regs), 0);
assert_eq!(solver.moves(),
&[mov(v10, gpr, r0, r2),
mov(v11, gpr, r1, r0),
mov(v10, gpr, r2, r1)]);
assert_eq!(
solver.moves(),
&[
mov(v10, gpr, r0, r2),
mov(v11, gpr, r1, r0),
mov(v10, gpr, r2, r1),
]
);
}
#[test]
@@ -862,11 +878,15 @@ mod tests {
solver.inputs_done();
assert!(solver.quick_solve().is_ok());
assert_eq!(solver.schedule_moves(&regs), 0);
assert_eq!(solver.moves(),
&[mov(v10, d, d0, d2),
mov(v11, s, s2, s0),
mov(v12, s, s3, s1),
mov(v10, d, d2, d1)]);
assert_eq!(
solver.moves(),
&[
mov(v10, d, d0, d2),
mov(v11, s, s2, s0),
mov(v12, s, s3, s1),
mov(v10, d, d2, d1),
]
);
// Same problem in the other direction: Swap (s0, s1) <-> d1.
//
@@ -879,10 +899,14 @@ mod tests {
solver.inputs_done();
assert!(solver.quick_solve().is_ok());
assert_eq!(solver.schedule_moves(&regs), 0);
assert_eq!(solver.moves(),
&[mov(v10, d, d1, d2),
mov(v12, s, s1, s3),
mov(v11, s, s0, s2),
mov(v10, d, d2, d0)]);
assert_eq!(
solver.moves(),
&[
mov(v10, d, d1, d2),
mov(v12, s, s1, s3),
mov(v11, s, s0, s2),
mov(v10, d, d2, d0),
]
);
}
}

138
lib/cretonne/src/regalloc/spilling.rs
View File

@@ -71,14 +71,16 @@ impl Spilling {
}
/// Run the spilling algorithm over `func`.
pub fn run(&mut self,
isa: &TargetIsa,
func: &mut Function,
domtree: &DominatorTree,
liveness: &mut Liveness,
virtregs: &VirtRegs,
topo: &mut TopoOrder,
tracker: &mut LiveValueTracker) {
pub fn run(
&mut self,
isa: &TargetIsa,
func: &mut Function,
domtree: &DominatorTree,
liveness: &mut Liveness,
virtregs: &VirtRegs,
topo: &mut TopoOrder,
tracker: &mut LiveValueTracker,
) {
dbg!("Spilling for:\n{}", func.display(isa));
let reginfo = isa.register_info();
let usable_regs = isa.allocatable_registers(func);
@@ -114,8 +116,10 @@ impl<'a> Context<'a> {
while let Some(inst) = self.cur.next_inst() {
if let Some(constraints) =
self.encinfo
.operand_constraints(self.cur.func.encodings[inst]) {
self.encinfo.operand_constraints(
self.cur.func.encodings[inst],
)
{
self.visit_inst(inst, ebb, constraints, tracker);
} else {
let (_throughs, kills) = tracker.process_ghost(inst);
@@ -150,11 +154,13 @@ impl<'a> Context<'a> {
}
fn visit_ebb_header(&mut self, ebb: Ebb, tracker: &mut LiveValueTracker) {
let (liveins, args) = tracker.ebb_top(ebb,
&self.cur.func.dfg,
self.liveness,
&self.cur.func.layout,
self.domtree);
let (liveins, args) = tracker.ebb_top(
ebb,
&self.cur.func.dfg,
self.liveness,
&self.cur.func.layout,
self.domtree,
);
// Count the live-in registers. These should already fit in registers; they did at the
// dominator.
@@ -167,16 +173,20 @@ impl<'a> Context<'a> {
if let Affinity::Reg(rci) = lv.affinity {
let rc = self.reginfo.rc(rci);
'try_take: while let Err(mask) = self.pressure.take_transient(rc) {
dbg!("Need {} reg for EBB argument {} from {} live-ins",
rc,
lv.value,
liveins.len());
dbg!(
"Need {} reg for EBB argument {} from {} live-ins",
rc,
lv.value,
liveins.len()
);
match self.spill_candidate(mask, liveins) {
Some(cand) => {
dbg!("Spilling live-in {} to make room for {} EBB argument {}",
cand,
rc,
lv.value);
dbg!(
"Spilling live-in {} to make room for {} EBB argument {}",
cand,
rc,
lv.value
);
self.spill_reg(cand);
}
None => {
@@ -199,11 +209,13 @@ impl<'a> Context<'a> {
self.pressure.preserve_transient();
}
fn visit_inst(&mut self,
inst: Inst,
ebb: Ebb,
constraints: &RecipeConstraints,
tracker: &mut LiveValueTracker) {
fn visit_inst(
&mut self,
inst: Inst,
ebb: Ebb,
constraints: &RecipeConstraints,
tracker: &mut LiveValueTracker,
) {
dbg!("Inst {}, {}", self.cur.display_inst(inst), self.pressure);
debug_assert_eq!(self.cur.current_inst(), Some(inst));
debug_assert_eq!(self.cur.current_ebb(), Some(ebb));
@@ -250,9 +262,11 @@ impl<'a> Context<'a> {
match self.spill_candidate(mask, throughs) {
Some(cand) => self.spill_reg(cand),
None => {
panic!("Ran out of {} registers for {}",
op.regclass,
self.cur.display_inst(inst))
panic!(
"Ran out of {} registers for {}",
op.regclass,
self.cur.display_inst(inst)
)
}
}
}
@@ -313,12 +327,16 @@ impl<'a> Context<'a> {
.argument_types
.iter()
.zip(args)
.enumerate() {
.enumerate()
{
if abi.location.is_reg() {
let (rci, spilled) = match self.liveness[arg].affinity {
Affinity::Reg(rci) => (rci, false),
Affinity::Stack => {
(self.cur.isa.regclass_for_abi_type(abi.value_type).into(), true)
(
self.cur.isa.regclass_for_abi_type(abi.value_type).into(),
true,
)
}
Affinity::None => panic!("Missing affinity for {}", arg),
};
@@ -373,17 +391,19 @@ impl<'a> Context<'a> {
// Spill a live register that is *not* used by the current instruction.
// Spilling a use wouldn't help.
match {
let args = self.cur.func.dfg.inst_args(inst);
self.spill_candidate(mask,
tracker.live().iter().filter(|lv| {
!args.contains(&lv.value)
}))
} {
let args = self.cur.func.dfg.inst_args(inst);
self.spill_candidate(
mask,
tracker.live().iter().filter(|lv| !args.contains(&lv.value)),
)
} {
Some(cand) => self.spill_reg(cand),
None => {
panic!("Ran out of {} registers when inserting copy before {}",
rc,
self.cur.display_inst(inst))
panic!(
"Ran out of {} registers when inserting copy before {}",
rc,
self.cur.display_inst(inst)
)
}
}
}
@@ -395,7 +415,8 @@ impl<'a> Context<'a> {
// Find a spill candidate from `candidates` whose top-level register class is in `mask`.
fn spill_candidate<'ii, II>(&self, mask: RegClassMask, candidates: II) -> Option<Value>
where II: IntoIterator<Item = &'ii LiveValue>
where
II: IntoIterator<Item = &'ii LiveValue>,
{
// Find the best viable spill candidate.
//
@@ -420,12 +441,13 @@ impl<'a> Context<'a> {
None
})
.min_by(|&a, &b| {
// Find the minimum candidate according to the RPO of their defs.
self.domtree
.rpo_cmp(self.cur.func.dfg.value_def(a),
self.cur.func.dfg.value_def(b),
&self.cur.func.layout)
})
// Find the minimum candidate according to the RPO of their defs.
self.domtree.rpo_cmp(
self.cur.func.dfg.value_def(a),
self.cur.func.dfg.value_def(b),
&self.cur.func.layout,
)
})
}
/// Spill `value` immediately by
@@ -447,10 +469,9 @@ impl<'a> Context<'a> {
}
// Assign a spill slot for the whole virtual register.
let ss = self.cur
.func
.stack_slots
.make_spill_slot(self.cur.func.dfg.value_type(value));
let ss = self.cur.func.stack_slots.make_spill_slot(
self.cur.func.dfg.value_type(value),
);
for &v in self.virtregs.congruence_class(&value) {
self.liveness.spill(v);
self.cur.func.locations[v] = ValueLoc::Stack(ss);
@@ -481,11 +502,12 @@ impl<'a> Context<'a> {
// Update live ranges.
self.liveness.create_dead(copy, inst, Affinity::Reg(rci));
self.liveness
.extend_locally(copy,
self.cur.func.layout.pp_ebb(inst),
self.cur.current_inst().expect("must be at an instruction"),
&self.cur.func.layout);
self.liveness.extend_locally(
copy,
self.cur.func.layout.pp_ebb(inst),
self.cur.current_inst().expect("must be at an instruction"),
&self.cur.func.layout,
);
copy
}

17
lib/cretonne/src/regalloc/virtregs.rs
View File

@@ -81,11 +81,12 @@ impl VirtRegs {
/// If `value` belongs to a virtual register, the congruence class is the values of the virtual
/// register. Otherwise it is just the value itself.
pub fn congruence_class<'a, 'b>(&'a self, value: &'b Value) -> &'b [Value]
where 'a: 'b
where
'a: 'b,
{
self.get(*value)
.map(|vr| self.values(vr))
.unwrap_or(ref_slice(value))
self.get(*value).map(|vr| self.values(vr)).unwrap_or(
ref_slice(value),
)
}
/// Check if `a` and `b` belong to the same congruence class.
@@ -126,9 +127,11 @@ impl VirtRegs {
.min()
.unwrap_or_else(|| self.vregs.push(Default::default()));
assert_eq!(values.len(),
singletons + cleared,
"Can't unify partial virtual registers");
assert_eq!(
values.len(),
singletons + cleared,
"Can't unify partial virtual registers"
);
self.vregs[vreg].extend(values.iter().cloned(), &mut self.pool);
for &v in values {