Make register copies for incompatible operands.

An instruction may have fixed operand constraints that make it
impossibly to use a single register value to satisfy two at a time.

Detect when the same value is used for multiple fixed register operands
and insert copies during the spilling pass.

filetests/regalloc/spill.cton

@@ -75,3 +75,32 @@ ebb0(v1: i32):
     ; check: call $fn0
     return
 }
+
+; The same value used for two function arguments.
+function %doubleuse(i32) {
+    fn0 = function %xx(i32, i32)
+ebb0(v0: i32):
+    ; check: $(c=$V) = copy $v0
+    call fn0(v0, v0)
+    ; check: call $fn0($v0, $c)
+    return
+}
+
+; The same value used as indirect callee and argument.
+function %doubleuse_icall1(i32) {
+    sig0 = signature(i32)
+ebb0(v0: i32):
+    ; not:copy
+    call_indirect sig0, v0(v0)
+    return
+}
+
+; The same value used as indirect callee and two arguments.
+function %doubleuse_icall2(i32) {
+    sig0 = signature(i32, i32)
+ebb0(v0: i32):
+    ; check: $(c=$V) = copy $v0
+    call_indirect sig0, v0(v0, v0)
+    ; check: call_indirect $sig0, $v0($v0, $c)
+    return
+}

lib/cretonne/src/ir/entities.rs

@@ -74,7 +74,7 @@ impl Ebb {
 }
 
 /// An opaque reference to an SSA value.
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
 pub struct Value(u32);
 entity_impl!(Value, "v");
 

lib/cretonne/src/regalloc/spilling.rs

@@ -4,12 +4,23 @@
 //! ensure that the register pressure never exceeds the number of available registers by moving
 //! some SSA values to spill slots on the stack. This is encoded in the affinity of the value's
 //! live range.
+//!
+//! Some instruction operand constraints may require additional registers to resolve. Since this
+//! can cause spilling, the spilling pass is also responsible for resolving those constraints by
+//! inserting copies. The extra constraints are:
+//!
+//! 1. A value used by a tied operand must be killed by the instruction. This is resolved by
+//!    inserting a copy to a temporary value when necessary.
+//! 2. When the same value is used more than once by an instruction, the operand constraints must
+//!    be compatible. Otherwise, the value must be copied into a new register for some of the
+//!    operands.
 
 use dominator_tree::DominatorTree;
-use ir::{DataFlowGraph, Layout, Cursor};
+use entity_map::EntityMap;
-use ir::{Function, Ebb, Inst, Value};
+use ir::{DataFlowGraph, Layout, Cursor, InstBuilder};
-use isa::{TargetIsa, RegInfo, EncInfo, RecipeConstraints, ConstraintKind};
+use ir::{Function, Ebb, Inst, Value, SigRef};
-use isa::registers::RegClassMask;
+use isa::registers::{RegClass, RegClassMask};
+use isa::{TargetIsa, RegInfo, EncInfo, Encoding, RecipeConstraints, ConstraintKind};
 use regalloc::affinity::Affinity;
 use regalloc::live_value_tracker::{LiveValue, LiveValueTracker};
 use regalloc::liveness::Liveness;
@@ -19,14 +30,19 @@ use topo_order::TopoOrder;
 /// Persistent data structures for the spilling pass.
 pub struct Spilling {
     spills: Vec<Value>,
+    reg_uses: Vec<RegUse>,
 }
 
 /// Context data structure that gets instantiated once per pass.
 struct Context<'a> {
+    isa: &'a TargetIsa,
     // Cached ISA information.
     reginfo: RegInfo,
     encinfo: EncInfo,
 
+    // References to parts of the current function.
+    encodings: &'a mut EntityMap<Inst, Encoding>,
+
     // References to contextual data structures we need.
     domtree: &'a DominatorTree,
     liveness: &'a mut Liveness,
@@ -39,12 +55,18 @@ struct Context<'a> {
     // pressure tracker, but they are still present in the live value tracker and their affinity
     // hasn't been changed yet.
     spills: &'a mut Vec<Value>,
+
+    // Uses of register values in the current instruction.
+    reg_uses: &'a mut Vec<RegUse>,
 }
 
 impl Spilling {
     /// Create a new spilling data structure.
     pub fn new() -> Spilling {
-        Spilling { spills: Vec::new() }
+        Spilling {
+            spills: Vec::new(),
+            reg_uses: Vec::new(),
+        }
     }
 
     /// Run the spilling algorithm over `func`.
@@ -59,36 +81,46 @@ impl Spilling {
         let reginfo = isa.register_info();
         let usable_regs = isa.allocatable_registers(func);
         let mut ctx = Context {
+            isa,
             reginfo: isa.register_info(),
             encinfo: isa.encoding_info(),
+            encodings: &mut func.encodings,
             domtree,
             liveness,
             topo,
             pressure: Pressure::new(&reginfo, &usable_regs),
             spills: &mut self.spills,
+            reg_uses: &mut self.reg_uses,
         };
-        ctx.run(func, tracker)
+        ctx.run(&mut func.layout, &mut func.dfg, tracker)
     }
 }
 
 impl<'a> Context<'a> {
-    fn run(&mut self, func: &mut Function, tracker: &mut LiveValueTracker) {
+    fn run(&mut self,
-        self.topo.reset(func.layout.ebbs());
+           layout: &mut Layout,
-        while let Some(ebb) = self.topo.next(&func.layout, self.domtree) {
+           dfg: &mut DataFlowGraph,
-            self.visit_ebb(ebb, func, tracker);
+           tracker: &mut LiveValueTracker) {
+        self.topo.reset(layout.ebbs());
+        while let Some(ebb) = self.topo.next(layout, self.domtree) {
+            self.visit_ebb(ebb, layout, dfg, tracker);
         }
     }
 
-    fn visit_ebb(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) {
+    fn visit_ebb(&mut self,
+                 ebb: Ebb,
+                 layout: &mut Layout,
+                 dfg: &mut DataFlowGraph,
+                 tracker: &mut LiveValueTracker) {
         dbg!("Spilling {}:", ebb);
-        self.visit_ebb_header(ebb, func, tracker);
+        self.visit_ebb_header(ebb, layout, dfg, tracker);
         tracker.drop_dead_args();
 
-        let mut pos = Cursor::new(&mut func.layout);
+        let mut pos = Cursor::new(layout);
         pos.goto_top(ebb);
         while let Some(inst) = pos.next_inst() {
-            if let Some(constraints) = self.encinfo.operand_constraints(func.encodings[inst]) {
+            if let Some(constraints) = self.encinfo.operand_constraints(self.encodings[inst]) {
-                self.visit_inst(inst, constraints, &mut pos, &mut func.dfg, tracker);
+                self.visit_inst(inst, constraints, &mut pos, dfg, tracker);
                 tracker.drop_dead(inst);
                 self.process_spills(tracker);
             }
@@ -118,9 +150,12 @@ impl<'a> Context<'a> {
         }
     }
 
-    fn visit_ebb_header(&mut self, ebb: Ebb, func: &mut Function, tracker: &mut LiveValueTracker) {
+    fn visit_ebb_header(&mut self,
-        let (liveins, args) =
+                        ebb: Ebb,
-            tracker.ebb_top(ebb, &func.dfg, self.liveness, &func.layout, self.domtree);
+                        layout: &mut Layout,
+                        dfg: &mut DataFlowGraph,
+                        tracker: &mut LiveValueTracker) {
+        let (liveins, args) = tracker.ebb_top(ebb, dfg, self.liveness, layout, self.domtree);
 
         // Count the live-in registers. These should already fit in registers; they did at the
         // dominator.
@@ -135,24 +170,32 @@ impl<'a> Context<'a> {
                   inst: Inst,
                   constraints: &RecipeConstraints,
                   pos: &mut Cursor,
-                  dfg: &DataFlowGraph,
+                  dfg: &mut DataFlowGraph,
                   tracker: &mut LiveValueTracker) {
         dbg!("Inst {}, {}", dfg.display_inst(inst), self.pressure);
         // TODO: Repair constraint violations by copying input values.
         //
         // - Tied use of value that is not killed.
-        // - Inconsistent uses of the same value.
+        // - Count pressure for register uses of spilled values too.
-        //
+
-        // Each inserted copy may increase register pressure. Fix by spilling something not used by
+        assert!(self.reg_uses.is_empty());
-        // the instruction.
+
-        //
+        // If the instruction has any fixed register operands, we may need to resolve register
-        // Count pressure for register uses of spilled values too.
+        // constraints.
-        //
+        if constraints.fixed_ins {
-        // Finally, reset pressure state to level from before the input adjustments, minus spills.
+            self.collect_reg_uses(inst, constraints, dfg);
-        //
+        }
-        // Spills should be removed from tracker. Otherwise they could be double-counted by
+
-        // free_regs below.
+        // Calls usually have fixed register uses.
         let call_sig = dfg.call_signature(inst);
+        if let Some(sig) = call_sig {
+            self.collect_abi_reg_uses(inst, sig, dfg);
+        }
+
+        if !self.reg_uses.is_empty() {
+            self.process_reg_uses(inst, pos, dfg, tracker);
+        }
+
 
         // Update the live value tracker with this instruction.
         let (throughs, kills, defs) = tracker.process_inst(inst, dfg, self.liveness);
@@ -190,6 +233,85 @@ impl<'a> Context<'a> {
         // This won't cause spilling.
         self.take_live_regs(defs);
     }
+    // Collect register uses from the fixed input constraints.
+    //
+    // We are assuming here that if a value is used both by a fixed register operand and a register
+    // class operand, they two are compatible. We are also assuming that two register class
+    // operands are always compatible.
+    fn collect_reg_uses(&mut self,
+                        inst: Inst,
+                        constraints: &RecipeConstraints,
+                        dfg: &DataFlowGraph) {
+        let args = dfg.inst_args(inst);
+        for (idx, (op, &arg)) in constraints.ins.iter().zip(args).enumerate() {
+            match op.kind {
+                ConstraintKind::FixedReg(_) => {
+                    self.reg_uses.push(RegUse::new(arg, idx));
+                }
+                _ => {}
+            }
+        }
+    }
+
+    // Collect register uses from the ABI input constraints.
+    fn collect_abi_reg_uses(&mut self, inst: Inst, sig: SigRef, dfg: &DataFlowGraph) {
+        let fixed_args = dfg[inst].opcode().constraints().fixed_value_arguments();
+        let args = dfg.inst_variable_args(inst);
+        for (idx, (abi, &arg)) in
+            dfg.signatures[sig]
+                .argument_types
+                .iter()
+                .zip(args)
+                .enumerate() {
+            if abi.location.is_reg() {
+                self.reg_uses.push(RegUse::new(arg, fixed_args + idx));
+            }
+        }
+    }
+
+    // Process multiple register uses to resolve potential conflicts.
+    //
+    // Look for multiple uses of the same value in `self.reg_uses` and insert copies as necessary.
+    // Trigger spilling if any of the temporaries cause the register pressure to become too high.
+    //
+    // Leave `self.reg_uses` empty.
+    fn process_reg_uses(&mut self,
+                        inst: Inst,
+                        pos: &mut Cursor,
+                        dfg: &mut DataFlowGraph,
+                        tracker: &LiveValueTracker) {
+        // We're looking for multiple uses of the same value, so start by sorting by value. The
+        // secondary `opidx` key makes it possible to use an unstable sort once that is available
+        // outside nightly Rust.
+        self.reg_uses.sort_by_key(|u| (u.value, u.opidx));
+
+        // We are assuming that `reg_uses` has an entry per fixed register operand, and that any
+        // non-fixed register operands are compatible with one of the fixed uses of the value.
+        for i in 1..self.reg_uses.len() {
+            let ru = self.reg_uses[i];
+            if self.reg_uses[i - 1].value != ru.value {
+                continue;
+            }
+
+            // We have two fixed uses of the same value. Make a copy.
+            let (copy, rc) = self.insert_copy(ru.value, pos, dfg);
+            dfg.inst_args_mut(inst)[ru.opidx as usize] = copy;
+
+            // Make sure the new copy doesn't blow the register pressure.
+            while let Err(mask) = self.pressure.take_transient(rc) {
+                dbg!("Copy of {} reg causes spill", rc);
+                // Spill a live register that is *not* used by the current instruction.
+                // Spilling a use wouldn't help.
+                let args = dfg.inst_args(inst);
+                self.spill_from(mask,
+                                tracker.live().iter().filter(|lv| !args.contains(&lv.value)),
+                                dfg,
+                                &pos.layout);
+            }
+        }
+        self.pressure.reset_transient();
+        self.reg_uses.clear()
+    }
 
     // Spill a candidate from `candidates` whose top-level register class is in `mask`.
     fn spill_from<'ii, II>(&mut self,
@@ -267,4 +389,57 @@ impl<'a> Context<'a> {
             self.spills.clear()
         }
     }
+
+    /// Insert a `copy value` before `pos` and give it a live range extending to `pos`.
+    ///
+    /// Returns the new local value created and its register class.
+    fn insert_copy(&mut self,
+                   value: Value,
+                   pos: &mut Cursor,
+                   dfg: &mut DataFlowGraph)
+                   -> (Value, RegClass) {
+        let copy = dfg.ins(pos).copy(value);
+        let inst = dfg.value_def(copy).unwrap_inst();
+        let ty = dfg.value_type(copy);
+
+        // Give it an encoding.
+        let encoding = self.isa
+            .encode(dfg, &dfg[inst], ty)
+            .expect("Can't encode copy");
+        *self.encodings.ensure(inst) = encoding;
+
+        // Update live ranges.
+        let rc = self.encinfo
+            .operand_constraints(encoding)
+            .expect("Bad copy encoding")
+            .outs
+            [0]
+                .regclass;
+        self.liveness
+            .create_dead(copy, inst, Affinity::Reg(rc.into()));
+        self.liveness
+            .extend_locally(copy,
+                            pos.layout.pp_ebb(inst),
+                            pos.current_inst().expect("must be at an instruction"),
+                            pos.layout);
+
+        (copy, rc)
+    }
+}
+
+// Struct representing a register use of a value.
+// Used to detect multiple uses of the same value with incompatible register constraints.
+#[derive(Clone, Copy)]
+struct RegUse {
+    value: Value,
+    opidx: u16,
+}
+
+impl RegUse {
+    fn new(value: Value, idx: usize) -> RegUse {
+        RegUse {
+            value,
+            opidx: idx as u16,
+        }
+    }
 }