diff --git a/cranelift/filetests/parser/tiny.cton b/cranelift/filetests/parser/tiny.cton index 9727e40177..13513a2b6c 100644 --- a/cranelift/filetests/parser/tiny.cton +++ b/cranelift/filetests/parser/tiny.cton @@ -117,6 +117,7 @@ function %stack() { ss2 = local 4 ss3 = incoming_arg 4, offset 8 ss4 = outgoing_arg 4 + ss5 = emergency_slot 4 ebb0: v1 = stack_load.i32 ss10 @@ -129,6 +130,7 @@ ebb0: ; nextln: $ss2 = local 4 ; nextln: $ss3 = incoming_arg 4, offset 8 ; nextln: $ss4 = outgoing_arg 4 +; nextln: $ss5 = emergency_slot 4 ; check: ebb0: ; nextln: $v1 = stack_load.i32 $ss10 diff --git a/cranelift/filetests/regalloc/schedule-moves.cton b/cranelift/filetests/regalloc/schedule-moves.cton index 79536aecf6..e0d9cf4e6e 100644 --- a/cranelift/filetests/regalloc/schedule-moves.cton +++ b/cranelift/filetests/regalloc/schedule-moves.cton @@ -1,12 +1,11 @@ test compile -set is_64bit=1 isa intel haswell function %pr165() native { ebb0: - v0 = iconst.i64 0x0102_0304_f1f2_f3f4 - v1 = iconst.i64 0x1102_0304_f1f2_f3f4 - v2 = iconst.i64 0x2102_0304_f1f2_f3f4 + v0 = iconst.i32 0x0102_0304 + v1 = iconst.i32 0x1102_0304 + v2 = iconst.i32 0x2102_0304 v20 = ishl v1, v0 v21 = ishl v2, v0 v22 = sshr v1, v0 @@ -17,3 +16,24 @@ ebb0: istore8 v1, v0+0x2710 return } + +; Same as above, but use so many registers that spilling is required. +; Note: This is also a candidate for using xchg instructions. +function %emergency_spill() native { +ebb0: + v0 = iconst.i32 0x0102_0304 + v1 = iconst.i32 0x1102_0304 + v2 = iconst.i32 0x2102_0304 + v3 = iconst.i32 0x3102_0304 + v4 = iconst.i32 0x4102_0304 + v20 = ishl v1, v0 + v21 = ishl v2, v3 + v22 = sshr v1, v0 + v23 = sshr v2, v0 + v24 = ushr v1, v0 + v25 = ushr v2, v0 + istore8 v0, v1+0x2710 + istore8 v1, v0+0x2710 + istore8 v3, v4+0x2710 + return +} diff --git a/lib/cretonne/src/ir/stackslot.rs b/lib/cretonne/src/ir/stackslot.rs index e5a4b10161..c1ad330e22 100644 --- a/lib/cretonne/src/ir/stackslot.rs +++ b/lib/cretonne/src/ir/stackslot.rs @@ -5,6 +5,7 @@ use entity::{PrimaryMap, Keys}; use ir::{Type, StackSlot}; +use packed_option::PackedOption; use std::fmt; use std::ops::Index; use std::str::FromStr; @@ -44,6 +45,12 @@ pub enum StackSlotKind { /// stack slots are used to represent individual arguments in the outgoing call frame. These /// stack slots are only valid while setting up a call. OutgoingArg, + + /// An emergency spill slot. + /// + /// Emergency slots are allocated late when the register's constraint solver needs extra space + /// to shuffle registers around. The are only used briefly, and can be reused. + EmergencySlot, } impl FromStr for StackSlotKind { @@ -56,6 +63,7 @@ impl FromStr for StackSlotKind { "spill_slot" => Ok(SpillSlot), "incoming_arg" => Ok(IncomingArg), "outgoing_arg" => Ok(OutgoingArg), + "emergency_slot" => Ok(EmergencySlot), _ => Err(()), } } @@ -69,6 +77,7 @@ impl fmt::Display for StackSlotKind { SpillSlot => "spill_slot", IncomingArg => "incoming_arg", OutgoingArg => "outgoing_arg", + EmergencySlot => "emergency_slot", }) } } @@ -135,6 +144,9 @@ pub struct StackSlots { /// All the outgoing stack slots, ordered by offset. outgoing: Vec, + /// All the emergency slots. + emergency: Vec, + /// The total size of the stack frame. /// /// This is the distance from the stack pointer in the current function to the stack pointer in @@ -152,6 +164,7 @@ impl StackSlots { StackSlots { slots: PrimaryMap::new(), outgoing: Vec::new(), + emergency: Vec::new(), frame_size: None, } } @@ -160,6 +173,7 @@ impl StackSlots { pub fn clear(&mut self) { self.slots.clear(); self.outgoing.clear(); + self.emergency.clear(); self.frame_size = None; } @@ -243,6 +257,43 @@ impl StackSlots { self.outgoing.insert(inspos, ss); ss } + + /// Get an emergency spill slot that can be used to store a `ty` value. + /// + /// This may allocate a new slot, or it may reuse an existing emergency spill slot, excluding + /// any slots in the `in_use` list. + pub fn get_emergency_slot( + &mut self, + ty: Type, + in_use: &[PackedOption], + ) -> StackSlot { + let size = ty.bytes(); + + // Find the smallest existing slot that can fit the type. + if let Some(&ss) = self.emergency + .iter() + .filter(|&&ss| self[ss].size >= size && !in_use.contains(&ss.into())) + .min_by_key(|&&ss| self[ss].size) + { + return ss; + } + + // Alternatively, use the largest available slot and make it larger. + if let Some(&ss) = self.emergency + .iter() + .filter(|&&ss| !in_use.contains(&ss.into())) + .max_by_key(|&&ss| self[ss].size) + { + self.slots[ss].size = size; + return ss; + } + + // No existing slot found. Make one and insert it into `emergency`. + let data = StackSlotData::new(StackSlotKind::EmergencySlot, size); + let ss = self.slots.push(data); + self.emergency.push(ss); + ss + } } #[cfg(test)] @@ -304,4 +355,31 @@ mod tests { assert_eq!(slot2.alignment(16), 8); assert_eq!(slot2.alignment(32), 8); } + + #[test] + fn emergency() { + let mut sss = StackSlots::new(); + + let ss0 = sss.get_emergency_slot(types::I32, &[]); + assert_eq!(sss[ss0].size, 4); + + // When a smaller size is requested, we should simply get the same slot back. + assert_eq!(sss.get_emergency_slot(types::I8, &[]), ss0); + assert_eq!(sss[ss0].size, 4); + assert_eq!(sss.get_emergency_slot(types::F32, &[]), ss0); + assert_eq!(sss[ss0].size, 4); + + // Ask for a larger size and the slot should grow. + assert_eq!(sss.get_emergency_slot(types::F64, &[]), ss0); + assert_eq!(sss[ss0].size, 8); + + // When one slot is in use, we should get a new one. + let ss1 = sss.get_emergency_slot(types::I32, &[None.into(), ss0.into()]); + assert_eq!(sss[ss0].size, 8); + assert_eq!(sss[ss1].size, 4); + + // Now we should get the smallest fit of the two available slots. + assert_eq!(sss.get_emergency_slot(types::F32, &[]), ss1); + assert_eq!(sss.get_emergency_slot(types::F64, &[]), ss0); + } } diff --git a/lib/cretonne/src/regalloc/coloring.rs b/lib/cretonne/src/regalloc/coloring.rs index 69e1086416..2aad6093a3 100644 --- a/lib/cretonne/src/regalloc/coloring.rs +++ b/lib/cretonne/src/regalloc/coloring.rs @@ -48,6 +48,7 @@ use ir::{Ebb, Inst, Value, Function, ValueLoc, SigRef}; use ir::{InstBuilder, ArgumentType, ArgumentLoc}; use isa::{RegUnit, RegClass, RegInfo, regs_overlap}; use isa::{TargetIsa, EncInfo, RecipeConstraints, OperandConstraint, ConstraintKind}; +use packed_option::PackedOption; use regalloc::RegDiversions; use regalloc::affinity::Affinity; use regalloc::allocatable_set::AllocatableSet; @@ -733,14 +734,47 @@ impl<'a> Context<'a> { fn shuffle_inputs(&mut self, regs: &mut AllocatableSet) { use regalloc::solver::Move::*; - self.solver.schedule_moves(regs); + let spills = self.solver.schedule_moves(regs); + + // The move operations returned by `schedule_moves` refer to emergency spill slots by + // consecutive indexes starting from 0. Map these to real stack slots. + // It is very unlikely (impossible?) that we would need more than one spill per top-level + // register class, so avoid allocation by using a fixed array here. + let mut slot = [PackedOption::default(); 8]; + assert!(spills <= slot.len(), "Too many spills ({})", spills); + for m in self.solver.moves() { match *m { Reg { value, from, to, .. } => { self.divert.regmove(value, from, to); self.cur.ins().regmove(value, from, to); } - Spill { .. } | Fill { .. } => unimplemented!(), + Spill { + value, + from, + to_slot, + .. + } => { + debug_assert_eq!(slot[to_slot].expand(), None, "Overwriting slot in use"); + let ss = self.cur.func.stack_slots.get_emergency_slot( + self.cur.func.dfg.value_type(value), + &slot[0..spills], + ); + slot[to_slot] = ss.into(); + self.divert.regspill(value, from, ss); + self.cur.ins().regspill(value, from, ss); + } + Fill { + value, + from_slot, + to, + .. + } => { + // These slots are single use, so mark `ss` as available again. + let ss = slot[from_slot].take().expect("Using unallocated slot"); + self.divert.regfill(value, ss, to); + self.cur.ins().regfill(value, ss, to); + } } } } diff --git a/lib/cretonne/src/regalloc/solver.rs b/lib/cretonne/src/regalloc/solver.rs index 5f6f27cbfd..ae5f23afb4 100644 --- a/lib/cretonne/src/regalloc/solver.rs +++ b/lib/cretonne/src/regalloc/solver.rs @@ -523,7 +523,13 @@ impl Solver { /// In either case, `to` will not be available for variables on the input side of the /// instruction. pub fn reassign_in(&mut self, value: Value, rc: RegClass, from: RegUnit, to: RegUnit) { - dbg!("reassign_in({}:{}, %{} -> %{})", value, rc, from, to); + dbg!( + "reassign_in({}:{}, {} -> {})", + value, + rc, + rc.info.display_regunit(from), + rc.info.display_regunit(to) + ); debug_assert!(!self.inputs_done); if self.regs_in.is_avail(rc, from) { // It looks like `value` was already removed from the register set. It must have been @@ -826,7 +832,9 @@ impl Solver { Move::with_assignment, )); - dbg!("collect_moves: {}", DisplayList(&self.moves)); + if !(self.moves.is_empty()) { + dbg!("collect_moves: {}", DisplayList(&self.moves)); + } } /// Try to schedule a sequence of `regmove` instructions that will shuffle registers into diff --git a/lib/cretonne/src/stack_layout.rs b/lib/cretonne/src/stack_layout.rs index 7486cee8a6..796850e059 100644 --- a/lib/cretonne/src/stack_layout.rs +++ b/lib/cretonne/src/stack_layout.rs @@ -56,7 +56,9 @@ pub fn layout_stack(frame: &mut StackSlots, alignment: StackSize) -> Result { + StackSlotKind::SpillSlot | + StackSlotKind::Local | + StackSlotKind::EmergencySlot => { // Determine the smallest alignment of any local or spill slot. min_align = slot.alignment(min_align); } diff --git a/misc/vim/syntax/cton.vim b/misc/vim/syntax/cton.vim index 3bc6a4a5da..56602e8ded 100644 --- a/misc/vim/syntax/cton.vim +++ b/misc/vim/syntax/cton.vim @@ -14,7 +14,7 @@ endif syn spell notoplevel syn keyword ctonHeader test isa set -syn keyword ctonDecl function jump_table incoming_arg outgoing_arg spill_slot local +syn keyword ctonDecl function jump_table incoming_arg outgoing_arg spill_slot local emergency_slot syn keyword ctonFilecheck check sameln nextln unordered not regex contained syn match ctonType /\<[bif]\d\+\(x\d\+\)\?\>/