diff --git a/fuzz/fuzz_targets/ion_checker.rs b/fuzz/fuzz_targets/ion_checker.rs index e3ce1dc..ea6b1be 100644 --- a/fuzz/fuzz_targets/ion_checker.rs +++ b/fuzz/fuzz_targets/ion_checker.rs @@ -21,6 +21,7 @@ impl Arbitrary for TestCase { reducible: false, block_params: true, always_local_uses: false, + reftypes: true, })?, }) } diff --git a/src/bitvec.rs b/src/bitvec.rs index ce3be7c..af5792e 100644 --- a/src/bitvec.rs +++ b/src/bitvec.rs @@ -48,6 +48,18 @@ impl BitVec { } } + pub fn assign(&mut self, other: &Self) { + if other.bits.len() > 0 { + self.ensure_idx(other.bits.len() - 1); + } + for i in 0..other.bits.len() { + self.bits[i] = other.bits[i]; + } + for i in other.bits.len()..self.bits.len() { + self.bits[i] = 0; + } + } + #[inline(always)] pub fn get(&mut self, idx: usize) -> bool { let word = idx / BITS_PER_WORD; @@ -59,16 +71,21 @@ impl BitVec { } } - pub fn or(&mut self, other: &Self) { + pub fn or(&mut self, other: &Self) -> bool { if other.bits.is_empty() { - return; + return false; } let last_idx = other.bits.len() - 1; self.ensure_idx(last_idx); + let mut changed = false; for (self_word, other_word) in self.bits.iter_mut().zip(other.bits.iter()) { + if *other_word & !*self_word != 0 { + changed = true; + } *self_word |= *other_word; } + changed } pub fn and(&mut self, other: &Self) { @@ -91,6 +108,29 @@ impl BitVec { } } +impl std::cmp::PartialEq for BitVec { + fn eq(&self, other: &Self) -> bool { + let limit = std::cmp::min(self.bits.len(), other.bits.len()); + for i in 0..limit { + if self.bits[i] != other.bits[i] { + return false; + } + } + for i in limit..self.bits.len() { + if self.bits[i] != 0 { + return false; + } + } + for i in limit..other.bits.len() { + if other.bits[i] != 0 { + return false; + } + } + true + } +} +impl std::cmp::Eq for BitVec {} + pub struct SetBitsIter<'a> { words: &'a [u64], word_idx: usize, diff --git a/src/checker.rs b/src/checker.rs index 5cdcb60..4e06c60 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -67,10 +67,10 @@ use crate::{ Allocation, AllocationKind, Block, Edit, Function, Inst, InstPosition, Operand, OperandKind, - OperandPolicy, OperandPos, Output, ProgPoint, VReg, + OperandPolicy, OperandPos, Output, PReg, ProgPoint, SpillSlot, VReg, }; -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::default::Default; use std::hash::Hash; use std::result::Result; @@ -127,6 +127,20 @@ pub enum CheckerError { alloc: Allocation, expected_alloc: Allocation, }, + AllocationIsNotStack { + inst: Inst, + op: Operand, + alloc: Allocation, + }, + ConflictedValueInStackmap { + inst: Inst, + slot: SpillSlot, + }, + NonRefValueInStackmap { + inst: Inst, + slot: SpillSlot, + vreg: VReg, + }, } /// Abstract state for an allocation. @@ -162,8 +176,10 @@ impl CheckerValue { (_, &CheckerValue::Unknown) => *self, (&CheckerValue::Conflicted, _) => *self, (_, &CheckerValue::Conflicted) => *other, - (&CheckerValue::Reg(r1, ref1), &CheckerValue::Reg(r2, ref2)) if r1 == r2 => { - CheckerValue::Reg(r1, ref1 || ref2) + (&CheckerValue::Reg(r1, ref1), &CheckerValue::Reg(r2, ref2)) + if r1 == r2 && ref1 == ref2 => + { + CheckerValue::Reg(r1, ref1) } _ => { log::debug!("{:?} and {:?} meet to Conflicted", self, other); @@ -192,7 +208,8 @@ impl std::fmt::Display for CheckerValue { match self { CheckerValue::Unknown => write!(f, "?"), CheckerValue::Conflicted => write!(f, "!"), - CheckerValue::Reg(r, _) => write!(f, "{}", r), + CheckerValue::Reg(r, false) => write!(f, "{}", r), + CheckerValue::Reg(r, true) => write!(f, "{}/ref", r), } } } @@ -305,13 +322,38 @@ impl CheckerState { self.check_val(inst, *op, *alloc, val, allocs)?; } } + &CheckerInst::Safepoint { inst, ref slots } => { + for &slot in slots { + let alloc = Allocation::stack(slot); + let val = self + .allocations + .get(&alloc) + .cloned() + .unwrap_or(Default::default()); + debug!( + "checker: checkinst {:?}: safepoint slot {}, checker value {:?}", + checkinst, slot, val + ); + + match val { + CheckerValue::Unknown => {} + CheckerValue::Conflicted => { + return Err(CheckerError::ConflictedValueInStackmap { inst, slot }); + } + CheckerValue::Reg(vreg, false) => { + return Err(CheckerError::NonRefValueInStackmap { inst, slot, vreg }); + } + CheckerValue::Reg(_, true) => {} + } + } + } _ => {} } Ok(()) } /// Update according to instruction. - fn update(&mut self, checkinst: &CheckerInst) { + fn update<'a, F: Function>(&mut self, checkinst: &CheckerInst, checker: &Checker<'a, F>) { match checkinst { &CheckerInst::Move { into, from } => { let val = self @@ -328,14 +370,19 @@ impl CheckerState { &CheckerInst::Op { ref operands, ref allocs, + ref clobbers, .. } => { for (op, alloc) in operands.iter().zip(allocs.iter()) { if op.kind() != OperandKind::Def { continue; } + let reftyped = checker.reftyped_vregs.contains(&op.vreg()); self.allocations - .insert(*alloc, CheckerValue::Reg(op.vreg(), false)); + .insert(*alloc, CheckerValue::Reg(op.vreg(), reftyped)); + } + for clobber in clobbers { + self.allocations.remove(&Allocation::reg(*clobber)); } } &CheckerInst::BlockParams { @@ -344,8 +391,20 @@ impl CheckerState { .. } => { for (vreg, alloc) in vregs.iter().zip(allocs.iter()) { + let reftyped = checker.reftyped_vregs.contains(vreg); self.allocations - .insert(*alloc, CheckerValue::Reg(*vreg, false)); + .insert(*alloc, CheckerValue::Reg(*vreg, reftyped)); + } + } + &CheckerInst::Safepoint { ref slots, .. } => { + for (alloc, value) in &mut self.allocations { + if let CheckerValue::Reg(_, true) = *value { + if alloc.is_reg() { + *value = CheckerValue::Conflicted; + } else if alloc.is_stack() && !slots.contains(&alloc.as_stack().unwrap()) { + *value = CheckerValue::Conflicted; + } + } } } } @@ -365,6 +424,11 @@ impl CheckerState { return Err(CheckerError::AllocationIsNotReg { inst, op, alloc }); } } + OperandPolicy::Stack => { + if alloc.kind() != AllocationKind::Stack { + return Err(CheckerError::AllocationIsNotStack { inst, op, alloc }); + } + } OperandPolicy::FixedReg(preg) => { if alloc != Allocation::reg(preg) { return Err(CheckerError::AllocationIsNotFixedReg { inst, op, alloc }); @@ -402,6 +466,7 @@ pub(crate) enum CheckerInst { inst: Inst, operands: Vec, allocs: Vec, + clobbers: Vec, }, /// The top of a block with blockparams. We define the given vregs @@ -411,6 +476,10 @@ pub(crate) enum CheckerInst { vregs: Vec, allocs: Vec, }, + + /// A safepoint, with the given SpillSlots specified as containing + /// reftyped values. All other reftyped values become invalid. + Safepoint { inst: Inst, slots: Vec }, } #[derive(Debug)] @@ -418,6 +487,7 @@ pub struct Checker<'a, F: Function> { f: &'a F, bb_in: HashMap, bb_insts: HashMap>, + reftyped_vregs: HashSet, } impl<'a, F: Function> Checker<'a, F> { @@ -428,6 +498,7 @@ impl<'a, F: Function> Checker<'a, F> { pub fn new(f: &'a F) -> Checker<'a, F> { let mut bb_in = HashMap::new(); let mut bb_insts = HashMap::new(); + let mut reftyped_vregs = HashSet::new(); for block in 0..f.blocks() { let block = Block::new(block); @@ -435,13 +506,31 @@ impl<'a, F: Function> Checker<'a, F> { bb_insts.insert(block, vec![]); } - Checker { f, bb_in, bb_insts } + for &vreg in f.reftype_vregs() { + reftyped_vregs.insert(vreg); + } + + Checker { + f, + bb_in, + bb_insts, + reftyped_vregs, + } } /// Build the list of checker instructions based on the given func /// and allocation results. pub fn prepare(&mut self, out: &Output) { debug!("checker: out = {:?}", out); + // Preprocess safepoint stack-maps into per-inst vecs. + let mut safepoint_slots: HashMap> = HashMap::new(); + for &(progpoint, slot) in &out.safepoint_slots { + safepoint_slots + .entry(progpoint.inst) + .or_insert_with(|| vec![]) + .push(slot); + } + // For each original instruction, create an `Op`. let mut last_inst = None; let mut insert_idx = 0; @@ -454,13 +543,23 @@ impl<'a, F: Function> Checker<'a, F> { // Any inserted edits before instruction. self.handle_edits(block, out, &mut insert_idx, ProgPoint::before(inst)); + // If this is a safepoint, then check the spillslots at this point. + if self.f.is_safepoint(inst) { + let slots = safepoint_slots.remove(&inst).unwrap_or_else(|| vec![]); + + let checkinst = CheckerInst::Safepoint { inst, slots }; + self.bb_insts.get_mut(&block).unwrap().push(checkinst); + } + // Instruction itself. let operands: Vec<_> = self.f.inst_operands(inst).iter().cloned().collect(); let allocs: Vec<_> = out.inst_allocs(inst).iter().cloned().collect(); + let clobbers: Vec<_> = self.f.inst_clobbers(inst).iter().cloned().collect(); let checkinst = CheckerInst::Op { inst, operands, allocs, + clobbers, }; debug!("checker: adding inst {:?}", checkinst); self.bb_insts.get_mut(&block).unwrap().push(checkinst); @@ -511,7 +610,7 @@ impl<'a, F: Function> Checker<'a, F> { let mut state = self.bb_in.get(&block).cloned().unwrap(); debug!("analyze: block {} has state {:?}", block.index(), state); for inst in self.bb_insts.get(&block).unwrap() { - state.update(inst); + state.update(inst, self); debug!("analyze: inst {:?} -> state {:?}", inst, state); } @@ -546,7 +645,7 @@ impl<'a, F: Function> Checker<'a, F> { debug!("Checker error: {:?}", e); errors.push(e); } - state.update(inst); + state.update(inst, self); if let Err(e) = state.check(InstPosition::After, inst) { debug!("Checker error: {:?}", e); errors.push(e); @@ -575,6 +674,9 @@ impl<'a, F: Function> Checker<'a, F> { } debug!(" {{ {} }}", s.join(", ")) } + for vreg in self.f.reftype_vregs() { + debug!(" REF: {}", vreg); + } for bb in 0..self.f.blocks() { let bb = Block::new(bb); debug!("block{}:", bb.index()); @@ -587,8 +689,15 @@ impl<'a, F: Function> Checker<'a, F> { inst, ref operands, ref allocs, + ref clobbers, } => { - debug!(" inst{}: {:?} ({:?})", inst.index(), operands, allocs); + debug!( + " inst{}: {:?} ({:?}) clobbers:{:?}", + inst.index(), + operands, + allocs, + clobbers + ); } &CheckerInst::Move { from, into } => { debug!(" {} -> {}", from, into); @@ -604,8 +713,15 @@ impl<'a, F: Function> Checker<'a, F> { } debug!(" blockparams: {}", args.join(", ")); } + &CheckerInst::Safepoint { ref slots, .. } => { + let mut slotargs = vec![]; + for &slot in slots { + slotargs.push(format!("{}", slot)); + } + debug!(" safepoint: {}", slotargs.join(", ")); + } } - state.update(inst); + state.update(inst, &self); print_state(&state); } } diff --git a/src/fuzzing/func.rs b/src/fuzzing/func.rs index ba38e98..67f0dbd 100644 --- a/src/fuzzing/func.rs +++ b/src/fuzzing/func.rs @@ -20,6 +20,7 @@ pub struct InstData { op: InstOpcode, operands: Vec, clobbers: Vec, + is_safepoint: bool, } impl InstData { @@ -32,6 +33,7 @@ impl InstData { op: InstOpcode::Op, operands, clobbers: vec![], + is_safepoint: false, } } pub fn branch(uses: &[usize]) -> InstData { @@ -43,6 +45,7 @@ impl InstData { op: InstOpcode::Branch, operands, clobbers: vec![], + is_safepoint: false, } } pub fn ret() -> InstData { @@ -50,6 +53,7 @@ impl InstData { op: InstOpcode::Ret, operands: vec![], clobbers: vec![], + is_safepoint: false, } } } @@ -62,6 +66,7 @@ pub struct Func { block_succs: Vec>, block_params: Vec>, num_vregs: usize, + reftype_vregs: Vec, } impl Function for Func { @@ -106,8 +111,12 @@ impl Function for Func { self.insts[insn.index()].op == InstOpcode::Branch } - fn is_safepoint(&self, _: Inst) -> bool { - false + fn is_safepoint(&self, insn: Inst) -> bool { + self.insts[insn.index()].is_safepoint + } + + fn reftype_vregs(&self) -> &[VReg] { + &self.reftype_vregs[..] } fn is_move(&self, _: Inst) -> Option<(VReg, VReg)> { @@ -153,6 +162,7 @@ impl FuncBuilder { insts: vec![], blocks: vec![], num_vregs: 0, + reftype_vregs: vec![], }, insts_per_block: vec![], } @@ -250,6 +260,7 @@ pub struct Options { pub reducible: bool, pub block_params: bool, pub always_local_uses: bool, + pub reftypes: bool, } impl std::default::Default for Options { @@ -262,6 +273,7 @@ impl std::default::Default for Options { reducible: false, block_params: true, always_local_uses: false, + reftypes: false, } } } @@ -355,6 +367,9 @@ impl Func { let vreg = VReg::new(builder.f.num_vregs, RegClass::Int); builder.f.num_vregs += 1; vregs.push(vreg); + if opts.reftypes && bool::arbitrary(u)? { + builder.f.reftype_vregs.push(vreg); + } } vregs_by_block.push(vregs.clone()); vregs_by_block_to_be_defined.push(vec![]); @@ -428,17 +443,28 @@ impl Func { op.kind(), OperandPos::After, ); + // Make sure reused input is a Reg. + let op = operands[reused]; + operands[reused] = + Operand::new(op.vreg(), OperandPolicy::Reg, op.kind(), OperandPos::Before); } else if opts.fixed_regs && bool::arbitrary(u)? { - // Pick an operand and make it a fixed reg. - let fixed_reg = PReg::new(u.int_in_range(0..=30)?, RegClass::Int); - let i = u.int_in_range(0..=(operands.len() - 1))?; - let op = operands[i]; - operands[i] = Operand::new( - op.vreg(), - OperandPolicy::FixedReg(fixed_reg), - op.kind(), - op.pos(), - ); + let mut fixed = vec![]; + for _ in 0..u.int_in_range(0..=operands.len() - 1)? { + // Pick an operand and make it a fixed reg. + let fixed_reg = PReg::new(u.int_in_range(0..=30)?, RegClass::Int); + if fixed.contains(&fixed_reg) { + break; + } + fixed.push(fixed_reg); + let i = u.int_in_range(0..=(operands.len() - 1))?; + let op = operands[i]; + operands[i] = Operand::new( + op.vreg(), + OperandPolicy::FixedReg(fixed_reg), + op.kind(), + op.pos(), + ); + } } else if opts.clobbers && bool::arbitrary(u)? { for _ in 0..u.int_in_range(0..=5)? { let reg = u.int_in_range(0..=30)?; @@ -448,6 +474,13 @@ impl Func { clobbers.push(PReg::new(reg, RegClass::Int)); } } + + let is_safepoint = opts.reftypes + && operands + .iter() + .all(|op| !builder.f.reftype_vregs.contains(&op.vreg())) + && bool::arbitrary(u)?; + let op = *u.choose(&[InstOpcode::Op, InstOpcode::Call])?; builder.add_inst( Block::new(block), @@ -455,6 +488,7 @@ impl Func { op, operands, clobbers, + is_safepoint, }, ); avail.push(vreg); @@ -493,6 +527,9 @@ impl Func { impl std::fmt::Debug for Func { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{{\n")?; + for vreg in self.reftype_vregs() { + write!(f, " REF: {}\n", vreg)?; + } for (i, blockrange) in self.blocks.iter().enumerate() { let succs = self.block_succs[i] .iter() @@ -513,6 +550,9 @@ impl std::fmt::Debug for Func { i, params, succs, preds )?; for inst in blockrange.iter() { + if self.is_safepoint(inst) { + write!(f, " -- SAFEPOINT --\n")?; + } write!( f, " inst{}: {:?} ops:{:?} clobber:{:?}\n", diff --git a/src/ion/mod.rs b/src/ion/mod.rs index 303c31a..bcd96c9 100644 --- a/src/ion/mod.rs +++ b/src/ion/mod.rs @@ -26,18 +26,13 @@ * * - reused-input reg: don't allocate register for input that is reused. * - * - more fuzzing: - * - test with *multiple* fixed-reg constraints on one vreg (same - * inst, different insts) - * * - modify CL to generate SSA VCode * - lower blockparams to blockparams directly * - use temps properly (`alloc_tmp()` vs `alloc_reg()`) * - * - produce stackmaps - * - stack constraint (also: unify this with stack-args? spillslot vs user stackslot?) - * - vreg reffyness - * - if reffy vreg, add to stackmap lists during reification scan + * - "Fixed-stack location": negative spillslot numbers? + * + * - Rematerialization */ #![allow(dead_code, unused_imports)] @@ -54,7 +49,7 @@ use crate::{ use log::debug; use smallvec::{smallvec, SmallVec}; use std::cmp::Ordering; -use std::collections::{BTreeMap, BinaryHeap}; +use std::collections::{BTreeMap, BinaryHeap, HashMap, HashSet, VecDeque}; use std::fmt::Debug; #[cfg(not(debug))] @@ -185,6 +180,8 @@ struct Use { next_use: UseIndex, } +const SLOT_NONE: usize = usize::MAX; + #[derive(Clone, Debug)] struct Def { operand: Operand, @@ -241,6 +238,7 @@ struct VRegData { def: DefIndex, blockparam: Block, first_range: LiveRangeIndex, + is_ref: bool, } #[derive(Clone, Debug)] @@ -307,7 +305,8 @@ struct Env<'a, F: Function> { pregs: Vec, allocation_queue: PrioQueue, hot_code: LiveRangeSet, - clobbers: Vec, // Sorted list of insts with clobbers. + clobbers: Vec, // Sorted list of insts with clobbers. + safepoints: Vec, // Sorted list of safepoint insts. spilled_bundles: Vec, spillslots: Vec, @@ -322,8 +321,8 @@ struct Env<'a, F: Function> { // will insert a copy from wherever the VReg's primary allocation // was to the approprate PReg. // - // (progpoint, copy-from-preg, copy-to-preg) - multi_fixed_reg_fixups: Vec<(ProgPoint, PRegIndex, PRegIndex)>, + // (progpoint, copy-from-preg, copy-to-preg, to-slot) + multi_fixed_reg_fixups: Vec<(ProgPoint, PRegIndex, PRegIndex, usize)>, inserted_moves: Vec, @@ -332,6 +331,7 @@ struct Env<'a, F: Function> { allocs: Vec, inst_alloc_offsets: Vec, num_spillslots: u32, + safepoint_slots: Vec<(ProgPoint, SpillSlot)>, stats: Stats, @@ -462,13 +462,16 @@ fn spill_weight_from_policy(policy: OperandPolicy) -> u32 { enum Requirement { Fixed(PReg), Register(RegClass), + Stack(RegClass), Any(RegClass), } impl Requirement { fn class(self) -> RegClass { match self { Requirement::Fixed(preg) => preg.class(), - Requirement::Register(class) | Requirement::Any(class) => class, + Requirement::Register(class) | Requirement::Any(class) | Requirement::Stack(class) => { + class + } } } @@ -478,6 +481,7 @@ impl Requirement { } match (self, other) { (other, Requirement::Any(_)) | (Requirement::Any(_), other) => Some(other), + (Requirement::Stack(_), Requirement::Stack(_)) => Some(self), (Requirement::Register(_), Requirement::Fixed(preg)) | (Requirement::Fixed(preg), Requirement::Register(_)) => { Some(Requirement::Fixed(preg)) @@ -491,6 +495,7 @@ impl Requirement { match op.policy() { OperandPolicy::FixedReg(preg) => Requirement::Fixed(preg), OperandPolicy::Reg | OperandPolicy::Reuse(_) => Requirement::Register(op.class()), + OperandPolicy::Stack => Requirement::Stack(op.class()), _ => Requirement::Any(op.class()), } } @@ -575,6 +580,7 @@ impl<'a, F: Function> Env<'a, F> { pregs: vec![], allocation_queue: PrioQueue::new(), clobbers: vec![], + safepoints: vec![], hot_code: LiveRangeSet::new(), spilled_bundles: vec![], spillslots: vec![], @@ -586,6 +592,7 @@ impl<'a, F: Function> Env<'a, F> { allocs: vec![], inst_alloc_offsets: vec![], num_spillslots: 0, + safepoint_slots: vec![], stats: Stats::default(), @@ -610,8 +617,12 @@ impl<'a, F: Function> Env<'a, F> { def: DefIndex::invalid(), first_range: LiveRangeIndex::invalid(), blockparam: Block::invalid(), + is_ref: false, }); } + for v in self.func.reftype_vregs() { + self.vregs[v.vreg()].is_ref = true; + } // Create allocations too. for inst in 0..self.func.insts() { let start = self.allocs.len() as u32; @@ -994,12 +1005,20 @@ impl<'a, F: Function> Env<'a, F> { if self.func.inst_clobbers(inst).len() > 0 { self.clobbers.push(inst); } + if self.func.is_safepoint(inst) { + self.safepoints.push(inst); + } // Mark clobbers with CodeRanges on PRegs. for i in 0..self.func.inst_clobbers(inst).len() { // don't borrow `self` let clobber = self.func.inst_clobbers(inst)[i]; + // Clobber range is at After point only: an + // instruction can still take an input in a reg + // that it later clobbers. (In other words, the + // clobber is like a normal def that never gets + // used.) let range = CodeRange { - from: ProgPoint::before(inst), + from: ProgPoint::after(inst), to: ProgPoint::before(inst.next()), }; self.add_liverange_to_preg(range, clobber); @@ -1089,7 +1108,7 @@ impl<'a, F: Function> Env<'a, F> { // If this is a branch, extend `pos` to // the end of the block. (Branch uses are // blockparams and need to be live at the - // end of the block. + // end of the block.) if self.func.is_branch(inst) { pos = self.cfginfo.block_exit[block.index()]; } @@ -1242,7 +1261,73 @@ impl<'a, F: Function> Env<'a, F> { self.liveins[block.index()] = live; } - // Do a cleanup pass: if there are any LiveRanges with + self.safepoints.sort(); + + // Insert safepoint virtual stack uses, if needed. + for vreg in self.func.reftype_vregs() { + let vreg = VRegIndex::new(vreg.vreg()); + let mut iter = self.vregs[vreg.index()].first_range; + let mut safepoint_idx = 0; + while iter.is_valid() { + let rangedata = &self.ranges[iter.index()]; + let range = rangedata.range; + while safepoint_idx < self.safepoints.len() + && ProgPoint::before(self.safepoints[safepoint_idx]) < range.from + { + safepoint_idx += 1; + } + while safepoint_idx < self.safepoints.len() + && range.contains_point(ProgPoint::before(self.safepoints[safepoint_idx])) + { + // Create a virtual use. + let pos = ProgPoint::before(self.safepoints[safepoint_idx]); + let operand = Operand::new( + self.vregs[vreg.index()].reg, + OperandPolicy::Stack, + OperandKind::Use, + OperandPos::Before, + ); + + // Create the actual use object. + let u = UseIndex(self.uses.len() as u32); + self.uses.push(Use { + operand, + pos, + slot: SLOT_NONE, + next_use: UseIndex::invalid(), + }); + + // Create/extend the LiveRange and add the use to the range. + let range = CodeRange { + from: pos, + to: pos.next(), + }; + let lr = self.add_liverange_to_vreg( + VRegIndex::new(operand.vreg().vreg()), + range, + &mut num_ranges, + ); + vreg_ranges[operand.vreg().vreg()] = lr; + + log::debug!( + "Safepoint-induced stack use of {:?} at {:?} -> {:?} -> {:?}", + operand, + pos, + u, + lr + ); + + self.insert_use_into_liverange_and_update_stats(lr, u); + safepoint_idx += 1; + } + if safepoint_idx >= self.safepoints.len() { + break; + } + iter = self.ranges[iter.index()].next_in_reg; + } + } + + // Do a fixed-reg cleanup pass: if there are any LiveRanges with // multiple uses (or defs) at the same ProgPoint and there is // more than one FixedReg constraint at that ProgPoint, we // need to record all but one of them in a special fixup list @@ -1264,11 +1349,13 @@ impl<'a, F: Function> Env<'a, F> { let mut first_preg: SmallVec<[PRegIndex; 16]> = smallvec![]; let mut extra_clobbers: SmallVec<[(PReg, Inst); 8]> = smallvec![]; let mut fixup_multi_fixed_vregs = |pos: ProgPoint, + slot: usize, op: &mut Operand, fixups: &mut Vec<( ProgPoint, PRegIndex, PRegIndex, + usize, )>| { if last_point.is_some() && Some(pos) != last_point { seen_fixed_for_vreg.clear(); @@ -1289,7 +1376,7 @@ impl<'a, F: Function> Env<'a, F> { { let orig_preg = first_preg[idx]; log::debug!(" -> duplicate; switching to policy Reg"); - fixups.push((pos, orig_preg, preg_idx)); + fixups.push((pos, orig_preg, preg_idx, slot)); *op = Operand::new(op.vreg(), OperandPolicy::Reg, op.kind(), op.pos()); extra_clobbers.push((preg, pos.inst)); } else { @@ -1302,8 +1389,10 @@ impl<'a, F: Function> Env<'a, F> { if self.ranges[iter.index()].def.is_valid() { let def_idx = self.vregs[vreg].def; let pos = self.defs[def_idx.index()].pos; + let slot = self.defs[def_idx.index()].slot; fixup_multi_fixed_vregs( pos, + slot, &mut self.defs[def_idx.index()].operand, &mut self.multi_fixed_reg_fixups, ); @@ -1312,8 +1401,10 @@ impl<'a, F: Function> Env<'a, F> { let mut use_iter = self.ranges[iter.index()].first_use; while use_iter.is_valid() { let pos = self.uses[use_iter.index()].pos; + let slot = self.uses[use_iter.index()].slot; fixup_multi_fixed_vregs( pos, + slot, &mut self.uses[use_iter.index()].operand, &mut self.multi_fixed_reg_fixups, ); @@ -1916,13 +2007,17 @@ impl<'a, F: Function> Env<'a, F> { let bundledata = &self.bundles[bundle.index()]; let first_range = &self.ranges[bundledata.first_range.index()]; + log::debug!("recompute bundle properties: bundle {:?}", bundle); + if first_range.vreg.is_invalid() { + log::debug!(" -> no vreg; minimal and fixed"); minimal = true; fixed = true; } else { if first_range.def.is_valid() { let def_data = &self.defs[first_range.def.index()]; if let OperandPolicy::FixedReg(_) = def_data.operand.policy() { + log::debug!(" -> fixed def {:?}", first_range.def); fixed = true; } } @@ -1930,6 +2025,7 @@ impl<'a, F: Function> Env<'a, F> { while use_iter.is_valid() { let use_data = &self.uses[use_iter.index()]; if let OperandPolicy::FixedReg(_) = use_data.operand.policy() { + log::debug!(" -> fixed use {:?}", use_iter); fixed = true; break; } @@ -1939,16 +2035,22 @@ impl<'a, F: Function> Env<'a, F> { // the range covers only one instruction. Note that it // could cover just one ProgPoint, i.e. X.Before..X.After, // or two ProgPoints, i.e. X.Before..X+1.Before. + log::debug!(" -> first range has range {:?}", first_range.range); + log::debug!( + " -> first range has next in bundle {:?}", + first_range.next_in_bundle + ); minimal = first_range.next_in_bundle.is_invalid() && first_range.range.from.inst == first_range.range.to.prev().inst; + log::debug!(" -> minimal: {}", minimal); } let spill_weight = if minimal { if fixed { - log::debug!(" -> fixed and minimal: 2000000"); + log::debug!(" -> fixed and minimal: spill weight 2000000"); 2_000_000 } else { - log::debug!(" -> non-fixed and minimal: 1000000"); + log::debug!(" -> non-fixed and minimal: spill weight 1000000"); 1_000_000 } } else { @@ -1957,15 +2059,20 @@ impl<'a, F: Function> Env<'a, F> { while range.is_valid() { let range_data = &self.ranges[range.index()]; if range_data.def.is_valid() { - log::debug!(" -> has def (2000)"); + log::debug!(" -> has def (spill weight +2000)"); total += 2000; } - log::debug!(" -> uses spill weight: {}", range_data.uses_spill_weight); + log::debug!(" -> uses spill weight: +{}", range_data.uses_spill_weight); total += range_data.uses_spill_weight; range = range_data.next_in_bundle; } if self.bundles[bundle.index()].prio > 0 { + log::debug!( + " -> dividing by prio {}; final weight {}", + self.bundles[bundle.index()].prio, + total / self.bundles[bundle.index()].prio + ); total / self.bundles[bundle.index()].prio } else { total @@ -2646,6 +2753,15 @@ impl<'a, F: Function> Env<'a, F> { lowest_cost_conflict_set.unwrap_or(smallvec![]) } + Requirement::Stack(_) => { + // If we must be on the stack, put ourselves on + // the spillset's list immediately. + self.spillsets[self.bundles[bundle.index()].spillset.index()] + .bundles + .push(bundle); + return; + } + Requirement::Any(_) => { // If a register is not *required*, spill now (we'll retry // allocation on spilled bundles later). @@ -2657,8 +2773,9 @@ impl<'a, F: Function> Env<'a, F> { log::debug!(" -> conflict set {:?}", conflicting_bundles); - // If we have already tried evictions once before and are still unsuccessful, give up - // and move on to splitting as long as this is not a minimal bundle. + // If we have already tried evictions once before and are + // still unsuccessful, give up and move on to splitting as + // long as this is not a minimal bundle. if attempts >= 2 && !self.minimal_bundle(bundle) { break; } @@ -3324,7 +3441,11 @@ impl<'a, F: Function> Env<'a, F> { debug_assert!(range.contains_point(usedata.pos)); let inst = usedata.pos.inst; let slot = usedata.slot; - self.set_alloc(inst, slot, alloc); + // Safepoints add virtual uses with no slots; + // avoid these. + if slot != SLOT_NONE { + self.set_alloc(inst, slot, alloc); + } use_iter = self.uses[use_iter.index()].next_use; } @@ -3425,7 +3546,7 @@ impl<'a, F: Function> Env<'a, F> { } // Handle multi-fixed-reg constraints by copying. - for (progpoint, from_preg, to_preg) in + for (progpoint, from_preg, to_preg, slot) in std::mem::replace(&mut self.multi_fixed_reg_fixups, vec![]) { log::debug!( @@ -3440,6 +3561,11 @@ impl<'a, F: Function> Env<'a, F> { Allocation::reg(self.pregs[from_preg.index()].reg), Allocation::reg(self.pregs[to_preg.index()].reg), ); + self.set_alloc( + progpoint.inst, + slot, + Allocation::reg(self.pregs[to_preg.index()].reg), + ); } // Handle outputs that reuse inputs: copy beforehand, then set @@ -3633,7 +3759,155 @@ impl<'a, F: Function> Env<'a, F> { self.edits.push((pos.to_index(), prio, edit)); } - fn compute_stackmaps(&mut self) {} + fn compute_stackmaps(&mut self) { + // For each ref-typed vreg, iterate through ranges and find + // safepoints in-range. Add the SpillSlot to the stackmap. + // + // Note that unlike in the rest of the allocator, we cannot + // overapproximate here: we cannot list a vreg's alloc at a + // certain program point in the metadata if it is not yet + // live. Because arbitrary block order and irreducible control + // flow could result in us encountering an (overapproximated, + // not actually live) vreg range for a reftyped value when + // scanning in block order, we need to do a fixpoint liveness + // analysis here for reftyped vregs only. We only perform this + // analysis if there are reftyped vregs present, so it will + // not add to allocation runtime otherwise. + + if self.func.reftype_vregs().is_empty() { + return; + } + + let mut reftype_vreg_map = BitVec::new(); + for vreg in self.func.reftype_vregs() { + reftype_vreg_map.set(vreg.vreg(), true); + } + + let mut live_reftypes_block_start: Vec = vec![]; + let mut live_reftypes_block_end: Vec = vec![]; + for _ in 0..self.func.blocks() { + live_reftypes_block_start.push(BitVec::new()); + live_reftypes_block_end.push(BitVec::new()); + } + + let mut safepoints_per_vreg: HashMap> = HashMap::new(); + for &vreg in self.func.reftype_vregs() { + safepoints_per_vreg.insert(vreg.vreg(), HashSet::new()); + } + + let mut workqueue = VecDeque::new(); + let mut workqueue_set = HashSet::new(); + let mut visited = HashSet::new(); + + // Backward analysis: start at return blocks. + for block in 0..self.func.blocks() { + let block = Block::new(block); + if self.func.is_ret(self.func.block_insns(block).last()) { + workqueue.push_back(block); + workqueue_set.insert(block); + } + } + + // While workqueue is not empty, scan a block backward. + while !workqueue.is_empty() { + let block = workqueue.pop_back().unwrap(); + workqueue_set.remove(&block); + visited.insert(block); + + let live = &mut live_reftypes_block_start[block.index()]; + live.assign(&live_reftypes_block_end[block.index()]); + + for inst in self.func.block_insns(block).rev().iter() { + for pos in &[OperandPos::After, OperandPos::Before] { + for op in self.func.inst_operands(inst) { + if !reftype_vreg_map.get(op.vreg().vreg()) { + continue; + } + if op.pos() != OperandPos::Both && op.pos() != *pos { + continue; + } + match op.kind() { + OperandKind::Def => { + live.set(op.vreg().vreg(), false); + } + OperandKind::Use => { + live.set(op.vreg().vreg(), true); + } + } + } + } + + if self.func.is_safepoint(inst) { + for vreg in live.iter() { + let safepoints = safepoints_per_vreg.get_mut(&vreg).unwrap(); + safepoints.insert(inst); + } + } + } + for blockparam in self.func.block_params(block) { + if !reftype_vreg_map.get(blockparam.vreg()) { + continue; + } + live.set(blockparam.vreg(), false); + } + + for &pred in self.func.block_preds(block) { + if live_reftypes_block_end[pred.index()].or(live) || !visited.contains(&pred) { + if !workqueue_set.contains(&pred) { + workqueue.push_back(pred); + workqueue_set.insert(pred); + } + } + } + } + + // Now we have `safepoints_per_vreg`. All we have to do is, + // for each vreg in this map, step through the LiveRanges + // along with a sorted list of safepoints; and for each + // safepoint in the current range, emit the allocation into + // the `safepoint_slots` list. + + log::debug!("safepoints_per_vreg = {:?}", safepoints_per_vreg); + + for vreg in self.func.reftype_vregs() { + log::debug!("generating safepoint info for vreg {}", vreg); + let vreg = VRegIndex::new(vreg.vreg()); + let mut safepoints: Vec = safepoints_per_vreg + .get(&vreg.index()) + .unwrap() + .iter() + .map(|&inst| ProgPoint::before(inst)) + .collect(); + safepoints.sort(); + log::debug!(" -> live over safepoints: {:?}", safepoints); + + let mut safepoint_idx = 0; + let mut iter = self.vregs[vreg.index()].first_range; + while iter.is_valid() { + let rangedata = &self.ranges[iter.index()]; + let range = rangedata.range; + let alloc = self.get_alloc_for_range(iter); + log::debug!(" -> range {:?}: alloc {}", range, alloc); + while safepoint_idx < safepoints.len() && safepoints[safepoint_idx] < range.to { + if safepoints[safepoint_idx] < range.from { + safepoint_idx += 1; + continue; + } + log::debug!(" -> covers safepoint {:?}", safepoints[safepoint_idx]); + + let slot = alloc + .as_stack() + .expect("Reference-typed value not in spillslot at safepoint"); + self.safepoint_slots.push((safepoints[safepoint_idx], slot)); + safepoint_idx += 1; + } + iter = rangedata.next_in_reg; + } + } + + self.safepoint_slots.sort(); + log::debug!("final safepoint slots info: {:?}", self.safepoint_slots); + } pub(crate) fn init(&mut self) -> Result<(), RegAllocError> { self.create_pregs_and_vregs(); @@ -3769,6 +4043,8 @@ pub fn run(func: &F, mach_env: &MachineEnv) -> Result (0, 0), OperandPolicy::Reg => (0, 1), + OperandPolicy::Stack => (0, 2), OperandPolicy::FixedReg(preg) => { assert_eq!(preg.class(), vreg.class()); - (preg.hw_enc() as u32, 2) + (preg.hw_enc() as u32, 3) } OperandPolicy::Reuse(which) => { assert!(which <= PReg::MAX); - (which as u32, 3) + (which as u32, 4) } }; let class_field = vreg.class() as u8 as u32; @@ -230,9 +233,9 @@ impl Operand { bits: vreg.vreg() as u32 | (preg_field << 20) | (class_field << 25) - | (policy_field << 26) + | (pos_field << 26) | (kind_field << 28) - | (pos_field << 29), + | (policy_field << 29), } } @@ -322,7 +325,7 @@ impl Operand { #[inline(always)] pub fn pos(self) -> OperandPos { - let pos_field = (self.bits >> 29) & 3; + let pos_field = (self.bits >> 26) & 3; match pos_field { 0 => OperandPos::Before, 1 => OperandPos::After, @@ -333,13 +336,14 @@ impl Operand { #[inline(always)] pub fn policy(self) -> OperandPolicy { - let policy_field = (self.bits >> 26) & 3; + let policy_field = (self.bits >> 29) & 7; let preg_field = ((self.bits >> 20) as usize) & PReg::MAX; match policy_field { 0 => OperandPolicy::Any, 1 => OperandPolicy::Reg, - 2 => OperandPolicy::FixedReg(PReg::new(preg_field, self.class())), - 3 => OperandPolicy::Reuse(preg_field), + 2 => OperandPolicy::Stack, + 3 => OperandPolicy::FixedReg(PReg::new(preg_field, self.class())), + 4 => OperandPolicy::Reuse(preg_field), _ => unreachable!(), } } @@ -357,15 +361,7 @@ impl Operand { impl std::fmt::Debug for Operand { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "Operand(vreg = {:?}, class = {:?}, kind = {:?}, pos = {:?}, policy = {:?})", - self.vreg().vreg(), - self.class(), - self.kind(), - self.pos(), - self.policy() - ) + std::fmt::Display::fmt(self, f) } } @@ -373,10 +369,14 @@ impl std::fmt::Display for Operand { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( f, - "{:?}@{:?}: {} {}", + "{:?}@{:?}: {}{} {}", self.kind(), self.pos(), self.vreg(), + match self.class() { + RegClass::Int => "i", + RegClass::Float => "f", + }, self.policy() ) } @@ -388,6 +388,8 @@ pub enum OperandPolicy { Any, /// Operand must be in a register. Register is read-only for Uses. Reg, + /// Operand must be on the stack. + Stack, /// Operand must be in a fixed register. FixedReg(PReg), /// On defs only: reuse a use's register. Which use is given by `preg` field. @@ -399,6 +401,7 @@ impl std::fmt::Display for OperandPolicy { match self { Self::Any => write!(f, "any"), Self::Reg => write!(f, "reg"), + Self::Stack => write!(f, "stack"), Self::FixedReg(preg) => write!(f, "fixed({})", preg), Self::Reuse(idx) => write!(f, "reuse({})", idx), } @@ -422,20 +425,21 @@ pub enum OperandPos { /// Operand. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Allocation { - /// Bit-pack in 31 bits: + /// Bit-pack in 32 bits. Note that `kind` overlaps with the + /// `policy` field in `Operand`, and we are careful to use + /// disjoint ranges of values in this field for each type. We also + /// leave the def-or-use bit (`kind` for `Operand`) unused here so + /// that the client may use it to mark `Allocation`s on + /// instructions as read or write when it edits instructions + /// (which is sometimes useful for post-allocation analyses). /// - /// op-or-alloc:1 kind:2 index:29 + /// kind:3 unused:1 index:28 bits: u32, } impl std::fmt::Debug for Allocation { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "Allocation(kind = {:?}, index = {})", - self.kind(), - self.index() - ) + std::fmt::Display::fmt(self, f) } } @@ -452,6 +456,7 @@ impl std::fmt::Display for Allocation { impl Allocation { #[inline(always)] pub(crate) fn new(kind: AllocationKind, index: usize) -> Self { + assert!(index < (1 << 28)); Self { bits: ((kind as u8 as u32) << 29) | (index as u32), } @@ -474,17 +479,32 @@ impl Allocation { #[inline(always)] pub fn kind(self) -> AllocationKind { - match (self.bits >> 29) & 3 { - 0 => AllocationKind::None, - 1 => AllocationKind::Reg, - 2 => AllocationKind::Stack, + match (self.bits >> 29) & 7 { + 5 => AllocationKind::None, + 6 => AllocationKind::Reg, + 7 => AllocationKind::Stack, _ => unreachable!(), } } + #[inline(always)] + pub fn is_none(self) -> bool { + self.kind() == AllocationKind::None + } + + #[inline(always)] + pub fn is_reg(self) -> bool { + self.kind() == AllocationKind::Reg + } + + #[inline(always)] + pub fn is_stack(self) -> bool { + self.kind() == AllocationKind::Stack + } + #[inline(always)] pub fn index(self) -> usize { - (self.bits & ((1 << 29) - 1)) as usize + (self.bits & ((1 << 28) - 1)) as usize } #[inline(always)] @@ -516,12 +536,14 @@ impl Allocation { } } +// N.B.: These values must be *disjoint* with the values used to +// encode `OperandPolicy`, because they share a 3-bit field. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(u8)] pub enum AllocationKind { - None = 0, - Reg = 1, - Stack = 2, + None = 5, + Reg = 6, + Stack = 7, } impl Allocation { @@ -535,6 +557,59 @@ impl Allocation { } } +/// A helper that wraps either an `Operand` or an `Allocation` and is +/// able to tell which it is based on the tag bits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OperandOrAllocation { + bits: u32, +} + +impl OperandOrAllocation { + pub fn from_operand(operand: Operand) -> Self { + Self { + bits: operand.bits(), + } + } + pub fn from_alloc(alloc: Allocation) -> Self { + Self { bits: alloc.bits() } + } + pub fn is_operand(&self) -> bool { + (self.bits >> 29) <= 4 + } + pub fn is_allocation(&self) -> bool { + (self.bits >> 29) >= 5 + } + pub fn as_operand(&self) -> Option { + if self.is_operand() { + Some(Operand::from_bits(self.bits)) + } else { + None + } + } + pub fn as_allocation(&self) -> Option { + if self.is_allocation() { + Some(Allocation::from_bits(self.bits & !(1 << 28))) + } else { + None + } + } + + pub fn kind(&self) -> OperandKind { + let kind_field = (self.bits >> 28) & 1; + match kind_field { + 0 => OperandKind::Def, + 1 => OperandKind::Use, + _ => unreachable!(), + } + } + + /// Replaces the Operand with an Allocation, keeping the def/use bit. + pub fn replace_with_alloc(&mut self, alloc: Allocation) { + self.bits &= 1 << 28; + self.bits |= alloc.bits; + } +} + /// A trait defined by the regalloc client to provide access to its /// machine-instruction / CFG representation. pub trait Function { @@ -576,7 +651,9 @@ pub trait Function { fn is_branch(&self, insn: Inst) -> bool; /// Determine whether an instruction is a safepoint and requires a stackmap. - fn is_safepoint(&self, insn: Inst) -> bool; + fn is_safepoint(&self, _: Inst) -> bool { + false + } /// Determine whether an instruction is a move; if so, return the /// vregs for (src, dst). @@ -598,6 +675,40 @@ pub trait Function { /// course better if it is exact. fn num_vregs(&self) -> usize; + /// Get the VRegs that are pointer/reference types. This has the + /// following effects for each such vreg: + /// + /// - At all safepoint instructions, the vreg will be in a + /// SpillSlot, not in a register. + /// - The vreg *may not* be used as a register operand on + /// safepoint instructions: this is because a vreg can only live + /// in one place at a time. The client should copy the value to an + /// integer-typed vreg and use this to pass a pointer as an input + /// to a safepoint instruction (such as a function call). + /// - At all safepoint instructions, all live vregs' locations + /// will be included in a list in the `Output` below, so that + /// pointer-inspecting/updating functionality (such as a moving + /// garbage collector) may observe and edit their values. + fn reftype_vregs(&self) -> &[VReg] { + &[] + } + + /// Get the VRegs for which we should generate value-location + /// metadata for debugging purposes. This can be used to generate + /// e.g. DWARF with valid prgram-point ranges for each value + /// expression in a way that is more efficient than a post-hoc + /// analysis of the allocator's output. + /// + /// Each tuple is (vreg, inclusive_start, exclusive_end, + /// label). In the `Output` there will be (label, inclusive_start, + /// exclusive_end, alloc)` tuples. The ranges may not exactly + /// match -- specifically, the returned metadata may cover only a + /// subset of the requested ranges -- if the value is not live for + /// the entire requested ranges. + fn debug_value_labels(&self) -> &[(Inst, Inst, VReg, u32)] { + &[] + } + // -------------- // Spills/reloads // -------------- @@ -736,6 +847,17 @@ pub struct Output { /// Allocation offset in `allocs` for each instruction. pub inst_alloc_offsets: Vec, + /// Safepoint records: at a given program point, a reference-typed value lives in the given SpillSlot. + pub safepoint_slots: Vec<(ProgPoint, SpillSlot)>, + + /// Debug info: a labeled value (as applied to vregs by + /// `Function::debug_value_labels()` on the input side) is located + /// in the given allocation from the first program point + /// (inclusive) to the second (exclusive). Guaranteed to be sorted + /// by label and program point, and the ranges are guaranteed to + /// be disjoint. + pub debug_locations: Vec<(u32, ProgPoint, ProgPoint, Allocation)>, + /// Internal stats from the allocator. pub stats: ion::Stats, }