From 9f7d6bb3b4d1284a9844b49f3103fb300433d4c8 Mon Sep 17 00:00:00 2001 From: T0b1 Date: Thu, 25 May 2023 02:30:08 +0200 Subject: [PATCH] WIP --- src/ion/fast_alloc2.rs | 491 +++++++++++++++++++++++++++++++---------- 1 file changed, 380 insertions(+), 111 deletions(-) diff --git a/src/ion/fast_alloc2.rs b/src/ion/fast_alloc2.rs index ba9d079..294eca0 100644 --- a/src/ion/fast_alloc2.rs +++ b/src/ion/fast_alloc2.rs @@ -651,7 +651,13 @@ impl<'a, F: Function> FastAlloc<'a, F> { } fn alloc_block_insts(&mut self, block: Block) -> Result<(), RegAllocError> { - let block_last_inst = self.func.block_insns(block).last().index(); + let block_after_pos = { + let block_last_inst = self.func.block_insns(block).last().index(); + let block_after_pos = self.cur_inst_pos + + (block_last_inst - self.func.block_insns(block).first().index()) + + 2; + block_after_pos + }; for inst in self.func.block_insns(block).iter() { let edit_start_idx = self.edits.len(); let clobbers = self.func.inst_clobbers(inst); @@ -681,7 +687,7 @@ impl<'a, F: Function> FastAlloc<'a, F> { } if req_refs_on_stack { - self.create_stackmap_for_reftypes(inst, block, block_last_inst); + self.create_stackmap_for_reftypes(inst, block, block_after_pos); } // TODO: hardcode operand patterns for ISA, e.g. Early Use 1, Early Use 2, Late Reuse(1) for x86 and Early Use 1, Early Use 2, Late Def 1 for ARM @@ -700,8 +706,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { // second pass: 'any'/stack uses // - preferred in reg // - TODO: these should come after fixed defs, no? - // third pass: fixed defs; allocate as given - // fourth pass: non-fixed uses and early defs + // fourth pass: fixed late and stack defs; allocate as given + // third pass: non-fixed uses and early defs (fixed, reg) // - allocate in reg if it does not interfere with fixed def/use // - spill vreg which is farthest away from being used again // - after it, process clobbers @@ -715,11 +721,13 @@ impl<'a, F: Function> FastAlloc<'a, F> { let mut op_lookup: SmallVec<[u8; 8]> = SmallVec::new(); let mut fixed_use_end = 0; let mut any_use_end = 0; - let mut fixed_def_end = 0; let mut nf_use_end = 0; + let mut fixed_def_end = 0; let mut nf_def_end = 0; let mut fixed_use_regs = PRegSet::empty(); + // TODO: fill this when iterating over the operands + // let mut fixed_def_regs = PRegSet::empty(); let mut regs_allocated = PRegSet::empty(); let mut late_def_disallow = PRegSet::empty(); @@ -731,37 +739,53 @@ impl<'a, F: Function> FastAlloc<'a, F> { op_lookup.insert(fixed_use_end, i as u8); fixed_use_end += 1; any_use_end += 1; - fixed_def_end += 1; nf_use_end += 1; + fixed_def_end += 1; nf_def_end += 1; } OperandConstraint::Any | OperandConstraint::Stack => { op_lookup.insert(any_use_end, i as u8); any_use_end += 1; - fixed_def_end += 1; nf_use_end += 1; + fixed_def_end += 1; nf_def_end += 1; } OperandConstraint::Reg => { op_lookup.insert(nf_use_end, i as u8); nf_use_end += 1; + fixed_def_end += 1; nf_def_end += 1; } _ => panic!("invalid"), }, OperandKind::Def => match op.constraint() { OperandConstraint::FixedReg(_) | OperandConstraint::Stack => { - op_lookup.insert(fixed_def_end, i as u8); - fixed_def_end += 1; - nf_use_end += 1; - nf_def_end += 1; + if op.pos() != OperandPos::Early + || op.constraint() == OperandConstraint::Stack + { + op_lookup.insert(fixed_def_end, i as u8); + fixed_def_end += 1; + nf_def_end += 1; + } else { + op_lookup.insert(nf_use_end, i as u8); + nf_use_end += 1; + fixed_def_end += 1; + nf_def_end += 1; + } } OperandConstraint::Any => { op_lookup.insert(op_lookup.len(), i as u8); } OperandConstraint::Reg | OperandConstraint::Reuse(_) => { - op_lookup.insert(nf_def_end, i as u8); - nf_def_end += 1; + if op.pos() == OperandPos::Early { + op_lookup.insert(nf_use_end, i as u8); + nf_use_end += 1; + fixed_def_end += 1; + nf_def_end += 1; + } else { + op_lookup.insert(nf_def_end, i as u8); + nf_def_end += 1; + } } }, } @@ -776,7 +800,10 @@ impl<'a, F: Function> FastAlloc<'a, F> { debug_assert_eq!(op.kind(), OperandKind::Use); if let Some(preg) = op.as_fixed_nonallocatable() { - panic!("TODO") + debug_assert!(self.pregs[preg.index()].vreg().is_none()); + self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); + trace!(" -> Allocated op {} to {}", op_idx, preg); + continue; } match op.constraint() { @@ -789,9 +816,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { op.vreg(), preg, ProgPoint::before(inst), - inst, block, - block_last_inst, + block_after_pos, ); fixed_use_regs.add(preg); @@ -820,12 +846,23 @@ impl<'a, F: Function> FastAlloc<'a, F> { OperandConstraint::Any => { match self.vregs[vreg].preg() { Some(preg) => { - self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); - regs_allocated.add(preg); - if op.pos() == OperandPos::Late { - late_def_disallow.add(preg); + if op.pos() == OperandPos::Late && clobbers.contains(preg) { + // Allocate stack slot for now + // TODO: try to find a reg in the future + let slot = match self.vregs[vreg].stack_slot() { + Some(slot) => slot, + None => self.alloc_stack_slot(op.vreg()), + }; + self.move_to_stack(op.vreg(), ProgPoint::before(inst)); + trace!(" -> Allocated op {} to slot {}", op_idx, slot); + } else { + self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); + regs_allocated.add(preg); + if op.pos() == OperandPos::Late { + late_def_disallow.add(preg); + } + trace!(" -> Allocated op {} to {}", op_idx, preg); } - trace!(" -> Allocated op {} to {}", op_idx, preg); } None => { let slot = self.vregs[vreg].stack_slot().unwrap(); @@ -853,7 +890,139 @@ impl<'a, F: Function> FastAlloc<'a, F> { } } - trace!("Third alloc pass: Fixed defs"); + trace!("Third alloc pass: Non-fixed uses and early defs"); + while op_lookup_idx < nf_use_end { + let op_idx = op_lookup[op_lookup_idx] as usize; + op_lookup_idx += 1; + let op = &operands[op_idx]; + debug_assert!(op.kind() == OperandKind::Use || op.pos() == OperandPos::Early); + debug_assert_eq!(op.constraint(), OperandConstraint::Reg); + + match op.constraint() { + OperandConstraint::Reg => { + if op.kind() == OperandKind::Use { + match self.vregs[op.vreg().vreg()].preg() { + Some(preg) => { + if op.pos() == OperandPos::Late && clobbers.contains(preg) { + // TODO: we need to make sure this value was not already used and assigned to its preg location since that may be invalid + + // find different preg + let mut reg_blacklist = regs_allocated; + reg_blacklist.union_from(clobbers); + let new_preg = self.find_free_reg( + op.vreg().class(), + reg_blacklist, + true, + block, + block_after_pos, + )?; + self.edits.push(( + ProgPoint::before(inst), + Edit::Move { + from: Allocation::reg(preg), + to: Allocation::reg(new_preg), + }, + )); + self.clear_preg(preg.index()); + self.assign_preg(new_preg, op.vreg()); + self.allocs[alloc_idx + op_idx] = Allocation::reg(new_preg); + regs_allocated.add(new_preg); + late_def_disallow.add(new_preg); + + trace!(" -> Allocated op {} to {}", op_idx, new_preg); + } else { + self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); + regs_allocated.add(preg); + + if op.pos() == OperandPos::Late { + late_def_disallow.add(preg); + } + trace!(" -> Allocated op {} to {}", op_idx, preg); + } + } + None => { + let mut reg_blacklist = regs_allocated; + if op.pos() == OperandPos::Late { + reg_blacklist.union_from(clobbers); + } + + let preg = self.find_free_reg( + op.vreg().class(), + reg_blacklist, + true, + block, + block_after_pos, + )?; + debug_assert!(self.pregs[preg.index()].vreg().is_none()); + debug_assert!(self.vregs[op.vreg().vreg()] + .stack_slot() + .is_none()); + self.move_vreg_to_preg( + op.vreg(), + preg, + ProgPoint::before(inst), + block, + block_after_pos, + ); + trace!(" -> Allocated op {} to {}", op_idx, preg); + } + } + self.vregs[op.vreg().vreg()].cur_use_idx -= 1; + } else { + let mut reg_blacklist = regs_allocated; + reg_blacklist.union_from(clobbers); + let preg = self.find_free_reg( + op.vreg().class(), + reg_blacklist, + true, + block, + block_after_pos, + )?; + debug_assert!(self.pregs[preg.index()].vreg().is_none()); + debug_assert!(self.vregs[op.vreg().vreg()].stack_slot().is_none()); + self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); + self.assign_preg(preg, op.vreg()); + regs_allocated.add(preg); + trace!(" -> Allocated op {} to {}", op_idx, preg); + } + } + OperandConstraint::FixedReg(preg) => { + debug_assert_eq!(op.kind(), OperandKind::Def); + if clobbers.contains(preg) { + return Err(RegAllocError::TooManyLiveRegs); + } + // TODO: should handle this + assert!(!regs_allocated.contains(preg)); + + self.allocate_preg_for_vreg( + preg, + op.vreg(), + ProgPoint::before(inst), + block, + block_after_pos, + ); + self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); + regs_allocated.add(preg); + trace!(" -> Allocated op {} to {}", op_idx, preg); + } + _ => unreachable!(), + } + } + + trace!("Handling clobbers"); + for preg in clobbers { + if let Some(vreg) = self.pregs[preg.index()].vreg() { + if self.vregs[vreg.vreg()].stack_slot().is_none() + && !self.vreg_killed(vreg.vreg(), block, block_after_pos, false) + { + self.alloc_and_move_to_stack(vreg, ProgPoint::before(inst)); + } + self.clear_preg(preg.index()); + trace!(" Cleared {}", preg); + } + } + + trace!("Fourth alloc pass: Fixed defs"); while op_lookup_idx < fixed_def_end { let op_idx = op_lookup[op_lookup_idx] as usize; op_lookup_idx += 1; @@ -880,9 +1049,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { preg, op.vreg(), ProgPoint::before(inst), - inst, block, - block_last_inst, + block_after_pos, ); if op.pos() == OperandPos::Early { @@ -896,73 +1064,6 @@ impl<'a, F: Function> FastAlloc<'a, F> { } } - trace!("Fourth alloc pass: Non-fixed uses and early defs"); - while op_lookup_idx < nf_use_end { - let op_idx = op_lookup[op_lookup_idx] as usize; - op_lookup_idx += 1; - let op = &operands[op_idx]; - debug_assert!(op.kind() == OperandKind::Use || op.pos() == OperandPos::Early); - debug_assert_eq!(op.constraint(), OperandConstraint::Reg); - - match op.constraint() { - OperandConstraint::Reg => { - if op.kind() == OperandKind::Use { - match self.vregs[op.vreg().vreg()].preg() { - Some(preg) => { - self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); - regs_allocated.add(preg); - - if op.pos() == OperandPos::Late { - late_def_disallow.add(preg); - } - trace!(" -> Allocated op {} to {}", op_idx, preg); - } - None => { - let preg = self.find_free_reg( - op.vreg().class(), - regs_allocated, - true, - inst, - block, - block_last_inst, - )?; - debug_assert!(self.pregs[preg.index()].vreg().is_none()); - debug_assert!(self.vregs[op.vreg().vreg()] - .stack_slot() - .is_none()); - self.move_vreg_to_preg( - op.vreg(), - preg, - ProgPoint::before(inst), - inst, - block, - block_last_inst, - ); - trace!(" -> Allocated op {} to {}", op_idx, preg); - } - } - self.vregs[op.vreg().vreg()].cur_use_idx -= 1; - } else { - let preg = self.find_free_reg( - op.vreg().class(), - regs_allocated, - true, - inst, - block, - block_last_inst, - )?; - debug_assert!(self.pregs[preg.index()].vreg().is_none()); - debug_assert!(self.vregs[op.vreg().vreg()].stack_slot().is_none()); - self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); - self.assign_preg(preg, op.vreg()); - regs_allocated.add(preg); - trace!(" -> Allocated op {} to {}", op_idx, preg); - } - } - _ => unreachable!(), - } - } - trace!("Fifth alloc pass: Non-fixed defs and reuses"); // need to handle reuses first let op_lookup_idx_bak = op_lookup_idx; @@ -1002,9 +1103,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { op.vreg().class(), late_def_disallow, true, - inst, block, - block_last_inst, + block_after_pos, )?; self.allocs[alloc_idx + op_idx] = Allocation::reg(preg); late_def_disallow.add(preg); @@ -1034,9 +1134,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { op.vreg().class(), reg_blacklist, false, - inst, block, - block_last_inst, + block_after_pos, ); if let Ok(preg) = preg { @@ -1051,13 +1150,181 @@ impl<'a, F: Function> FastAlloc<'a, F> { trace!(" -> Allocated op {} to slot {}", op_idx, slot); } } - todo!("") + + debug_assert!(!self.allocs[alloc_idx..alloc_idx + operands.len()] + .iter() + .any(|a| a.is_none())); + + self.cur_inst_pos += 1; } - todo!("") + + Ok(()) } fn alloc_block_edge(&mut self, block: Block) -> Result<(), RegAllocError> { - todo!("") + trace!("Allocating block edges for {}", block.index()); + // Three cases: Single successor nonallocated, single successor allocated and multiple successors unallocated + let succs = self.func.block_succs(block); + let res = if succs.len() == 1 { + let succ = succs[0]; + if self.blocks[succ.index()].regs_allocated() { + self.alloc_block_edge_single_allocated(block, succ) + } else { + self.alloc_block_edge_single_unallocated(block, succ) + } + } else { + self.alloc_block_edge_multiple(block, succs) + }; + self.cur_inst_pos += 1; + return res; + } + + fn alloc_block_edge_single_unallocated( + &mut self, + block: Block, + succ: Block, + ) -> Result<(), RegAllocError> { + assert_ne!(block, succ); + trace!(" -> Block only has a single unallocated edge. Copying allocations"); + + // move allocations if possible or duplicate values + // if the outgoing vreg is killed, simply copy the allocation + // otherwise keep the vreg that is closest to being used in a preg + // and move the other value to stack except if the outgoing vreg already has a stack slot + // TODO: evaluate if it makes sense to keep the preg for the outgoing vreg even if it has a stack slot + + let succ_params = self.func.block_params(succ); + let last_inst = self.func.block_insns(block).last(); + debug_assert!(self.func.is_branch(last_inst)); + let out_vregs = self.func.branch_blockparams(block, last_inst, 0); + + // prevent these from being allocated as tmp regs + let mut pregs_allocated = PRegSet::empty(); + let mut tmp_reg = None; + for i in 0..succ_params.len() { + let succ_param = succ_params[i]; + let out_vreg = out_vregs[i]; + + if self.vreg_killed(out_vreg.vreg(), block, self.cur_inst_pos + 1, false) { + trace!(" -> {} mapping to {} is killed", out_vreg, succ_param); + if let Some(slot) = self.vregs[out_vreg.vreg()].stack_slot() { + trace!(" -> Transferring slot {} to {}", slot, succ_param); + self.vregs[succ_param.vreg()].set_stack_slot(slot); + if cfg!(debug_assertions) { + self.vregs[out_vreg.vreg()].clear_stack_slot(); + } + } + if let Some(preg) = self.vregs[out_vreg.vreg()].preg() { + trace!(" -> Transferring {} to {}", preg, succ_param); + self.vregs[succ_param.vreg()].set_preg(preg); + if cfg!(debug_assertions) { + self.vregs[out_vreg.vreg()].clear_preg(); + } + self.pregs[preg.index()].set_vreg(succ_param); + pregs_allocated.add(preg); + } + self.vregs[out_vreg.vreg()].cur_use_idx -= 1; + continue; + } + + trace!(" -> {} mapping to {} is not killed", out_vreg, succ_param); + // the out_vreg might only be alive until the end of the block but used by another param or still be live after the block + if let Some(preg) = self.vregs[out_vreg.vreg()].preg() { + // TODO: we could do some fancy calculation here which of the block params should get to stay in the preg but rn we dont + if self.next_use(out_vreg.vreg()).unwrap() + > self.next_use(succ_param.vreg()).unwrap() + || self.vregs[out_vreg.vreg()].stack_slot().is_some() + { + trace!(" -> Transferring preg from {} to {}", out_vreg, succ_param); + + if self.vregs[out_vreg.vreg()].stack_slot().is_none() { + // save to stack if necessary + self.alloc_and_move_to_stack(out_vreg, ProgPoint::before(last_inst)); + } + + self.vregs[out_vreg.vreg()].clear_preg(); + self.vregs[succ_param.vreg()].set_preg(preg); + self.pregs[preg.index()].set_vreg(succ_param); + pregs_allocated.add(preg); + + // don't need to care about stack slot + self.vregs[out_vreg.vreg()].cur_use_idx -= 1; + continue; + } + + let slot = self.alloc_stack_slot(succ_param); + trace!(" -> Transferring preg from {} to slot {}", preg, slot); + self.edits.push(( + ProgPoint::before(last_inst), + Edit::Move { + from: Allocation::reg(preg), + to: Allocation::stack(SpillSlot::new(slot as usize)), + }, + )); + + self.vregs[out_vreg.vreg()].cur_use_idx -= 1; + continue; + } + + let dst_slot = self.alloc_stack_slot(succ_param) as usize; + let src_slot = self.vregs[out_vreg.vreg()].stack_slot().unwrap() as usize; + trace!( + " -> Copying {} from slot {} to slot {} for {}", + out_vreg, + src_slot, + dst_slot, + succ_param + ); + // out_vreg is on the stack so create a stack slot for succ_param + let preg = match tmp_reg { + Some(preg) => preg, + None => { + let preg = self.find_free_reg( + succ_param.class(), + pregs_allocated, + true, + block, + self.cur_inst_pos + 1, + )?; + tmp_reg = Some(preg); + preg + } + }; + + self.edits.push(( + ProgPoint::before(last_inst), + Edit::Move { + from: Allocation::stack(SpillSlot::new(src_slot)), + to: Allocation::reg(preg), + }, + )); + self.edits.push(( + ProgPoint::before(last_inst), + Edit::Move { + to: Allocation::stack(SpillSlot::new(dst_slot)), + from: Allocation::reg(preg), + }, + )); + self.vregs[out_vreg.vreg()].cur_use_idx -= 1; + } + + Ok(()) + } + + fn alloc_block_edge_single_allocated( + &mut self, + block: Block, + succ: Block, + ) -> Result<(), RegAllocError> { + todo!() + } + + fn alloc_block_edge_multiple( + &mut self, + block: Block, + succs: &[Block], + ) -> Result<(), RegAllocError> { + todo!() } fn run(&mut self) -> Result<(), RegAllocError> { @@ -1095,7 +1362,7 @@ impl<'a, F: Function> FastAlloc<'a, F> { Ok(()) } - fn create_stackmap_for_reftypes(&mut self, inst: Inst, block: Block, block_last_inst: usize) { + fn create_stackmap_for_reftypes(&mut self, inst: Inst, block: Block, block_after_pos: usize) { // make sure all reftypes have a valid stackslot self.move_reftype_to_stack(inst); let pos = ProgPoint::before(inst); @@ -1107,7 +1374,7 @@ impl<'a, F: Function> FastAlloc<'a, F> { continue; } - if self.vreg_killed(vreg, inst, block, block_last_inst, true) { + if self.vreg_killed(vreg, block, block_after_pos, true) { continue; } @@ -1126,9 +1393,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { reg_class: RegClass, reg_blacklist: PRegSet, spill: bool, - inst: Inst, block: Block, - block_last_inst: usize, + block_after_pos: usize, ) -> Result { todo!("") } @@ -1141,9 +1407,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { vreg: VReg, preg: PReg, pos: ProgPoint, - inst: Inst, block: Block, - block_last_inst: usize, + block_after_pos: usize, ) { todo!("") } @@ -1217,9 +1482,8 @@ impl<'a, F: Function> FastAlloc<'a, F> { preg: PReg, vreg: VReg, pos: ProgPoint, - inst: Inst, block: Block, - block_last_inst: usize, + block_after_pos: usize, ) { todo!("") } @@ -1254,22 +1518,27 @@ impl<'a, F: Function> FastAlloc<'a, F> { } } + fn next_use(&self, vreg: usize) -> Option { + let use_idx = self.vregs[vreg].cur_use_idx; + if use_idx as usize >= self.vregs[vreg].uses.len() { + return None; + } + return Some(self.vregs[vreg].uses[use_idx as usize]); + } + fn vreg_killed( &self, vreg: usize, - inst: Inst, block: Block, - block_last_inst: usize, + block_after_pos: usize, save_on_current_use: bool, ) -> bool { let info = &self.vregs[vreg]; - let block_after_pos = self.cur_inst_pos + (block_last_inst - inst.index()) + 2; let cur_use_idx = info.cur_use_idx as usize; trace!( - "Checking live-status of v{} in {:?} at inst {:?}; CurPos: {}, SaveOnCurrent: {}, Liveout {}, block_after: {}", + "Checking live-status of v{} in {:?} at CurPos: {}; SaveOnCurrent: {}, Liveout {}, block_after: {}", vreg, block, - inst, self.cur_inst_pos, save_on_current_use, self.liveouts[block.index()].get(vreg),