T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cranelift: fix regalloc2 integration bug wrt blockparam branch args. (#4042)

Browse Source 
Previously, the block successor accumulation and the blockparam branch
arg setup were decoupled. The lowering backend implicitly specified
the order of successor edges via its `MachTerminator` enum on the last
instruction in the block, while the `Lower` toplevel
machine-independent driver set up blockparam branch args in the edge
order seen in CLIF.

In some cases, these orders did not match -- for example, when the
conditional branch depended on an FP condition that was implemented by
swapping taken/not-taken edges and inverting the condition code.

This PR refactors the successor handling to be centralized in `Lower`
rather than flow through the terminator `MachInst`, and adds a
successor block and its blockparam args at the same time, ensuring the
orders match.
This commit is contained in:
Chris Fallin
2022-04-18 09:53:57 -07:00
committed by GitHub
parent 7cf5f05830
commit 5774e068b7
7 changed files with 44 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -98,7 +98,6 @@ pub struct CallIndInfo {
pub struct JTSequenceInfo { pub struct JTSequenceInfo {
pub targets: Vec<BranchTarget>, pub targets: Vec<BranchTarget>,
pub default_target: BranchTarget, pub default_target: BranchTarget,
pub targets_for_term: Vec<MachLabel>, // needed for MachTerminator.
} }
fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize { fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {
@@ -1091,17 +1090,13 @@ impl MachInst for Inst {
} }
} }
fn is_term<'a>(&'a self) -> MachTerminator<'a> { fn is_term(&self) -> MachTerminator {
match self { match self {
&Inst::Ret { .. } | &Inst::EpiloguePlaceholder => MachTerminator::Ret, &Inst::Ret { .. } | &Inst::EpiloguePlaceholder => MachTerminator::Ret,
&Inst::Jump { dest } => MachTerminator::Uncond(dest.as_label().unwrap()), &Inst::Jump { .. } => MachTerminator::Uncond,
&Inst::CondBr { &Inst::CondBr { .. } => MachTerminator::Cond,
taken, not_taken, .. &Inst::IndirectBr { .. } => MachTerminator::Indirect,
} => MachTerminator::Cond(taken.as_label().unwrap(), not_taken.as_label().unwrap()), &Inst::JTSequence { .. } => MachTerminator::Indirect,
&Inst::IndirectBr { ref targets, .. } => MachTerminator::Indirect(&targets[..]),
&Inst::JTSequence { ref info, .. } => {
MachTerminator::Indirect(&info.targets_for_term[..])
}
_ => MachTerminator::None, _ => MachTerminator::None,
} }
} }

2
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -2739,7 +2739,6 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
.map(|bix| BranchTarget::Label(*bix)) .map(|bix| BranchTarget::Label(*bix))
.collect(); .collect();
let default_target = BranchTarget::Label(targets[0]); let default_target = BranchTarget::Label(targets[0]);
let targets_for_term: Vec<MachLabel> = targets.to_vec();
ctx.emit(Inst::JTSequence { ctx.emit(Inst::JTSequence {
ridx, ridx,
rtmp1, rtmp1,
@@ -2747,7 +2746,6 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
info: Box::new(JTSequenceInfo { info: Box::new(JTSequenceInfo {
targets: jt_targets, targets: jt_targets,
default_target, default_target,
targets_for_term,
}), }),
}); });
} }

12
cranelift/codegen/src/isa/s390x/inst/mod.rs
View File

@@ -733,19 +733,17 @@ impl MachInst for Inst {
} }
} }
fn is_term<'a>(&'a self) -> MachTerminator<'a> { fn is_term(&self) -> MachTerminator {
match self { match self {
&Inst::Ret { .. } | &Inst::EpiloguePlaceholder => MachTerminator::Ret, &Inst::Ret { .. } | &Inst::EpiloguePlaceholder => MachTerminator::Ret,
&Inst::Jump { dest } => MachTerminator::Uncond(dest), &Inst::Jump { .. } => MachTerminator::Uncond,
&Inst::CondBr { &Inst::CondBr { .. } => MachTerminator::Cond,
taken, not_taken, ..
} => MachTerminator::Cond(taken, not_taken),
&Inst::OneWayCondBr { .. } => { &Inst::OneWayCondBr { .. } => {
// Explicitly invisible to CFG processing. // Explicitly invisible to CFG processing.
MachTerminator::None MachTerminator::None
} }
&Inst::IndirectBr { ref targets, .. } => MachTerminator::Indirect(&targets[..]), &Inst::IndirectBr { .. } => MachTerminator::Indirect,
&Inst::JTSequence { ref targets, .. } => MachTerminator::Indirect(&targets[..]), &Inst::JTSequence { .. } => MachTerminator::Indirect,
_ => MachTerminator::None, _ => MachTerminator::None,
} }
} }

13
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -2119,18 +2119,13 @@ impl MachInst for Inst {
} }
} }
fn is_term<'a>(&'a self) -> MachTerminator<'a> { fn is_term(&self) -> MachTerminator {
match self { match self {
// Interesting cases. // Interesting cases.
&Self::Ret { .. } | &Self::EpiloguePlaceholder => MachTerminator::Ret, &Self::Ret { .. } | &Self::EpiloguePlaceholder => MachTerminator::Ret,
&Self::JmpKnown { dst } => MachTerminator::Uncond(dst), &Self::JmpKnown { .. } => MachTerminator::Uncond,
&Self::JmpCond { &Self::JmpCond { .. } => MachTerminator::Cond,
taken, not_taken, .. &Self::JmpTableSeq { .. } => MachTerminator::Indirect,
} => MachTerminator::Cond(taken, not_taken),
&Self::JmpTableSeq {
ref targets_for_term,
..
} => MachTerminator::Indirect(&targets_for_term[..]),
// All other cases are boring. // All other cases are boring.
_ => MachTerminator::None, _ => MachTerminator::None,
} }

20
cranelift/codegen/src/machinst/lower.rs
View File

@@ -745,6 +745,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
fn lower_clif_branches<B: LowerBackend<MInst = I>>( fn lower_clif_branches<B: LowerBackend<MInst = I>>(
&mut self, &mut self,
backend: &B, backend: &B,
// Lowered block index:
bindex: BlockIndex,
// Original CLIF block:
block: Block, block: Block,
branches: &SmallVec<[Inst; 2]>, branches: &SmallVec<[Inst; 2]>,
targets: &SmallVec<[MachLabel; 2]>, targets: &SmallVec<[MachLabel; 2]>,
@@ -762,12 +765,15 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
let loc = self.srcloc(branches[0]); let loc = self.srcloc(branches[0]);
self.finish_ir_inst(loc); self.finish_ir_inst(loc);
// Add block param outputs for current block. // Add block param outputs for current block.
self.lower_branch_blockparam_args(block); self.lower_branch_blockparam_args(bindex);
Ok(()) Ok(())
} }
fn lower_branch_blockparam_args(&mut self, block: Block) { fn lower_branch_blockparam_args(&mut self, block: BlockIndex) {
visit_block_succs(self.f, block, |inst, _succ| { for succ_idx in 0..self.vcode.block_order().succ_indices(block).len() {
// Avoid immutable borrow by explicitly indexing.
let (inst, succ) = self.vcode.block_order().succ_indices(block)[succ_idx];
// Get branch args and convert to Regs.
let branch_args = self.f.dfg.inst_variable_args(inst); let branch_args = self.f.dfg.inst_variable_args(inst);
let mut branch_arg_vregs: SmallVec<[Reg; 16]> = smallvec![]; let mut branch_arg_vregs: SmallVec<[Reg; 16]> = smallvec![];
for &arg in branch_args { for &arg in branch_args {
@@ -778,8 +784,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
branch_arg_vregs.push(vreg.into()); branch_arg_vregs.push(vreg.into());
} }
} }
self.vcode.add_branch_args_for_succ(&branch_arg_vregs[..]); self.vcode.add_succ(succ, &branch_arg_vregs[..]);
}); }
self.finish_ir_inst(SourceLoc::default()); self.finish_ir_inst(SourceLoc::default());
} }
@@ -849,7 +855,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
if let Some(bb) = lb.orig_block() { if let Some(bb) = lb.orig_block() {
self.collect_branches_and_targets(bindex, bb, &mut branches, &mut targets); self.collect_branches_and_targets(bindex, bb, &mut branches, &mut targets);
if branches.len() > 0 { if branches.len() > 0 {
self.lower_clif_branches(backend, bb, &branches, &targets)?; self.lower_clif_branches(backend, bindex, bb, &branches, &targets)?;
self.finish_ir_inst(self.srcloc(branches[0])); self.finish_ir_inst(self.srcloc(branches[0]));
} }
} else { } else {
@@ -875,7 +881,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
.add_block_param(vreg, self.vcode.get_vreg_type(vreg)); .add_block_param(vreg, self.vcode.get_vreg_type(vreg));
} }
} }
self.vcode.add_branch_args_for_succ(&branch_arg_vregs[..]); self.vcode.add_succ(succ, &branch_arg_vregs[..]);
self.emit(I::gen_jump(MachLabel::from_block(succ))); self.emit(I::gen_jump(MachLabel::from_block(succ)));
self.finish_ir_inst(SourceLoc::default()); self.finish_ir_inst(SourceLoc::default());

14
cranelift/codegen/src/machinst/mod.rs
View File

@@ -94,7 +94,7 @@ pub trait MachInst: Clone + Debug {
/// Is this a terminator (branch or ret)? If so, return its type /// Is this a terminator (branch or ret)? If so, return its type
/// (ret/uncond/cond) and target if applicable. /// (ret/uncond/cond) and target if applicable.
fn is_term<'a>(&'a self) -> MachTerminator<'a>; fn is_term(&self) -> MachTerminator;
/// Returns true if the instruction is an epilogue placeholder. /// Returns true if the instruction is an epilogue placeholder.
fn is_epilogue_placeholder(&self) -> bool; fn is_epilogue_placeholder(&self) -> bool;
@@ -221,18 +221,22 @@ pub trait MachInstLabelUse: Clone + Copy + Debug + Eq {
/// Describes a block terminator (not call) in the vcode, when its branches /// Describes a block terminator (not call) in the vcode, when its branches
/// have not yet been finalized (so a branch may have two targets). /// have not yet been finalized (so a branch may have two targets).
///
/// Actual targets are not included: the single-source-of-truth for
/// those is the VCode itself, which holds, for each block, successors
/// and outgoing branch args per successor.
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub enum MachTerminator<'a> { pub enum MachTerminator {
/// Not a terminator. /// Not a terminator.
None, None,
/// A return instruction. /// A return instruction.
Ret, Ret,
/// An unconditional branch to another block. /// An unconditional branch to another block.
Uncond(MachLabel), Uncond,
/// A conditional branch to one of two other blocks. /// A conditional branch to one of two other blocks.
Cond(MachLabel, MachLabel), Cond,
/// An indirect branch with known possible targets. /// An indirect branch with known possible targets.
Indirect(&'a [MachLabel]), Indirect,
} }
/// A trait describing the ability to encode a MachInst into binary machine code. /// A trait describing the ability to encode a MachInst into binary machine code.

36
cranelift/codegen/src/machinst/vcode.rs
View File

@@ -371,7 +371,7 @@ impl<I: VCodeInst> VCodeBuilder<I> {
self.vcode.block_params.push(param.into()); self.vcode.block_params.push(param.into());
} }
pub fn add_branch_args_for_succ(&mut self, args: &[Reg]) { fn add_branch_args_for_succ(&mut self, args: &[Reg]) {
let start = self.vcode.branch_block_args.len(); let start = self.vcode.branch_block_args.len();
self.vcode self.vcode
.branch_block_args .branch_block_args
@@ -385,34 +385,16 @@ impl<I: VCodeInst> VCodeBuilder<I> {
/// Push an instruction for the current BB and current IR inst /// Push an instruction for the current BB and current IR inst
/// within the BB. /// within the BB.
pub fn push(&mut self, insn: I) { pub fn push(&mut self, insn: I) {
match insn.is_term() {
MachTerminator::None | MachTerminator::Ret => {}
MachTerminator::Uncond(target) => {
self.vcode
.block_succs_preds
.push(BlockIndex::new(target.get() as usize));
}
MachTerminator::Cond(true_branch, false_branch) => {
self.vcode
.block_succs_preds
.push(BlockIndex::new(true_branch.get() as usize));
self.vcode
.block_succs_preds
.push(BlockIndex::new(false_branch.get() as usize));
}
MachTerminator::Indirect(targets) => {
for target in targets {
self.vcode
.block_succs_preds
.push(BlockIndex::new(target.get() as usize));
}
}
}
self.vcode.insts.push(insn); self.vcode.insts.push(insn);
self.vcode.srclocs.push(self.cur_srcloc); self.vcode.srclocs.push(self.cur_srcloc);
} }
/// Add a successor block with branch args.
pub fn add_succ(&mut self, block: BlockIndex, args: &[Reg]) {
self.vcode.block_succs_preds.push(block);
self.add_branch_args_for_succ(args);
}
/// Set the current source location. /// Set the current source location.
pub fn set_srcloc(&mut self, srcloc: SourceLoc) { pub fn set_srcloc(&mut self, srcloc: SourceLoc) {
self.cur_srcloc = srcloc; self.cur_srcloc = srcloc;
@@ -1191,9 +1173,7 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
fn is_branch(&self, insn: InsnIndex) -> bool { fn is_branch(&self, insn: InsnIndex) -> bool {
match self.insts[insn.index()].is_term() { match self.insts[insn.index()].is_term() {
MachTerminator::Cond(..) MachTerminator::Cond | MachTerminator::Uncond | MachTerminator::Indirect => true,
| MachTerminator::Uncond(..)
| MachTerminator::Indirect(..) => true,
_ => false, _ => false,
} }
} }