diff --git a/cranelift/filetests/regalloc/reload-208.cton b/cranelift/filetests/regalloc/reload-208.cton new file mode 100644 index 0000000000..29c6e09840 --- /dev/null +++ b/cranelift/filetests/regalloc/reload-208.cton @@ -0,0 +1,94 @@ +test regalloc +set is_64bit +isa intel haswell + +; regex: V=v\d+ + +; Filed as https://github.com/stoklund/cretonne/issues/208 +; +; The verifier complains about a branch argument that is not in the same virtual register as the +; corresponding EBB argument. +; +; The problem was the reload pass rewriting EBB arguments on "brnz v9, ebb3(v9)" + +function %pr208(i64 vmctx [%rdi]) native { + gv0 = vmctx-8 + heap0 = static gv0, min 0, bound 0x5000, guard 0x0040_0000 + sig0 = (i64 vmctx [%rdi]) -> i32 [%rax] native + sig1 = (i64 vmctx [%rdi], i32 [%rsi]) native + fn0 = sig0 u0:1 + fn1 = sig1 u0:3 + +ebb0(v0: i64): + v1 = iconst.i32 0 + v2 = call fn0(v0) + v20 = iconst.i32 0x4ffe + v16 = icmp uge v2, v20 + brz v16, ebb5 + trap heap_oob + +ebb5: + v17 = uextend.i64 v2 + v18 = iadd_imm.i64 v0, -8 + v19 = load.i64 v18 + v3 = iadd v19, v17 + v4 = load.i32 v3 + v21 = iconst.i32 0 + v5 = icmp eq v4, v21 + v6 = bint.i32 v5 + brnz v6, ebb2 + jump ebb3(v4) + +ebb3(v7: i32): + call fn1(v0, v7) + v26 = iconst.i32 0x4ffe + v22 = icmp uge v7, v26 + brz v22, ebb6 + trap heap_oob + +ebb6: + v23 = uextend.i64 v7 + v24 = iadd_imm.i64 v0, -8 + v25 = load.i64 v24 + v8 = iadd v25, v23 + v9 = load.i32 v8+56 + ; check: $v9 = spill + ; check: brnz $V, $ebb3($v9) + brnz v9, ebb3(v9) + jump ebb4 + +ebb4: + jump ebb2 + +ebb2: + v10 = iconst.i32 0 + v31 = iconst.i32 0x4ffe + v27 = icmp uge v10, v31 + brz v27, ebb7 + trap heap_oob + +ebb7: + v28 = uextend.i64 v10 + v29 = iadd_imm.i64 v0, -8 + v30 = load.i64 v29 + v11 = iadd v30, v28 + v12 = load.i32 v11+12 + call fn1(v0, v12) + v13 = iconst.i32 0 + v36 = iconst.i32 0x4ffe + v32 = icmp uge v13, v36 + brz v32, ebb8 + trap heap_oob + +ebb8: + v33 = uextend.i64 v13 + v34 = iadd_imm.i64 v0, -8 + v35 = load.i64 v34 + v14 = iadd v35, v33 + v15 = load.i32 v14+12 + call fn1(v0, v15) + jump ebb1 + +ebb1: + return +} diff --git a/lib/cretonne/src/regalloc/reload.rs b/lib/cretonne/src/regalloc/reload.rs index ab9798d4c9..8ed95628fd 100644 --- a/lib/cretonne/src/regalloc/reload.rs +++ b/lib/cretonne/src/regalloc/reload.rs @@ -90,6 +90,7 @@ impl Reload { /// This represents a stack value that is used by the current instruction where a register is /// needed. struct ReloadCandidate { + argidx: usize, value: Value, regclass: RegClass, } @@ -205,9 +206,10 @@ impl<'a> Context<'a> { assert!(self.candidates.is_empty()); self.find_candidates(inst, constraints); - // Insert fill instructions before `inst`. - while let Some(cand) = self.candidates.pop() { - if let Some(_reload) = self.reloads.get_mut(cand.value) { + // Insert fill instructions before `inst` and replace `cand.value` with the filled value. + for cand in self.candidates.iter_mut() { + if let Some(reload) = self.reloads.get(cand.value) { + cand.value = reload.reg; continue; } @@ -218,6 +220,7 @@ impl<'a> Context<'a> { stack: cand.value, reg: reg, }); + cand.value = reg; // Create a live range for the new reload. let affinity = Affinity::Reg(cand.regclass.into()); @@ -230,10 +233,16 @@ impl<'a> Context<'a> { ); } - // Rewrite arguments. - for arg in self.cur.func.dfg.inst_args_mut(inst) { - if let Some(reload) = self.reloads.get(*arg) { - *arg = reload.reg; + // Rewrite instruction arguments. + // + // Only rewrite those arguments that were identified as candidates. This leaves EBB + // arguments on branches as-is without rewriting them. A spilled EBB argument needs to stay + // spilled because the matching EBB parameter is going to be in the same virtual register + // and therefore the same stack slot as the EBB argument value. + if !self.candidates.is_empty() { + let args = self.cur.func.dfg.inst_args_mut(inst); + while let Some(cand) = self.candidates.pop() { + args[cand.argidx] = cand.value; } } @@ -295,10 +304,11 @@ impl<'a> Context<'a> { fn find_candidates(&mut self, inst: Inst, constraints: &RecipeConstraints) { let args = self.cur.func.dfg.inst_args(inst); - for (op, &arg) in constraints.ins.iter().zip(args) { + for (argidx, (op, &arg)) in constraints.ins.iter().zip(args).enumerate() { if op.kind != ConstraintKind::Stack { if self.liveness[arg].affinity.is_stack() { self.candidates.push(ReloadCandidate { + argidx, value: arg, regclass: op.regclass, }) @@ -307,10 +317,11 @@ impl<'a> Context<'a> { } // If we only have the fixed arguments, we're done now. - if args.len() == constraints.ins.len() { + let offset = constraints.ins.len(); + if args.len() == offset { return; } - let var_args = &args[constraints.ins.len()..]; + let var_args = &args[offset..]; // Handle ABI arguments. if let Some(sig) = self.cur.func.dfg.call_signature(inst) { @@ -318,6 +329,7 @@ impl<'a> Context<'a> { self.candidates, &self.cur.func.dfg.signatures[sig].params, var_args, + offset, self.cur.isa, self.liveness, ); @@ -326,6 +338,7 @@ impl<'a> Context<'a> { self.candidates, &self.cur.func.signature.returns, var_args, + offset, self.cur.isa, self.liveness, ); @@ -358,15 +371,17 @@ fn handle_abi_args( candidates: &mut Vec, abi_types: &[AbiParam], var_args: &[Value], + offset: usize, isa: &TargetIsa, liveness: &Liveness, ) { assert_eq!(abi_types.len(), var_args.len()); - for (abi, &arg) in abi_types.iter().zip(var_args) { + for ((abi, &arg), argidx) in abi_types.iter().zip(var_args).zip(offset..) { if abi.location.is_reg() { let lv = liveness.get(arg).expect("Missing live range for ABI arg"); if lv.affinity.is_stack() { candidates.push(ReloadCandidate { + argidx, value: arg, regclass: isa.regclass_for_abi_type(abi.value_type), });