diff --git a/cranelift/filetests/regalloc/fallthrough-return.clif b/cranelift/filetests/regalloc/fallthrough-return.clif new file mode 100644 index 0000000000..557710eb5a --- /dev/null +++ b/cranelift/filetests/regalloc/fallthrough-return.clif @@ -0,0 +1,23 @@ +test regalloc +target x86_64 + +; Test that fallthrough returns are visited by reload and coloring. + +function %foo() -> f64 { + fn0 = %bar() + +ebb0: + v0 = f64const 0.0 + call fn0() + fallthrough_return v0 +} +; check: fill v0 + +function %foo() -> f64 { + fn0 = %bar() -> f64, f64 + +ebb0: + v0, v1 = call fn0() + fallthrough_return v1 +} +; check: regmove v1, %xmm1 -> %xmm0 diff --git a/lib/codegen/meta-python/base/instructions.py b/lib/codegen/meta-python/base/instructions.py index 498a75727f..718bb33f45 100644 --- a/lib/codegen/meta-python/base/instructions.py +++ b/lib/codegen/meta-python/base/instructions.py @@ -844,7 +844,7 @@ vsplit = Instruction( the lanes from ``x``. The result may be two scalars if ``x`` only had two lanes. """, - ins=x, outs=(lo, hi)) + ins=x, outs=(lo, hi), is_ghost=True) Any128 = TypeVar( 'Any128', 'Any scalar or vector type with as most 128 lanes', @@ -864,7 +864,7 @@ vconcat = Instruction( It is possible to form a vector by concatenating two scalars. """, - ins=(x, y), outs=a) + ins=(x, y), outs=a, is_ghost=True) c = Operand('c', TxN.as_bool(), doc='Controlling vector') x = Operand('x', TxN, doc='Value to use where `c` is true') @@ -2009,7 +2009,7 @@ isplit = Instruction( Returns the low half of `x` and the high half of `x` as two independent values. """, - ins=x, outs=(lo, hi)) + ins=x, outs=(lo, hi), is_ghost=True) NarrowInt = TypeVar( @@ -2029,6 +2029,6 @@ iconcat = Instruction( the same number of lanes as the inputs, but the lanes are twice the size. """, - ins=(lo, hi), outs=a) + ins=(lo, hi), outs=a, is_ghost=True) GROUP.close() diff --git a/lib/codegen/meta-python/cdsl/instructions.py b/lib/codegen/meta-python/cdsl/instructions.py index f158ecbd4b..30bdc306e6 100644 --- a/lib/codegen/meta-python/cdsl/instructions.py +++ b/lib/codegen/meta-python/cdsl/instructions.py @@ -92,6 +92,7 @@ class Instruction(object): :param is_indirect_branch: This is an indirect branch instruction. :param is_call: This is a call instruction. :param is_return: This is a return instruction. + :param is_ghost: This is a ghost instruction. :param can_trap: This instruction can trap. :param can_load: This instruction can load from memory. :param can_store: This instruction can store to memory. @@ -108,6 +109,7 @@ class Instruction(object): 'True for all indirect branch or jump instructions.', 'is_call': 'Is this a call instruction?', 'is_return': 'Is this a return instruction?', + 'is_ghost': 'Is this a ghost instruction?', 'can_load': 'Can this instruction read from memory?', 'can_store': 'Can this instruction write to memory?', 'can_trap': 'Can this instruction cause a trap?', diff --git a/lib/codegen/src/regalloc/coloring.rs b/lib/codegen/src/regalloc/coloring.rs index 3c1a11b3fb..f0b96e79c9 100644 --- a/lib/codegen/src/regalloc/coloring.rs +++ b/lib/codegen/src/regalloc/coloring.rs @@ -165,8 +165,9 @@ impl<'a> Context<'a> { self.cur.goto_top(ebb); while let Some(inst) = self.cur.next_inst() { self.cur.use_srcloc(inst); - let enc = self.cur.func.encodings[inst]; - if let Some(constraints) = self.encinfo.operand_constraints(enc) { + if !self.cur.func.dfg[inst].opcode().is_ghost() { + let enc = self.cur.func.encodings[inst]; + let constraints = self.encinfo.operand_constraints(enc); if self.visit_inst(inst, constraints, tracker, &mut regs) { self.replace_global_defines(inst, tracker); // Restore cursor location after `replace_global_defines` moves it. @@ -290,7 +291,7 @@ impl<'a> Context<'a> { fn visit_inst( &mut self, inst: Inst, - constraints: &RecipeConstraints, + constraints: Option<&RecipeConstraints>, tracker: &mut LiveValueTracker, regs: &mut AvailableRegs, ) -> bool { @@ -306,7 +307,9 @@ impl<'a> Context<'a> { // Program the solver with register constraints for the input side. self.solver.reset(®s.input); - self.program_input_constraints(inst, constraints.ins); + if let Some(constraints) = constraints { + self.program_input_constraints(inst, constraints.ins); + } let call_sig = self.cur.func.dfg.call_signature(inst); if let Some(sig) = call_sig { program_input_abi( @@ -390,14 +393,16 @@ impl<'a> Context<'a> { // Program the fixed output constraints before the general defines. This allows us to // detect conflicts between fixed outputs and tied operands where the input value hasn't // been converted to a solver variable. - if constraints.fixed_outs { - self.program_fixed_outputs( - constraints.outs, - defs, - throughs, - &mut replace_global_defines, - ®s.global, - ); + if let Some(constraints) = constraints { + if constraints.fixed_outs { + self.program_fixed_outputs( + constraints.outs, + defs, + throughs, + &mut replace_global_defines, + ®s.global, + ); + } } if let Some(sig) = call_sig { self.program_output_abi( @@ -408,13 +413,15 @@ impl<'a> Context<'a> { ®s.global, ); } - self.program_output_constraints( - inst, - constraints.outs, - defs, - &mut replace_global_defines, - ®s.global, - ); + if let Some(constraints) = constraints { + self.program_output_constraints( + inst, + constraints.outs, + defs, + &mut replace_global_defines, + ®s.global, + ); + } // Finally, we've fully programmed the constraint solver. // We expect a quick solution in most cases. @@ -440,12 +447,14 @@ impl<'a> Context<'a> { // Tied defs are not part of the solution above. // Copy register assignments from tied inputs to tied outputs. - if constraints.tied_ops { - for (op, lv) in constraints.outs.iter().zip(defs) { - if let ConstraintKind::Tied(num) = op.kind { - let arg = self.cur.func.dfg.inst_args(inst)[num as usize]; - let reg = self.divert.reg(arg, &self.cur.func.locations); - self.cur.func.locations[lv.value] = ValueLoc::Reg(reg); + if let Some(constraints) = constraints { + if constraints.tied_ops { + for (op, lv) in constraints.outs.iter().zip(defs) { + if let ConstraintKind::Tied(num) = op.kind { + let arg = self.cur.func.dfg.inst_args(inst)[num as usize]; + let reg = self.divert.reg(arg, &self.cur.func.locations); + self.cur.func.locations[lv.value] = ValueLoc::Reg(reg); + } } } } diff --git a/lib/codegen/src/regalloc/reload.rs b/lib/codegen/src/regalloc/reload.rs index 22a6ac5963..c767993578 100644 --- a/lib/codegen/src/regalloc/reload.rs +++ b/lib/codegen/src/regalloc/reload.rs @@ -125,8 +125,8 @@ impl<'a> Context<'a> { // visit_ebb_header() places us at the first interesting instruction in the EBB. while let Some(inst) = self.cur.current_inst() { - let encoding = self.cur.func.encodings[inst]; - if encoding.is_legal() { + if !self.cur.func.dfg[inst].opcode().is_ghost() { + let encoding = self.cur.func.encodings[inst]; self.visit_inst(ebb, inst, encoding, tracker); tracker.drop_dead(inst); } else { @@ -200,10 +200,7 @@ impl<'a> Context<'a> { self.cur.use_srcloc(inst); // Get the operand constraints for `inst` that we are trying to satisfy. - let constraints = self - .encinfo - .operand_constraints(encoding) - .expect("Missing instruction encoding"); + let constraints = self.encinfo.operand_constraints(encoding); // Identify reload candidates. debug_assert!(self.candidates.is_empty()); @@ -240,27 +237,32 @@ impl<'a> Context<'a> { // v2 = spill v7 // // That way, we don't need to rewrite all future uses of v2. - for (lv, op) in defs.iter().zip(constraints.outs) { - if lv.affinity.is_stack() && op.kind != ConstraintKind::Stack { - if let InstructionData::Unary { - opcode: Opcode::Copy, - arg, - } = self.cur.func.dfg[inst] - { - self.cur.func.dfg.replace(inst).spill(arg); - let ok = self.cur.func.update_encoding(inst, self.cur.isa).is_ok(); - debug_assert!(ok); - } else { - let value_type = self.cur.func.dfg.value_type(lv.value); - let reg = self.cur.func.dfg.replace_result(lv.value, value_type); - self.liveness.create_dead(reg, inst, Affinity::new(op)); - self.insert_spill(ebb, lv.value, reg); + if let Some(constraints) = constraints { + for (lv, op) in defs.iter().zip(constraints.outs) { + if lv.affinity.is_stack() && op.kind != ConstraintKind::Stack { + if let InstructionData::Unary { + opcode: Opcode::Copy, + arg, + } = self.cur.func.dfg[inst] + { + self.cur.func.dfg.replace(inst).spill(arg); + let ok = self.cur.func.update_encoding(inst, self.cur.isa).is_ok(); + debug_assert!(ok); + } else { + let value_type = self.cur.func.dfg.value_type(lv.value); + let reg = self.cur.func.dfg.replace_result(lv.value, value_type); + self.liveness.create_dead(reg, inst, Affinity::new(op)); + self.insert_spill(ebb, lv.value, reg); + } } } } // Same thing for spilled call return values. - let retvals = &defs[constraints.outs.len()..]; + let retvals = &defs[self.cur.func.dfg[inst] + .opcode() + .constraints() + .fixed_results()..]; if !retvals.is_empty() { let sig = self .cur @@ -342,21 +344,26 @@ impl<'a> Context<'a> { // Find reload candidates for `inst` and add them to `self.candidates`. // // These are uses of spilled values where the operand constraint requires a register. - fn find_candidates(&mut self, inst: Inst, constraints: &RecipeConstraints) { + fn find_candidates(&mut self, inst: Inst, constraints: Option<&RecipeConstraints>) { let args = self.cur.func.dfg.inst_args(inst); - for (argidx, (op, &arg)) in constraints.ins.iter().zip(args).enumerate() { - if op.kind != ConstraintKind::Stack && self.liveness[arg].affinity.is_stack() { - self.candidates.push(ReloadCandidate { - argidx, - value: arg, - regclass: op.regclass, - }) + if let Some(constraints) = constraints { + for (argidx, (op, &arg)) in constraints.ins.iter().zip(args).enumerate() { + if op.kind != ConstraintKind::Stack && self.liveness[arg].affinity.is_stack() { + self.candidates.push(ReloadCandidate { + argidx, + value: arg, + regclass: op.regclass, + }) + } } } // If we only have the fixed arguments, we're done now. - let offset = constraints.ins.len(); + let offset = self.cur.func.dfg[inst] + .opcode() + .constraints() + .fixed_value_arguments(); if args.len() == offset { return; }