diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 9a4f64896e..ffcb8273b2 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -631,14 +631,11 @@ impl AArch64ABIBody { rn: stack_reg(), rm: stack_limit, }); - insts.push(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(8), - // Here `Hs` == "higher or same" when interpreting the two - // operands as unsigned integers. - kind: CondBrKind::Cond(Cond::Hs), - }); - insts.push(Inst::Udf { + insts.push(Inst::TrapIf { trap_info: (ir::SourceLoc::default(), ir::TrapCode::StackOverflow), + // Here `Lo` == "less than" when interpreting the two + // operands as unsigned integers. + kind: CondBrKind::Cond(Cond::Lo), }); } } diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index a075401555..9802349f85 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -1490,12 +1490,20 @@ impl MachInstEmit for Inst { } sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero())); } - &Inst::OneWayCondBr { target, kind } => { + &Inst::TrapIf { kind, trap_info } => { + // condbr KIND, LABEL let off = sink.cur_offset(); - if let Some(l) = target.as_label() { - sink.use_label_at_offset(off, l, LabelUse::Branch19); - } - sink.put4(enc_conditional_br(target, kind)); + let label = sink.get_label(); + sink.put4(enc_conditional_br( + BranchTarget::Label(label), + kind.invert(), + )); + sink.use_label_at_offset(off, label, LabelUse::Branch19); + // udf + let trap = Inst::Udf { trap_info }; + trap.emit(sink, flags, state); + // LABEL: + sink.bind_label(label); } &Inst::IndirectBr { rn, .. } => { sink.put4(enc_br(rn)); @@ -1534,6 +1542,17 @@ impl MachInstEmit for Inst { // emission time, because we cannot allow the regalloc to insert spills/reloads in // the middle; we depend on hardcoded PC-rel addressing below. + // Branch to default when condition code from prior comparison indicates. + let br = enc_conditional_br(info.default_target, CondBrKind::Cond(Cond::Hs)); + // No need to inform the sink's branch folding logic about this branch, because it + // will not be merged with any other branch, flipped, or elided (it is not preceded + // or succeeded by any other branch). Just emit it with the label use. + let default_br_offset = sink.cur_offset(); + if let BranchTarget::Label(l) = info.default_target { + sink.use_label_at_offset(default_br_offset, l, LabelUse::Branch19); + } + sink.put4(br); + // Save index in a tmp (the live range of ridx only goes to start of this // sequence; rtmp1 or rtmp2 may overwrite it). let inst = Inst::gen_move(rtmp2, ridx, I64); diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 01786f13af..16bec07ac3 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -2681,149 +2681,148 @@ fn test_aarch64_binemit() { )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Zero(xreg(8)), - }, - "080200B4", - "cbz x8, 64", - )); - insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::NotZero(xreg(8)), }, - "080200B5", - "cbnz x8, 64", + "480000B40000A0D4", + "cbz x8, 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Eq), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Zero(xreg(8)), }, - "00020054", - "b.eq 64", + "480000B50000A0D4", + "cbnz x8, 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Ne), }, - "01020054", - "b.ne 64", + "400000540000A0D4", + "b.eq 8 ; udf", )); - insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Hs), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Eq), }, - "02020054", - "b.hs 64", + "410000540000A0D4", + "b.ne 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Lo), }, - "03020054", - "b.lo 64", + "420000540000A0D4", + "b.hs 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Mi), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Hs), }, - "04020054", - "b.mi 64", + "430000540000A0D4", + "b.lo 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Pl), }, - "05020054", - "b.pl 64", + "440000540000A0D4", + "b.mi 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Vs), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Mi), }, - "06020054", - "b.vs 64", + "450000540000A0D4", + "b.pl 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Vc), }, - "07020054", - "b.vc 64", + "460000540000A0D4", + "b.vs 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Hi), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Vs), }, - "08020054", - "b.hi 64", + "470000540000A0D4", + "b.vc 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Ls), }, - "09020054", - "b.ls 64", + "480000540000A0D4", + "b.hi 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Ge), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Hi), }, - "0A020054", - "b.ge 64", + "490000540000A0D4", + "b.ls 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Lt), }, - "0B020054", - "b.lt 64", + "4A0000540000A0D4", + "b.ge 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Gt), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Ge), }, - "0C020054", - "b.gt 64", + "4B0000540000A0D4", + "b.lt 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Le), }, - "0D020054", - "b.le 64", + "4C0000540000A0D4", + "b.gt 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), - kind: CondBrKind::Cond(Cond::Al), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Gt), }, - "0E020054", - "b.al 64", + "4D0000540000A0D4", + "b.le 8 ; udf", )); insns.push(( - Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(64), + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), kind: CondBrKind::Cond(Cond::Nv), }, - "0F020054", - "b.nv 64", + "4E0000540000A0D4", + "b.al 8 ; udf", + )); + insns.push(( + Inst::TrapIf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + kind: CondBrKind::Cond(Cond::Al), + }, + "4F0000540000A0D4", + "b.nv 8 ; udf", )); insns.push(( diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 9d229b8df8..5c3086efce 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -342,6 +342,7 @@ pub struct CallIndInfo { #[derive(Clone, Debug)] pub struct JTSequenceInfo { pub targets: Vec, + pub default_target: BranchTarget, pub targets_for_term: Vec, // needed for MachTerminator. } @@ -825,20 +826,17 @@ pub enum Inst { kind: CondBrKind, }, - /// A one-way conditional branch, invisible to the CFG processing; used *only* as part of - /// straight-line sequences in code to be emitted. + /// A conditional trap: execute a `udf` if the condition is true. This is + /// one VCode instruction because it uses embedded control flow; it is + /// logically a single-in, single-out region, but needs to appear as one + /// unit to the register allocator. /// - /// In more detail: - /// - This branch is lowered to a branch at the machine-code level, but does not end a basic - /// block, and does not create edges in the CFG seen by regalloc. - /// - Thus, it is *only* valid to use as part of a single-in, single-out sequence that is - /// lowered from a single CLIF instruction. For example, certain arithmetic operations may - /// use these branches to handle certain conditions, such as overflows, traps, etc. - /// - /// See, e.g., the lowering of `trapif` (conditional trap) for an example. - OneWayCondBr { - target: BranchTarget, + /// The `CondBrKind` gives the conditional-branch condition that will + /// *execute* the embedded `Inst`. (In the emitted code, we use the inverse + /// of this condition in a branch that skips the trap instruction.) + TrapIf { kind: CondBrKind, + trap_info: (SourceLoc, TrapCode), }, /// An indirect branch through a register, augmented with set of all @@ -1354,7 +1352,7 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_defs(&*info.defs); collector.add_use(info.rn); } - &Inst::CondBr { ref kind, .. } | &Inst::OneWayCondBr { ref kind, .. } => match kind { + &Inst::CondBr { ref kind, .. } => match kind { CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => { collector.add_use(*rt); } @@ -1366,6 +1364,12 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { &Inst::Nop0 | Inst::Nop4 => {} &Inst::Brk => {} &Inst::Udf { .. } => {} + &Inst::TrapIf { ref kind, .. } => match kind { + CondBrKind::Zero(rt) | CondBrKind::NotZero(rt) => { + collector.add_use(*rt); + } + CondBrKind::Cond(_) => {} + }, &Inst::Adr { rd, .. } => { collector.add_def(rd); } @@ -1957,13 +1961,16 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { } map_use(mapper, &mut info.rn); } - &mut Inst::CondBr { ref mut kind, .. } | &mut Inst::OneWayCondBr { ref mut kind, .. } => { + &mut Inst::CondBr { ref mut kind, .. } => { map_br(mapper, kind); } &mut Inst::IndirectBr { ref mut rn, .. } => { map_use(mapper, rn); } &mut Inst::Nop0 | &mut Inst::Nop4 | &mut Inst::Brk | &mut Inst::Udf { .. } => {} + &mut Inst::TrapIf { ref mut kind, .. } => { + map_br(mapper, kind); + } &mut Inst::Adr { ref mut rd, .. } => { map_def(mapper, rd); } @@ -2034,10 +2041,6 @@ impl MachInst for Inst { &Inst::CondBr { taken, not_taken, .. } => MachTerminator::Cond(taken.as_label().unwrap(), not_taken.as_label().unwrap()), - &Inst::OneWayCondBr { .. } => { - // Explicitly invisible to CFG processing. - MachTerminator::None - } &Inst::IndirectBr { ref targets, .. } => MachTerminator::Indirect(&targets[..]), &Inst::JTSequence { ref info, .. } => { MachTerminator::Indirect(&info.targets_for_term[..]) @@ -2887,32 +2890,26 @@ impl ShowWithRRU for Inst { } } } - &Inst::OneWayCondBr { - ref target, - ref kind, - } => { - let target = target.show_rru(mb_rru); - match kind { - &CondBrKind::Zero(reg) => { - let reg = reg.show_rru(mb_rru); - format!("cbz {}, {}", reg, target) - } - &CondBrKind::NotZero(reg) => { - let reg = reg.show_rru(mb_rru); - format!("cbnz {}, {}", reg, target) - } - &CondBrKind::Cond(c) => { - let c = c.show_rru(mb_rru); - format!("b.{} {}", c, target) - } - } - } &Inst::IndirectBr { rn, .. } => { let rn = rn.show_rru(mb_rru); format!("br {}", rn) } &Inst::Brk => "brk #0".to_string(), &Inst::Udf { .. } => "udf".to_string(), + &Inst::TrapIf { ref kind, .. } => match kind { + &CondBrKind::Zero(reg) => { + let reg = reg.show_rru(mb_rru); + format!("cbnz {}, 8 ; udf", reg) + } + &CondBrKind::NotZero(reg) => { + let reg = reg.show_rru(mb_rru); + format!("cbz {}, 8 ; udf", reg) + } + &CondBrKind::Cond(c) => { + let c = c.invert().show_rru(mb_rru); + format!("b.{} 8 ; udf", c) + } + }, &Inst::Adr { rd, off } => { let rd = rd.show_rru(mb_rru); format!("adr {}, pc+{}", rd, off) @@ -2929,15 +2926,26 @@ impl ShowWithRRU for Inst { let ridx = ridx.show_rru(mb_rru); let rtmp1 = rtmp1.show_rru(mb_rru); let rtmp2 = rtmp2.show_rru(mb_rru); + let default_target = info.default_target.show_rru(mb_rru); format!( concat!( + "b.hs {} ; ", "adr {}, pc+16 ; ", "ldrsw {}, [{}, {}, LSL 2] ; ", "add {}, {}, {} ; ", "br {} ; ", "jt_entries {:?}" ), - rtmp1, rtmp2, rtmp1, ridx, rtmp1, rtmp1, rtmp2, rtmp1, info.targets + default_target, + rtmp1, + rtmp2, + rtmp1, + ridx, + rtmp1, + rtmp1, + rtmp2, + rtmp1, + info.targets ) } &Inst::LoadConst64 { rd, const_data } => { diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index fe2a24a614..5dc6030e13 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -324,14 +324,11 @@ pub(crate) fn lower_insn_to_regs>( // msub rd, rd, rm, rn ; rd = rn - rd * rm // Check for divide by 0. - let branch_size = 8; - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(branch_size), - kind: CondBrKind::NotZero(rm), - }); - let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero); - ctx.emit(Inst::Udf { trap_info }); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Zero(rm), + }); ctx.emit(Inst::AluRRRR { alu_op: ALUOp::MSub64, @@ -342,17 +339,17 @@ pub(crate) fn lower_insn_to_regs>( }); } else { if div_op == ALUOp::SDiv64 { - // cbz rm, #20 + // cbnz rm, #8 + // udf ; divide by zero // cmn rm, 1 // ccmp rn, 1, #nzcv, eq - // b.vc 12 + // b.vc #8 // udf ; signed overflow - // udf ; divide by zero // Check for divide by 0. - let branch_size = 20; - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(branch_size), + let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero); + ctx.emit(Inst::TrapIf { + trap_info, kind: CondBrKind::Zero(rm), }); @@ -378,27 +375,22 @@ pub(crate) fn lower_insn_to_regs>( nzcv: NZCV::new(false, false, false, false), cond: Cond::Eq, }); - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(12), - kind: CondBrKind::Cond(Cond::Vc), - }); - let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); - ctx.emit(Inst::Udf { trap_info }); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Cond(Cond::Vs), + }); } else { // cbnz rm, #8 // udf ; divide by zero // Check for divide by 0. - let branch_size = 8; - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(branch_size), - kind: CondBrKind::NotZero(rm), + let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Zero(rm), }); } - - let trap_info = (ctx.srcloc(insn), TrapCode::IntegerDivisionByZero); - ctx.emit(Inst::Udf { trap_info }); } } @@ -1366,15 +1358,10 @@ pub(crate) fn lower_insn_to_regs>( cond }; - // Branch around the break instruction with inverted cond. Go straight to lowered - // one-target form; this is logically part of a single-in single-out template lowering. - let cond = cond.invert(); - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(8), + ctx.emit(Inst::TrapIf { + trap_info, kind: CondBrKind::Cond(cond), }); - - ctx.emit(Inst::Udf { trap_info }) } Opcode::Safepoint => { @@ -1753,12 +1740,11 @@ pub(crate) fn lower_insn_to_regs>( } else { ctx.emit(Inst::FpuCmp64 { rn, rm: rn }); } - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(8), - kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Ordered)), - }); let trap_info = (ctx.srcloc(insn), TrapCode::BadConversionToInteger); - ctx.emit(Inst::Udf { trap_info }); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Unordered)), + }); let tmp = ctx.alloc_tmp(RegClass::V128, I128); @@ -1794,12 +1780,11 @@ pub(crate) fn lower_insn_to_regs>( rn, rm: tmp.to_reg(), }); - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(8), - kind: CondBrKind::Cond(lower_fp_condcode(low_cond)), - }); let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); - ctx.emit(Inst::Udf { trap_info }); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()), + }); // <= high_bound lower_constant_f32(ctx, tmp, high_bound); @@ -1807,12 +1792,11 @@ pub(crate) fn lower_insn_to_regs>( rn, rm: tmp.to_reg(), }); - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(8), - kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan)), - }); let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); - ctx.emit(Inst::Udf { trap_info }); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()), + }); } else { // From float64. let (low_bound, low_cond, high_bound) = match (signed, out_bits) { @@ -1837,12 +1821,11 @@ pub(crate) fn lower_insn_to_regs>( rn, rm: tmp.to_reg(), }); - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(8), - kind: CondBrKind::Cond(lower_fp_condcode(low_cond)), - }); let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); - ctx.emit(Inst::Udf { trap_info }); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Cond(lower_fp_condcode(low_cond).invert()), + }); // <= high_bound lower_constant_f64(ctx, tmp, high_bound); @@ -1850,12 +1833,11 @@ pub(crate) fn lower_insn_to_regs>( rn, rm: tmp.to_reg(), }); - ctx.emit(Inst::OneWayCondBr { - target: BranchTarget::ResolvedOffset(8), - kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan)), - }); let trap_info = (ctx.srcloc(insn), TrapCode::IntegerOverflow); - ctx.emit(Inst::Udf { trap_info }); + ctx.emit(Inst::TrapIf { + trap_info, + kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::LessThan).invert()), + }); }; // Do the conversion. @@ -2349,7 +2331,8 @@ pub(crate) fn lower_branch>( let rtmp1 = ctx.alloc_tmp(RegClass::I64, I32); let rtmp2 = ctx.alloc_tmp(RegClass::I64, I32); - // Bounds-check and branch to default. + // Bounds-check, leaving condition codes for JTSequence's + // branch to default target below. if let Some(imm12) = Imm12::maybe_from_u64(jt_size as u64) { ctx.emit(Inst::AluRRImm12 { alu_op: ALUOp::SubS32, @@ -2366,14 +2349,10 @@ pub(crate) fn lower_branch>( rm: rtmp1.to_reg(), }); } - let default_target = BranchTarget::Label(targets[0]); - ctx.emit(Inst::OneWayCondBr { - target: default_target.clone(), - kind: CondBrKind::Cond(Cond::Hs), // unsigned >= - }); // Emit the compound instruction that does: // + // b.hs default // adr rA, jt // ldrsw rB, [rA, rIndex, UXTW 2] // add rA, rA, rB @@ -2392,6 +2371,7 @@ pub(crate) fn lower_branch>( .skip(1) .map(|bix| BranchTarget::Label(*bix)) .collect(); + let default_target = BranchTarget::Label(targets[0]); let targets_for_term: Vec = targets.to_vec(); ctx.emit(Inst::JTSequence { ridx, @@ -2399,6 +2379,7 @@ pub(crate) fn lower_branch>( rtmp2, info: Box::new(JTSequenceInfo { targets: jt_targets, + default_target, targets_for_term: targets_for_term, }), }); diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 5f54b1ba77..3a3578ceb5 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -1390,7 +1390,9 @@ mod test { inst.emit(&mut buf, &flags, &mut state); buf.bind_label(label(1)); - let inst = Inst::Nop4; + let inst = Inst::Udf { + trap_info: (SourceLoc::default(), TrapCode::Interrupt), + }; inst.emit(&mut buf, &flags, &mut state); buf.bind_label(label(2)); @@ -1403,14 +1405,13 @@ mod test { let mut buf2 = MachBuffer::new(); let mut state = Default::default(); - let inst = Inst::OneWayCondBr { - kind: CondBrKind::Zero(xreg(0)), - target: BranchTarget::ResolvedOffset(8), + let inst = Inst::TrapIf { + kind: CondBrKind::NotZero(xreg(0)), + trap_info: (SourceLoc::default(), TrapCode::Interrupt), }; inst.emit(&mut buf2, &flags, &mut state); let inst = Inst::Nop4; inst.emit(&mut buf2, &flags, &mut state); - inst.emit(&mut buf2, &flags, &mut state); let buf2 = buf2.finish(); diff --git a/cranelift/filetests/filetests/vcode/aarch64/arithmetic.clif b/cranelift/filetests/filetests/vcode/aarch64/arithmetic.clif index 98ec613214..9fe09c2166 100644 --- a/cranelift/filetests/filetests/vcode/aarch64/arithmetic.clif +++ b/cranelift/filetests/filetests/vcode/aarch64/arithmetic.clif @@ -76,12 +76,10 @@ block0(v0: i64, v1: i64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: sdiv x2, x0, x1 -; nextln: cbz x1, 20 +; nextln: cbnz x1, 8 ; udf ; nextln: adds xzr, x1, #1 ; nextln: ccmp x0, #1, #nzcv, eq -; nextln: b.vc 12 -; nextln: udf -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: mov x0, x2 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -98,12 +96,10 @@ block0(v0: i64): ; nextln: mov fp, sp ; nextln: movz x2, #2 ; nextln: sdiv x1, x0, x2 -; nextln: cbz x2, 20 +; nextln: cbnz x2, 8 ; udf ; nextln: adds xzr, x2, #1 ; nextln: ccmp x0, #1, #nzcv, eq -; nextln: b.vc 12 -; nextln: udf -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: mov x0, x1 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -118,8 +114,7 @@ block0(v0: i64, v1: i64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: udiv x0, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -135,8 +130,7 @@ block0(v0: i64): ; nextln: mov fp, sp ; nextln: movz x1, #2 ; nextln: udiv x0, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -150,8 +144,7 @@ block0(v0: i64, v1: i64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: sdiv x2, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: msub x0, x2, x1, x0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -166,8 +159,7 @@ block0(v0: i64, v1: i64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: udiv x2, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: msub x0, x2, x1, x0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -185,12 +177,10 @@ block0(v0: i32, v1: i32): ; nextln: sxtw x3, w0 ; nextln: sxtw x2, w1 ; nextln: sdiv x0, x3, x2 -; nextln: cbz x2, 20 +; nextln: cbnz x2, 8 ; udf ; nextln: adds wzr, w2, #1 ; nextln: ccmp w3, #1, #nzcv, eq -; nextln: b.vc 12 -; nextln: udf -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -204,16 +194,15 @@ block0(v0: i32): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp -; nextln: sxtw x1, w0 -; nextln: movz x0, #2 -; nextln: sxtw x2, w0 -; nextln: sdiv x0, x1, x2 -; nextln: cbz x2, 20 +; nextln: sxtw x0, w0 +; nextln: movz x1, #2 +; nextln: sxtw x2, w1 +; nextln: sdiv x1, x0, x2 +; nextln: cbnz x2, 8 ; udf ; nextln: adds wzr, w2, #1 -; nextln: ccmp w1, #1, #nzcv, eq -; nextln: b.vc 12 -; nextln: udf -; nextln: udf +; nextln: ccmp w0, #1, #nzcv, eq +; nextln: b.vc 8 ; udf +; nextln: mov x0, x1 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -229,8 +218,7 @@ block0(v0: i32, v1: i32): ; nextln: mov w0, w0 ; nextln: mov w1, w1 ; nextln: udiv x0, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -248,8 +236,7 @@ block0(v0: i32): ; nextln: mov w0, w0 ; nextln: movz x1, #2 ; nextln: udiv x0, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -265,8 +252,7 @@ block0(v0: i32, v1: i32): ; nextln: sxtw x0, w0 ; nextln: sxtw x1, w1 ; nextln: sdiv x2, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: msub x0, x2, x1, x0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -283,8 +269,7 @@ block0(v0: i32, v1: i32): ; nextln: mov w0, w0 ; nextln: mov w1, w1 ; nextln: udiv x2, x0, x1 -; nextln: cbnz x1, 8 -; nextln: udf +; nextln: cbnz x1, 8 ; udf ; nextln: msub x0, x2, x1, x0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 diff --git a/cranelift/filetests/filetests/vcode/aarch64/floating-point.clif b/cranelift/filetests/filetests/vcode/aarch64/floating-point.clif index 7fb8ec07ed..8303450b23 100644 --- a/cranelift/filetests/filetests/vcode/aarch64/floating-point.clif +++ b/cranelift/filetests/filetests/vcode/aarch64/floating-point.clif @@ -426,16 +426,13 @@ block0(v0: f32): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp s0, s0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -1 ; nextln: fcmp s0, s1 -; nextln: b.gt 8 -; nextln: udf +; nextln: b.gt 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 4294967300 ; nextln: fcmp s0, s1 -; nextln: b.mi 8 -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzu w0, s0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -450,16 +447,13 @@ block0(v0: f32): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp s0, s0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -2147483600 ; nextln: fcmp s0, s1 -; nextln: b.ge 8 -; nextln: udf +; nextln: b.ge 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 2147483600 ; nextln: fcmp s0, s1 -; nextln: b.mi -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzs w0, s0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -474,16 +468,13 @@ block0(v0: f32): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp s0, s0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -1 ; nextln: fcmp s0, s1 -; nextln: b.gt 8 -; nextln: udf +; nextln: b.gt 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 18446744000000000000 ; nextln: fcmp s0, s1 -; nextln: b.mi 8 -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzu x0, s0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -498,16 +489,13 @@ block0(v0: f32): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp s0, s0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 -9223372000000000000 ; nextln: fcmp s0, s1 -; nextln: b.ge 8 -; nextln: udf +; nextln: b.ge 8 ; udf ; nextln: ldr s1, pc+8 ; b 8 ; data.f32 9223372000000000000 ; nextln: fcmp s0, s1 -; nextln: b.mi 8 -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzs x0, s0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -522,16 +510,13 @@ block0(v0: f64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp d0, d0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -1 ; nextln: fcmp d0, d1 -; nextln: b.gt 8 -; nextln: udf +; nextln: b.gt 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 4294967296 ; nextln: fcmp d0, d1 -; nextln: b.mi 8 -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzu w0, d0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -546,16 +531,13 @@ block0(v0: f64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp d0, d0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -2147483649 ; nextln: fcmp d0, d1 -; nextln: b.gt 8 -; nextln: udf +; nextln: b.gt 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 2147483648 ; nextln: fcmp d0, d1 -; nextln: b.mi 8 -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzs w0, d0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -570,16 +552,13 @@ block0(v0: f64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp d0, d0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -1 ; nextln: fcmp d0, d1 -; nextln: b.gt 8 -; nextln: udf +; nextln: b.gt 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 18446744073709552000 ; nextln: fcmp d0, d1 -; nextln: b.mi 8 -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzu x0, d0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -594,16 +573,13 @@ block0(v0: f64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: fcmp d0, d0 -; nextln: b.vc 8 -; nextln: udf +; nextln: b.vc 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 -9223372036854776000 ; nextln: fcmp d0, d1 -; nextln: b.ge 8 -; nextln: udf +; nextln: b.ge 8 ; udf ; nextln: ldr d1, pc+8 ; b 12 ; data.f64 9223372036854776000 ; nextln: fcmp d0, d1 -; nextln: b.mi 8 -; nextln: udf +; nextln: b.mi 8 ; udf ; nextln: fcvtzs x0, d0 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 diff --git a/cranelift/filetests/filetests/vcode/aarch64/jumptable.clif b/cranelift/filetests/filetests/vcode/aarch64/jumptable.clif index da4daae571..1e8922b1a4 100644 --- a/cranelift/filetests/filetests/vcode/aarch64/jumptable.clif +++ b/cranelift/filetests/filetests/vcode/aarch64/jumptable.clif @@ -29,8 +29,7 @@ block5(v5: i64): } ; check: subs wzr, w0, #3 -; nextln: b.hs -; nextln: adr x1, pc+16 ; ldrsw x2, [x1, x0, LSL 2] ; add x1, x1, x2 ; br x1 ; jt_entries +; nextln: b.hs label1 ; adr x1, pc+16 ; ldrsw x2, [x1, x0, LSL 2] ; add x1, x1, x2 ; br x1 ; jt_entries ; check: movz x1, #1 ; nextln: b diff --git a/cranelift/filetests/filetests/vcode/aarch64/stack-limit.clif b/cranelift/filetests/filetests/vcode/aarch64/stack-limit.clif index ffb8b9b599..8ad0393669 100644 --- a/cranelift/filetests/filetests/vcode/aarch64/stack-limit.clif +++ b/cranelift/filetests/filetests/vcode/aarch64/stack-limit.clif @@ -43,8 +43,7 @@ block0(v0: i64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: subs xzr, sp, x0 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: ldr x16 ; nextln: blr x16 ; nextln: mov sp, fp @@ -67,8 +66,7 @@ block0(v0: i64): ; nextln: ldur x16, [x0] ; nextln: ldur x16, [x16, #4] ; nextln: subs xzr, sp, x16 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: ldr x16 ; nextln: blr x16 ; nextln: mov sp, fp @@ -86,8 +84,7 @@ block0(v0: i64): ; nextln: mov fp, sp ; nextln: add x16, x0, #176 ; nextln: subs xzr, sp, x16 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: sub sp, sp, #176 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -102,14 +99,12 @@ block0(v0: i64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp ; nextln: subs xzr, sp, x0 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: movz x17, #6784 ; nextln: movk x17, #6, LSL #16 ; nextln: add x16, x0, x17, UXTX ; nextln: subs xzr, sp, x16 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: ldr x16, 8 ; b 12 ; data 400000 ; nextln: sub sp, sp, x16, UXTX ; nextln: mov sp, fp @@ -132,8 +127,7 @@ block0(v0: i64): ; nextln: ldur x16, [x16, #4] ; nextln: add x16, x16, #32 ; nextln: subs xzr, sp, x16 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: sub sp, sp, #32 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 @@ -154,14 +148,12 @@ block0(v0: i64): ; nextln: ldur x16, [x0] ; nextln: ldur x16, [x16, #4] ; nextln: subs xzr, sp, x16 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: movz x17, #6784 ; nextln: movk x17, #6, LSL #16 ; nextln: add x16, x16, x17, UXTX ; nextln: subs xzr, sp, x16 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: ldr x16, 8 ; b 12 ; data 400000 ; nextln: sub sp, sp, x16, UXTX ; nextln: mov sp, fp @@ -182,8 +174,7 @@ block0(v0: i64): ; nextln: movz x16, #6784 ; movk x16, #6, LSL #16 ; add x16, x0, x16, UXTX ; ldr x16, [x16] ; nextln: add x16, x16, #32 ; nextln: subs xzr, sp, x16 -; nextln: b.hs 8 -; nextln: udf +; nextln: b.hs 8 ; udf ; nextln: sub sp, sp, #32 ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 diff --git a/cranelift/filetests/filetests/vcode/aarch64/traps.clif b/cranelift/filetests/filetests/vcode/aarch64/traps.clif index ab0ca07ae4..237af98919 100644 --- a/cranelift/filetests/filetests/vcode/aarch64/traps.clif +++ b/cranelift/filetests/filetests/vcode/aarch64/traps.clif @@ -17,8 +17,7 @@ block0(v0: i64): } ; check: subs xzr, x0, #42 -; nextln: b.ne 8 -; nextln: udf +; nextln: b.ne 8 ; udf function %h() { block0: