From e94ebc2263dd95bd5f34eef1af5aaf2b1957b68b Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 17 Dec 2021 12:37:17 -0600 Subject: [PATCH] aarch64: Translate rot{r,l} to ISLE (#3614) This commit translates the `rotl` and `rotr` lowerings already existing to ISLE. The port was relatively straightforward with the biggest changing being the instructions generated around i128 rotl/rotr primarily due to register changes. --- .../codegen/src/isa/aarch64/inst/imms.rs | 2 +- cranelift/codegen/src/isa/aarch64/lower.isle | 142 +++++++ cranelift/codegen/src/isa/aarch64/lower.rs | 229 ---------- .../codegen/src/isa/aarch64/lower/isle.rs | 17 + .../lower/isle/generated_code.manifest | 2 +- .../isa/aarch64/lower/isle/generated_code.rs | 394 ++++++++++++++++++ .../codegen/src/isa/aarch64/lower_inst.rs | 243 +---------- .../filetests/isa/aarch64/shift-rotate.clif | 93 +++-- cranelift/filetests/src/subtest.rs | 7 +- 9 files changed, 610 insertions(+), 519 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rs index bc3ebac7de..beed2f40de 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs @@ -583,7 +583,7 @@ impl ImmLogic { } /// An immediate for shift instructions. -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug)] pub struct ImmShift { /// 6-bit shift amount. pub imm: u8, diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 64ae824c91..501fa89c86 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -894,3 +894,145 @@ (tst64_imm amt (u64_into_imm_logic $I64 64)) (csel (Cond.Ne) hi_rshift maybe_lo) (csel (Cond.Ne) hi_sign hi_rshift)))) + +;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; General 8/16-bit case. +(rule (lower (has_type (fits_in_16 ty) (rotl x y))) + (let ((neg_shift Reg (alu_rrr (ALUOp.Sub32) (zero_reg) (put_in_reg y)))) + (value_reg (small_rotr ty (put_in_reg_zext32 x) neg_shift)))) + +;; Specialization for the 8/16-bit case when the rotation amount is an immediate. +(rule (lower (has_type (fits_in_16 ty) (rotl x (def_inst (iconst (imm_shift_from_imm64 +;; +;; and masked_amt, amt, +;; sub tmp_sub, masked_amt, +;; sub neg_amt, zero, tmp_sub ; neg +;; lsr val_rshift, val, masked_amt +;; lsl val_lshift, val, neg_amt +;; orr rd, val_lshift val_rshift +(decl small_rotr (Type Reg Reg) Reg) +(rule (small_rotr ty val amt) + (let ( + (masked_amt Reg (alu_rr_imm_logic (ALUOp.And32) amt (rotr_mask ty))) + (tmp_sub Reg (alu_rr_imm12 (ALUOp.Sub32) masked_amt (u8_into_imm12 (ty_bits ty)))) + (neg_amt Reg (alu_rrr (ALUOp.Sub32) (zero_reg) tmp_sub)) + (val_rshift Reg (alu_rrr (ALUOp.Lsr32) val masked_amt)) + (val_lshift Reg (alu_rrr (ALUOp.Lsl32) val neg_amt)) + ) + (alu_rrr (ALUOp.Orr32) val_lshift val_rshift))) + +(decl rotr_mask (Type) ImmLogic) +(extern constructor rotr_mask rotr_mask) + +;; For a constant amount, we can instead do: +;; +;; rotr rd, val, #amt +;; +;; => +;; +;; lsr val_rshift, val, # +;; lsl val_lshift, val, +;; orr rd, val_lshift, val_rshift +(decl small_rotr_imm (Type Reg ImmShift) Reg) +(rule (small_rotr_imm ty val amt) + (let ( + (val_rshift Reg (alu_rr_imm_shift (ALUOp.Lsr32) val amt)) + (val_lshift Reg (alu_rr_imm_shift (ALUOp.Lsl32) val (rotr_opposite_amount ty amt))) + ) + (alu_rrr (ALUOp.Orr32) val_lshift val_rshift))) + +(decl rotr_opposite_amount (Type ImmShift) ImmShift) +(extern constructor rotr_opposite_amount rotr_opposite_amount) + +;; General 128-bit case. +;; +;; TODO: much better codegen is possible with a constant amount. +(rule (lower (has_type $I128 (rotr x y))) + (let ( + (val ValueRegs (put_in_regs x)) + (amt Reg (value_regs_get (put_in_regs y) 0)) + (neg_amt Reg (alu_rrr (ALUOp.Sub64) (imm $I64 128) amt)) + (rshift ValueRegs (lower_ushr128 val amt)) + (lshift ValueRegs (lower_shl128 val neg_amt)) + (hi Reg (alu_rrr (ALUOp.Orr64) (value_regs_get rshift 1) (value_regs_get lshift 1))) + (lo Reg (alu_rrr (ALUOp.Orr64) (value_regs_get rshift 0) (value_regs_get lshift 0))) + ) + (value_regs lo hi))) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index eeb6635d65..8046004516 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -80,15 +80,6 @@ impl ResultRSEImm12 { } } -/// A lowering result: register or immediate shift amount (arg to a shift op). -/// An SSA value can always be lowered into one of these options; the register form is the -/// fallback. -#[derive(Clone, Debug)] -pub(crate) enum ResultRegImmShift { - Reg(Reg), - ImmShift(ImmShift), -} - //============================================================================ // Lowering: convert instruction inputs to forms that we can use. @@ -460,21 +451,6 @@ pub(crate) fn put_input_in_rse_imm12>( ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode)) } -pub(crate) fn put_input_in_reg_immshift>( - ctx: &mut C, - input: InsnInput, - shift_width_bits: usize, -) -> ResultRegImmShift { - if let Some(imm_value) = input_to_const(ctx, input) { - let imm_value = imm_value & ((shift_width_bits - 1) as u64); - if let Some(immshift) = ImmShift::maybe_from_u64(imm_value) { - return ResultRegImmShift::ImmShift(immshift); - } - } - - ResultRegImmShift::Reg(put_input_in_reg(ctx, input, NarrowValueMode::None)) -} - //============================================================================ // ALU instruction constructors. @@ -1557,211 +1533,6 @@ pub(crate) fn lower_load< f(ctx, rd, elem_ty, mem) } -pub(crate) fn emit_shl_i128>( - ctx: &mut C, - src: ValueRegs, - dst: ValueRegs>, - amt: Reg, -) { - let src_lo = src.regs()[0]; - let src_hi = src.regs()[1]; - let dst_lo = dst.regs()[0]; - let dst_hi = dst.regs()[1]; - - // mvn inv_amt, amt - // lsr tmp1, src_lo, #1 - // lsl tmp2, src_hi, amt - // lsr tmp1, tmp1, inv_amt - // lsl tmp3, src_lo, amt - // tst amt, #0x40 - // orr tmp2, tmp2, tmp1 - // csel dst_hi, tmp3, tmp2, ne - // csel dst_lo, xzr, tmp3, ne - - let xzr = writable_zero_reg(); - let inv_amt = ctx.alloc_tmp(I64).only_reg().unwrap(); - let tmp1 = ctx.alloc_tmp(I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(I64).only_reg().unwrap(); - let tmp3 = ctx.alloc_tmp(I64).only_reg().unwrap(); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::OrrNot32, - rd: inv_amt, - rn: xzr.to_reg(), - rm: amt, - }); - - ctx.emit(Inst::AluRRImmShift { - alu_op: ALUOp::Lsr64, - rd: tmp1, - rn: src_lo, - immshift: ImmShift::maybe_from_u64(1).unwrap(), - }); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Lsl64, - rd: tmp2, - rn: src_hi, - rm: amt, - }); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Lsr64, - rd: tmp1, - rn: tmp1.to_reg(), - rm: inv_amt.to_reg(), - }); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Lsl64, - rd: tmp3, - rn: src_lo, - rm: amt, - }); - - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::AndS64, - rd: xzr, - rn: amt, - imml: ImmLogic::maybe_from_u64(64, I64).unwrap(), - }); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Orr64, - rd: tmp2, - rn: tmp2.to_reg(), - rm: tmp1.to_reg(), - }); - - ctx.emit(Inst::CSel { - cond: Cond::Ne, - rd: dst_hi, - rn: tmp3.to_reg(), - rm: tmp2.to_reg(), - }); - - ctx.emit(Inst::CSel { - cond: Cond::Ne, - rd: dst_lo, - rn: xzr.to_reg(), - rm: tmp3.to_reg(), - }); -} - -pub(crate) fn emit_shr_i128>( - ctx: &mut C, - src: ValueRegs, - dst: ValueRegs>, - amt: Reg, - is_signed: bool, -) { - let src_lo = src.regs()[0]; - let src_hi = src.regs()[1]; - let dst_lo = dst.regs()[0]; - let dst_hi = dst.regs()[1]; - - // mvn inv_amt, amt - // lsl tmp1, src_lo, #1 - // lsr tmp2, src_hi, amt - // lsl tmp1, tmp1, inv_amt - // lsr/asr tmp3, src_lo, amt - // tst amt, #0x40 - // orr tmp2, tmp2, tmp1 - // - // if signed: - // asr tmp4, src_hi, #63 - // csel dst_hi, tmp4, tmp3, ne - // else: - // csel dst_hi, xzr, tmp3, ne - // - // csel dst_lo, tmp3, tmp2, ne - - let xzr = writable_zero_reg(); - let inv_amt = ctx.alloc_tmp(I64).only_reg().unwrap(); - let tmp1 = ctx.alloc_tmp(I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(I64).only_reg().unwrap(); - let tmp3 = ctx.alloc_tmp(I64).only_reg().unwrap(); - let tmp4 = ctx.alloc_tmp(I64).only_reg().unwrap(); - - let shift_op = if is_signed { - ALUOp::Asr64 - } else { - ALUOp::Lsr64 - }; - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::OrrNot32, - rd: inv_amt, - rn: xzr.to_reg(), - rm: amt, - }); - - ctx.emit(Inst::AluRRImmShift { - alu_op: ALUOp::Lsl64, - rd: tmp1, - rn: src_hi, - immshift: ImmShift::maybe_from_u64(1).unwrap(), - }); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Lsr64, - rd: tmp2, - rn: src_lo, - rm: amt, - }); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Lsl64, - rd: tmp1, - rn: tmp1.to_reg(), - rm: inv_amt.to_reg(), - }); - - ctx.emit(Inst::AluRRR { - alu_op: shift_op, - rd: tmp3, - rn: src_hi, - rm: amt, - }); - - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::AndS64, - rd: xzr, - rn: amt, - imml: ImmLogic::maybe_from_u64(64, I64).unwrap(), - }); - - if is_signed { - ctx.emit(Inst::AluRRImmShift { - alu_op: ALUOp::Asr64, - rd: tmp4, - rn: src_hi, - immshift: ImmShift::maybe_from_u64(63).unwrap(), - }); - } - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Orr64, - rd: tmp2, - rn: tmp2.to_reg(), - rm: tmp1.to_reg(), - }); - - ctx.emit(Inst::CSel { - cond: Cond::Ne, - rd: dst_hi, - rn: if is_signed { tmp4 } else { xzr }.to_reg(), - rm: tmp3.to_reg(), - }); - - ctx.emit(Inst::CSel { - cond: Cond::Ne, - rd: dst_lo, - rn: tmp3.to_reg(), - rm: tmp2.to_reg(), - }); -} - pub(crate) fn emit_clz_i128>( ctx: &mut C, src: ValueRegs, diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 84b7a6074e..51df016418 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -24,6 +24,7 @@ use crate::{ machinst::{ty_bits, InsnOutput, LowerCtx}, }; use std::boxed::Box; +use std::convert::TryFrom; use std::vec::Vec; type BoxCallInfo = Box; @@ -262,4 +263,20 @@ where fn u64_into_imm_logic(&mut self, ty: Type, val: u64) -> ImmLogic { ImmLogic::maybe_from_u64(val, ty).unwrap() } + + fn negate_imm_shift(&mut self, ty: Type, mut imm: ImmShift) -> ImmShift { + let size = u8::try_from(ty.bits()).unwrap(); + imm.imm = size.wrapping_sub(imm.value()); + imm.imm &= size - 1; + imm + } + + fn rotr_mask(&mut self, ty: Type) -> ImmLogic { + ImmLogic::maybe_from_u64((ty.bits() - 1) as u64, I32).unwrap() + } + + fn rotr_opposite_amount(&mut self, ty: Type, val: ImmShift) -> ImmShift { + let amount = val.value() & u8::try_from(ty.bits() - 1).unwrap(); + ImmShift::maybe_from_u64(u64::from(ty.bits()) - u64::from(amount)).unwrap() + } } diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index a56afee6e1..0de7cc8720 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602 src/prelude.isle 15c8dd937171bd0f619179e219422d43af0eb0ef9a6e88f23b2aa55776712e27342309dd3a4441876b2dfec7f16ce7fe13b3a926ace89b25cfc9577e7b1d1578 src/isa/aarch64/inst.isle a6921329a85a252b8059657056ee0c4477304dff461bd58c39133a2870666b23a34c911be55e25e7887074cb54b640ff09998730af09b3c1ba792f434309af24 -src/isa/aarch64/lower.isle 1fb105feeabfd582e680900a5c3dd82950045761dc82ff30e92107cdac479ff046ce10853489d158c6b98961c7eab5d274ede3b0181c06f4ae67bf4556d6130d +src/isa/aarch64/lower.isle 3cc84f8e3907818da7e0cbb4afe7f269da7090f3d504c74ecf02ef57463b281f4a320e52656d9397720ec8e4af98c1ecea05c4ffbe3f0c4126af4ed95d2734cc diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index e4aac7e1e7..2b6233b7ae 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -84,6 +84,9 @@ pub trait Context { fn sink_atomic_load(&mut self, arg0: &SinkableAtomicLoad) -> Reg; fn safe_divisor_from_imm64(&mut self, arg0: Imm64) -> Option; fn shift_mask(&mut self, arg0: Type) -> ImmLogic; + fn negate_imm_shift(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; + fn rotr_mask(&mut self, arg0: Type) -> ImmLogic; + fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } /// Internal type ProducesFlags: defined at src/prelude.isle line 263. @@ -2269,6 +2272,105 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let closure12 = || { + let expr0_0: Type = I32; + return Some(expr0_0); + }; + if let Some(pattern12_0) = closure12() { + if let Some(pattern13_0) = + C::imm_shift_from_imm64(ctx, pattern10_1, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 928. + let expr0_0 = ALUOp::RotR32; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0: Type = I32; + let expr3_0 = + C::negate_imm_shift(ctx, expr2_0, pattern13_0); + let expr4_0 = constructor_alu_rr_imm_shift( + ctx, &expr0_0, expr1_0, expr3_0, + )?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + } + // Rule at src/isa/aarch64/lower.isle line 918. + let expr0_0 = ALUOp::Sub32; + let expr1_0 = C::zero_reg(ctx); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = ALUOp::RotR32; + let expr5_0 = C::put_in_reg(ctx, pattern7_0); + let expr6_0 = constructor_alu_rrr(ctx, &expr4_0, expr5_0, expr3_0)?; + let expr7_0 = C::value_reg(ctx, expr6_0); + return Some(expr7_0); + } + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let closure12 = || { + let expr0_0: Type = I32; + return Some(expr0_0); + }; + if let Some(pattern12_0) = closure12() { + if let Some(pattern13_0) = + C::imm_shift_from_imm64(ctx, pattern10_1, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 972. + let expr0_0 = ALUOp::RotR32; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm_shift( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + } + } + // Rule at src/isa/aarch64/lower.isle line 960. + let expr0_0 = ALUOp::RotR32; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + _ => {} + } + } + } if pattern2_0 == I64 { let pattern4_0 = C::inst_data(ctx, pattern0_0); if let &InstructionData::Binary { @@ -2363,6 +2465,93 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let closure12 = || { + let expr0_0: Type = I64; + return Some(expr0_0); + }; + if let Some(pattern12_0) = closure12() { + if let Some(pattern13_0) = + C::imm_shift_from_imm64(ctx, pattern10_1, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 932. + let expr0_0 = ALUOp::RotR64; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0: Type = I64; + let expr3_0 = + C::negate_imm_shift(ctx, expr2_0, pattern13_0); + let expr4_0 = constructor_alu_rr_imm_shift( + ctx, &expr0_0, expr1_0, expr3_0, + )?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + } + } + } + } + // Rule at src/isa/aarch64/lower.isle line 923. + let expr0_0 = ALUOp::Sub64; + let expr1_0 = C::zero_reg(ctx); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = ALUOp::RotR64; + let expr5_0 = C::put_in_reg(ctx, pattern7_0); + let expr6_0 = constructor_alu_rrr(ctx, &expr4_0, expr5_0, expr3_0)?; + let expr7_0 = C::value_reg(ctx, expr6_0); + return Some(expr7_0); + } + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let closure12 = || { + let expr0_0: Type = I64; + return Some(expr0_0); + }; + if let Some(pattern12_0) = closure12() { + if let Some(pattern13_0) = + C::imm_shift_from_imm64(ctx, pattern10_1, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 976. + let expr0_0 = ALUOp::RotR64; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = constructor_alu_rr_imm_shift( + ctx, + &expr0_0, + expr1_0, + pattern13_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + } + } + // Rule at src/isa/aarch64/lower.isle line 964. + let expr0_0 = ALUOp::RotR64; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); // Rule at src/isa/aarch64/lower.isle line 706. @@ -2529,6 +2718,66 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 941. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = C::put_in_regs(ctx, pattern7_1); + let expr2_0: usize = 0; + let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0); + let expr4_0 = ALUOp::Sub64; + let expr5_0: Type = I64; + let expr6_0: u64 = 128; + let expr7_0 = constructor_imm(ctx, expr5_0, expr6_0)?; + let expr8_0 = constructor_alu_rrr(ctx, &expr4_0, expr7_0, expr3_0)?; + let expr9_0 = constructor_lower_shl128(ctx, expr0_0, expr3_0)?; + let expr10_0 = constructor_lower_ushr128(ctx, expr0_0, expr8_0)?; + let expr11_0 = ALUOp::Orr64; + let expr12_0: usize = 0; + let expr13_0 = C::value_regs_get(ctx, expr9_0, expr12_0); + let expr14_0: usize = 0; + let expr15_0 = C::value_regs_get(ctx, expr10_0, expr14_0); + let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr13_0, expr15_0)?; + let expr17_0 = ALUOp::Orr64; + let expr18_0: usize = 1; + let expr19_0 = C::value_regs_get(ctx, expr9_0, expr18_0); + let expr20_0: usize = 1; + let expr21_0 = C::value_regs_get(ctx, expr10_0, expr20_0); + let expr22_0 = constructor_alu_rrr(ctx, &expr17_0, expr19_0, expr21_0)?; + let expr23_0 = C::value_regs(ctx, expr16_0, expr22_0); + return Some(expr23_0); + } + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 1028. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = C::put_in_regs(ctx, pattern7_1); + let expr2_0: usize = 0; + let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0); + let expr4_0 = ALUOp::Sub64; + let expr5_0: Type = I64; + let expr6_0: u64 = 128; + let expr7_0 = constructor_imm(ctx, expr5_0, expr6_0)?; + let expr8_0 = constructor_alu_rrr(ctx, &expr4_0, expr7_0, expr3_0)?; + let expr9_0 = constructor_lower_ushr128(ctx, expr0_0, expr3_0)?; + let expr10_0 = constructor_lower_shl128(ctx, expr0_0, expr8_0)?; + let expr11_0 = ALUOp::Orr64; + let expr12_0: usize = 1; + let expr13_0 = C::value_regs_get(ctx, expr9_0, expr12_0); + let expr14_0: usize = 1; + let expr15_0 = C::value_regs_get(ctx, expr10_0, expr14_0); + let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr13_0, expr15_0)?; + let expr17_0 = ALUOp::Orr64; + let expr18_0: usize = 0; + let expr19_0 = C::value_regs_get(ctx, expr9_0, expr18_0); + let expr20_0: usize = 0; + let expr21_0 = C::value_regs_get(ctx, expr10_0, expr20_0); + let expr22_0 = constructor_alu_rrr(ctx, &expr17_0, expr19_0, expr21_0)?; + let expr23_0 = C::value_regs(ctx, expr22_0, expr16_0); + return Some(expr23_0); + } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); @@ -3278,6 +3527,100 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let closure12 = || { + return Some(pattern3_0); + }; + if let Some(pattern12_0) = closure12() { + if let Some(pattern13_0) = + C::imm_shift_from_imm64(ctx, pattern10_1, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 906. + let expr0_0 = + constructor_put_in_reg_zext32(ctx, pattern7_0)?; + let expr1_0 = + C::negate_imm_shift(ctx, pattern3_0, pattern13_0); + let expr2_0 = constructor_small_rotr_imm( + ctx, pattern3_0, expr0_0, expr1_0, + )?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + } + } + // Rule at src/isa/aarch64/lower.isle line 901. + let expr0_0 = ALUOp::Sub32; + let expr1_0 = C::zero_reg(ctx); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?; + let expr5_0 = constructor_small_rotr(ctx, pattern3_0, expr4_0, expr3_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); + } + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = &pattern10_0 { + let closure12 = || { + return Some(pattern3_0); + }; + if let Some(pattern12_0) = closure12() { + if let Some(pattern13_0) = + C::imm_shift_from_imm64(ctx, pattern10_1, pattern12_0) + { + // Rule at src/isa/aarch64/lower.isle line 968. + let expr0_0 = + constructor_put_in_reg_zext32(ctx, pattern7_0)?; + let expr1_0 = constructor_small_rotr_imm( + ctx, + pattern3_0, + expr0_0, + pattern13_0, + )?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + } + } + } + } + // Rule at src/isa/aarch64/lower.isle line 956. + let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_small_rotr(ctx, pattern3_0, expr0_0, expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} + } + } + } if let Some(pattern3_0) = C::fits_in_32(ctx, pattern2_0) { let pattern4_0 = C::inst_data(ctx, pattern0_0); if let &InstructionData::Binary { @@ -4615,3 +4958,54 @@ pub fn constructor_lower_sshr128( let expr31_0 = constructor_with_flags_2(ctx, &expr26_0, &expr28_0, &expr30_0)?; return Some(expr31_0); } + +// Generated as internal constructor for term small_rotr. +pub fn constructor_small_rotr( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/lower.isle line 992. + let expr0_0 = ALUOp::And32; + let expr1_0 = C::rotr_mask(ctx, pattern0_0); + let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern2_0, expr1_0)?; + let expr3_0 = ALUOp::Sub32; + let expr4_0 = C::ty_bits(ctx, pattern0_0); + let expr5_0 = C::u8_into_imm12(ctx, expr4_0); + let expr6_0 = constructor_alu_rr_imm12(ctx, &expr3_0, expr2_0, expr5_0)?; + let expr7_0 = ALUOp::Sub32; + let expr8_0 = C::zero_reg(ctx); + let expr9_0 = constructor_alu_rrr(ctx, &expr7_0, expr8_0, expr6_0)?; + let expr10_0 = ALUOp::Lsr32; + let expr11_0 = constructor_alu_rrr(ctx, &expr10_0, pattern1_0, expr2_0)?; + let expr12_0 = ALUOp::Lsl32; + let expr13_0 = constructor_alu_rrr(ctx, &expr12_0, pattern1_0, expr9_0)?; + let expr14_0 = ALUOp::Orr32; + let expr15_0 = constructor_alu_rrr(ctx, &expr14_0, expr13_0, expr11_0)?; + return Some(expr15_0); +} + +// Generated as internal constructor for term small_rotr_imm. +pub fn constructor_small_rotr_imm( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: ImmShift, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/lower.isle line 1015. + let expr0_0 = ALUOp::Lsr32; + let expr1_0 = constructor_alu_rr_imm_shift(ctx, &expr0_0, pattern1_0, pattern2_0)?; + let expr2_0 = ALUOp::Lsl32; + let expr3_0 = C::rotr_opposite_amount(ctx, pattern0_0, pattern2_0); + let expr4_0 = constructor_alu_rr_imm_shift(ctx, &expr2_0, pattern1_0, expr3_0)?; + let expr5_0 = ALUOp::Orr32; + let expr6_0 = constructor_alu_rrr(ctx, &expr5_0, expr4_0, expr1_0)?; + return Some(expr6_0); +} diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 1d143b7316..7d28f396f1 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -90,248 +90,7 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => implemented_in_isle(ctx), - Opcode::Rotr | Opcode::Rotl => { - // aarch64 doesn't have a left-rotate instruction, but a left rotation of K places is - // effectively a right rotation of N - K places, if N is the integer's bit size. We - // implement left rotations with this trick. - // - // For a 32-bit or 64-bit rotate-right, we can use the ROR instruction directly. - // - // For a < 32-bit rotate-right, we synthesize this as: - // - // rotr rd, rn, rm - // - // => - // - // zero-extend rn, <32-or-64> - // and tmp_masked_rm, rm, - // sub tmp1, tmp_masked_rm, - // sub tmp1, zero, tmp1 ; neg - // lsr tmp2, rn, tmp_masked_rm - // lsl rd, rn, tmp1 - // orr rd, rd, tmp2 - // - // For a constant amount, we can instead do: - // - // zero-extend rn, <32-or-64> - // lsr tmp2, rn, # - // lsl rd, rn, - // orr rd, rd, tmp2 - - let is_rotl = op == Opcode::Rotl; - - let ty = ty.unwrap(); - let ty_bits_size = ty_bits(ty) as u8; - - if ty.is_vector() { - return Err(CodegenError::Unsupported(format!( - "{}: Unsupported type: {:?}", - op, ty - ))); - } - - // TODO: We can do much better codegen if we have a constant amt - if ty == I128 { - let dst = get_output_reg(ctx, outputs[0]); - let src = put_input_in_regs(ctx, inputs[0]); - let amt_src = put_input_in_regs(ctx, inputs[1]).regs()[0]; - - let tmp = ctx.alloc_tmp(I128); - let inv_amt = ctx.alloc_tmp(I64).only_reg().unwrap(); - - lower_constant_u64(ctx, inv_amt, 128); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Sub64, - rd: inv_amt, - rn: inv_amt.to_reg(), - rm: amt_src, - }); - - if is_rotl { - // rotl - // (shl.i128 tmp, amt) - // (ushr.i128 dst, 128-amt) - - emit_shl_i128(ctx, src, tmp, amt_src); - emit_shr_i128( - ctx, - src, - dst, - inv_amt.to_reg(), - /* is_signed = */ false, - ); - } else { - // rotr - // (ushr.i128 tmp, amt) - // (shl.i128 dst, 128-amt) - - emit_shr_i128(ctx, src, tmp, amt_src, /* is_signed = */ false); - emit_shl_i128(ctx, src, dst, inv_amt.to_reg()); - } - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Orr64, - rd: dst.regs()[0], - rn: dst.regs()[0].to_reg(), - rm: tmp.regs()[0].to_reg(), - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Orr64, - rd: dst.regs()[1], - rn: dst.regs()[1].to_reg(), - rm: tmp.regs()[1].to_reg(), - }); - - return Ok(()); - } - - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let rn = put_input_in_reg( - ctx, - inputs[0], - if ty_bits_size <= 32 { - NarrowValueMode::ZeroExtend32 - } else { - NarrowValueMode::ZeroExtend64 - }, - ); - let rm = put_input_in_reg_immshift(ctx, inputs[1], ty_bits(ty)); - - if ty_bits_size == 32 || ty_bits_size == 64 { - let alu_op = choose_32_64(ty, ALUOp::RotR32, ALUOp::RotR64); - match rm { - ResultRegImmShift::ImmShift(mut immshift) => { - if is_rotl { - immshift.imm = ty_bits_size.wrapping_sub(immshift.value()); - } - immshift.imm &= ty_bits_size - 1; - ctx.emit(Inst::AluRRImmShift { - alu_op, - rd, - rn, - immshift, - }); - } - - ResultRegImmShift::Reg(rm) => { - let rm = if is_rotl { - // Really ty_bits_size - rn, but the upper bits of the result are - // ignored (because of the implicit masking done by the instruction), - // so this is equivalent to negating the input. - let alu_op = choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64); - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - ctx.emit(Inst::AluRRR { - alu_op, - rd: tmp, - rn: zero_reg(), - rm, - }); - tmp.to_reg() - } else { - rm - }; - ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm }); - } - } - } else { - debug_assert!(ty_bits_size < 32); - - match rm { - ResultRegImmShift::Reg(reg) => { - let reg = if is_rotl { - // Really ty_bits_size - rn, but the upper bits of the result are - // ignored (because of the implicit masking done by the instruction), - // so this is equivalent to negating the input. - let tmp = ctx.alloc_tmp(I32).only_reg().unwrap(); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Sub32, - rd: tmp, - rn: zero_reg(), - rm: reg, - }); - tmp.to_reg() - } else { - reg - }; - - // Explicitly mask the rotation count. - let tmp_masked_rm = ctx.alloc_tmp(I32).only_reg().unwrap(); - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::And32, - rd: tmp_masked_rm, - rn: reg, - imml: ImmLogic::maybe_from_u64((ty_bits_size - 1) as u64, I32).unwrap(), - }); - let tmp_masked_rm = tmp_masked_rm.to_reg(); - - let tmp1 = ctx.alloc_tmp(I32).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(I32).only_reg().unwrap(); - ctx.emit(Inst::AluRRImm12 { - alu_op: ALUOp::Sub32, - rd: tmp1, - rn: tmp_masked_rm, - imm12: Imm12::maybe_from_u64(ty_bits_size as u64).unwrap(), - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Sub32, - rd: tmp1, - rn: zero_reg(), - rm: tmp1.to_reg(), - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Lsr32, - rd: tmp2, - rn, - rm: tmp_masked_rm, - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Lsl32, - rd, - rn, - rm: tmp1.to_reg(), - }); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Orr32, - rd, - rn: rd.to_reg(), - rm: tmp2.to_reg(), - }); - } - - ResultRegImmShift::ImmShift(mut immshift) => { - if is_rotl { - immshift.imm = ty_bits_size.wrapping_sub(immshift.value()); - } - immshift.imm &= ty_bits_size - 1; - - let tmp1 = ctx.alloc_tmp(I32).only_reg().unwrap(); - ctx.emit(Inst::AluRRImmShift { - alu_op: ALUOp::Lsr32, - rd: tmp1, - rn, - immshift: immshift.clone(), - }); - - let amount = immshift.value() & (ty_bits_size - 1); - let opp_shift = - ImmShift::maybe_from_u64(ty_bits_size as u64 - amount as u64).unwrap(); - ctx.emit(Inst::AluRRImmShift { - alu_op: ALUOp::Lsl32, - rd, - rn, - immshift: opp_shift, - }); - - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Orr32, - rd, - rn: rd.to_reg(), - rm: tmp1.to_reg(), - }); - } - } - } - } + Opcode::Rotr | Opcode::Rotl => implemented_in_isle(ctx), Opcode::Bitrev | Opcode::Clz | Opcode::Cls | Opcode::Ctz => { let ty = ty.unwrap(); diff --git a/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clif b/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clif index f46277a0e9..4608fc5ae5 100644 --- a/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clif +++ b/cranelift/filetests/filetests/isa/aarch64/shift-rotate.clif @@ -12,30 +12,32 @@ block0(v0: i128, v1: i128): return v2 } -; check: movz x3, #128 -; nextln: sub x5, x3, x2 -; nextln: orn w4, wzr, w2 -; nextln: lsl x6, x1, #1 +; check: mov x4, x1 +; nextln: orr x1, xzr, #128 +; nextln: sub x1, x1, x2 ; nextln: lsr x3, x0, x2 -; nextln: lsl x6, x6, x4 -; nextln: lsr x4, x1, x2 +; nextln: lsr x5, x4, x2 +; nextln: orn w6, wzr, w2 +; nextln: lsl x7, x4, #1 +; nextln: lsl x6, x7, x6 +; nextln: orr x6, x3, x6 ; nextln: ands xzr, x2, #64 -; nextln: orr x2, x3, x6 -; nextln: csel x3, xzr, x4, ne -; nextln: csel x4, x4, x2, ne -; nextln: orn w2, wzr, w5 -; nextln: lsr x6, x0, #1 -; nextln: lsl x1, x1, x5 -; nextln: lsr x2, x6, x2 -; nextln: lsl x0, x0, x5 -; nextln: ands xzr, x5, #64 -; nextln: orr x1, x1, x2 -; nextln: csel x1, x0, x1, ne -; nextln: csel x0, xzr, x0, ne -; nextln: orr x0, x0, x4 -; nextln: orr x1, x1, x3 +; nextln: csel x3, xzr, x5, ne +; nextln: csel x2, x5, x6, ne +; nextln: lsl x5, x0, x1 +; nextln: lsl x4, x4, x1 +; nextln: orn w6, wzr, w1 +; nextln: lsr x0, x0, #1 +; nextln: lsr x0, x0, x6 +; nextln: orr x0, x4, x0 +; nextln: ands xzr, x1, #64 +; nextln: csel x1, x5, x0, ne +; nextln: csel x0, xzr, x5, ne +; nextln: orr x1, x3, x1 +; nextln: orr x0, x2, x0 ; nextln: ret + function %f0(i64, i64) -> i64 { block0(v0: i64, v1: i64): v2 = rotr.i64 v0, v1 @@ -94,28 +96,29 @@ block0(v0: i128, v1: i128): return v2 } -; check: movz x3, #128 -; nextln: sub x5, x3, x2 -; nextln: orn w4, wzr, w2 -; nextln: lsr x6, x0, #1 -; nextln: lsl x3, x1, x2 -; nextln: lsr x6, x6, x4 -; nextln: lsl x4, x0, x2 +; check: mov x4, x0 +; nextln: orr x0, xzr, #128 +; nextln: sub x0, x0, x2 +; nextln: lsl x3, x4, x2 +; nextln: lsl x5, x1, x2 +; nextln: orn w6, wzr, w2 +; nextln: lsr x7, x4, #1 +; nextln: lsr x6, x7, x6 +; nextln: orr x5, x5, x6 ; nextln: ands xzr, x2, #64 -; nextln: orr x2, x3, x6 -; nextln: csel x3, x4, x2, ne -; nextln: csel x4, xzr, x4, ne -; nextln: orn w2, wzr, w5 -; nextln: lsl x6, x1, #1 -; nextln: lsr x0, x0, x5 -; nextln: lsl x2, x6, x2 -; nextln: lsr x1, x1, x5 -; nextln: ands xzr, x5, #64 -; nextln: orr x2, x0, x2 -; nextln: csel x0, xzr, x1, ne -; nextln: csel x1, x1, x2, ne -; nextln: orr x1, x1, x4 -; nextln: orr x0, x0, x3 +; nextln: csel x2, x3, x5, ne +; nextln: csel x3, xzr, x3, ne +; nextln: lsr x5, x4, x0 +; nextln: lsr x4, x1, x0 +; nextln: orn w6, wzr, w0 +; nextln: lsl x1, x1, #1 +; nextln: lsl x1, x1, x6 +; nextln: orr x1, x5, x1 +; nextln: ands xzr, x0, #64 +; nextln: csel x0, xzr, x4, ne +; nextln: csel x1, x4, x1, ne +; nextln: orr x1, x3, x1 +; nextln: orr x0, x2, x0 ; nextln: mov x2, x0 ; nextln: mov x0, x1 ; nextln: mov x1, x2 @@ -147,8 +150,8 @@ block0(v0: i16, v1: i16): return v2 } -; check: uxth w0, w0 -; nextln: sub w1, wzr, w1 +; check: sub w1, wzr, w1 +; nextln: uxth w0, w0 ; nextln: and w1, w1, #15 ; nextln: sub w2, w1, #16 ; nextln: sub w2, wzr, w2 @@ -163,8 +166,8 @@ block0(v0: i8, v1: i8): return v2 } -; check: uxtb w0, w0 -; nextln: sub w1, wzr, w1 +; check: sub w1, wzr, w1 +; nextln: uxtb w0, w0 ; nextln: and w1, w1, #7 ; nextln: sub w2, w1, #8 ; nextln: sub w2, wzr, w2 diff --git a/cranelift/filetests/src/subtest.rs b/cranelift/filetests/src/subtest.rs index 7fd54267d6..f1af598fee 100644 --- a/cranelift/filetests/src/subtest.rs +++ b/cranelift/filetests/src/subtest.rs @@ -81,7 +81,12 @@ pub fn run_filecheck(text: &str, context: &Context) -> anyhow::Result<()> { let (_, explain) = checker .explain(text, NO_VARIABLES) .context("filecheck explain failed")?; - anyhow::bail!("filecheck failed:\n{}{}", checker, explain); + anyhow::bail!( + "filecheck failed for function on line {}:\n{}{}", + context.details.location.line_number, + checker, + explain + ); } }