T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Port flag-based ops to ISLE (AArch64) (#4942)

Browse Source 
Ported the existing implementations of the following opcodes for AArch64
to ISLE:
- `Trueif`
- `Trueff`
- `Trapif`
- `Trapff`
- `Select`
- `Selectif`
- `SelectifSpectreGuard`

Copyright (c) 2022 Arm Limited
This commit is contained in:
Damian Heaton
2022-09-22 23:44:32 +01:00
committed by GitHub
parent 89abd80c3c
commit 3f8cccfb59
12 changed files with 1641 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
cranelift/codegen/src/isa/aarch64/inst.isle
View File

@@ -2077,6 +2077,14 @@
(MInst.FpuCSel64 dst if_true if_false cond)
dst)))
;; Helper for emitting `MInst.VecCSel` instructions.
(decl vec_csel (Cond Reg Reg) ConsumesFlags)
(rule (vec_csel cond if_true if_false)
(let ((dst WritableReg (temp_writable_reg $I8X16)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.VecCSel dst if_true if_false cond)
dst)))
;; Helper for emitting `MInst.FpuRound` instructions.
(decl fpu_round (FpuRoundMode Reg) Reg)
(rule (fpu_round op rn)
@@ -2353,6 +2361,11 @@
(rule (aarch64_fence)
(SideEffectNoResult.Inst (MInst.Fence)))
;; Helper for generating `csdb` instructions.
(decl csdb () SideEffectNoResult)
(rule (csdb)
(SideEffectNoResult.Inst (MInst.Csdb)))
;; Helper for generating `brk` instructions.
(decl brk () SideEffectNoResult)
(rule (brk)
@@ -2641,6 +2654,15 @@
(rule (aarch64_storep64 amode flags val1 val2)
(SideEffectNoResult.Inst (MInst.StoreP64 val1 val2 amode flags)))
;; Helper for generating a `trapif` instruction.
(decl trap_if (ProducesFlags TrapCode Cond) InstOutput)
(rule (trap_if flags trap_code cond)
(side_effect
(with_flags_side_effect flags
(ConsumesFlags.ConsumesFlagsSideEffect
(MInst.TrapIf (cond_br_cond cond) trap_code)))))
;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Type of extension performed by an immediate helper
@@ -3378,7 +3400,9 @@
;; ccmp lhs_hi, rhs_hi, #0, eq
(decl lower_icmp_i128_eq_ne (Value Value) ProducesFlags)
(rule (lower_icmp_i128_eq_ne lhs rhs)
(let ((lhs_lo Reg (value_regs_get lhs 0))
(let ((lhs ValueRegs (put_in_regs lhs))
(rhs ValueRegs (put_in_regs rhs))
(lhs_lo Reg (value_regs_get lhs 0))
(lhs_hi Reg (value_regs_get lhs 1))
(rhs_lo Reg (value_regs_get rhs 0))
(rhs_hi Reg (value_regs_get rhs 1))
@@ -3399,6 +3423,8 @@
(rule -1 (lower_icmp_into_reg cond lhs rhs $I128 out_ty)
(let ((unsigned_cond Cond (cond_code (intcc_unsigned cond)))
(cond Cond (cond_code cond))
(lhs ValueRegs (put_in_regs lhs))
(rhs ValueRegs (put_in_regs rhs))
(lhs_lo Reg (value_regs_get lhs 0))
(lhs_hi Reg (value_regs_get lhs 1))
(rhs_lo Reg (value_regs_get rhs 0))
@@ -3477,3 +3503,30 @@
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) (zero_reg) dst)))
;; Helpers for generating select instruction sequences.
(decl lower_select (ProducesFlags Cond Type Value Value) ValueRegs)
(rule (lower_select flags cond (ty_scalar_float ty) rn rm)
(with_flags flags (fpu_csel ty cond rn rm)))
(rule (lower_select flags cond (ty_vec128 ty) rn rm)
(with_flags flags (vec_csel cond rn rm)))
(rule (lower_select flags cond ty rn rm)
(if (ty_vec64 ty))
(with_flags flags (fpu_csel $F64 cond rn rm)))
(rule (lower_select flags cond $I128 rn rm)
(let ((dst_lo WritableReg (temp_writable_reg $I64))
(dst_hi WritableReg (temp_writable_reg $I64))
(rn ValueRegs (put_in_regs rn))
(rm ValueRegs (put_in_regs rm))
(rn_lo Reg (value_regs_get rn 0))
(rn_hi Reg (value_regs_get rn 1))
(rm_lo Reg (value_regs_get rm 0))
(rm_hi Reg (value_regs_get rm 1)))
(with_flags flags
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSel dst_lo cond rn_lo rm_lo)
(MInst.CSel dst_hi cond rn_hi rm_hi)
(value_regs dst_lo dst_hi)))))
(rule (lower_select flags cond ty rn rm)
(if (ty_int_bool_ref_scalar_64 ty))
(with_flags flags (csel cond rn rm)))

112
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -1738,11 +1738,111 @@
(rule (lower (trap trap_code))
(side_effect (udf trap_code)))
;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trapif cc insn @ (iadd_ifcout x y) trap_code))
;; The flags must not have been clobbered by any other instruction, as
;; verified by the CLIF validator; so we can simply use the flags here.
(let ((insn ProducesFlags (flags_to_producesflags insn)))
(trap_if insn trap_code (cond_code cc))))
;; Verification ensures the input is always a single-def ifcmp.
(rule (lower (trapif cc insn @ (ifcmp x @ (value_type ty) y) trap_code))
(let ((cond Cond (cond_code cc)))
(trap_if (lower_icmp_into_flags cc x y ty) trap_code cond)))
;;;; Rules for `trapff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures the input is always a single-def ffcmp.
(rule (lower (trapff cc insn @ (ffcmp x @ (value_type ty) y) trap_code))
(let ((cond Cond (fp_cond_code cc)))
(trap_if (fpu_cmp (scalar_size ty) x y) trap_code cond)))
;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (resumable_trap trap_code))
(side_effect (udf trap_code)))
;;;; Rules for `trueif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures the input is always a single-def ifcmp.
(rule (lower (has_type out_ty
(trueif cc insn @ (ifcmp x @ (value_type in_ty) y))))
(lower_icmp_into_reg cc x y in_ty out_ty))
;;;; Rules for `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures the input is always a single-def ffcmp.
(rule (lower (has_type ty (trueff cc insn @ (ffcmp x @ (value_type in_ty) y))))
(with_flags_reg
(fpu_cmp (scalar_size in_ty) x y)
(materialize_bool_result (ty_bits ty) (fp_cond_code cc))))
;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty
(select _flags @ (icmp cc x @ (value_type in_ty) y) rn rm)))
(let ((cond Cond (cond_code cc)))
(lower_select
(lower_icmp_into_flags cc x y in_ty)
cond ty rn rm)))
(rule (lower (has_type ty
(select _flags @ (bint (icmp cc x @ (value_type in_ty) y)) rn rm)))
(let ((cond Cond (cond_code cc)))
(lower_select
(lower_icmp_into_flags cc x y in_ty)
cond ty rn rm)))
(rule (lower (has_type ty
(select _flags @ (fcmp cc x @ (value_type in_ty) y) rn rm)))
(let ((cond Cond (fp_cond_code cc)))
(lower_select
(fpu_cmp (scalar_size in_ty) x y)
cond ty rn rm)))
(rule (lower (has_type ty
(select _flags @ (bint (fcmp cc x @ (value_type in_ty) y)) rn rm)))
(let ((cond Cond (fp_cond_code cc)))
(lower_select
(fpu_cmp (scalar_size in_ty) x y)
cond ty rn rm)))
(rule -1 (lower (has_type ty (select rcond @ (value_type (fits_in_32 _)) rn rm)))
(let ((rcond Reg (put_in_reg_zext32 rcond)))
(lower_select
(cmp (OperandSize.Size32) rcond (zero_reg))
(Cond.Ne) ty rn rm)))
(rule -2 (lower (has_type ty (select rcond rn rm)))
(let ((rcond Reg (put_in_reg_zext64 rcond)))
(lower_select
(cmp (OperandSize.Size64) rcond (zero_reg))
(Cond.Ne) ty rn rm)))
;;;; Rules for `selectif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures that the input is always a single-def ifcmp.
(rule (lower (has_type ty
(selectif cc flags @ (ifcmp x @ (value_type in_ty) y)
if_true if_false)))
(let ((cond Cond (cond_code cc)))
(lower_select
(lower_icmp_into_flags cc x y in_ty)
cond ty if_true if_false)))
;;;; Rules for `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty
(selectif_spectre_guard cc
flags @ (ifcmp x @ (value_type in_ty) y) if_true if_false)))
(let ((cond Cond (cond_code cc))
(dst ValueRegs (lower_select
(lower_icmp_into_flags cc x y in_ty)
cond ty if_true if_false))
(_ InstOutput (side_effect (csdb))))
dst))
;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (ty_vec128 _) (vconst (u128_from_constant x))))
@@ -2395,18 +2495,6 @@
(rule (lower (fvpromote_low val))
(vec_rr_long (VecRRLongOp.Fcvtl32) val $false))
;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TODO: requires icmp/fcmp first.
;;; Rules for `selectif` / `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TODO: requires icmp/fcmp first.
;;; Rules for `trueif` / `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TODO: requires icmp/fcmp first.
;;; Rules for `brz`/`brnz`/`brif`/`brff`/`bricmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TODO: requires icmp/fcmp first.

8
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -1178,15 +1178,12 @@ pub(crate) enum IcmpOutput {
/// Lowers the comparison into a cond code, discarding the results. The cond code emitted can
/// be checked in the resulting [IcmpResult].
CondCode,
/// Materializes the results into a register. This may overwrite any flags previously set.
Register(Writable<Reg>),
}
impl IcmpOutput {
pub fn reg(&self) -> Option<Writable<Reg>> {
match self {
IcmpOutput::CondCode => None,
IcmpOutput::Register(reg) => Some(*reg),
}
}
}
@@ -1197,15 +1194,12 @@ pub(crate) enum IcmpResult {
/// The result was output into the given [Cond]. Callers may perform operations using this [Cond]
/// and its inverse, other [Cond]'s are not guaranteed to be correct.
CondCode(Cond),
/// The result was materialized into the output register.
Register,
}
impl IcmpResult {
pub fn unwrap_cond(&self) -> Cond {
match self {
IcmpResult::CondCode(c) => *c,
_ => panic!("Unwrapped cond, but IcmpResult was {:?}", self),
}
}
}
@@ -1364,8 +1358,6 @@ pub(crate) fn lower_icmp(
}
Ok(match output {
// We currently never emit a different register than what was asked for
IcmpOutput::Register(_) => IcmpResult::Register,
IcmpOutput::CondCode => IcmpResult::CondCode(out_condcode),
})
}

159
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -24,7 +24,6 @@ pub(crate) fn lower_insn_to_regs(
isa_flags: &aarch64_settings::Flags,
) -> CodegenResult<()> {
let op = ctx.data(insn).opcode();
let inputs = insn_inputs(ctx, insn);
let outputs = insn_outputs(ctx, insn);
let ty = if outputs.len() > 0 {
Some(ctx.output_ty(insn, 0))
@@ -140,131 +139,15 @@ pub(crate) fn lower_insn_to_regs(
// Nothing.
}
Opcode::Select => {
let flag_input = inputs[0];
let cond = if let Some(icmp_insn) =
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint)
{
let condcode = ctx.data(icmp_insn).cond_code().unwrap();
lower_icmp(ctx, icmp_insn, condcode, IcmpOutput::CondCode)?.unwrap_cond()
} else if let Some(fcmp_insn) =
maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint)
{
let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode);
lower_fcmp_or_ffcmp_to_flags(ctx, fcmp_insn);
cond
} else {
let (size, narrow_mode) = if ty_bits(ctx.input_ty(insn, 0)) > 32 {
(OperandSize::Size64, NarrowValueMode::ZeroExtend64)
} else {
(OperandSize::Size32, NarrowValueMode::ZeroExtend32)
};
Opcode::Select => implemented_in_isle(ctx),
let rcond = put_input_in_reg(ctx, inputs[0], narrow_mode);
// cmp rcond, #0
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::SubS,
size,
rd: writable_zero_reg(),
rn: rcond,
rm: zero_reg(),
});
Cond::Ne
};
// csel.cond rd, rn, rm
let ty = ctx.output_ty(insn, 0);
let bits = ty_bits(ty);
let is_float = ty_has_float_or_vec_representation(ty);
let dst = get_output_reg(ctx, outputs[0]);
let lhs = put_input_in_regs(ctx, inputs[1]);
let rhs = put_input_in_regs(ctx, inputs[2]);
let rd = dst.regs()[0];
let rn = lhs.regs()[0];
let rm = rhs.regs()[0];
match (is_float, bits) {
(true, 32) => ctx.emit(Inst::FpuCSel32 { cond, rd, rn, rm }),
(true, 64) => ctx.emit(Inst::FpuCSel64 { cond, rd, rn, rm }),
(true, 128) => ctx.emit(Inst::VecCSel { cond, rd, rn, rm }),
(false, 128) => {
ctx.emit(Inst::CSel {
cond,
rd: dst.regs()[0],
rn: lhs.regs()[0],
rm: rhs.regs()[0],
});
ctx.emit(Inst::CSel {
cond,
rd: dst.regs()[1],
rn: lhs.regs()[1],
rm: rhs.regs()[1],
});
}
(false, bits) if bits <= 64 => ctx.emit(Inst::CSel { cond, rd, rn, rm }),
_ => {
return Err(CodegenError::Unsupported(format!(
"Select: Unsupported type: {:?}",
ty
)));
}
}
}
Opcode::Selectif | Opcode::SelectifSpectreGuard => {
let condcode = ctx.data(insn).cond_code().unwrap();
// Verification ensures that the input is always a
// single-def ifcmp.
let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
let cond = lower_icmp(ctx, ifcmp_insn, condcode, IcmpOutput::CondCode)?.unwrap_cond();
// csel.COND rd, rn, rm
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
let ty = ctx.output_ty(insn, 0);
let bits = ty_bits(ty);
let is_float = ty_has_float_or_vec_representation(ty);
if is_float && bits == 32 {
ctx.emit(Inst::FpuCSel32 { cond, rd, rn, rm });
} else if is_float && bits == 64 {
ctx.emit(Inst::FpuCSel64 { cond, rd, rn, rm });
} else if !is_float && bits <= 64 {
ctx.emit(Inst::CSel { cond, rd, rn, rm });
} else {
return Err(CodegenError::Unsupported(format!(
"{}: Unsupported type: {:?}",
op, ty
)));
}
if op == Opcode::SelectifSpectreGuard {
ctx.emit(Inst::Csdb);
}
}
Opcode::Selectif | Opcode::SelectifSpectreGuard => implemented_in_isle(ctx),
Opcode::Bitselect | Opcode::Vselect => implemented_in_isle(ctx),
Opcode::Trueif => {
let condcode = ctx.data(insn).cond_code().unwrap();
// Verification ensures that the input is always a
// single-def ifcmp.
let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
lower_icmp(ctx, ifcmp_insn, condcode, IcmpOutput::Register(rd))?;
}
Opcode::Trueif => implemented_in_isle(ctx),
Opcode::Trueff => {
let condcode = ctx.data(insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode);
let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap();
lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn);
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
materialize_bool_result(ctx, insn, rd, cond);
}
Opcode::Trueff => implemented_in_isle(ctx),
Opcode::IsNull | Opcode::IsInvalid => implemented_in_isle(ctx),
@@ -296,39 +179,7 @@ pub(crate) fn lower_insn_to_regs(
Opcode::Trap | Opcode::ResumableTrap => implemented_in_isle(ctx),
Opcode::Trapif | Opcode::Trapff => {
let trap_code = ctx.data(insn).trap_code().unwrap();
let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() {
let condcode = ctx.data(insn).cond_code().unwrap();
let cond = lower_condcode(condcode);
// The flags must not have been clobbered by any other
// instruction between the iadd_ifcout and this instruction, as
// verified by the CLIF validator; so we can simply use the
// flags here.
cond
} else if op == Opcode::Trapif {
let condcode = ctx.data(insn).cond_code().unwrap();
// Verification ensures that the input is always a single-def ifcmp.
let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
lower_icmp(ctx, ifcmp_insn, condcode, IcmpOutput::CondCode)?.unwrap_cond()
} else {
let condcode = ctx.data(insn).fp_cond_code().unwrap();
let cond = lower_fp_condcode(condcode);
// Verification ensures that the input is always a
// single-def ffcmp.
let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap();
lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn);
cond
};
ctx.emit(Inst::TrapIf {
trap_code,
kind: CondBrKind::Cond(cond),
});
}
Opcode::Trapif | Opcode::Trapff => implemented_in_isle(ctx),
Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
panic!("trapz / trapnz / resumable_trapnz should have been removed by legalization!");