From 3f8cccfb59495fcc630f237f25c3ff8fe9f44d1e Mon Sep 17 00:00:00 2001
From: Damian Heaton <87125748+dheaton-arm@users.noreply.github.com>
Date: Thu, 22 Sep 2022 23:44:32 +0100
Subject: [PATCH] Port flag-based ops to ISLE (AArch64) (#4942)

Ported the existing implementations of the following opcodes for AArch64
to ISLE:
- `Trueif`
- `Trueff`
- `Trapif`
- `Trapff`
- `Select`
- `Selectif`
- `SelectifSpectreGuard`

Copyright (c) 2022 Arm Limited
---
 cranelift/codegen/src/isa/aarch64/inst.isle   |  55 +-
 cranelift/codegen/src/isa/aarch64/lower.isle  | 112 ++-
 cranelift/codegen/src/isa/aarch64/lower.rs    |   8 -
 .../codegen/src/isa/aarch64/lower_inst.rs     | 159 +---
 .../filetests/isa/aarch64/condops.clif        | 737 +++++++++++++++++-
 .../filetests/isa/aarch64/heap_addr.clif      |   6 +-
 .../filetests/isa/aarch64/traps.clif          |   1 +
 .../filetests/filetests/runtests/select.clif  |   1 +
 .../runtests/selectif-spectre-guard.clif      | 316 ++++++++
 .../filetests/runtests/selectif.clif          | 316 ++++++++
 .../filetests/runtests/trueif-ff.clif         | 110 +++
 cranelift/interpreter/src/step.rs             |  10 +-
 12 files changed, 1641 insertions(+), 190 deletions(-)
 create mode 100644 cranelift/filetests/filetests/runtests/selectif-spectre-guard.clif
 create mode 100644 cranelift/filetests/filetests/runtests/selectif.clif
 create mode 100644 cranelift/filetests/filetests/runtests/trueif-ff.clif

diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index f81bfd63bf..6c2e47b424 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -2077,6 +2077,14 @@
          (MInst.FpuCSel64 dst if_true if_false cond)
          dst)))
 
+;; Helper for emitting `MInst.VecCSel` instructions.
+(decl vec_csel (Cond Reg Reg) ConsumesFlags)
+(rule (vec_csel cond if_true if_false)
+      (let ((dst WritableReg (temp_writable_reg $I8X16)))
+        (ConsumesFlags.ConsumesFlagsReturnsReg
+         (MInst.VecCSel dst if_true if_false cond)
+         dst)))
+
 ;; Helper for emitting `MInst.FpuRound` instructions.
 (decl fpu_round (FpuRoundMode Reg) Reg)
 (rule (fpu_round op rn)
@@ -2353,6 +2361,11 @@
 (rule (aarch64_fence)
       (SideEffectNoResult.Inst (MInst.Fence)))
 
+;; Helper for generating `csdb` instructions.
+(decl csdb () SideEffectNoResult)
+(rule (csdb)
+      (SideEffectNoResult.Inst (MInst.Csdb)))
+
 ;; Helper for generating `brk` instructions.
 (decl brk () SideEffectNoResult)
 (rule (brk)
@@ -2641,6 +2654,15 @@
 (rule (aarch64_storep64 amode flags val1 val2)
       (SideEffectNoResult.Inst (MInst.StoreP64 val1 val2 amode flags)))
 
+;; Helper for generating a `trapif` instruction.
+
+(decl trap_if (ProducesFlags TrapCode Cond) InstOutput)
+(rule (trap_if flags trap_code cond)
+      (side_effect
+       (with_flags_side_effect flags
+        (ConsumesFlags.ConsumesFlagsSideEffect
+         (MInst.TrapIf (cond_br_cond cond) trap_code)))))
+
 ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Type of extension performed by an immediate helper
@@ -3378,7 +3400,9 @@
 ;; ccmp lhs_hi, rhs_hi, #0, eq
 (decl lower_icmp_i128_eq_ne (Value Value) ProducesFlags)
 (rule (lower_icmp_i128_eq_ne lhs rhs)
-      (let ((lhs_lo Reg (value_regs_get lhs 0))
+      (let ((lhs ValueRegs (put_in_regs lhs))
+            (rhs ValueRegs (put_in_regs rhs))
+            (lhs_lo Reg (value_regs_get lhs 0))
             (lhs_hi Reg (value_regs_get lhs 1))
             (rhs_lo Reg (value_regs_get rhs 0))
             (rhs_hi Reg (value_regs_get rhs 1))
@@ -3399,6 +3423,8 @@
 (rule -1 (lower_icmp_into_reg cond lhs rhs $I128 out_ty)
       (let ((unsigned_cond Cond (cond_code (intcc_unsigned cond)))
             (cond Cond (cond_code cond))
+            (lhs ValueRegs (put_in_regs lhs))
+            (rhs ValueRegs (put_in_regs rhs))
             (lhs_lo Reg (value_regs_get lhs 0))
             (lhs_hi Reg (value_regs_get lhs 1))
             (rhs_lo Reg (value_regs_get rhs 0))
@@ -3477,3 +3503,30 @@
       (let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
             (dst Reg (value_regs_get dst 0)))
        (cmp (OperandSize.Size64) (zero_reg) dst)))
+
+;; Helpers for generating select instruction sequences.
+(decl lower_select (ProducesFlags Cond Type Value Value) ValueRegs)
+(rule (lower_select flags cond (ty_scalar_float ty) rn rm)
+      (with_flags flags (fpu_csel ty cond rn rm)))
+(rule (lower_select flags cond (ty_vec128 ty) rn rm)
+      (with_flags flags (vec_csel cond rn rm)))
+(rule (lower_select flags cond ty rn rm)
+      (if (ty_vec64 ty))
+      (with_flags flags (fpu_csel $F64 cond rn rm)))
+(rule (lower_select flags cond $I128 rn rm)
+      (let ((dst_lo WritableReg (temp_writable_reg $I64))
+            (dst_hi WritableReg (temp_writable_reg $I64))
+            (rn ValueRegs (put_in_regs rn))
+            (rm ValueRegs (put_in_regs rm))
+            (rn_lo Reg (value_regs_get rn 0))
+            (rn_hi Reg (value_regs_get rn 1))
+            (rm_lo Reg (value_regs_get rm 0))
+            (rm_hi Reg (value_regs_get rm 1)))
+       (with_flags flags
+        (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
+         (MInst.CSel dst_lo cond rn_lo rm_lo)
+         (MInst.CSel dst_hi cond rn_hi rm_hi)
+         (value_regs dst_lo dst_hi)))))
+(rule (lower_select flags cond ty rn rm)
+      (if (ty_int_bool_ref_scalar_64 ty))
+      (with_flags flags (csel cond rn rm)))
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 664b04037b..3d861bb4ed 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -1738,11 +1738,111 @@
 (rule (lower (trap trap_code))
       (side_effect (udf trap_code)))
 
+;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (trapif cc insn @ (iadd_ifcout x y) trap_code))
+      ;; The flags must not have been clobbered by any other instruction, as
+      ;; verified by the CLIF validator; so we can simply use the flags here.
+      (let ((insn ProducesFlags (flags_to_producesflags insn)))
+       (trap_if insn trap_code (cond_code cc))))
+
+;; Verification ensures the input is always a single-def ifcmp.
+(rule (lower (trapif cc insn @ (ifcmp x @ (value_type ty) y) trap_code))
+      (let ((cond Cond (cond_code cc)))
+       (trap_if (lower_icmp_into_flags cc x y ty) trap_code cond)))
+
+;;;; Rules for `trapff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures the input is always a single-def ffcmp.
+(rule (lower (trapff cc insn @ (ffcmp x @ (value_type ty) y) trap_code))
+      (let ((cond Cond (fp_cond_code cc)))
+       (trap_if (fpu_cmp (scalar_size ty) x y) trap_code cond)))
+
 ;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (resumable_trap trap_code))
       (side_effect (udf trap_code)))
 
+;;;; Rules for `trueif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures the input is always a single-def ifcmp.
+(rule (lower (has_type out_ty
+       (trueif cc insn @ (ifcmp x @ (value_type in_ty) y))))
+      (lower_icmp_into_reg cc x y in_ty out_ty))
+
+;;;; Rules for `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures the input is always a single-def ffcmp.
+(rule (lower (has_type ty (trueff cc insn @ (ffcmp x @ (value_type in_ty) y))))
+      (with_flags_reg
+       (fpu_cmp (scalar_size in_ty) x y)
+       (materialize_bool_result (ty_bits ty) (fp_cond_code cc))))
+
+;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty
+       (select _flags @ (icmp cc x @ (value_type in_ty) y) rn rm)))
+      (let ((cond Cond (cond_code cc)))
+       (lower_select
+        (lower_icmp_into_flags cc x y in_ty)
+        cond ty rn rm)))
+
+(rule (lower (has_type ty
+       (select _flags @ (bint (icmp cc x @ (value_type in_ty) y)) rn rm)))
+      (let ((cond Cond (cond_code cc)))
+       (lower_select
+        (lower_icmp_into_flags cc x y in_ty)
+        cond ty rn rm)))
+
+(rule (lower (has_type ty
+       (select _flags @ (fcmp cc x @ (value_type in_ty) y) rn rm)))
+      (let ((cond Cond (fp_cond_code cc)))
+       (lower_select
+        (fpu_cmp (scalar_size in_ty) x y)
+        cond ty rn rm)))
+
+(rule (lower (has_type ty
+       (select _flags @ (bint (fcmp cc x @ (value_type in_ty) y)) rn rm)))
+      (let ((cond Cond (fp_cond_code cc)))
+       (lower_select
+        (fpu_cmp (scalar_size in_ty) x y)
+        cond ty rn rm)))
+
+(rule -1 (lower (has_type ty (select rcond @ (value_type (fits_in_32 _)) rn rm)))
+      (let ((rcond Reg (put_in_reg_zext32 rcond)))
+       (lower_select
+        (cmp (OperandSize.Size32) rcond (zero_reg))
+        (Cond.Ne) ty rn rm)))
+
+(rule -2 (lower (has_type ty (select rcond rn rm)))
+      (let ((rcond Reg (put_in_reg_zext64 rcond)))
+       (lower_select
+        (cmp (OperandSize.Size64) rcond (zero_reg))
+        (Cond.Ne) ty rn rm)))
+
+;;;; Rules for `selectif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Verification ensures that the input is always a single-def ifcmp.
+(rule (lower (has_type ty
+       (selectif cc flags @ (ifcmp x @ (value_type in_ty) y)
+        if_true if_false)))
+      (let ((cond Cond (cond_code cc)))
+       (lower_select
+        (lower_icmp_into_flags cc x y in_ty)
+        cond ty if_true if_false)))
+
+;;;; Rules for `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty
+       (selectif_spectre_guard cc
+        flags @ (ifcmp x @ (value_type in_ty) y) if_true if_false)))
+      (let ((cond Cond (cond_code cc))
+            (dst ValueRegs (lower_select
+                      (lower_icmp_into_flags cc x y in_ty)
+                      cond ty if_true if_false))
+            (_ InstOutput (side_effect (csdb))))
+       dst))
+
 ;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (has_type (ty_vec128 _) (vconst (u128_from_constant x))))
@@ -2395,18 +2495,6 @@
 (rule (lower (fvpromote_low val))
       (vec_rr_long (VecRRLongOp.Fcvtl32) val $false))
 
-;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; TODO: requires icmp/fcmp first.
-
-;;; Rules for `selectif` / `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; TODO: requires icmp/fcmp first.
-
-;;; Rules for `trueif` / `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; TODO: requires icmp/fcmp first.
-
 ;;; Rules for `brz`/`brnz`/`brif`/`brff`/`bricmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; TODO: requires icmp/fcmp first.
diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs
index eaa01d89dd..773991b693 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower.rs
@@ -1178,15 +1178,12 @@ pub(crate) enum IcmpOutput {
     /// Lowers the comparison into a cond code, discarding the results. The cond code emitted can
     /// be checked in the resulting [IcmpResult].
     CondCode,
-    /// Materializes the results into a register. This may overwrite any flags previously set.
-    Register(Writable<Reg>),
 }
 
 impl IcmpOutput {
     pub fn reg(&self) -> Option<Writable<Reg>> {
         match self {
             IcmpOutput::CondCode => None,
-            IcmpOutput::Register(reg) => Some(*reg),
         }
     }
 }
@@ -1197,15 +1194,12 @@ pub(crate) enum IcmpResult {
     /// The result was output into the given [Cond]. Callers may perform operations using this [Cond]
     /// and its inverse, other [Cond]'s are not guaranteed to be correct.
     CondCode(Cond),
-    /// The result was materialized into the output register.
-    Register,
 }
 
 impl IcmpResult {
     pub fn unwrap_cond(&self) -> Cond {
         match self {
             IcmpResult::CondCode(c) => *c,
-            _ => panic!("Unwrapped cond, but IcmpResult was {:?}", self),
         }
     }
 }
@@ -1364,8 +1358,6 @@ pub(crate) fn lower_icmp(
     }
 
     Ok(match output {
-        // We currently never emit a different register than what was asked for
-        IcmpOutput::Register(_) => IcmpResult::Register,
         IcmpOutput::CondCode => IcmpResult::CondCode(out_condcode),
     })
 }
diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
index a507cd331c..580aea9ec2 100644
--- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
@@ -24,7 +24,6 @@ pub(crate) fn lower_insn_to_regs(
     isa_flags: &aarch64_settings::Flags,
 ) -> CodegenResult<()> {
     let op = ctx.data(insn).opcode();
-    let inputs = insn_inputs(ctx, insn);
     let outputs = insn_outputs(ctx, insn);
     let ty = if outputs.len() > 0 {
         Some(ctx.output_ty(insn, 0))
@@ -140,131 +139,15 @@ pub(crate) fn lower_insn_to_regs(
             // Nothing.
         }
 
-        Opcode::Select => {
-            let flag_input = inputs[0];
-            let cond = if let Some(icmp_insn) =
-                maybe_input_insn_via_conv(ctx, flag_input, Opcode::Icmp, Opcode::Bint)
-            {
-                let condcode = ctx.data(icmp_insn).cond_code().unwrap();
-                lower_icmp(ctx, icmp_insn, condcode, IcmpOutput::CondCode)?.unwrap_cond()
-            } else if let Some(fcmp_insn) =
-                maybe_input_insn_via_conv(ctx, flag_input, Opcode::Fcmp, Opcode::Bint)
-            {
-                let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap();
-                let cond = lower_fp_condcode(condcode);
-                lower_fcmp_or_ffcmp_to_flags(ctx, fcmp_insn);
-                cond
-            } else {
-                let (size, narrow_mode) = if ty_bits(ctx.input_ty(insn, 0)) > 32 {
-                    (OperandSize::Size64, NarrowValueMode::ZeroExtend64)
-                } else {
-                    (OperandSize::Size32, NarrowValueMode::ZeroExtend32)
-                };
+        Opcode::Select => implemented_in_isle(ctx),
 
-                let rcond = put_input_in_reg(ctx, inputs[0], narrow_mode);
-                // cmp rcond, #0
-                ctx.emit(Inst::AluRRR {
-                    alu_op: ALUOp::SubS,
-                    size,
-                    rd: writable_zero_reg(),
-                    rn: rcond,
-                    rm: zero_reg(),
-                });
-                Cond::Ne
-            };
-
-            // csel.cond rd, rn, rm
-            let ty = ctx.output_ty(insn, 0);
-            let bits = ty_bits(ty);
-            let is_float = ty_has_float_or_vec_representation(ty);
-
-            let dst = get_output_reg(ctx, outputs[0]);
-            let lhs = put_input_in_regs(ctx, inputs[1]);
-            let rhs = put_input_in_regs(ctx, inputs[2]);
-
-            let rd = dst.regs()[0];
-            let rn = lhs.regs()[0];
-            let rm = rhs.regs()[0];
-
-            match (is_float, bits) {
-                (true, 32) => ctx.emit(Inst::FpuCSel32 { cond, rd, rn, rm }),
-                (true, 64) => ctx.emit(Inst::FpuCSel64 { cond, rd, rn, rm }),
-                (true, 128) => ctx.emit(Inst::VecCSel { cond, rd, rn, rm }),
-                (false, 128) => {
-                    ctx.emit(Inst::CSel {
-                        cond,
-                        rd: dst.regs()[0],
-                        rn: lhs.regs()[0],
-                        rm: rhs.regs()[0],
-                    });
-                    ctx.emit(Inst::CSel {
-                        cond,
-                        rd: dst.regs()[1],
-                        rn: lhs.regs()[1],
-                        rm: rhs.regs()[1],
-                    });
-                }
-                (false, bits) if bits <= 64 => ctx.emit(Inst::CSel { cond, rd, rn, rm }),
-                _ => {
-                    return Err(CodegenError::Unsupported(format!(
-                        "Select: Unsupported type: {:?}",
-                        ty
-                    )));
-                }
-            }
-        }
-
-        Opcode::Selectif | Opcode::SelectifSpectreGuard => {
-            let condcode = ctx.data(insn).cond_code().unwrap();
-            // Verification ensures that the input is always a
-            // single-def ifcmp.
-            let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
-            let cond = lower_icmp(ctx, ifcmp_insn, condcode, IcmpOutput::CondCode)?.unwrap_cond();
-
-            // csel.COND rd, rn, rm
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-            let ty = ctx.output_ty(insn, 0);
-            let bits = ty_bits(ty);
-            let is_float = ty_has_float_or_vec_representation(ty);
-            if is_float && bits == 32 {
-                ctx.emit(Inst::FpuCSel32 { cond, rd, rn, rm });
-            } else if is_float && bits == 64 {
-                ctx.emit(Inst::FpuCSel64 { cond, rd, rn, rm });
-            } else if !is_float && bits <= 64 {
-                ctx.emit(Inst::CSel { cond, rd, rn, rm });
-            } else {
-                return Err(CodegenError::Unsupported(format!(
-                    "{}: Unsupported type: {:?}",
-                    op, ty
-                )));
-            }
-
-            if op == Opcode::SelectifSpectreGuard {
-                ctx.emit(Inst::Csdb);
-            }
-        }
+        Opcode::Selectif | Opcode::SelectifSpectreGuard => implemented_in_isle(ctx),
 
         Opcode::Bitselect | Opcode::Vselect => implemented_in_isle(ctx),
 
-        Opcode::Trueif => {
-            let condcode = ctx.data(insn).cond_code().unwrap();
-            // Verification ensures that the input is always a
-            // single-def ifcmp.
-            let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            lower_icmp(ctx, ifcmp_insn, condcode, IcmpOutput::Register(rd))?;
-        }
+        Opcode::Trueif => implemented_in_isle(ctx),
 
-        Opcode::Trueff => {
-            let condcode = ctx.data(insn).fp_cond_code().unwrap();
-            let cond = lower_fp_condcode(condcode);
-            let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap();
-            lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            materialize_bool_result(ctx, insn, rd, cond);
-        }
+        Opcode::Trueff => implemented_in_isle(ctx),
 
         Opcode::IsNull | Opcode::IsInvalid => implemented_in_isle(ctx),
 
@@ -296,39 +179,7 @@ pub(crate) fn lower_insn_to_regs(
 
         Opcode::Trap | Opcode::ResumableTrap => implemented_in_isle(ctx),
 
-        Opcode::Trapif | Opcode::Trapff => {
-            let trap_code = ctx.data(insn).trap_code().unwrap();
-
-            let cond = if maybe_input_insn(ctx, inputs[0], Opcode::IaddIfcout).is_some() {
-                let condcode = ctx.data(insn).cond_code().unwrap();
-                let cond = lower_condcode(condcode);
-                // The flags must not have been clobbered by any other
-                // instruction between the iadd_ifcout and this instruction, as
-                // verified by the CLIF validator; so we can simply use the
-                // flags here.
-                cond
-            } else if op == Opcode::Trapif {
-                let condcode = ctx.data(insn).cond_code().unwrap();
-
-                // Verification ensures that the input is always a single-def ifcmp.
-                let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
-                lower_icmp(ctx, ifcmp_insn, condcode, IcmpOutput::CondCode)?.unwrap_cond()
-            } else {
-                let condcode = ctx.data(insn).fp_cond_code().unwrap();
-                let cond = lower_fp_condcode(condcode);
-
-                // Verification ensures that the input is always a
-                // single-def ffcmp.
-                let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap();
-                lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn);
-                cond
-            };
-
-            ctx.emit(Inst::TrapIf {
-                trap_code,
-                kind: CondBrKind::Cond(cond),
-            });
-        }
+        Opcode::Trapif | Opcode::Trapff => implemented_in_isle(ctx),
 
         Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
             panic!("trapz / trapnz / resumable_trapnz should have been removed by legalization!");
diff --git a/cranelift/filetests/filetests/isa/aarch64/condops.clif b/cranelift/filetests/filetests/isa/aarch64/condops.clif
index ac1e6e61d6..8e422a0da8 100644
--- a/cranelift/filetests/filetests/isa/aarch64/condops.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/condops.clif
@@ -2,6 +2,48 @@ test compile precise-output
 set unwind_info=false
 target aarch64
 
+function %f(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   ret
+
 function %f(i8, i64, i64) -> i64 {
 block0(v0: i8, v1: i64, v2: i64):
   v3 = iconst.i8 42
@@ -11,11 +53,690 @@ block0(v0: i8, v1: i64, v2: i64):
 }
 
 ; block0:
-;   uxtb w6, w0
-;   subs wzr, w6, #42
+;   uxtb w5, w0
+;   subs wzr, w5, #42
 ;   csel x0, x1, x2, eq
 ;   ret
 
+function %f(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w8, w0
+;   subs wzr, w8, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   ret
+
+function %f(i16, i8, i8) -> i8 {
+block0(v0: i16, v1: i8, v2: i8):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i16, i16, i16) -> i16 {
+block0(v0: i16, v1: i16, v2: i16):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i16, i32, i32) -> i32 {
+block0(v0: i16, v1: i32, v2: i32):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i16, i64, i64) -> i64 {
+block0(v0: i16, v1: i64, v2: i64):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i16, i128, i128) -> i128 {
+block0(v0: i16, v1: i128, v2: i128):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w8, w0
+;   subs wzr, w8, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   ret
+
+function %f(i32, i8, i8) -> i8 {
+block0(v0: i32, v1: i8, v2: i8):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i32, i16, i16) -> i16 {
+block0(v0: i32, v1: i16, v2: i16):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i32, i64, i64) -> i64 {
+block0(v0: i32, v1: i64, v2: i64):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i32, i128, i128) -> i128 {
+block0(v0: i32, v1: i128, v2: i128):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   ret
+
+function %f(i64, i8, i8) -> i8 {
+block0(v0: i64, v1: i8, v2: i8):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i64, i16, i16) -> i16 {
+block0(v0: i64, v1: i16, v2: i16):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i64, i32, i32) -> i32 {
+block0(v0: i64, v1: i32, v2: i32):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   ret
+
+function %f(i64, i128, i128) -> i128 {
+block0(v0: i64, v1: i128, v2: i128):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   ret
+
+function %f(i128, i8, i8) -> i8 {
+block0(v0: i128, v1: i8, v2: i8):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   ret
+
+function %f(i128, i16, i16) -> i16 {
+block0(v0: i128, v1: i16, v2: i16):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   ret
+
+function %f(i128, i32, i32) -> i32 {
+block0(v0: i128, v1: i32, v2: i32):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   ret
+
+function %f(i128, i64, i64) -> i64 {
+block0(v0: i128, v1: i64, v2: i64):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   ret
+
+function %f(i128, i128, i128) -> i128 {
+block0(v0: i128, v1: i128, v2: i128):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x9, #42
+;   movz x11, #0
+;   subs xzr, x0, x9
+;   ccmp x1, x11, #nzcv, eq
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   ret
+
+function %f(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxtb w8, w0
+;   subs wzr, w8, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   csdb
+;   ret
+
+function %f(i16, i8, i8) -> i8 {
+block0(v0: i16, v1: i8, v2: i8):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i16, i16, i16) -> i16 {
+block0(v0: i16, v1: i16, v2: i16):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i16, i32, i32) -> i32 {
+block0(v0: i16, v1: i32, v2: i32):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i16, i64, i64) -> i64 {
+block0(v0: i16, v1: i64, v2: i64):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w5, w0
+;   subs wzr, w5, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i16, i128, i128) -> i128 {
+block0(v0: i16, v1: i128, v2: i128):
+  v3 = iconst.i16 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   uxth w8, w0
+;   subs wzr, w8, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   csdb
+;   ret
+
+function %f(i32, i8, i8) -> i8 {
+block0(v0: i32, v1: i8, v2: i8):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i32, i16, i16) -> i16 {
+block0(v0: i32, v1: i16, v2: i16):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i32, i64, i64) -> i64 {
+block0(v0: i32, v1: i64, v2: i64):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i32, i128, i128) -> i128 {
+block0(v0: i32, v1: i128, v2: i128):
+  v3 = iconst.i32 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs wzr, w0, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   csdb
+;   ret
+
+function %f(i64, i8, i8) -> i8 {
+block0(v0: i64, v1: i8, v2: i8):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i64, i16, i16) -> i16 {
+block0(v0: i64, v1: i16, v2: i16):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i64, i32, i32) -> i32 {
+block0(v0: i64, v1: i32, v2: i32):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x1, x2, eq
+;   csdb
+;   ret
+
+function %f(i64, i128, i128) -> i128 {
+block0(v0: i64, v1: i128, v2: i128):
+  v3 = iconst.i64 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   subs xzr, x0, #42
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   csdb
+;   ret
+
+function %f(i128, i8, i8) -> i8 {
+block0(v0: i128, v1: i8, v2: i8):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   csdb
+;   ret
+
+function %f(i128, i16, i16) -> i16 {
+block0(v0: i128, v1: i16, v2: i16):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   csdb
+;   ret
+
+function %f(i128, i32, i32) -> i32 {
+block0(v0: i128, v1: i32, v2: i32):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   csdb
+;   ret
+
+function %f(i128, i64, i64) -> i64 {
+block0(v0: i128, v1: i64, v2: i64):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x6, #42
+;   movz x8, #0
+;   subs xzr, x0, x6
+;   ccmp x1, x8, #nzcv, eq
+;   csel x0, x2, x3, eq
+;   csdb
+;   ret
+
+function %f(i128, i128, i128) -> i128 {
+block0(v0: i128, v1: i128, v2: i128):
+  v3 = iconst.i128 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 eq v4, v1, v2
+  return v5
+}
+
+; block0:
+;   movz x9, #42
+;   movz x11, #0
+;   subs xzr, x0, x9
+;   ccmp x1, x11, #nzcv, eq
+;   csel x0, x2, x4, eq
+;   csel x1, x3, x5, eq
+;   csdb
+;   ret
+
 function %g(i8) -> b1 {
 block0(v0: i8):
   v3 = iconst.i8 42
@@ -25,8 +746,8 @@ block0(v0: i8):
 }
 
 ; block0:
-;   uxtb w4, w0
-;   subs wzr, w4, #42
+;   uxtb w3, w0
+;   subs wzr, w3, #42
 ;   cset x0, eq
 ;   ret
 
@@ -49,8 +770,8 @@ block0(v0: b1, v1: i8, v2: i8):
 }
 
 ; block0:
-;   and w6, w0, #1
-;   subs wzr, w6, wzr
+;   and w5, w0, #1
+;   subs wzr, w5, wzr
 ;   csel x0, x1, x2, ne
 ;   ret
 
@@ -74,8 +795,8 @@ block0(v0: b1, v1: i128, v2: i128):
 }
 
 ; block0:
-;   and w10, w0, #1
-;   subs wzr, w10, wzr
+;   and w8, w0, #1
+;   subs wzr, w8, wzr
 ;   csel x0, x2, x4, ne
 ;   csel x1, x3, x5, ne
 ;   ret
diff --git a/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif b/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
index fdcd552066..0c81d67829 100644
--- a/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
@@ -21,8 +21,8 @@ block0(v0: i64, v1: i32):
 ;   b.ls label1 ; b label2
 ; block1:
 ;   add x11, x0, x1, UXTW
-;   subs xzr, x9, x10
 ;   movz x12, #0
+;   subs xzr, x9, x10
 ;   csel x0, x12, x11, hi
 ;   csdb
 ;   ret
@@ -44,9 +44,9 @@ block0(v0: i64, v1: i32):
 ;   b.ls label1 ; b label2
 ; block1:
 ;   add x9, x0, x1, UXTW
+;   movz x8, #0
 ;   subs xzr, x7, #65536
-;   movz x10, #0
-;   csel x0, x10, x9, hi
+;   csel x0, x8, x9, hi
 ;   csdb
 ;   ret
 ; block2:
diff --git a/cranelift/filetests/filetests/isa/aarch64/traps.clif b/cranelift/filetests/filetests/isa/aarch64/traps.clif
index abd1985864..41864bdd02 100644
--- a/cranelift/filetests/filetests/isa/aarch64/traps.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/traps.clif
@@ -17,6 +17,7 @@ block0(v0: i64, v1: i64):
 }
 
 ; block0:
+;   adds x3, x0, x1
 ;   b.hs 8 ; udf
 ;   ret
 
diff --git a/cranelift/filetests/filetests/runtests/select.clif b/cranelift/filetests/filetests/runtests/select.clif
index 0aa575af1f..e93f082f65 100644
--- a/cranelift/filetests/filetests/runtests/select.clif
+++ b/cranelift/filetests/filetests/runtests/select.clif
@@ -1,5 +1,6 @@
 test interpret
 test run
+target aarch64
 target s390x
 target x86_64
 
diff --git a/cranelift/filetests/filetests/runtests/selectif-spectre-guard.clif b/cranelift/filetests/filetests/runtests/selectif-spectre-guard.clif
new file mode 100644
index 0000000000..8edc967dc7
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/selectif-spectre-guard.clif
@@ -0,0 +1,316 @@
+;; the interpreter does not support `selectif_spectre_guard`.
+test run
+set enable_llvm_abi_extensions=true
+target aarch64
+target s390x
+target x86_64
+
+function %selectif_spectre_guard_i8_eq(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i8_eq(0, 32, 255) == 255
+; run: %selectif_spectre_guard_i8_eq(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i8_eq(42, 32, 255) == 32
+
+function %selectif_spectre_guard_i16_eq(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i16_eq(0, 32, 65535) == 65535
+; run: %selectif_spectre_guard_i16_eq(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i16_eq(42, 32, 65535) == 32
+
+function %selectif_spectre_guard_i32_eq(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i32_eq(0, 32, 4294967295) == 4294967295
+; run: %selectif_spectre_guard_i32_eq(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i32_eq(42, 32, 4294967295) == 32
+
+function %selectif_spectre_guard_i64_eq(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i64_eq(0, 32, 18446744073709551615) == 18446744073709551615
+; run: %selectif_spectre_guard_i64_eq(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i64_eq(42, 32, 18446744073709551615) == 32
+
+function %selectif_spectre_guard_i128_eq(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i128_eq(0, 32, 19000000000000000000) == 19000000000000000000
+; run: %selectif_spectre_guard_i128_eq(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i128_eq(42, 32, 19000000000000000000) == 32
+
+function %selectif_spectre_guard_i8_ult(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i8_ult(0, 32, 255) == 32
+; run: %selectif_spectre_guard_i8_ult(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i8_ult(42, 32, 255) == 255
+
+function %selectif_spectre_guard_i16_ult(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i16_ult(0, 32, 65535) == 32
+; run: %selectif_spectre_guard_i16_ult(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i16_ult(42, 32, 65535) == 65535
+
+function %selectif_spectre_guard_i32_ult(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i32_ult(0, 32, 4294967295) == 32
+; run: %selectif_spectre_guard_i32_ult(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i32_ult(42, 32, 4294967295) == 4294967295
+
+function %selectif_spectre_guard_i64_ult(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i64_ult(0, 32, 18446744073709551615) == 32
+; run: %selectif_spectre_guard_i64_ult(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i64_ult(42, 32, 18446744073709551615) == 18446744073709551615
+
+function %selectif_spectre_guard_i128_ult(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i128_ult(0, 32, 19000000000000000000) == 32
+; run: %selectif_spectre_guard_i128_ult(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i128_ult(42, 32, 19000000000000000000) == 19000000000000000000
+
+function %selectif_spectre_guard_i8_ule(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i8_ule(0, 32, 255) == 32
+; run: %selectif_spectre_guard_i8_ule(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i8_ule(42, 32, 255) == 32
+
+function %selectif_spectre_guard_i16_ule(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i16_ule(0, 32, 65535) == 32
+; run: %selectif_spectre_guard_i16_ule(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i16_ule(42, 32, 65535) == 32
+
+function %selectif_spectre_guard_i32_ule(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i32_ule(0, 32, 4294967295) == 32
+; run: %selectif_spectre_guard_i32_ule(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i32_ule(42, 32, 4294967295) == 32
+
+function %selectif_spectre_guard_i64_ule(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i64_ule(0, 32, 18446744073709551615) == 32
+; run: %selectif_spectre_guard_i64_ule(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i64_ule(42, 32, 18446744073709551615) == 32
+
+function %selectif_spectre_guard_i128_ule(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i128_ule(0, 32, 19000000000000000000) == 32
+; run: %selectif_spectre_guard_i128_ule(255, 32, -1) == -1
+; run: %selectif_spectre_guard_i128_ule(42, 32, 19000000000000000000) == 32
+
+function %selectif_spectre_guard_i8_slt(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i8_slt(0, 32, 255) == 32
+; run: %selectif_spectre_guard_i8_slt(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i8_slt(42, 32, 255) == 255
+
+function %selectif_spectre_guard_i16_slt(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i16_slt(0, 32, 65535) == 32
+; run: %selectif_spectre_guard_i16_slt(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i16_slt(42, 32, 65535) == 65535
+
+function %selectif_spectre_guard_i32_slt(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i32_slt(0, 32, 4294967295) == 32
+; run: %selectif_spectre_guard_i32_slt(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i32_slt(42, 32, 4294967295) == 4294967295
+
+function %selectif_spectre_guard_i64_slt(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i64_slt(0, 32, 18446744073709551615) == 32
+; run: %selectif_spectre_guard_i64_slt(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i64_slt(42, 32, 18446744073709551615) == 18446744073709551615
+
+function %selectif_spectre_guard_i128_slt(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i128_slt(0, 32, 19000000000000000000) == 32
+; run: %selectif_spectre_guard_i128_slt(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i128_slt(42, 32, 19000000000000000000) == 19000000000000000000
+
+function %selectif_spectre_guard_i8_sle(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i8 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i8_sle(0, 32, 127) == 32
+; run: %selectif_spectre_guard_i8_sle(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i8_sle(127, 32, -1) == -1
+; run: %selectif_spectre_guard_i8_sle(127, 32, 127) == 127
+; run: %selectif_spectre_guard_i8_sle(42, 32, 127) == 32
+
+function %selectif_spectre_guard_i16_sle(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i16 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i16_sle(0, 32, 65535) == 32
+; run: %selectif_spectre_guard_i16_sle(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i16_sle(127, 32, -1) == -1
+; run: %selectif_spectre_guard_i16_sle(127, 32, 65535) == 65535
+; run: %selectif_spectre_guard_i16_sle(42, 32, 65535) == 32
+
+function %selectif_spectre_guard_i32_sle(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i32 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i32_sle(0, 32, 4294967295) == 32
+; run: %selectif_spectre_guard_i32_sle(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i32_sle(127, 32, -1) == -1
+; run: %selectif_spectre_guard_i32_sle(127, 32, 4294967295) == 4294967295
+; run: %selectif_spectre_guard_i32_sle(42, 32, 4294967295) == 32
+
+function %selectif_spectre_guard_i64_sle(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i64 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i64_sle(0, 32, 18446744073709551615) == 32
+; run: %selectif_spectre_guard_i64_sle(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i64_sle(127, 32, -1) == -1
+; run: %selectif_spectre_guard_i64_sle(127, 32, 18446744073709551615) == 18446744073709551615
+; run: %selectif_spectre_guard_i64_sle(42, 32, 18446744073709551615) == 32
+
+function %selectif_spectre_guard_i128_sle(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif_spectre_guard.i128 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_spectre_guard_i128_sle(0, 32, 19000000000000000000) == 32
+; run: %selectif_spectre_guard_i128_sle(-128, 32, -1) == 32
+; run: %selectif_spectre_guard_i128_sle(127, 32, -1) == -1
+; run: %selectif_spectre_guard_i128_sle(127, 32, 19000000000000000000) == 19000000000000000000
+; run: %selectif_spectre_guard_i128_sle(42, 32, 19000000000000000000) == 32
diff --git a/cranelift/filetests/filetests/runtests/selectif.clif b/cranelift/filetests/filetests/runtests/selectif.clif
new file mode 100644
index 0000000000..3437b971a5
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/selectif.clif
@@ -0,0 +1,316 @@
+test interpret
+test run
+set enable_llvm_abi_extensions=true
+target aarch64
+target x86_64
+; `selectif` panics on s390x.
+
+function %selectif_i8_eq(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i8_eq(0, 32, 255) == 255
+; run: %selectif_i8_eq(255, 32, -1) == -1
+; run: %selectif_i8_eq(42, 32, 255) == 32
+
+function %selectif_i16_eq(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i16_eq(0, 32, 65535) == 65535
+; run: %selectif_i16_eq(255, 32, -1) == -1
+; run: %selectif_i16_eq(42, 32, 65535) == 32
+
+function %selectif_i32_eq(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i32_eq(0, 32, 4294967295) == 4294967295
+; run: %selectif_i32_eq(255, 32, -1) == -1
+; run: %selectif_i32_eq(42, 32, 4294967295) == 32
+
+function %selectif_i64_eq(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i64_eq(0, 32, 18446744073709551615) == 18446744073709551615
+; run: %selectif_i64_eq(255, 32, -1) == -1
+; run: %selectif_i64_eq(42, 32, 18446744073709551615) == 32
+
+function %selectif_i128_eq(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 eq v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i128_eq(0, 32, 19000000000000000000) == 19000000000000000000
+; run: %selectif_i128_eq(255, 32, -1) == -1
+; run: %selectif_i128_eq(42, 32, 19000000000000000000) == 32
+
+function %selectif_i8_ult(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i8_ult(0, 32, 255) == 32
+; run: %selectif_i8_ult(255, 32, -1) == -1
+; run: %selectif_i8_ult(42, 32, 255) == 255
+
+function %selectif_i16_ult(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i16_ult(0, 32, 65535) == 32
+; run: %selectif_i16_ult(255, 32, -1) == -1
+; run: %selectif_i16_ult(42, 32, 65535) == 65535
+
+function %selectif_i32_ult(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i32_ult(0, 32, 4294967295) == 32
+; run: %selectif_i32_ult(255, 32, -1) == -1
+; run: %selectif_i32_ult(42, 32, 4294967295) == 4294967295
+
+function %selectif_i64_ult(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i64_ult(0, 32, 18446744073709551615) == 32
+; run: %selectif_i64_ult(255, 32, -1) == -1
+; run: %selectif_i64_ult(42, 32, 18446744073709551615) == 18446744073709551615
+
+function %selectif_i128_ult(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 ult v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i128_ult(0, 32, 19000000000000000000) == 32
+; run: %selectif_i128_ult(255, 32, -1) == -1
+; run: %selectif_i128_ult(42, 32, 19000000000000000000) == 19000000000000000000
+
+function %selectif_i8_ule(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i8_ule(0, 32, 255) == 32
+; run: %selectif_i8_ule(255, 32, -1) == -1
+; run: %selectif_i8_ule(42, 32, 255) == 32
+
+function %selectif_i16_ule(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i16_ule(0, 32, 65535) == 32
+; run: %selectif_i16_ule(255, 32, -1) == -1
+; run: %selectif_i16_ule(42, 32, 65535) == 32
+
+function %selectif_i32_ule(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i32_ule(0, 32, 4294967295) == 32
+; run: %selectif_i32_ule(255, 32, -1) == -1
+; run: %selectif_i32_ule(42, 32, 4294967295) == 32
+
+function %selectif_i64_ule(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i64_ule(0, 32, 18446744073709551615) == 32
+; run: %selectif_i64_ule(255, 32, -1) == -1
+; run: %selectif_i64_ule(42, 32, 18446744073709551615) == 32
+
+function %selectif_i128_ule(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 ule v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i128_ule(0, 32, 19000000000000000000) == 32
+; run: %selectif_i128_ule(255, 32, -1) == -1
+; run: %selectif_i128_ule(42, 32, 19000000000000000000) == 32
+
+function %selectif_i8_slt(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i8_slt(0, 32, 255) == 32
+; run: %selectif_i8_slt(-128, 32, -1) == 32
+; run: %selectif_i8_slt(42, 32, 255) == 255
+
+function %selectif_i16_slt(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i16_slt(0, 32, 65535) == 32
+; run: %selectif_i16_slt(-128, 32, -1) == 32
+; run: %selectif_i16_slt(42, 32, 65535) == 65535
+
+function %selectif_i32_slt(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i32_slt(0, 32, 4294967295) == 32
+; run: %selectif_i32_slt(-128, 32, -1) == 32
+; run: %selectif_i32_slt(42, 32, 4294967295) == 4294967295
+
+function %selectif_i64_slt(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i64_slt(0, 32, 18446744073709551615) == 32
+; run: %selectif_i64_slt(-128, 32, -1) == 32
+; run: %selectif_i64_slt(42, 32, 18446744073709551615) == 18446744073709551615
+
+function %selectif_i128_slt(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 slt v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i128_slt(0, 32, 19000000000000000000) == 32
+; run: %selectif_i128_slt(-128, 32, -1) == 32
+; run: %selectif_i128_slt(42, 32, 19000000000000000000) == 19000000000000000000
+
+function %selectif_i8_sle(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i8 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i8_sle(0, 32, 127) == 32
+; run: %selectif_i8_sle(-128, 32, -1) == 32
+; run: %selectif_i8_sle(127, 32, -1) == -1
+; run: %selectif_i8_sle(127, 32, 127) == 127
+; run: %selectif_i8_sle(42, 32, 127) == 32
+
+function %selectif_i16_sle(i8, i16, i16) -> i16 {
+block0(v0: i8, v1: i16, v2: i16):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i16 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i16_sle(0, 32, 65535) == 32
+; run: %selectif_i16_sle(-128, 32, -1) == 32
+; run: %selectif_i16_sle(127, 32, -1) == -1
+; run: %selectif_i16_sle(127, 32, 65535) == 65535
+; run: %selectif_i16_sle(42, 32, 65535) == 32
+
+function %selectif_i32_sle(i8, i32, i32) -> i32 {
+block0(v0: i8, v1: i32, v2: i32):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i32 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i32_sle(0, 32, 4294967295) == 32
+; run: %selectif_i32_sle(-128, 32, -1) == 32
+; run: %selectif_i32_sle(127, 32, -1) == -1
+; run: %selectif_i32_sle(127, 32, 4294967295) == 4294967295
+; run: %selectif_i32_sle(42, 32, 4294967295) == 32
+
+function %selectif_i64_sle(i8, i64, i64) -> i64 {
+block0(v0: i8, v1: i64, v2: i64):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i64 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i64_sle(0, 32, 18446744073709551615) == 32
+; run: %selectif_i64_sle(-128, 32, -1) == 32
+; run: %selectif_i64_sle(127, 32, -1) == -1
+; run: %selectif_i64_sle(127, 32, 18446744073709551615) == 18446744073709551615
+; run: %selectif_i64_sle(42, 32, 18446744073709551615) == 32
+
+function %selectif_i128_sle(i8, i128, i128) -> i128 {
+block0(v0: i8, v1: i128, v2: i128):
+  v3 = iconst.i8 42
+  v4 = ifcmp v0, v3
+  v5 = selectif.i128 sle v4, v1, v2
+  return v5
+}
+
+; run: %selectif_i128_sle(0, 32, 19000000000000000000) == 32
+; run: %selectif_i128_sle(-128, 32, -1) == 32
+; run: %selectif_i128_sle(127, 32, -1) == -1
+; run: %selectif_i128_sle(127, 32, 19000000000000000000) == 19000000000000000000
+; run: %selectif_i128_sle(42, 32, 19000000000000000000) == 32
diff --git a/cranelift/filetests/filetests/runtests/trueif-ff.clif b/cranelift/filetests/filetests/runtests/trueif-ff.clif
new file mode 100644
index 0000000000..ceef624b54
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/trueif-ff.clif
@@ -0,0 +1,110 @@
+test interpret
+test run
+target aarch64
+; `true{if,ff}` not implemented on x86_64, and panics on s390x.
+
+function %trueif_i8_eq(i8, i8) -> b1 {
+block0(v0: i8, v1: i8):
+  v2 = ifcmp v0, v1
+  v3 = trueif eq v2
+  return v3
+}
+
+; run: %trueif_i8_eq(42, 42) == true
+; run: %trueif_i8_eq(-1, 255) == true
+; run: %trueif_i8_eq(255, 0) == false
+; run: %trueif_i8_eq(32, 64) == false
+
+function %trueif_i16_eq(i16, i16) -> b1 {
+block0(v0: i16, v1: i16):
+  v2 = ifcmp v0, v1
+  v3 = trueif eq v2
+  return v3
+}
+
+; run: %trueif_i16_eq(42, 42) == true
+; run: %trueif_i16_eq(-1, 65535) == true
+; run: %trueif_i16_eq(65535, 0) == false
+; run: %trueif_i16_eq(32, 64) == false
+
+function %trueif_i32_eq(i32, i32) -> b1 {
+block0(v0: i32, v1: i32):
+  v2 = ifcmp v0, v1
+  v3 = trueif eq v2
+  return v3
+}
+
+; run: %trueif_i32_eq(42, 42) == true
+; run: %trueif_i32_eq(-1, 4294967295) == true
+; run: %trueif_i32_eq(4294967295, 0) == false
+; run: %trueif_i32_eq(32, 64) == false
+
+function %trueif_i64_eq(i64, i64) -> b1 {
+block0(v0: i64, v1: i64):
+  v2 = ifcmp v0, v1
+  v3 = trueif eq v2
+  return v3
+}
+
+; run: %trueif_i64_eq(42, 42) == true
+; run: %trueif_i64_eq(-1, 18446744073709551615) == true
+; run: %trueif_i64_eq(18446744073709551615, 0) == false
+; run: %trueif_i64_eq(32, 64) == false
+
+function %trueif_i128_eq(i128, i128) -> b1 {
+block0(v0: i128, v1: i128):
+  v2 = ifcmp v0, v1
+  v3 = trueif eq v2
+  return v3
+}
+
+; run: %trueif_i128_eq(42, 42) == true
+; run: %trueif_i128_eq(-1, 18446744073709551615) == false
+; run: %trueif_i128_eq(19000000000000000000, 0) == false
+; run: %trueif_i128_eq(32, 64) == false
+
+function %trueff_f32_eq(f32, f32) -> b1 {
+block0(v0: f32, v1: f32):
+  v2 = ffcmp v0, v1
+  v3 = trueff eq v2
+  return v3
+}
+
+; run: %trueff_f32_eq(0x42.0, 0x42.0) == true
+; run: %trueff_f32_eq(-0x1.0, -0x1.0) == true
+; run: %trueff_f32_eq(0x1.0, 0x0.0) == false
+
+function %trueff_f64_eq(f64, f64) -> b1 {
+block0(v0: f64, v1: f64):
+  v2 = ffcmp v0, v1
+  v3 = trueff eq v2
+  return v3
+}
+
+; run: %trueff_f64_eq(0x42.0, 0x42.0) == true
+; run: %trueff_f64_eq(-0x1.0, -0x1.0) == true
+; run: %trueff_f64_eq(0x1.0, 0x0.0) == false
+
+function %trueff_f32_ne(f32, f32) -> b1 {
+block0(v0: f32, v1: f32):
+  v2 = ffcmp v0, v1
+  v3 = trueff ne v2
+  return v3
+}
+
+; run: %trueff_f32_ne(0x42.0, 0x42.0) == false
+; run: %trueff_f32_ne(-0x1.0, -0x1.0) == false
+; run: %trueff_f32_ne(0x1.0, 0x0.0) == true
+; run: %trueff_f32_ne(NaN, NaN) == true
+
+function %trueff_f64_ne(f64, f64) -> b1 {
+block0(v0: f64, v1: f64):
+  v2 = ffcmp v0, v1
+  v3 = trueff ne v2
+  return v3
+}
+
+; run: %trueff_f64_ne(0x42.0, 0x42.0) == false
+; run: %trueff_f64_ne(-0x1.0, -0x1.0) == false
+; run: %trueff_f64_ne(0x1.0, 0x0.0) == true
+; run: %trueff_f64_ne(NaN, NaN) == true
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index fc23932536..66fe43017d 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -942,15 +942,17 @@ where
         Opcode::Nearest => assign(Value::nearest(arg(0)?)?),
         Opcode::IsNull => unimplemented!("IsNull"),
         Opcode::IsInvalid => unimplemented!("IsInvalid"),
+        // `ctrl_ty` is `INVALID` for `Trueif` and `Trueff`, but both should
+        // return a 1-bit boolean value.
         Opcode::Trueif => choose(
             state.has_iflag(inst.cond_code().unwrap()),
-            Value::bool(true, ctrl_ty)?,
-            Value::bool(false, ctrl_ty)?,
+            Value::bool(true, types::B1)?,
+            Value::bool(false, types::B1)?,
         ),
         Opcode::Trueff => choose(
             state.has_fflag(inst.fp_cond_code().unwrap()),
-            Value::bool(true, ctrl_ty)?,
-            Value::bool(false, ctrl_ty)?,
+            Value::bool(true, types::B1)?,
+            Value::bool(false, types::B1)?,
         ),
         Opcode::Bitcast
         | Opcode::RawBitcast