T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cranelift: Remove booleans (#5031)

Browse Source 
Remove the boolean types from cranelift, and the associated instructions breduce, bextend, bconst, and bint. Standardize on using 1/0 for the return value from instructions that produce scalar boolean results, and -1/0 for boolean vector elements.

Fixes #3205

Co-authored-by: Afonso Bordado <afonso360@users.noreply.github.com>
Co-authored-by: Ulrich Weigand <ulrich.weigand@de.ibm.com>
Co-authored-by: Chris Fallin <chris@cfallin.org>
This commit is contained in:
Trevor Elliott
2022-10-17 16:00:27 -07:00
committed by GitHub
parent 766ecb561e
commit 32a7593c94
242 changed files with 7695 additions and 10010 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
cranelift/codegen/src/isa/aarch64/inst.isle
View File

@@ -1156,12 +1156,6 @@
(rule (scalar_size $I64) (ScalarSize.Size64))
(rule (scalar_size $I128) (ScalarSize.Size128))
(rule (scalar_size $B8) (ScalarSize.Size8))
(rule (scalar_size $B16) (ScalarSize.Size16))
(rule (scalar_size $B32) (ScalarSize.Size32))
(rule (scalar_size $B64) (ScalarSize.Size64))
(rule (scalar_size $B128) (ScalarSize.Size128))
(rule (scalar_size $F32) (ScalarSize.Size32))
(rule (scalar_size $F64) (ScalarSize.Size64))
@@ -1947,19 +1941,13 @@
;; Helper for materializing a boolean value into a register from
;; flags.
(decl materialize_bool_result (u8 Cond) ConsumesFlags)
(rule (materialize_bool_result 1 cond)
(decl materialize_bool_result (Cond) ConsumesFlags)
(rule (materialize_bool_result cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.CSet dst cond)
dst)))
(rule -1 (materialize_bool_result _ty_bits cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.CSetm dst cond)
dst)))
(decl cmn_imm (OperandSize Reg Imm12) ProducesFlags)
(rule (cmn_imm size src1 src2)
(ProducesFlags.ProducesFlagsSideEffect
@@ -2224,6 +2212,18 @@
(MInst.CSel dst cond if_true if_false)
dst)))
;; Helper for constructing `cset` instructions.
(decl cset (Cond) ConsumesFlags)
(rule (cset cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg (MInst.CSet dst cond) dst)))
;; Helper for constructing `csetm` instructions.
(decl csetm (Cond) ConsumesFlags)
(rule (csetm cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg (MInst.CSetm dst cond) dst)))
;; Helper for generating a `CSNeg` instruction.
;;
;; Note that this doesn't actually emit anything, instead it produces a
@@ -2244,21 +2244,14 @@
(produces_flags_append inst_input (MInst.CCmp size rn rm nzcv cond)))
;; Helper for generating `MInst.CCmpImm` instructions.
(decl ccmp_imm (OperandSize u8 Reg UImm5 NZCV Cond) ConsumesFlags)
(rule 1 (ccmp_imm size 1 rn imm nzcv cond)
(decl ccmp_imm (OperandSize Reg UImm5 NZCV Cond) ConsumesFlags)
(rule 1 (ccmp_imm size rn imm nzcv cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CCmpImm size rn imm nzcv cond)
(MInst.CSet dst cond)
(value_reg dst))))
(rule (ccmp_imm size _ty_bits rn imm nzcv cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CCmpImm size rn imm nzcv cond)
(MInst.CSetm dst cond)
(value_reg dst))))
;; Helpers for generating `add` instructions.
(decl add (Type Reg Reg) Reg)
@@ -3381,11 +3374,11 @@
;; Integers <= 64-bits.
(rule -2 (lower_icmp_into_reg cond rn rm in_ty out_ty)
(if (ty_int_bool_ref_scalar_64 in_ty))
(if (ty_int_ref_scalar_64 in_ty))
(let ((cc Cond (cond_code cond)))
(with_flags
(lower_icmp cond rn rm in_ty)
(materialize_bool_result (ty_bits out_ty) cc))))
(materialize_bool_result cc))))
(rule 1 (lower_icmp cond rn rm (fits_in_16 ty))
(if (signed_cond_code cond))
@@ -3398,23 +3391,23 @@
(let ((rn Reg (put_in_reg_zext32 rn)))
(cmp_extend (operand_size ty) rn rm (lower_icmp_extend ty $false))))
(rule -3 (lower_icmp cond rn (imm12_from_value rm) ty)
(if (ty_int_bool_ref_scalar_64 ty))
(if (ty_int_ref_scalar_64 ty))
(cmp_imm (operand_size ty) rn rm))
(rule -4 (lower_icmp cond rn rm ty)
(if (ty_int_bool_ref_scalar_64 ty))
(if (ty_int_ref_scalar_64 ty))
(cmp (operand_size ty) rn rm))
;; 128-bit integers.
(rule (lower_icmp_into_reg cond @ (IntCC.Equal) rn rm $I128 out_ty)
(rule (lower_icmp_into_reg cond @ (IntCC.Equal) rn rm $I128 $I8)
(let ((cc Cond (cond_code cond)))
(with_flags
(lower_icmp cond rn rm $I128)
(materialize_bool_result (ty_bits out_ty) cc))))
(rule (lower_icmp_into_reg cond @ (IntCC.NotEqual) rn rm $I128 out_ty)
(materialize_bool_result cc))))
(rule (lower_icmp_into_reg cond @ (IntCC.NotEqual) rn rm $I128 $I8)
(let ((cc Cond (cond_code cond)))
(with_flags
(lower_icmp cond rn rm $I128)
(materialize_bool_result (ty_bits out_ty) cc))))
(materialize_bool_result cc))))
;; cmp lhs_lo, rhs_lo
;; ccmp lhs_hi, rhs_hi, #0, eq
@@ -3440,7 +3433,7 @@
;; cmp lhs_hi, rhs_hi
;; cset tmp2, cond
;; csel dst, tmp1, tmp2, eq
(rule -1 (lower_icmp_into_reg cond lhs rhs $I128 out_ty)
(rule -1 (lower_icmp_into_reg cond lhs rhs $I128 $I8)
(let ((unsigned_cond Cond (cond_code (intcc_unsigned cond)))
(cond Cond (cond_code cond))
(lhs ValueRegs (put_in_regs lhs))
@@ -3449,78 +3442,100 @@
(lhs_hi Reg (value_regs_get lhs 1))
(rhs_lo Reg (value_regs_get rhs 0))
(rhs_hi Reg (value_regs_get rhs 1))
(tmp1 ValueRegs
(with_flags (cmp (OperandSize.Size64) lhs_lo rhs_lo)
(materialize_bool_result
(ty_bits out_ty) unsigned_cond)))
(tmp1 Reg (value_regs_get tmp1 0))
(dst ValueRegs
(with_flags (cmp (OperandSize.Size64) lhs_hi rhs_hi)
(lower_icmp_i128_consumer cond (ty_bits out_ty)
tmp1 lhs_hi rhs_hi))))
dst))
(tmp1 Reg (with_flags_reg (cmp (OperandSize.Size64) lhs_lo rhs_lo)
(materialize_bool_result unsigned_cond))))
(with_flags (cmp (OperandSize.Size64) lhs_hi rhs_hi)
(lower_icmp_i128_consumer cond tmp1))))
(decl lower_icmp_i128_consumer (Cond u8 Reg Reg Reg) ConsumesFlags)
(rule (lower_icmp_i128_consumer cond 1 tmp1 lhs_hi rhs_hi)
(decl lower_icmp_i128_consumer (Cond Reg) ConsumesFlags)
(rule (lower_icmp_i128_consumer cond tmp1)
(let ((tmp2 WritableReg (temp_writable_reg $I64))
(dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSet tmp2 cond)
(MInst.CSel dst (Cond.Eq) tmp1 tmp2)
(value_reg dst))))
(rule (lower_icmp_i128_consumer cond 128 tmp1 lhs_hi rhs_hi)
(let ((tmp2 WritableReg (temp_writable_reg $I64))
(dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSetm tmp2 cond)
(MInst.CSel dst (Cond.Eq) tmp1 tmp2)
(value_regs dst dst))))
(rule -1 (lower_icmp_i128_consumer cond _out_ty_bits tmp1 lhs_hi rhs_hi)
(let ((tmp2 WritableReg (temp_writable_reg $I64))
(dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSetm tmp2 cond)
(MInst.CSel dst (Cond.Eq) tmp1 tmp2)
(value_reg dst))))
(decl lower_bmask (Type Type ValueRegs) ValueRegs)
;; For conversions that fit in a register, we can use csetm.
;;
;; cmp val, #0
;; csetm res, ne
(rule 0
(lower_bmask (fits_in_64 _) (fits_in_64 _) val)
(with_flags_reg
(cmp64_imm (value_regs_get val 0) (u8_into_imm12 0))
(csetm (Cond.Ne))))
;; For conversions from a 128-bit value into a 64-bit or smaller one, we or the
;; two registers of the 128-bit value together, and then recurse with the
;; combined value as a 64-bit test.
;;
;; orr val, lo, hi
;; cmp val, #0
;; csetm res, ne
(rule 1
(lower_bmask (fits_in_64 ty) $I128 val)
(let ((lo Reg (value_regs_get val 0))
(hi Reg (value_regs_get val 1))
(combined Reg (orr $I64 lo hi)))
(lower_bmask ty $I64 (value_reg combined))))
;; For converting from a smaller type into i128, duplicate the result of
;; converting to i64.
(rule 2
(lower_bmask $I128 (fits_in_64 ty) val)
(let ((res ValueRegs (lower_bmask $I64 ty val))
(res Reg (value_regs_get res 0)))
(value_regs res res)))
;; For conversions to a 128-bit mask, we duplicate the result of converting to
;; an I64.
(rule 3
(lower_bmask $I128 $I128 val)
(let ((res ValueRegs (lower_bmask $I64 $I128 val))
(res Reg (value_regs_get res 0)))
(value_regs res res)))
;; Exceptional `lower_icmp_into_flags` rules.
;; We need to guarantee that the flags for `cond` are correct, so we
;; compare `dst` with 1.
(rule (lower_icmp_into_flags cond @ (IntCC.SignedGreaterThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Sign) 1))) ;; mov tmp, #1
(cmp (OperandSize.Size64) dst tmp)))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedGreaterThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Zero) 1)))
(cmp (OperandSize.Size64) dst tmp)))
(rule (lower_icmp_into_flags cond @ (IntCC.SignedLessThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Sign) 1)))
(cmp (OperandSize.Size64) tmp dst)))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedLessThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Zero) 1)))
(cmp (OperandSize.Size64) tmp dst)))
;; For strict comparisons, we compare with 0.
(rule (lower_icmp_into_flags cond @ (IntCC.SignedGreaterThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) dst (zero_reg))))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedGreaterThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) dst (zero_reg))))
(rule (lower_icmp_into_flags cond @ (IntCC.SignedLessThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) (zero_reg) dst)))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedLessThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $I8))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) (zero_reg) dst)))
@@ -3548,7 +3563,7 @@
(MInst.CSel dst_hi cond rn_hi rm_hi)
(value_regs dst_lo dst_hi)))))
(rule 1 (lower_select flags cond ty rn rm)
(if (ty_int_bool_ref_scalar_64 ty))
(if (ty_int_ref_scalar_64 ty))
(with_flags flags (csel cond rn rm)))
;; Helper for emitting `MInst.Jump` instructions.

3
cranelift/codegen/src/isa/aarch64/inst/imms.rs
View File

@@ -221,9 +221,6 @@ impl UImm12Scaled {
/// Create a UImm12Scaled from a raw offset and the known scale type, if
/// possible.
pub fn maybe_from_i64(value: i64, scale_ty: Type) -> Option<UImm12Scaled> {
// Ensure the type is at least one byte.
let scale_ty = if scale_ty == B1 { B8 } else { scale_ty };
let scale = scale_ty.bytes();
assert!(scale.is_power_of_two());
let scale = scale as i64;

30
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -1,9 +1,7 @@
//! This module defines aarch64-specific machine instruction types.
use crate::binemit::{Addend, CodeOffset, Reloc};
use crate::ir::types::{
B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64,
};
use crate::ir::types::{F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64};
use crate::ir::{types, ExternalName, MemFlags, Opcode, Type};
use crate::isa::CallConv;
use crate::machinst::*;
@@ -440,22 +438,22 @@ impl Inst {
/// Generic constructor for a load (zero-extending where appropriate).
pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst {
match ty {
B1 | B8 | I8 => Inst::ULoad8 {
I8 => Inst::ULoad8 {
rd: into_reg,
mem,
flags,
},
B16 | I16 => Inst::ULoad16 {
I16 => Inst::ULoad16 {
rd: into_reg,
mem,
flags,
},
B32 | I32 | R32 => Inst::ULoad32 {
I32 | R32 => Inst::ULoad32 {
rd: into_reg,
mem,
flags,
},
B64 | I64 | R64 => Inst::ULoad64 {
I64 | R64 => Inst::ULoad64 {
rd: into_reg,
mem,
flags,
@@ -491,22 +489,22 @@ impl Inst {
/// Generic constructor for a store.
pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst {
match ty {
B1 | B8 | I8 => Inst::Store8 {
I8 => Inst::Store8 {
rd: from_reg,
mem,
flags,
},
B16 | I16 => Inst::Store16 {
I16 => Inst::Store16 {
rd: from_reg,
mem,
flags,
},
B32 | I32 | R32 => Inst::Store32 {
I32 | R32 => Inst::Store32 {
rd: from_reg,
mem,
flags,
},
B64 | I64 | R64 => Inst::Store64 {
I64 | R64 => Inst::Store64 {
rd: from_reg,
mem,
flags,
@@ -1209,9 +1207,7 @@ impl MachInst for Inst {
match ty {
F64 => Inst::load_fp_constant64(to_reg.unwrap(), value as u64, alloc_tmp),
F32 => Inst::load_fp_constant32(to_reg.unwrap(), value as u32, alloc_tmp),
B1 | B8 | B16 | B32 | B64 | I8 | I16 | I32 | I64 | R32 | R64 => {
Inst::load_constant(to_reg.unwrap(), value as u64)
}
I8 | I16 | I32 | I64 | R32 | R64 => Inst::load_constant(to_reg.unwrap(), value as u64),
I128 => Inst::load_constant128(to_regs, value),
_ => panic!("Cannot generate constant for type: {}", ty),
}
@@ -1236,17 +1232,11 @@ impl MachInst for Inst {
I16 => Ok((&[RegClass::Int], &[I16])),
I32 => Ok((&[RegClass::Int], &[I32])),
I64 => Ok((&[RegClass::Int], &[I64])),
B1 => Ok((&[RegClass::Int], &[B1])),
B8 => Ok((&[RegClass::Int], &[B8])),
B16 => Ok((&[RegClass::Int], &[B16])),
B32 => Ok((&[RegClass::Int], &[B32])),
B64 => Ok((&[RegClass::Int], &[B64])),
R32 => panic!("32-bit reftype pointer should never be seen on AArch64"),
R64 => Ok((&[RegClass::Int], &[R64])),
F32 => Ok((&[RegClass::Float], &[F32])),
F64 => Ok((&[RegClass::Float], &[F64])),
I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
B128 => Ok((&[RegClass::Int, RegClass::Int], &[B64, B64])),
_ if ty.is_vector() => {
assert!(ty.bits() <= 128);
Ok((&[RegClass::Float], &[I8X16]))

203
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -19,14 +19,6 @@
(rule (lower (has_type ty (iconst (u64_from_imm64 n))))
(imm ty (ImmExtend.Zero) n))
;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty (bconst $false)))
(imm ty (ImmExtend.Zero) 0))
(rule (lower (has_type ty (bconst $true)))
(imm ty (ImmExtend.Zero) 1))
;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty (null)))
@@ -142,10 +134,10 @@
(rule (lower (has_type $F64X2 (scalar_to_vector x)))
(fpu_extend x (ScalarSize.Size64)))
(rule -1 (lower (scalar_to_vector x @ (value_type (ty_int_bool_64 _))))
(rule -1 (lower (scalar_to_vector x @ (value_type $I64)))
(mov_to_fpu x (ScalarSize.Size64)))
(rule -2 (lower (scalar_to_vector x @ (value_type (int_bool_fits_in_32 _))))
(rule -2 (lower (scalar_to_vector x @ (value_type (int_fits_in_32 _))))
(mov_to_fpu (put_in_reg_zext32 x) (ScalarSize.Size32)))
;;;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -159,18 +151,17 @@
;; 0 when all input elements are true, i.e. non-zero, or a NaN otherwise
;; (either -1 or -2 when represented as an integer); NaNs are the only
;; floating-point numbers that compare unequal to themselves.
(rule (lower (has_type out_ty (vall_true x @ (value_type (multi_lane 64 2)))))
(rule (lower (vall_true x @ (value_type (multi_lane 64 2))))
(let ((x1 Reg (cmeq0 x (VectorSize.Size64x2)))
(x2 Reg (addp x1 x1 (VectorSize.Size64x2))))
(with_flags (fpu_cmp (ScalarSize.Size64) x2 x2)
(materialize_bool_result (ty_bits out_ty) (Cond.Eq)))))
(materialize_bool_result (Cond.Eq)))))
(rule (lower (has_type out_ty (vall_true x @ (value_type (multi_lane 32 2)))))
(rule (lower (vall_true x @ (value_type (multi_lane 32 2))))
(let ((x1 Reg (mov_from_vec x 0 (ScalarSize.Size64))))
(with_flags (cmp_rr_shift (OperandSize.Size64) (zero_reg) x1 32)
(ccmp_imm
(OperandSize.Size32)
(ty_bits out_ty)
x1
(u8_into_uimm5 0)
(nzcv $false $true $false $false)
@@ -183,18 +174,18 @@
;; mov xm, vn.d[0]
;; cmp xm, #0
;; cset xm, ne
(rule -1 (lower (has_type out_ty (vall_true x @ (value_type (lane_fits_in_32 ty)))))
(rule -1 (lower (vall_true x @ (value_type (lane_fits_in_32 ty))))
(if (not_vec32x2 ty))
(let ((x1 Reg (vec_lanes (VecLanesOp.Uminv) x (vector_size ty)))
(x2 Reg (mov_from_vec x1 0 (ScalarSize.Size64))))
(with_flags (cmp_imm (OperandSize.Size64) x2 (u8_into_imm12 0))
(materialize_bool_result (ty_bits out_ty) (Cond.Ne)))))
(materialize_bool_result (Cond.Ne)))))
;;;; Rules for `vany_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type out_ty (vany_true x @ (value_type in_ty))))
(rule (lower (vany_true x @ (value_type in_ty)))
(with_flags (vanytrue x in_ty)
(materialize_bool_result (ty_bits out_ty) (Cond.Ne))))
(materialize_bool_result (Cond.Ne))))
;;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1536,60 +1527,11 @@
(rule -1 (lower (has_type ty (cls x)))
(a64_cls ty x))
;;;; Rules for `bint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; Rules for `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Booleans are stored as all-zeroes (0) or all-ones (-1). We AND
;; out the LSB to give a 0 / 1-valued integer result.
(rule 1 (lower (has_type $I128 (bint x)))
(let ((val ValueRegs x)
(in_lo Reg (value_regs_get val 0))
(dst_lo Reg (and_imm $I32 in_lo (u64_into_imm_logic $I32 1)))
(dst_hi Reg (imm $I64 (ImmExtend.Zero) 0)))
(value_regs dst_lo dst_hi)))
(rule (lower (bint x))
(and_imm $I32 x (u64_into_imm_logic $I32 1)))
;;;; Rules for `bmask`/`bextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Bextend and Bmask both simply sign-extend. This works for:
;; - Bextend, because booleans are stored as 0 / -1, so we
;; sign-extend the -1 to a -1 in the wider width.
;; - Bmask, because the resulting integer mask value must be
;; all-ones (-1) if the argument is true.
;; Use a common helper to type cast bools to either bool or integer types.
(decl cast_bool (Type Type Value) InstOutput)
(rule (lower (has_type out_ty (bextend x @ (value_type in_ty))))
(cast_bool in_ty out_ty x))
;; Bmask tests the value against zero, and uses `csetm` to assert the result.
(rule (lower (has_type out_ty (bmask x @ (value_type in_ty))))
(cast_bool in_ty out_ty x))
;; If the target has the same or a smaller size than the source, it's a no-op.
(rule (cast_bool $B8 $I8 x) x)
(rule (cast_bool $B16 (fits_in_16 _out) x) x)
(rule (cast_bool $B32 (fits_in_32 _out) x) x)
(rule (cast_bool $B64 (fits_in_64 _out) x) x)
;; Casting between 128 bits is a noop
(rule -1 (cast_bool (ty_int_bool_128 _in) (ty_int_bool_128 _out) x)
x)
;; Converting from 128 bits to anything below we just ignore the top register
(rule -2 (cast_bool (ty_int_bool_128 _in) (fits_in_64 _out) x)
(value_regs_get x 0))
;; Extend to 64 bits first, then this will be all 0s or all 1s and we can
;; duplicate to both halves of 128 bits
(rule -3 (cast_bool in (ty_int_bool_128 _out) x)
(let ((tmp Reg (extend x $true (ty_bits in) 64)))
(value_regs tmp tmp)))
;; Values that fit in a single register are sign extended normally
(rule -4 (cast_bool (fits_in_64 in) (fits_in_64 out) x)
(extend x $true (ty_bits in) (ty_bits out)))
(lower_bmask out_ty in_ty x))
;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1648,7 +1590,7 @@
;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty (bitselect c x y)))
(if (ty_int_bool_ref_scalar_64 ty))
(if (ty_int_ref_scalar_64 ty))
(let ((tmp1 Reg (and_reg ty x c))
(tmp2 Reg (bic ty y c)))
(orr ty tmp1 tmp2)))
@@ -1661,22 +1603,15 @@
(rule (lower (has_type (ty_vec128 ty) (vselect c x y)))
(bsl ty c x y))
;;;; Rules for `ireduce` / `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; T -> I{64,32,16,8}: We can simply pass through the value: values
;; are always stored with high bits undefined, so we can just leave
;; them be.
(rule (lower (has_type ty (ireduce src)))
(if (ty_int_bool_ref_scalar_64 ty))
(if (ty_int_ref_scalar_64 ty))
(value_regs_get src 0))
;; Likewise for breduce.
(rule (lower (has_type ty (breduce src)))
(if (ty_int_bool_ref_scalar_64 ty))
(value_regs_get src 0))
;;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule 4 (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond_not_eq cond) x y)))
@@ -1706,9 +1641,7 @@
(rule 0 (lower (has_type out_ty
(fcmp cond x @ (value_type (ty_scalar_float in_ty)) y)))
(with_flags (fpu_cmp (scalar_size in_ty) x y)
(materialize_bool_result
(ty_bits out_ty)
(fp_cond_code cond))))
(materialize_bool_result (fp_cond_code cond))))
(rule -1 (lower (has_type out_ty (fcmp cond x @ (value_type in_ty) y)))
(if (ty_vector_float in_ty))
@@ -1740,8 +1673,8 @@
(vec_size VectorSize (vector_size ty)))
(value_reg (int_cmp_zero_swap cond rn vec_size))))
(rule -1 (lower (has_type out_ty (icmp cond x @ (value_type in_ty) y)))
(lower_icmp_into_reg cond x y in_ty out_ty))
(rule -1 (lower (icmp cond x @ (value_type in_ty) y))
(lower_icmp_into_reg cond x y in_ty $I8))
;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1783,10 +1716,10 @@
;;;; Rules for `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures the input is always a single-def ffcmp.
(rule (lower (has_type ty (trueff cc insn @ (ffcmp x @ (value_type in_ty) y))))
(rule (lower (trueff cc insn @ (ffcmp x @ (value_type in_ty) y)))
(with_flags_reg
(fpu_cmp (scalar_size in_ty) x y)
(materialize_bool_result (ty_bits ty) (fp_cond_code cc))))
(materialize_bool_result (fp_cond_code cc))))
;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1797,13 +1730,6 @@
(lower_icmp_into_flags cc x y in_ty)
cond ty rn rm)))
(rule (lower (has_type ty
(select _flags @ (bint (icmp cc x @ (value_type in_ty) y)) rn rm)))
(let ((cond Cond (cond_code cc)))
(lower_select
(lower_icmp_into_flags cc x y in_ty)
cond ty rn rm)))
(rule (lower (has_type ty
(select _flags @ (fcmp cc x @ (value_type in_ty) y) rn rm)))
(let ((cond Cond (fp_cond_code cc)))
@@ -1811,20 +1737,19 @@
(fpu_cmp (scalar_size in_ty) x y)
cond ty rn rm)))
(rule (lower (has_type ty
(select _flags @ (bint (fcmp cc x @ (value_type in_ty) y)) rn rm)))
(let ((cond Cond (fp_cond_code cc)))
(rule -1 (lower (has_type ty (select rcond @ (value_type $I8) rn rm)))
(let ((rcond Reg rcond))
(lower_select
(fpu_cmp (scalar_size in_ty) x y)
cond ty rn rm)))
(tst_imm $I32 rcond (u64_into_imm_logic $I32 255))
(Cond.Ne) ty rn rm)))
(rule -1 (lower (has_type ty (select rcond @ (value_type (fits_in_32 _)) rn rm)))
(rule -2 (lower (has_type ty (select rcond @ (value_type (fits_in_32 _)) rn rm)))
(let ((rcond Reg (put_in_reg_zext32 rcond)))
(lower_select
(cmp (OperandSize.Size32) rcond (zero_reg))
(Cond.Ne) ty rn rm)))
(rule -2 (lower (has_type ty (select rcond rn rm)))
(rule -3 (lower (has_type ty (select rcond rn rm)))
(let ((rcond Reg (put_in_reg_zext64 rcond)))
(lower_select
(cmp (OperandSize.Size64) rcond (zero_reg))
@@ -1865,18 +1790,12 @@
;;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule -1 (lower (has_type ty (splat x @ (value_type in_ty))))
(if (ty_int_bool_ref_scalar_64 in_ty))
(if (ty_int_ref_scalar_64 in_ty))
(vec_dup x (vector_size ty)))
(rule -2 (lower (has_type ty (splat x @ (value_type (ty_scalar_float _)))))
(vec_dup_from_fpu x (vector_size ty)))
(rule (lower (has_type ty (splat (bconst (u64_from_bool n)))))
(splat_const n (vector_size ty)))
(rule (lower (has_type ty (splat (breduce (bconst (u64_from_bool n))))))
(splat_const n (vector_size ty)))
(rule (lower (has_type ty (splat (f32const (u64_from_ieee32 n)))))
(splat_const n (vector_size ty)))
@@ -2089,17 +2008,15 @@
;;;; Rules for `IsNull` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type out_ty (is_null x @ (value_type ty))))
(rule (lower (is_null x @ (value_type ty)))
(with_flags (cmp_imm (operand_size ty) x (u8_into_imm12 0))
(materialize_bool_result
(ty_bits out_ty) (Cond.Eq))))
(materialize_bool_result (Cond.Eq))))
;;;; Rules for `IsInvalid` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type out_ty (is_invalid x @ (value_type ty))))
(rule (lower (is_invalid x @ (value_type ty)))
(with_flags (cmn_imm (operand_size ty) x (u8_into_imm12 1))
(materialize_bool_result
(ty_bits out_ty) (Cond.Eq))))
(materialize_bool_result (Cond.Eq))))
;;;; Rules for `Debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2325,18 +2242,18 @@
; GPR => SIMD&FP
(rule 4 (lower (has_type (ty_float_or_vec _) (bitcast x @ (value_type in_ty))))
(if (ty_int_bool_ref_scalar_64 in_ty))
(if (ty_int_ref_scalar_64 in_ty))
(mov_to_fpu x (scalar_size in_ty)))
; SIMD&FP => GPR
(rule 3 (lower (has_type out_ty (bitcast x @ (value_type (fits_in_64 (ty_float_or_vec _))))))
(if (ty_int_bool_ref_scalar_64 out_ty))
(if (ty_int_ref_scalar_64 out_ty))
(mov_from_vec x 0 (scalar_size out_ty)))
; GPR <=> GPR
(rule 2 (lower (has_type out_ty (bitcast x @ (value_type in_ty))))
(if (ty_int_bool_ref_scalar_64 out_ty))
(if (ty_int_bool_ref_scalar_64 in_ty))
(if (ty_int_ref_scalar_64 out_ty))
(if (ty_int_ref_scalar_64 in_ty))
x)
(rule 1 (lower (has_type $I128 (bitcast x @ (value_type $I128)))) x)
@@ -2352,7 +2269,7 @@
(rule 2 (lower (has_type (ty_scalar_float _) (extractlane val (u8_from_uimm8 0))))
val)
(rule 0 (lower (has_type (ty_int_bool ty)
(rule 0 (lower (has_type (ty_int ty)
(extractlane val
(u8_from_uimm8 lane))))
(mov_from_vec val lane (scalar_size ty)))
@@ -2365,7 +2282,7 @@
;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule 1 (lower (insertlane vec @ (value_type vty)
val @ (value_type (ty_int_bool _))
val @ (value_type (ty_int _))
(u8_from_uimm8 lane)))
(mov_to_vec vec val lane (vector_size vty)))
@@ -2507,7 +2424,7 @@
;;; Rules for `brz`/`brnz`/`brif`/`brff`/`bricmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `brz` following `icmp`, possibly converted via `bint`.
;; `brz` following `icmp`
(rule (lower_branch (brz (icmp cc x @ (value_type ty) y) _ _) targets)
(let ((cond Cond (cond_code cc))
(cond Cond (invert_cond cond)) ;; negate for `brz`
@@ -2517,16 +2434,7 @@
(with_flags_side_effect (lower_icmp_into_flags cc x y ty)
(cond_br taken not_taken
(cond_br_cond cond))))))
(rule (lower_branch (brz (bint (icmp cc x @ (value_type ty) y)) _ _) targets)
(let ((cond Cond (cond_code cc))
(cond Cond (invert_cond cond)) ;; negate for `brz`
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (lower_icmp_into_flags cc x y ty)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; `brnz` following `icmp`, possibly converted via `bint`.
;; `brnz` following `icmp`
(rule (lower_branch (brnz (icmp cc x @ (value_type ty) y) _ _) targets)
(let ((cond Cond (cond_code cc))
(taken BranchTarget (branch_target targets 0))
@@ -2535,15 +2443,7 @@
(with_flags_side_effect (lower_icmp_into_flags cc x y ty)
(cond_br taken not_taken
(cond_br_cond cond))))))
(rule (lower_branch (brnz (bint (icmp cc x @ (value_type ty) y)) _ _) targets)
(let ((cond Cond (cond_code cc))
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (lower_icmp_into_flags cc x y ty)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; `brz` following `fcmp`, possibly converted via `bint`.
;; `brz` following `fcmp`
(rule (lower_branch (brz (fcmp cc x @ (value_type (ty_scalar_float ty)) y) _ _) targets)
(let ((cond Cond (fp_cond_code cc))
(cond Cond (invert_cond cond)) ;; negate for `brz`
@@ -2553,16 +2453,7 @@
(with_flags_side_effect (fpu_cmp (scalar_size ty) x y)
(cond_br taken not_taken
(cond_br_cond cond))))))
(rule (lower_branch (brz (bint (fcmp cc x @ (value_type (ty_scalar_float ty)) y)) _ _) targets)
(let ((cond Cond (fp_cond_code cc))
(cond Cond (invert_cond cond)) ;; negate for `brz`
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (fpu_cmp (scalar_size ty) x y)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; `brnz` following `fcmp`, possibly converted via `bint`.
;; `brnz` following `fcmp`
(rule (lower_branch (brnz (fcmp cc x @ (value_type (ty_scalar_float ty)) y) _ _) targets)
(let ((cond Cond (fp_cond_code cc))
(taken BranchTarget (branch_target targets 0))
@@ -2571,14 +2462,6 @@
(with_flags_side_effect (fpu_cmp (scalar_size ty) x y)
(cond_br taken not_taken
(cond_br_cond cond))))))
(rule (lower_branch (brnz (bint (fcmp cc x @ (value_type (ty_scalar_float ty)) y)) _ _) targets)
(let ((cond Cond (fp_cond_code cc))
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (fpu_cmp (scalar_size ty) x y)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; standard `brz`
(rule -1 (lower_branch (brz c @ (value_type $I128) _ _) targets)
(let ((flags ProducesFlags (flags_to_producesflags c))
@@ -2592,7 +2475,7 @@
(with_flags_side_effect flags
(cond_br taken not_taken (cond_br_zero rt))))))
(rule -2 (lower_branch (brz c @ (value_type ty) _ _) targets)
(if (ty_int_bool_ref_scalar_64 ty))
(if (ty_int_ref_scalar_64 ty))
(let ((flags ProducesFlags (flags_to_producesflags c))
(rt Reg (put_in_reg_zext64 c))
(taken BranchTarget (branch_target targets 0))
@@ -2613,7 +2496,7 @@
(with_flags_side_effect flags
(cond_br taken not_taken (cond_br_not_zero rt))))))
(rule -2 (lower_branch (brnz c @ (value_type ty) _ _) targets)
(if (ty_int_bool_ref_scalar_64 ty))
(if (ty_int_ref_scalar_64 ty))
(let ((flags ProducesFlags (flags_to_producesflags c))
(rt Reg (put_in_reg_zext64 c))
(taken BranchTarget (branch_target targets 0))

1
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -165,7 +165,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
fn integral_ty(&mut self, ty: Type) -> Option<Type> {
match ty {
I8 | I16 | I32 | I64 | R64 => Some(ty),
ty if ty.is_bool() => Some(ty),
_ => None,
}
}

8
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -39,7 +39,7 @@ pub(crate) fn lower_insn_to_regs(
};
match op {
Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(ctx),
Opcode::Iconst | Opcode::Null => implemented_in_isle(ctx),
Opcode::F32const => {
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
@@ -163,11 +163,9 @@ pub(crate) fn lower_insn_to_regs(
Opcode::Copy => implemented_in_isle(ctx),
Opcode::Breduce | Opcode::Ireduce => implemented_in_isle(ctx),
Opcode::Ireduce => implemented_in_isle(ctx),
Opcode::Bextend | Opcode::Bmask => implemented_in_isle(ctx),
Opcode::Bint => implemented_in_isle(ctx),
Opcode::Bmask => implemented_in_isle(ctx),
Opcode::Bitcast => implemented_in_isle(ctx),

70
cranelift/codegen/src/isa/riscv64/inst.isle
View File

@@ -1659,11 +1659,6 @@
(result Reg (alu_rrr (AluOPRRR.Or) tmp_x tmp_y)))
result))
(decl gen_bint (Reg) Reg)
(rule
(gen_bint r)
(alu_rr_imm12 (AluOPRRI.Andi) r (imm12_const 1)))
(decl gen_int_select (Type IntSelectOP ValueRegs ValueRegs) ValueRegs)
(rule
(gen_int_select ty op x y)
@@ -1729,12 +1724,6 @@
(_ Unit (emit (MInst.FcvtToInt is_sat result tmp rs is_signed in_type out_type))))
result))
;;;; in_type out_type
;;;; out_type is returned.
(decl pure valid_bextend_ty (Type Type) Type)
(extern constructor valid_bextend_ty valid_bextend_ty)
;;; some float binary operation
;;; 1. need move into x reister.
;;; 2. do the operation.
@@ -1907,14 +1896,29 @@
(decl lower_brz_or_nz (IntCC ValueRegs VecMachLabel Type) InstOutput)
(extern constructor lower_brz_or_nz lower_brz_or_nz)
;; Normalize a value by masking to its bit-size.
(decl normalize_value (Type ValueRegs) ValueRegs)
(rule (normalize_value $I8 r)
(value_reg (alu_rr_imm12 (AluOPRRI.Andi) r (imm12_const 255))))
(rule (normalize_value $I16 r)
(value_reg (alu_rrr (AluOPRRR.And) r (imm $I16 65535))))
(rule (normalize_value $I32 r)
(value_reg (alu_rr_imm12 (AluOPRRI.Andi) r (imm12_const -1))))
(rule (normalize_value $I64 r) r)
(rule (normalize_value $I128 r) r)
(rule (normalize_value $F32 r) r)
(rule (normalize_value $F64 r) r)
;;;;;
(rule
(lower_branch (brz v @ (value_type ty) _ _) targets)
(lower_brz_or_nz (IntCC.Equal) v targets ty))
(lower_brz_or_nz (IntCC.Equal) (normalize_value ty v) targets ty))
;;;;
(rule
(lower_branch (brnz v @ (value_type ty) _ _) targets)
(lower_brz_or_nz (IntCC.NotEqual) v targets ty))
(lower_brz_or_nz (IntCC.NotEqual) (normalize_value ty v) targets ty))
;;;
(rule
@@ -2082,3 +2086,43 @@
(decl umulh (Reg Reg) Reg)
(rule (umulh a b)
(alu_rrr (AluOPRRR.Mulhu) a b))
;;;; Helpers for bmask ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl lower_bmask (Type Type ValueRegs) ValueRegs)
;; Produces -1 if the 64-bit value is non-zero, and 0 otherwise.
(rule
0
(lower_bmask (fits_in_64 _) (fits_in_64 _) val)
(let ((input Reg val)
(zero Reg (zero_reg))
(ones Reg (load_imm12 -1)))
(value_reg (gen_select_reg (IntCC.Equal) zero input zero ones))))
;; Bitwise-or the two registers that make up the 128-bit value, then recurse as
;; though it was a 64-bit value.
(rule
1
(lower_bmask (fits_in_64 ty) $I128 val)
(let ((lo Reg (value_regs_get val 0))
(hi Reg (value_regs_get val 1))
(combined Reg (alu_rrr (AluOPRRR.Or) lo hi)))
(lower_bmask ty $I64 (value_reg combined))))
;; Conversion of one 64-bit value to a 128-bit one. Duplicate the result of the
;; bmask of the 64-bit value into both result registers of the i128.
(rule
2
(lower_bmask $I128 (fits_in_64 _) val)
(let ((res ValueRegs (lower_bmask $I64 $I64 val)))
(value_regs (value_regs_get res 0) (value_regs_get res 0))))
;; Conversion of one 64-bit value to a 128-bit one. Duplicate the result of
;; bmasking the 128-bit value to a 64-bit value into both registers of the
;; 128-bit result.
(rule
3
(lower_bmask $I128 $I128 val)
(let ((res ValueRegs (lower_bmask $I64 $I128 val)))
(value_regs (value_regs_get res 0) (value_regs_get res 0))))

6
cranelift/codegen/src/isa/riscv64/inst/args.rs
View File

@@ -1189,10 +1189,8 @@ impl LoadOP {
return if t == F32 { Self::Flw } else { Self::Fld };
}
match t {
B1 | B8 => Self::Lbu,
B16 => Self::Lhu,
B32 | R32 => Self::Lwu,
B64 | R64 | I64 => Self::Ld,
R32 => Self::Lwu,
R64 | I64 => Self::Ld,
I8 => Self::Lb,
I16 => Self::Lh,

7
cranelift/codegen/src/isa/riscv64/inst/emit.rs
View File

@@ -1039,9 +1039,8 @@ impl MachInstEmit for Inst {
&Inst::CondBr {
taken,
not_taken,
kind,
mut kind,
} => {
let mut kind = kind;
kind.rs1 = allocs.next(kind.rs1);
kind.rs2 = allocs.next(kind.rs2);
match taken {
@@ -1385,13 +1384,13 @@ impl MachInstEmit for Inst {
.for_each(|i| i.emit(&[], sink, emit_info, state));
sink.bind_label(label_true);
Inst::load_imm12(rd, Imm12::from_bits(-1)).emit(&[], sink, emit_info, state);
Inst::load_imm12(rd, Imm12::TRUE).emit(&[], sink, emit_info, state);
Inst::Jal {
dest: BranchTarget::offset(Inst::INSTRUCTION_SIZE * 2),
}
.emit(&[], sink, emit_info, state);
sink.bind_label(label_false);
Inst::load_imm12(rd, Imm12::from_bits(0)).emit(&[], sink, emit_info, state);
Inst::load_imm12(rd, Imm12::FALSE).emit(&[], sink, emit_info, state);
}
&Inst::AtomicCas {
offset,

31
cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs
View File

@@ -572,16 +572,6 @@ fn test_riscv64_binemit() {
"lb a0,100(a1)",
0x6458503,
));
insns.push(TestUnit::new(
Inst::Load {
rd: writable_a0(),
op: LoadOP::Lbu,
flags: MemFlags::new(),
from: AMode::RegOffset(a1(), 100, B8),
},
"lbu a0,100(a1)",
0x645c503,
));
insns.push(TestUnit::new(
Inst::Load {
rd: writable_a0(),
@@ -593,17 +583,6 @@ fn test_riscv64_binemit() {
0x6459503,
));
insns.push(TestUnit::new(
Inst::Load {
rd: writable_a0(),
op: LoadOP::Lhu,
flags: MemFlags::new(),
from: AMode::RegOffset(a1(), 100, B16),
},
"lhu a0,100(a1)",
0x645d503,
));
insns.push(TestUnit::new(
Inst::Load {
rd: writable_a0(),
@@ -615,16 +594,6 @@ fn test_riscv64_binemit() {
0x645a503,
));
insns.push(TestUnit::new(
Inst::Load {
rd: writable_a0(),
op: LoadOP::Lwu,
flags: MemFlags::new(),
from: AMode::RegOffset(a1(), 100, B32),
},
"lwu a0,100(a1)",
0x645e503,
));
insns.push(TestUnit::new(
Inst::Load {
rd: writable_a0(),

2
cranelift/codegen/src/isa/riscv64/inst/imms.rs
View File

@@ -12,7 +12,7 @@ pub struct Imm12 {
impl Imm12 {
pub(crate) const FALSE: Self = Self { bits: 0 };
pub(crate) const TRUE: Self = Self { bits: -1 };
pub(crate) const TRUE: Self = Self { bits: 1 };
pub fn maybe_from_u64(val: u64) -> Option<Imm12> {
let sign_bit = 1 << 11;
if val == 0 {

19
cranelift/codegen/src/isa/riscv64/inst/mod.rs
View File

@@ -6,9 +6,7 @@
use crate::binemit::{Addend, CodeOffset, Reloc};
pub use crate::ir::condcodes::IntCC;
use crate::ir::types::{
B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, IFLAGS, R32, R64,
};
use crate::ir::types::{F32, F64, FFLAGS, I128, I16, I32, I64, I8, IFLAGS, R32, R64};
pub use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, Type, ValueLabel};
use crate::isa::CallConv;
@@ -691,14 +689,11 @@ impl MachInst for Inst {
fn gen_constant<F: FnMut(Type) -> Writable<Reg>>(
to_regs: ValueRegs<Writable<Reg>>,
mut value: u128,
value: u128,
ty: Type,
mut alloc_tmp: F,
) -> SmallVec<[Inst; 4]> {
if ty.is_bool() && value != 0 {
value = !0;
}
if (ty.bits() <= 64 && (ty.is_bool() || ty.is_int())) || ty == R32 || ty == R64 {
if (ty.bits() <= 64 && ty.is_int()) || ty == R32 || ty == R64 {
return Inst::load_constant_u64(to_regs.only_reg().unwrap(), value as u64);
};
match ty {
@@ -708,7 +703,7 @@ impl MachInst for Inst {
F64 => {
Inst::load_fp_constant64(to_regs.only_reg().unwrap(), value as u64, alloc_tmp(I64))
}
I128 | B128 => {
I128 => {
let mut insts = SmallInstVec::new();
insts.extend(Inst::load_constant_u64(
to_regs.regs()[0],
@@ -736,17 +731,11 @@ impl MachInst for Inst {
I16 => Ok((&[RegClass::Int], &[I16])),
I32 => Ok((&[RegClass::Int], &[I32])),
I64 => Ok((&[RegClass::Int], &[I64])),
B1 => Ok((&[RegClass::Int], &[B1])),
B8 => Ok((&[RegClass::Int], &[B8])),
B16 => Ok((&[RegClass::Int], &[B16])),
B32 => Ok((&[RegClass::Int], &[B32])),
B64 => Ok((&[RegClass::Int], &[B64])),
R32 => panic!("32-bit reftype pointer should never be seen on riscv64"),
R64 => Ok((&[RegClass::Int], &[R64])),
F32 => Ok((&[RegClass::Float], &[F32])),
F64 => Ok((&[RegClass::Float], &[F64])),
I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
B128 => Ok((&[RegClass::Int, RegClass::Int], &[B64, B64])),
IFLAGS => Ok((&[RegClass::Int], &[IFLAGS])),
FFLAGS => Ok((&[RegClass::Int], &[FFLAGS])),
_ => Err(CodegenError::Unsupported(format!(

2
cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs
View File

@@ -143,7 +143,7 @@ mod tests {
assert_eq!(
format!("{:?}", fde),
"FrameDescriptionEntry { address: Constant(4321), length: 12, lsda: None, instructions: [] }"
"FrameDescriptionEntry { address: Constant(4321), length: 16, lsda: None, instructions: [] }"
);
}

74
cranelift/codegen/src/isa/riscv64/lower.isle
View File

@@ -9,15 +9,6 @@
(rule (lower (has_type ty (iconst (u64_from_imm64 n))))
(imm ty n))
;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty (bconst $false)))
(imm ty 0))
(rule (lower (has_type ty (bconst $true)))
(imm ty 1))
;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty (null)))
@@ -194,8 +185,6 @@
(rule 1 (lower (has_type (fits_in_64 ty) (band (imm12_from_value x) y)))
(alu_rr_imm12 (AluOPRRI.Andi) y x))
(rule (lower (has_type $B128 (band x y)))
(lower_b128_binary (AluOPRRR.And) x y))
(rule (lower (has_type $I128 (band x y)))
(lower_b128_binary (AluOPRRR.And) x y))
@@ -215,8 +204,6 @@
(rule 1 (lower (has_type (fits_in_64 ty) (bor (imm12_from_value x) y)))
(alu_rr_imm12 (AluOPRRI.Ori) y x))
(rule (lower (has_type $B128 (bor x y)))
(lower_b128_binary (AluOPRRR.Or) x y))
(rule (lower (has_type $I128 (bor x y)))
(lower_b128_binary (AluOPRRR.Or) x y))
(rule (lower (has_type $F32 (bor x y)))
@@ -235,8 +222,6 @@
(rule 1 (lower (has_type (fits_in_64 ty) (bxor (imm12_from_value x) y)))
(alu_rr_imm12 (AluOPRRI.Xori) y x))
(rule (lower (has_type $B128 (bxor x y)))
(lower_b128_binary (AluOPRRR.Xor) x y))
(rule (lower (has_type $I128 (bxor x y)))
(lower_b128_binary (AluOPRRR.Xor) x y))
(rule (lower (has_type $F32 (bxor x y)))
@@ -251,8 +236,6 @@
(rule (lower (has_type $I128 (bnot x)))
(bnot_128 x))
(rule (lower (has_type $B128 (bnot x)))
(bnot_128 x))
(rule
(lower (has_type $F32 (bnot x)))
(lower_float_bnot x $F32)
@@ -556,11 +539,6 @@
(rule (lower (has_type ty (copy x)))
(gen_move2 x ty ty))
;;;;; Rules for `breduce`;;;;;;;;;;;;;;;;;
(rule
(lower (has_type ty (breduce x)))
(gen_move2 (value_regs_get x 0) ty ty))
;;;;; Rules for `ireduce`;;;;;;;;;;;;;;;;;
(rule
(lower (has_type ty (ireduce x)))
@@ -623,8 +601,8 @@
;;;;; Rules for `select`;;;;;;;;;
(rule
(lower (has_type ty (select c x y)))
(gen_select ty c x y)
(lower (has_type ty (select c @ (value_type cty) x y)))
(gen_select ty (normalize_value cty c) x y)
)
;;;;; Rules for `bitselect`;;;;;;;;;
@@ -633,15 +611,6 @@
(lower (has_type ty (bitselect c x y)))
(gen_bitselect ty c x y))
;;;;; Rules for `bint`;;;;;;;;;
(rule
(lower (has_type (fits_in_64 ty) (bint (valueregs_2_reg x))))
(gen_bint x))
(rule 1
(lower (has_type $I128 (bint (valueregs_2_reg x))))
(let ((tmp Reg (gen_bint x)))
(value_regs tmp (load_u64_constant 0))))
;;;;; Rules for `isplit`;;;;;;;;;
(rule
(lower (isplit x))
@@ -733,10 +702,6 @@
(rule 1
(lower (has_type $I128 (load flags p offset)))
(gen_load_128 p offset flags))
;;;; for B128
(rule 1
(lower (has_type $B128 (load flags p offset)))
(gen_load_128 p offset flags))
;;;;; Rules for `istore8`;;;;;;;;;
(rule
@@ -762,11 +727,6 @@
(lower (store flags x @ (value_type $I128 ) p offset))
(gen_store_128 p offset flags x))
;;; special for B128
(rule 1
(lower (store flags x @ (value_type $B128 ) p offset))
(gen_store_128 p offset flags x))
(decl gen_icmp (IntCC ValueRegs ValueRegs Type) Reg)
(rule
(gen_icmp cc x y ty)
@@ -923,34 +883,8 @@
;;;;; Rules for `bmask`;;;;;;;;;
(rule
;; because we encode bool all 1s.
;; move is just ok.
(lower (has_type (fits_in_64 ty) (bmask x @ (value_type ity))))
(gen_move2 (value_regs_get x 0) ity ty))
;;; for i128
(rule 1
;; because we encode bool all 1s.
;; move is just ok.
(lower (has_type $I128 (bmask x @ (value_type ity))))
(value_regs (gen_move2 (value_regs_get x 0) $I64 $I64) (gen_move2 (value_regs_get x 0) $I64 $I64)))
;;;;; Rules for `bextend`;;;;;;;;;
(rule
;; because we encode bool all 1s.
;; move is just ok.
(lower (has_type ty (bextend x @ (value_type ity))))
;;extra checks.
(if-let _ (valid_bextend_ty ity ty))
(gen_moves x ity ty))
;;; for B128
(rule 1
;; because we encode bool all 1s.
;; move is just ok.
(lower (has_type ty (bextend x @ (value_type ity))))
;;extra checks.
(if-let $B128 (valid_bextend_ty ity ty))
(value_regs (gen_moves x $I64 $I64) (gen_moves x $I64 $I64)))
(lower (has_type oty (bmask x @ (value_type ity))))
(lower_bmask oty ity x))
;; N.B.: the Ret itself is generated by the ABI.
(rule (lower (return args))

40
cranelift/codegen/src/isa/riscv64/lower/isle.rs
View File

@@ -71,13 +71,6 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
}
}
fn valid_bextend_ty(&mut self, from: Type, to: Type) -> Option<Type> {
if from.is_bool() && to.is_bool() && from.bits() < to.bits() {
Some(to)
} else {
None
}
}
fn lower_br_fcmp(
&mut self,
cc: &FloatCC,
@@ -155,7 +148,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
}
}
fn int_zero_reg(&mut self, ty: Type) -> ValueRegs {
assert!(ty.is_int() || ty.is_bool(), "{:?}", ty);
assert!(ty.is_int(), "{:?}", ty);
if ty.bits() == 128 {
ValueRegs::two(self.zero_reg(), self.zero_reg())
} else {
@@ -190,7 +183,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
Imm12::from_bits(imm.as_i16() & (x as i16))
}
fn alloc_vec_writable(&mut self, ty: Type) -> VecWritableReg {
if ty.is_int() || ty.is_bool() || ty == R32 || ty == R64 {
if ty.is_int() || ty == R32 || ty == R64 {
if ty.bits() <= 64 {
vec![self.temp_writable_reg(I64)]
} else {
@@ -203,26 +196,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
}
}
fn imm(&mut self, ty: Type, mut val: u64) -> Reg {
// Boolean types
// Boolean values are either true or false.
// The b1 type represents an abstract boolean value. It can only exist as an SSA value, and can't be directly stored in memory. It can, however, be converted into an integer with value 0 or 1 by the bint instruction (and converted back with icmp_imm with 0).
// Several larger boolean types are also defined, primarily to be used as SIMD element types. They can be stored in memory, and are represented as either all zero bits or all one bits.
// b1
// b8
// b16
// b32
// b64
// ///////////////////////////////////////////////////////////
// "represented as either all zero bits or all one bits."
// \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
if ty.is_bool() && val != 0 {
// need all be one
val = !0;
}
fn imm(&mut self, ty: Type, val: u64) -> Reg {
let tmp = self.temp_writable_reg(ty);
self.emit_list(&MInst::load_constant_u64(tmp, val));
tmp.to_reg()
@@ -296,7 +270,11 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
rd.to_reg()
}
fn imm12_const(&mut self, val: i32) -> Imm12 {
Imm12::maybe_from_u64(val as u64).unwrap()
if let Some(res) = Imm12::maybe_from_u64(val as u64) {
res
} else {
panic!("Unable to make an Imm12 value from {}", val)
}
}
fn imm12_const_add(&mut self, val: i32, add: i32) -> Imm12 {
Imm12::maybe_from_u64((val + add) as u64).unwrap()
@@ -526,7 +504,7 @@ fn construct_dest<F: std::ops::FnMut(Type) -> WritableReg>(
mut alloc: F,
ty: Type,
) -> WritableValueRegs {
if ty.is_bool() || ty.is_int() {
if ty.is_int() {
if ty.bits() == 128 {
WritableValueRegs::two(alloc(I64), alloc(I64))
} else {

1
cranelift/codegen/src/isa/s390x/abi.rs
View File

@@ -94,7 +94,6 @@ pub type S390xCallee = Callee<S390xMachineDeps>;
fn in_int_reg(ty: Type) -> bool {
match ty {
types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => true,
types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
_ => false,
}
}

22
cranelift/codegen/src/isa/s390x/inst.isle
View File

@@ -3129,7 +3129,6 @@
dst))
;; Sign-extend a register from a smaller `Type` into a 32-bit register.
;; This handles both integer and boolean input types (except $B1).
(decl sext32_reg (Type Reg) Reg)
(rule (sext32_reg ty src)
(let ((dst WritableReg (temp_writable_reg $I32))
@@ -3137,7 +3136,6 @@
dst))
;; Zero-extend a register from a smaller `Type` into a 64-bit register.
;; This handles both integer and boolean input types (except $B1).
(decl zext64_reg (Type Reg) Reg)
(rule (zext64_reg ty src)
(let ((dst WritableReg (temp_writable_reg $I64))
@@ -3145,7 +3143,6 @@
dst))
;; Sign-extend a register from a smaller `Type` into a 64-bit register.
;; This handles both integer and boolean input types (except $B1).
(decl sext64_reg (Type Reg) Reg)
(rule (sext64_reg ty src)
(let ((dst WritableReg (temp_writable_reg $I64))
@@ -3477,14 +3474,19 @@
(_ Unit (emit_consumer (emit_cmov_imm ty dst cond imm_true))))
dst))
;; Lower a boolean condition to a boolean type. The value used to represent
;; "true" is -1 for all result types except for $B1, which uses 1.
;; Lower a boolean condition to the values 1/0. This rule is only used in the
;; context of instructions that return $I8 results.
(decl lower_bool (Type ProducesBool) Reg)
(rule (lower_bool $B1 cond) (select_bool_imm $B1 cond 1 0))
(rule (lower_bool $B8 cond) (select_bool_imm $B8 cond -1 0))
(rule (lower_bool $B16 cond) (select_bool_imm $B16 cond -1 0))
(rule (lower_bool $B32 cond) (select_bool_imm $B32 cond -1 0))
(rule (lower_bool $B64 cond) (select_bool_imm $B64 cond -1 0))
(rule (lower_bool $I8 cond) (select_bool_imm $I8 cond 1 0))
;; Lower a boolean condition to the values -1/0.
(decl lower_bool_to_mask (Type ProducesBool) Reg)
(rule 0 (lower_bool_to_mask (fits_in_64 ty) producer)
(select_bool_imm ty producer -1 0))
(rule 1 (lower_bool_to_mask $I128 producer)
(let ((res Reg (lower_bool_to_mask $I64 producer)))
(mov_to_vec128 $I128 res res)))
;; Emit a conditional branch based on a boolean condition.
(decl cond_br_bool (ProducesBool MachLabel MachLabel) SideEffectNoResult)

38
cranelift/codegen/src/isa/s390x/inst/mod.rs
View File

@@ -397,10 +397,10 @@ impl Inst {
/// Generic constructor for a load (zero-extending where appropriate).
pub fn gen_load(into_reg: Writable<Reg>, mem: MemArg, ty: Type) -> Inst {
match ty {
types::B1 | types::B8 | types::I8 => Inst::Load64ZExt8 { rd: into_reg, mem },
types::B16 | types::I16 => Inst::Load64ZExt16 { rd: into_reg, mem },
types::B32 | types::I32 => Inst::Load64ZExt32 { rd: into_reg, mem },
types::B64 | types::I64 | types::R64 => Inst::Load64 { rd: into_reg, mem },
types::I8 => Inst::Load64ZExt8 { rd: into_reg, mem },
types::I16 => Inst::Load64ZExt16 { rd: into_reg, mem },
types::I32 => Inst::Load64ZExt32 { rd: into_reg, mem },
types::I64 | types::R64 => Inst::Load64 { rd: into_reg, mem },
types::F32 => Inst::VecLoadLaneUndef {
size: 32,
rd: into_reg,
@@ -414,7 +414,7 @@ impl Inst {
lane_imm: 0,
},
_ if ty.is_vector() && ty.bits() == 128 => Inst::VecLoad { rd: into_reg, mem },
types::B128 | types::I128 => Inst::VecLoad { rd: into_reg, mem },
types::I128 => Inst::VecLoad { rd: into_reg, mem },
_ => unimplemented!("gen_load({})", ty),
}
}
@@ -422,10 +422,10 @@ impl Inst {
/// Generic constructor for a store.
pub fn gen_store(mem: MemArg, from_reg: Reg, ty: Type) -> Inst {
match ty {
types::B1 | types::B8 | types::I8 => Inst::Store8 { rd: from_reg, mem },
types::B16 | types::I16 => Inst::Store16 { rd: from_reg, mem },
types::B32 | types::I32 => Inst::Store32 { rd: from_reg, mem },
types::B64 | types::I64 | types::R64 => Inst::Store64 { rd: from_reg, mem },
types::I8 => Inst::Store8 { rd: from_reg, mem },
types::I16 => Inst::Store16 { rd: from_reg, mem },
types::I32 => Inst::Store32 { rd: from_reg, mem },
types::I64 | types::R64 => Inst::Store64 { rd: from_reg, mem },
types::F32 => Inst::VecStoreLane {
size: 32,
rd: from_reg,
@@ -439,7 +439,7 @@ impl Inst {
lane_imm: 0,
},
_ if ty.is_vector() && ty.bits() == 128 => Inst::VecStore { rd: from_reg, mem },
types::B128 | types::I128 => Inst::VecStore { rd: from_reg, mem },
types::I128 => Inst::VecStore { rd: from_reg, mem },
_ => unimplemented!("gen_store({})", ty),
}
}
@@ -1086,7 +1086,7 @@ impl MachInst for Inst {
.only_reg()
.expect("multi-reg values not supported yet");
match ty {
types::I128 | types::B128 => {
types::I128 => {
let mut ret = SmallVec::new();
ret.push(Inst::load_vec_constant(to_reg, value));
ret
@@ -1112,14 +1112,8 @@ impl MachInst for Inst {
));
ret
}
types::I64 | types::B64 | types::R64 => Inst::load_constant64(to_reg, value as u64),
types::B1
| types::I8
| types::B8
| types::I16
| types::B16
| types::I32
| types::B32 => Inst::load_constant32(to_reg, value as u32),
types::I64 | types::R64 => Inst::load_constant64(to_reg, value as u64),
types::I8 | types::I16 | types::I32 => Inst::load_constant32(to_reg, value as u32),
_ => unreachable!(),
}
}
@@ -1140,17 +1134,11 @@ impl MachInst for Inst {
types::I16 => Ok((&[RegClass::Int], &[types::I16])),
types::I32 => Ok((&[RegClass::Int], &[types::I32])),
types::I64 => Ok((&[RegClass::Int], &[types::I64])),
types::B1 => Ok((&[RegClass::Int], &[types::B1])),
types::B8 => Ok((&[RegClass::Int], &[types::B8])),
types::B16 => Ok((&[RegClass::Int], &[types::B16])),
types::B32 => Ok((&[RegClass::Int], &[types::B32])),
types::B64 => Ok((&[RegClass::Int], &[types::B64])),
types::R32 => panic!("32-bit reftype pointer should never be seen on s390x"),
types::R64 => Ok((&[RegClass::Int], &[types::R64])),
types::F32 => Ok((&[RegClass::Float], &[types::F32])),
types::F64 => Ok((&[RegClass::Float], &[types::F64])),
types::I128 => Ok((&[RegClass::Float], &[types::I128])),
types::B128 => Ok((&[RegClass::Float], &[types::B128])),
_ if ty.is_vector() && ty.bits() == 128 => Ok((&[RegClass::Float], &[types::I8X16])),
// FIXME: We don't really have IFLAGS, but need to allow it here
// for now to support the SelectifSpectreGuard instruction.

115
cranelift/codegen/src/isa/s390x/lower.isle
View File

@@ -16,14 +16,6 @@
(imm ty n))
;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty (bconst $false)))
(imm ty 0))
(rule (lower (has_type ty (bconst $true)))
(imm ty 1))
;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (f32const (u64_from_ieee32 x)))
@@ -1163,92 +1155,10 @@
(vec_select ty y z x))
;;;; Rules for `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; Rules for `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Up to 64-bit source type: Always a no-op.
(rule 1 (lower (breduce x @ (value_type (fits_in_64 _ty))))
x)
;; 128-bit source type: Extract the low half.
(rule (lower (breduce x @ (value_type (vr128_ty _ty))))
(vec_extract_lane $I64X2 x 1 (zero_reg)))
;;;; Rules for `bextend` and `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Use a common helper to type cast bools to either bool or integer types.
(decl cast_bool (Type Value) Reg)
(rule (lower (has_type ty (bextend x)))
(cast_bool ty x))
(rule (lower (has_type ty (bmask x)))
(cast_bool ty x))
;; If the target has the same or a smaller size than the source, it's a no-op.
(rule 8 (cast_bool $B1 x @ (value_type $B1)) x)
(rule 8 (cast_bool $B1 x @ (value_type $B8)) x)
(rule 8 (cast_bool $B8 x @ (value_type $B8)) x)
(rule 8 (cast_bool $I8 x @ (value_type $B8)) x)
(rule 7 (cast_bool (fits_in_16 _ty) x @ (value_type $B16)) x)
(rule 6 (cast_bool (fits_in_32 _ty) x @ (value_type $B32)) x)
(rule 5 (cast_bool (fits_in_64 _ty) x @ (value_type $B64)) x)
(rule 4 (cast_bool (vr128_ty _ty) x @ (value_type $B128)) x)
(rule 5 (cast_bool (fits_in_64 _ty) x @ (value_type $B128))
(vec_extract_lane $I64X2 x 1 (zero_reg)))
;; Single-bit values are sign-extended via a pair of shifts.
(rule 0 (cast_bool (gpr32_ty ty) x @ (value_type $B1))
(ashr_imm $I32 (lshl_imm $I32 x 31) 31))
(rule 1 (cast_bool (gpr64_ty ty) x @ (value_type $B1))
(ashr_imm $I64 (lshl_imm $I64 x 63) 63))
(rule 4 (cast_bool (vr128_ty ty) x @ (value_type $B1))
(let ((gpr Reg (ashr_imm $I64 (lshl_imm $I64 x 63) 63)))
(mov_to_vec128 ty gpr gpr)))
;; Other values are just sign-extended normally.
(rule 0 (cast_bool (gpr32_ty _ty) x @ (value_type $B8))
(sext32_reg $I8 x))
(rule 0 (cast_bool (gpr32_ty _ty) x @ (value_type $B16))
(sext32_reg $I16 x))
(rule 1(cast_bool (gpr64_ty _ty) x @ (value_type $B8))
(sext64_reg $I8 x))
(rule 1(cast_bool (gpr64_ty _ty) x @ (value_type $B16))
(sext64_reg $I16 x))
(rule 1(cast_bool (gpr64_ty _ty) x @ (value_type $B32))
(sext64_reg $I32 x))
(rule 3 (cast_bool (vr128_ty ty) x @ (value_type (gpr32_ty src_ty)))
(let ((x_ext Reg (sext64_reg src_ty x)))
(mov_to_vec128 ty x_ext x_ext)))
(rule 2 (cast_bool (vr128_ty ty) x @ (value_type (gpr64_ty src_ty)))
(mov_to_vec128 ty x x))
;;;; Rules for `bint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Mask with 1 to get a 0/1 result (8- or 16-bit result types).
(rule 5 (lower (has_type (fits_in_16 ty) (bint x @ (value_type (fits_in_64 _)))))
(and_uimm16shifted ty x (uimm16shifted 1 0)))
;; Mask with 1 to get a 0/1 result (32-bit result types).
(rule 4 (lower (has_type (fits_in_32 ty) (bint x @ (value_type (fits_in_64 _)))))
(and_uimm32shifted ty x (uimm32shifted 1 0)))
;; Mask with 1 to get a 0/1 result (64-bit result types).
(rule 3 (lower (has_type (fits_in_64 ty) (bint x @ (value_type (fits_in_64 _)))))
(and_reg ty x (imm ty 1)))
;; Mask with 1 to get a 0/1 result (128-bit result types).
(rule 1 (lower (has_type (vr128_ty ty) (bint x @ (value_type (fits_in_64 _)))))
(let ((x_ext Reg (and_uimm16shifted $I8 x (uimm16shifted 1 0))))
(vec_insert_lane $I8X16 (vec_imm ty 0) x_ext 15 (zero_reg))))
;; Mask with 1 to get a 0/1 result (128-bit source types).
(rule 2 (lower (has_type (fits_in_64 ty) (bint x @ (value_type (vr128_ty _)))))
(let ((x_gpr Reg (vec_extract_lane $I8X16 x 15 (zero_reg))))
(and_uimm16shifted ty x_gpr (uimm16shifted 1 0))))
;; Mask with 1 to get a 0/1 result (128-bit source and result types).
(rule 0 (lower (has_type (vr128_ty ty) (bint x @ (value_type (vr128_ty _)))))
(vec_and ty x (vec_imm ty 1)))
(lower_bool_to_mask ty (value_nonzero x)))
;;;; Rules for `bitrev` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1864,7 +1774,7 @@
(rule 1 (lower (insertlane x @ (value_type ty)
y @ (value_type in_ty)
(u8_from_uimm8 idx)))
(if (ty_int_bool_ref_scalar_64 in_ty))
(if (ty_int_ref_scalar_64 in_ty))
(vec_insert_lane ty x y (be_lane_idx ty idx) (zero_reg)))
;; Insert vector lane from floating-point register.
@@ -1980,7 +1890,7 @@
;; Extract vector lane to general-purpose register.
(rule 1 (lower (has_type out_ty
(extractlane x @ (value_type ty) (u8_from_uimm8 idx))))
(if (ty_int_bool_ref_scalar_64 out_ty))
(if (ty_int_ref_scalar_64 out_ty))
(vec_extract_lane ty x (be_lane_idx ty idx) (zero_reg)))
;; Extract vector lane to floating-point register.
@@ -2037,7 +1947,7 @@
;; Load replicated value from general-purpose register.
(rule 1 (lower (has_type ty (splat x @ (value_type in_ty))))
(if (ty_int_bool_ref_scalar_64 in_ty))
(if (ty_int_ref_scalar_64 in_ty))
(vec_replicate_lane ty (vec_insert_lane_undef ty x 0 (zero_reg)) 0))
;; Load replicated value from floating-point register.
@@ -2097,7 +2007,7 @@
;; Load scalar value from general-purpose register.
(rule 1 (lower (has_type ty (scalar_to_vector
x @ (value_type in_ty))))
(if (ty_int_bool_ref_scalar_64 in_ty))
(if (ty_int_ref_scalar_64 in_ty))
(vec_insert_lane ty (vec_imm ty 0) x (be_lane_idx ty 0) (zero_reg)))
;; Load scalar value from floating-point register.
@@ -3783,14 +3693,14 @@
;;;; Rules for `is_null` and `is_invalid` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Null references are represented by the constant value 0.
(rule (lower (has_type $B1 (is_null x @ (value_type $R64))))
(lower_bool $B1 (bool (icmps_simm16 $I64 x 0)
(rule (lower (has_type $I8 (is_null x @ (value_type $R64))))
(lower_bool $I8 (bool (icmps_simm16 $I64 x 0)
(intcc_as_cond (IntCC.Equal)))))
;; Invalid references are represented by the constant value -1.
(rule (lower (has_type $B1 (is_invalid x @ (value_type $R64))))
(lower_bool $B1 (bool (icmps_simm16 $I64 x -1)
(rule (lower (has_type $I8 (is_invalid x @ (value_type $R64))))
(lower_bool $I8 (bool (icmps_simm16 $I64 x -1)
(intcc_as_cond (IntCC.Equal)))))
@@ -3798,10 +3708,9 @@
;; Return a `ProducesBool` to capture the fact that the input value is nonzero.
;; In the common case where that input is the result of an `icmp` or `fcmp`
;; instruction (possibly via an intermediate `bint`), directly use that compare.
;; Note that it is not safe to sink memory loads here, see the `icmp` comment.
;; instruction, directly use that compare. Note that it is not safe to sink
;; memory loads here, see the `icmp` comment.
(decl value_nonzero (Value) ProducesBool)
(rule (value_nonzero (bint val)) (value_nonzero val))
(rule (value_nonzero (icmp int_cc x y)) (icmp_val $false int_cc x y))
(rule (value_nonzero (fcmp float_cc x y)) (fcmp_val float_cc x y))
(rule -1 (value_nonzero val @ (value_type (gpr32_ty ty)))

4
cranelift/codegen/src/isa/s390x/lower.rs
View File

@@ -45,7 +45,6 @@ impl LowerBackend for S390xBackend {
Opcode::Nop
| Opcode::Copy
| Opcode::Iconst
| Opcode::Bconst
| Opcode::F32const
| Opcode::F64const
| Opcode::Vconst
@@ -100,10 +99,7 @@ impl LowerBackend for S390xBackend {
| Opcode::BxorNot
| Opcode::Bitselect
| Opcode::Vselect
| Opcode::Breduce
| Opcode::Bextend
| Opcode::Bmask
| Opcode::Bint
| Opcode::Bitrev
| Opcode::Clz
| Opcode::Cls

6
cranelift/codegen/src/isa/s390x/lower/isle.rs
View File

@@ -252,7 +252,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
#[inline]
fn gpr32_ty(&mut self, ty: Type) -> Option<Type> {
match ty {
I8 | I16 | I32 | B1 | B8 | B16 | B32 => Some(ty),
I8 | I16 | I32 => Some(ty),
_ => None,
}
}
@@ -260,7 +260,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
#[inline]
fn gpr64_ty(&mut self, ty: Type) -> Option<Type> {
match ty {
I64 | B64 | R64 => Some(ty),
I64 | R64 => Some(ty),
_ => None,
}
}
@@ -268,7 +268,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
#[inline]
fn vr128_ty(&mut self, ty: Type) -> Option<Type> {
match ty {
I128 | B128 => Some(ty),
I128 => Some(ty),
_ if ty.is_vector() && ty.bits() == 128 => Some(ty),
_ => None,
}

8
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -260,13 +260,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
// For integer-typed values, we always load a full 64 bits (and we always spill a full 64
// bits as well -- see `Inst::store()`).
let ty = match ty {
types::B1
| types::B8
| types::I8
| types::B16
| types::I16
| types::B32
| types::I32 => types::I64,
types::I8 | types::I16 | types::I32 => types::I64,
_ => ty,
};
Inst::load(ty, mem, into_reg, ExtKind::None)

13
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -2217,17 +2217,11 @@ impl MachInst for Inst {
types::I16 => Ok((&[RegClass::Int], &[types::I16])),
types::I32 => Ok((&[RegClass::Int], &[types::I32])),
types::I64 => Ok((&[RegClass::Int], &[types::I64])),
types::B1 => Ok((&[RegClass::Int], &[types::B1])),
types::B8 => Ok((&[RegClass::Int], &[types::B8])),
types::B16 => Ok((&[RegClass::Int], &[types::B16])),
types::B32 => Ok((&[RegClass::Int], &[types::B32])),
types::B64 => Ok((&[RegClass::Int], &[types::B64])),
types::R32 => panic!("32-bit reftype pointer should never be seen on x86-64"),
types::R64 => Ok((&[RegClass::Int], &[types::R64])),
types::F32 => Ok((&[RegClass::Float], &[types::F32])),
types::F64 => Ok((&[RegClass::Float], &[types::F64])),
types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
types::B128 => Ok((&[RegClass::Int, RegClass::Int], &[types::B64, types::B64])),
_ if ty.is_vector() => {
assert!(ty.bits() <= 128);
Ok((&[RegClass::Float], &[types::I8X16]))
@@ -2326,15 +2320,10 @@ impl MachInst for Inst {
} else {
// Must be an integer type.
debug_assert!(
ty == types::B1
|| ty == types::I8
|| ty == types::B8
ty == types::I8
|| ty == types::I16
|| ty == types::B16
|| ty == types::I32
|| ty == types::B32
|| ty == types::I64
|| ty == types::B64
|| ty == types::R32
|| ty == types::R64
);

137
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -22,30 +22,6 @@
(value_regs (imm $I64 x)
(imm $I64 0)))
;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `b64` and smaller.
(rule (lower (has_type (fits_in_64 ty)
(bconst $false)))
(imm ty 0))
(rule (lower (has_type (fits_in_64 ty)
(bconst $true)))
(imm ty 1))
;; `b128`
(rule 1 (lower (has_type $B128
(bconst $false)))
(value_regs (imm $B64 0)
(imm $B64 0)))
(rule 1 (lower (has_type $B128
(bconst $true)))
(value_regs (imm $B64 1)
(imm $B64 0)))
;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (f32const (u64_from_ieee32 x)))
@@ -303,7 +279,7 @@
(band x y)))
(sse_and ty x y))
;; `{i,b}128`.
;; `i128`.
(rule 6 (lower (has_type $I128 (band x y)))
(let ((x_regs ValueRegs x)
@@ -315,17 +291,6 @@
(value_gprs (x64_and $I64 x_lo y_lo)
(x64_and $I64 x_hi y_hi))))
(rule 6 (lower (has_type $B128 (band x y)))
;; Booleans are always `0` or `1`, so we only need to do the `and` on the
;; low half. The high half is always zero but, rather than generate a new
;; zero, we just reuse `x`'s high half which is already zero.
(let ((x_regs ValueRegs x)
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_lo Gpr (lo_gpr y)))
(value_gprs (x64_and $I64 x_lo y_lo)
x_hi)))
;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `{i,b}64` and smaller.
@@ -381,17 +346,6 @@
(rule 6 (lower (has_type $I128 (bor x y)))
(or_i128 x y))
(rule 6 (lower (has_type $B128 (bor x y)))
;; Booleans are always `0` or `1`, so we only need to do the `or` on the
;; low half. The high half is always zero but, rather than generate a new
;; zero, we just reuse `x`'s high half which is already zero.
(let ((x_regs ValueRegs x)
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_lo Gpr (lo_gpr y)))
(value_gprs (x64_or $I64 x_lo y_lo)
x_hi)))
;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `{i,b}64` and smaller.
@@ -439,17 +393,6 @@
(value_gprs (x64_xor $I64 x_lo y_lo)
(x64_xor $I64 x_hi y_hi))))
(rule 6 (lower (has_type $B128 (bxor x y)))
;; Booleans are always `0` or `1`, so we only need to do the `xor` on the
;; low half. The high half is always zero but, rather than generate a new
;; zero, we just reuse `x`'s high half which is already zero.
(let ((x_regs ValueRegs x)
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_lo Gpr (lo_gpr y)))
(value_gprs (x64_xor $I64 x_lo y_lo)
x_hi)))
;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `i64` and smaller.
@@ -1240,9 +1183,6 @@
(rule (lower (has_type $I128 (bnot x)))
(i128_not x))
(rule (lower (has_type $B128 (bnot x)))
(i128_not x))
;; Special case for vector-types where bit-negation is an xor against an
;; all-one value
(rule -1 (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x)))
@@ -1450,35 +1390,35 @@
(lower_icmp_bool (emit_cmp cc a b)))
;; Peephole optimization for `x < 0`, when x is a signed 64 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedLessThan) x @ (value_type $I64) (u64_from_iconst 0))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThan) x @ (value_type $I64) (u64_from_iconst 0))))
(x64_shr $I64 x (Imm8Reg.Imm8 63)))
;; Peephole optimization for `0 > x`, when x is a signed 64 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I64))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I64))))
(x64_shr $I64 x (Imm8Reg.Imm8 63)))
;; Peephole optimization for `0 <= x`, when x is a signed 64 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I64))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I64))))
(x64_shr $I64 (x64_not $I64 x) (Imm8Reg.Imm8 63)))
;; Peephole optimization for `x >= 0`, when x is a signed 64 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I64) (u64_from_iconst 0))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I64) (u64_from_iconst 0))))
(x64_shr $I64 (x64_not $I64 x) (Imm8Reg.Imm8 63)))
;; Peephole optimization for `x < 0`, when x is a signed 32 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedLessThan) x @ (value_type $I32) (u64_from_iconst 0))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThan) x @ (value_type $I32) (u64_from_iconst 0))))
(x64_shr $I32 x (Imm8Reg.Imm8 31)))
;; Peephole optimization for `0 > x`, when x is a signed 32 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I32))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I32))))
(x64_shr $I32 x (Imm8Reg.Imm8 31)))
;; Peephole optimization for `0 <= x`, when x is a signed 32 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I32))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I32))))
(x64_shr $I32 (x64_not $I64 x) (Imm8Reg.Imm8 31)))
;; Peephole optimization for `x >= 0`, when x is a signed 32 bit value
(rule 2 (lower (has_type $B1 (icmp (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I32) (u64_from_iconst 0))))
(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I32) (u64_from_iconst 0))))
(x64_shr $I32 (x64_not $I64 x) (Imm8Reg.Imm8 31)))
;; For XMM-held values, we lower to `PCMP*` instructions, sometimes more than
@@ -1710,14 +1650,7 @@
;; Finally, we lower `select` from a condition value `c`. These rules are meant
;; to be the final, default lowerings if no other patterns matched above.
(rule -1 (lower (has_type ty (select c @ (value_type $B1) x y)))
(let ((size OperandSize (raw_operand_size_of_type $B1))
;; N.B.: disallow load-op fusion, see above. TODO:
;; https://github.com/bytecodealliance/wasmtime/issues/3953.
(gpr_c Gpr (put_in_gpr c)))
(with_flags (x64_test size (RegMemImm.Imm 1) gpr_c) (cmove_from_values ty (CC.NZ) x y))))
(rule -2 (lower (has_type ty (select c @ (value_type (fits_in_64 a_ty)) x y)))
(rule -1 (lower (has_type ty (select c @ (value_type (fits_in_64 a_ty)) x y)))
(let ((size OperandSize (raw_operand_size_of_type a_ty))
;; N.B.: disallow load-op fusion, see above. TODO:
;; https://github.com/bytecodealliance/wasmtime/issues/3953.
@@ -2125,7 +2058,7 @@
(uextend src @ (has_type $I32 (uload32 _ _ _)))))
src)
;; Rules for `sextend` / `bextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl generic_sextend (Value Type Type) InstOutput)
@@ -2140,17 +2073,17 @@
(x64_sar $I64 src (Imm8Reg.Imm8 63)))
;; I64 -> I128.
(rule 3 (generic_sextend src (ty_int_bool_64 _) (ty_int_bool_128 _))
(rule 3 (generic_sextend src $I64 $I128)
(value_regs src (spread_sign_bit src)))
;; I{8,16,32} -> I128.
(rule 2 (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_128 _))
(rule 2 (generic_sextend src (fits_in_32 src_ty) $I128)
(let ((lo Gpr (extend_to_gpr src $I64 (ExtendKind.Sign)))
(hi Gpr (spread_sign_bit lo)))
(value_regs lo hi)))
;; I{8,16,32} -> I64.
(rule 1 (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_64 _))
(rule 1 (generic_sextend src (fits_in_32 src_ty) $I64)
(extend_to_gpr src $I64 (ExtendKind.Sign)))
;; I8 -> I{16,32}, I16 -> I32.
@@ -2162,13 +2095,7 @@
(sextend src @ (value_type src_ty))))
(generic_sextend src src_ty dst_ty))
;; Bools are stored as 0/-1 so extends must sign-extend as well.
(rule (lower
(has_type dst_ty
(bextend src @ (value_type src_ty))))
(generic_sextend src src_ty dst_ty))
;; Rules for `ireduce` / `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; T -> T is always a no-op, even I128 -> I128.
(rule (lower (has_type ty (ireduce src @ (value_type ty))))
@@ -2180,28 +2107,6 @@
(rule 1 (lower (has_type (fits_in_64 ty) (ireduce src)))
(value_regs_get_gpr src 0))
;; Likewise for breduce.
(rule (lower (has_type ty (breduce src @ (value_type ty))))
src)
(rule 1 (lower (has_type (fits_in_64 ty) (breduce src)))
(value_regs_get_gpr src 0))
;; Rules for `bint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Booleans are stored as all-zeroes (0) or all-ones (-1). We AND out
;; the LSB to give a 0 / 1-valued integer result.
(rule (lower (has_type (fits_in_64 ty)
(bint src)))
(x64_and ty src (RegMemImm.Imm 1)))
(rule 1 (lower (has_type $I128
(bint src)))
(value_regs
(x64_and $I64 src (RegMemImm.Imm 1))
(imm $I64 0)))
;; Rules for `debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (debugtrap))
@@ -2505,7 +2410,7 @@
(x64_movzx (ext_mode (ty_bits_u16 ty) 64) (to_amode flags address offset)))
;; But if we know that both the `from` and `to` are 64 bits, we simply load with
;; no extension.
(rule -1 (lower (has_type (ty_int_bool_ref_64 ty) (load flags address offset)))
(rule -1 (lower (has_type (ty_int_ref_64 ty) (load flags address offset)))
(x64_mov (to_amode flags address offset)))
;; Also, certain scalar loads have a specific `from` width and extension kind
;; (signed -> `sx`, zeroed -> `zx`). We overwrite the high bits of the 64-bit
@@ -2538,8 +2443,8 @@
(rule -2 (lower (has_type (ty_vec128 ty) (load flags address offset)))
(x64_movdqu (to_amode flags address offset)))
;; We can load an I128/B128 by doing two 64-bit loads.
(rule -3 (lower (has_type (ty_int_bool_128 _)
;; We can load an I128 by doing two 64-bit loads.
(rule -3 (lower (has_type $I128
(load flags address offset)))
(let ((addr_lo Amode (to_amode flags address offset))
(addr_hi Amode (amode_offset addr_lo 8))
@@ -2623,9 +2528,9 @@
(side_effect
(x64_xmm_movrm (SseOpcode.Movdqu) (to_amode flags address offset) value)))
;; Stores of I128/B128 values: store the two 64-bit halves separately.
;; Stores of I128 values: store the two 64-bit halves separately.
(rule 0 (lower (store flags
value @ (value_type (ty_int_bool_128 _))
value @ (value_type $I128)
address
offset))
(let ((value_reg ValueRegs value)
@@ -2918,8 +2823,6 @@
(decl cmp_zero_int_bool_ref (Value) ProducesFlags)
(rule 1 (cmp_zero_int_bool_ref val @ (value_type $B1))
(x64_test (OperandSize.Size8) (RegMemImm.Imm 1) val))
(rule (cmp_zero_int_bool_ref val @ (value_type ty))
(let ((size OperandSize (raw_operand_size_of_type ty))
(src Gpr val))

5
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -22,7 +22,6 @@ use target_lexicon::Triple;
fn is_int_or_ref_ty(ty: Type) -> bool {
match ty {
types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => true,
types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
types::R32 => panic!("shouldn't have 32-bits refs on x64"),
_ => false,
}
@@ -328,7 +327,6 @@ fn lower_insn_to_regs(
let op = ctx.data(insn).opcode();
match op {
Opcode::Iconst
| Opcode::Bconst
| Opcode::F32const
| Opcode::F64const
| Opcode::Null
@@ -369,10 +367,7 @@ fn lower_insn_to_regs(
| Opcode::IsInvalid
| Opcode::Uextend
| Opcode::Sextend
| Opcode::Breduce
| Opcode::Bextend
| Opcode::Ireduce
| Opcode::Bint
| Opcode::Debugtrap
| Opcode::WideningPairwiseDotProductS
| Opcode::Fadd

3
cranelift/codegen/src/isa/x64/lower/isle.rs
View File

@@ -549,7 +549,7 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
#[inline]
fn type_register_class(&mut self, ty: Type) -> Option<RegisterClass> {
if is_int_or_ref_ty(ty) || ty == I128 || ty == B128 {
if is_int_or_ref_ty(ty) || ty == I128 {
Some(RegisterClass::Gpr {
single_register: ty != I128,
})
@@ -564,7 +564,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
fn ty_int_bool_or_ref(&mut self, ty: Type) -> Option<()> {
match ty {
types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => Some(()),
types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => Some(()),
types::R32 => panic!("shouldn't have 32-bits refs on x64"),
_ => None,
}