T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x64: port fcmp to ISLE (#3967)

Browse Source 
* x64: port scalar `fcmp` to ISLE

Implement the CLIF lowering for the `fcmp` to ISLE. This adds a new
type-matcher, `ty_scalar_float`, for detecting uses of `F32` and `F64`.

* isle: rename `vec128` to `ty_vec12`

This refactoring changes the name of the `vec128` matcher function to
follow the `ty_*` convention of the other type matchers. It also makes
the helper an inline function call.

* x64: port vector `fcmp` to ISLE
This commit is contained in:
Andrew Brown
2022-03-29 15:41:49 -07:00
committed by GitHub
parent 819b61b661
commit 5d8dd648d7
13 changed files with 746 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -145,22 +145,22 @@
;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (vec128 ty) (uadd_sat x y)))
(rule (lower (has_type (ty_vec128 ty) (uadd_sat x y)))
(uqadd x y (vector_size ty)))
;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (vec128 ty) (sadd_sat x y)))
(rule (lower (has_type (ty_vec128 ty) (sadd_sat x y)))
(sqadd x y (vector_size ty)))
;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (vec128 ty) (usub_sat x y)))
(rule (lower (has_type (ty_vec128 ty) (usub_sat x y)))
(uqsub x y (vector_size ty)))
;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (vec128 ty) (ssub_sat x y)))
(rule (lower (has_type (ty_vec128 ty) (ssub_sat x y)))
(sqsub x y (vector_size ty)))
;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -170,7 +170,7 @@
(sub ty (zero_reg) x))
;; vectors.
(rule (lower (has_type (vec128 ty) (ineg x)))
(rule (lower (has_type (ty_vec128 ty) (ineg x)))
(neg x (vector_size ty)))
;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -208,7 +208,7 @@
(value_regs dst_lo dst_hi)))
;; Case for i8x16, i16x8, and i32x4.
(rule (lower (has_type (vec128 ty @ (not_i64x2)) (imul x y)))
(rule (lower (has_type (ty_vec128 ty @ (not_i64x2)) (imul x y)))
(mul x y (vector_size ty)))
;; Special lowering for i64x2.
@@ -575,7 +575,7 @@
(value_regs new_lo new_hi)))
;; Implementation of `bnot` for vector types.
(rule (lower (has_type (vec128 ty) (bnot x)))
(rule (lower (has_type (ty_vec128 ty) (bnot x)))
(not x (vector_size ty)))
;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -588,7 +588,7 @@
(rule (lower (has_type $I128 (band x y))) (i128_alu_bitop (ALUOp.And) $I64 x y))
(rule (lower (has_type (vec128 ty) (band x y)))
(rule (lower (has_type (ty_vec128 ty) (band x y)))
(and_vec x y (vector_size ty)))
;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -601,7 +601,7 @@
(rule (lower (has_type $I128 (bor x y))) (i128_alu_bitop (ALUOp.Orr) $I64 x y))
(rule (lower (has_type (vec128 ty) (bor x y)))
(rule (lower (has_type (ty_vec128 ty) (bor x y)))
(orr_vec x y (vector_size ty)))
;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -614,7 +614,7 @@
(rule (lower (has_type $I128 (bxor x y))) (i128_alu_bitop (ALUOp.Eor) $I64 x y))
(rule (lower (has_type (vec128 ty) (bxor x y)))
(rule (lower (has_type (ty_vec128 ty) (bxor x y)))
(eor_vec x y (vector_size ty)))
;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -627,7 +627,7 @@
(rule (lower (has_type $I128 (band_not x y))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))
(rule (lower (has_type (vec128 ty) (band_not x y)))
(rule (lower (has_type (ty_vec128 ty) (band_not x y)))
(bic_vec x y (vector_size ty)))
;;;; Rules for `bor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -691,7 +691,7 @@
(csel (Cond.Ne) lo_lshift maybe_hi)))))
;; Shift for vector types.
(rule (lower (has_type (vec128 ty) (ishl x y)))
(rule (lower (has_type (ty_vec128 ty) (ishl x y)))
(let ((size VectorSize (vector_size ty))
(shift Reg (vec_dup y size)))
(sshl x shift size)))
@@ -749,7 +749,7 @@
(lower_ushr128 x (value_regs_get y 0)))
;; Vector shifts.
(rule (lower (has_type (vec128 ty) (ushr x y)))
(rule (lower (has_type (ty_vec128 ty) (ushr x y)))
(let ((size VectorSize (vector_size ty))
(shift Reg (vec_dup (sub $I32 (zero_reg) y) size)))
(ushl x shift size)))
@@ -798,7 +798,7 @@
;; Vector shifts.
;;
;; Note that right shifts are implemented with a negative left shift.
(rule (lower (has_type (vec128 ty) (sshr x y)))
(rule (lower (has_type (ty_vec128 ty) (sshr x y)))
(let ((size VectorSize (vector_size ty))
(shift Reg (vec_dup (sub $I32 (zero_reg) y) size)))
(sshl x shift size)))

4
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle b2bc986bcbbbb77
src/prelude.isle 74d9514ac948e163
src/isa/aarch64/inst.isle 19ccefb6a496d392
src/isa/aarch64/lower.isle 90ead921762336d2
src/isa/aarch64/lower.isle d88b62dd6b40622

33
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

@@ -56,7 +56,8 @@ pub trait Context {
fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
fn vec128(&mut self, arg0: Type) -> Option<Type>;
fn ty_scalar_float(&mut self, arg0: Type) -> Option<Type>;
fn ty_vec128(&mut self, arg0: Type) -> Option<Type>;
fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>;
@@ -129,13 +130,13 @@ pub trait Context {
fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 397.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
/// Internal type ProducesFlags: defined at src/prelude.isle line 419.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -143,7 +144,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 430.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -1085,7 +1086,7 @@ pub fn constructor_side_effect<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 398.
// Rule at src/prelude.isle line 402.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -1103,7 +1104,7 @@ pub fn constructor_safepoint<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 404.
// Rule at src/prelude.isle line 408.
let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -1122,7 +1123,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 442.
// Rule at src/prelude.isle line 446.
return Some(pattern1_1);
}
return None;
@@ -1139,7 +1140,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 447.
// Rule at src/prelude.isle line 451.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1149,7 +1150,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 449.
// Rule at src/prelude.isle line 453.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1178,7 +1179,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 456.
// Rule at src/prelude.isle line 460.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1208,7 +1209,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 481.
// Rule at src/prelude.isle line 485.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1219,7 +1220,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 487.
// Rule at src/prelude.isle line 491.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1232,7 +1233,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 499.
// Rule at src/prelude.isle line 503.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1253,7 +1254,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 475.
// Rule at src/prelude.isle line 479.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1273,7 +1274,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 516.
// Rule at src/prelude.isle line 520.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -6477,7 +6478,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
_ => {}
}
}
if let Some(pattern3_0) = C::vec128(ctx, pattern2_0) {
if let Some(pattern3_0) = C::ty_vec128(ctx, pattern2_0) {
let pattern4_0 = C::inst_data(ctx, pattern0_0);
match &pattern4_0 {
&InstructionData::Binary {

2
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle b2bc986bcbbbb77
src/prelude.isle 74d9514ac948e163
src/isa/s390x/inst.isle d91a16074ab186a8
src/isa/s390x/lower.isle 1cc5a12adc8c75f9

31
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs generated
View File

@@ -56,7 +56,8 @@ pub trait Context {
fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
fn vec128(&mut self, arg0: Type) -> Option<Type>;
fn ty_scalar_float(&mut self, arg0: Type) -> Option<Type>;
fn ty_vec128(&mut self, arg0: Type) -> Option<Type>;
fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
fn value_slice_empty(&mut self, arg0: ValueSlice) -> Option<()>;
@@ -152,13 +153,13 @@ pub trait Context {
fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 397.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
/// Internal type ProducesFlags: defined at src/prelude.isle line 419.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -166,7 +167,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 430.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -956,7 +957,7 @@ pub fn constructor_side_effect<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 398.
// Rule at src/prelude.isle line 402.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -974,7 +975,7 @@ pub fn constructor_safepoint<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 404.
// Rule at src/prelude.isle line 408.
let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -993,7 +994,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 442.
// Rule at src/prelude.isle line 446.
return Some(pattern1_1);
}
return None;
@@ -1010,7 +1011,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 447.
// Rule at src/prelude.isle line 451.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1020,7 +1021,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 449.
// Rule at src/prelude.isle line 453.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1049,7 +1050,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 456.
// Rule at src/prelude.isle line 460.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1079,7 +1080,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 481.
// Rule at src/prelude.isle line 485.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1090,7 +1091,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 487.
// Rule at src/prelude.isle line 491.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1103,7 +1104,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 499.
// Rule at src/prelude.isle line 503.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1124,7 +1125,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 475.
// Rule at src/prelude.isle line 479.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1144,7 +1145,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 516.
// Rule at src/prelude.isle line 520.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

30
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -2092,7 +2092,11 @@
imm
size))
;; Helper for creating `cmpps` instructions.
;; Helpers for creating `cmpp*` instructions.
(decl x64_cmpp (Type Xmm XmmMem FcmpImm) Xmm)
(rule (x64_cmpp $F32X4 x y imm) (x64_cmpps x y imm))
(rule (x64_cmpp $F64X2 x y imm) (x64_cmppd x y imm))
(decl x64_cmpps (Xmm XmmMem FcmpImm) Xmm)
(rule (x64_cmpps src1 src2 imm)
(xmm_rm_r_imm (SseOpcode.Cmpps)
@@ -2101,6 +2105,17 @@
(encode_fcmp_imm imm)
(OperandSize.Size32)))
;; Note that `Size32` is intentional despite this being used for 64-bit
;; operations, since this presumably induces the correct encoding of the
;; instruction.
(decl x64_cmppd (Xmm XmmMem FcmpImm) Xmm)
(rule (x64_cmppd src1 src2 imm)
(xmm_rm_r_imm (SseOpcode.Cmppd)
src1
src2
(encode_fcmp_imm imm)
(OperandSize.Size32)))
;; Helper for creating `pinsrb` instructions.
(decl x64_pinsrb (Xmm GprMem u8) Xmm)
(rule (x64_pinsrb src1 src2 lane)
@@ -2321,19 +2336,6 @@
(operand_size_of_type_32_64 (lane_type ty))))))
dst))
;; Helper for creating `cmppd` instructions.
;;
;; Note that `Size32` is intentional despite this being used for 64-bit
;; operations, since this presumably induces the correct encoding of the
;; instruction.
(decl x64_cmppd (Xmm XmmMem FcmpImm) Xmm)
(rule (x64_cmppd src1 src2 imm)
(xmm_rm_r_imm (SseOpcode.Cmppd)
src1
src2
(encode_fcmp_imm imm)
(OperandSize.Size32)))
;; Helper for creating `MInst.GprToXmm` instructions.
(decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm)
(rule (gpr_to_xmm op src size)

129
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -1454,22 +1454,22 @@
;; lane will be filled with all 1s or all 0s according to the comparison,
;; whereas for GPR-held values, the result will be simply 0 or 1 (upper bits
;; unset).
(rule (lower (icmp (IntCC.Equal) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.Equal) a @ (value_type (ty_vec128 ty)) b))
(x64_pcmpeq ty a b))
;; To lower a not-equals comparison, we perform an equality comparison
;; (PCMPEQ*) and then invert the bits (PXOR with all 1s).
(rule (lower (icmp (IntCC.NotEqual) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.NotEqual) a @ (value_type (ty_vec128 ty)) b))
(let ((checked Xmm (x64_pcmpeq ty a b))
(all_ones Xmm (vector_all_ones ty)))
(x64_pxor checked all_ones)))
;; Signed comparisons have a single-instruction lowering, unlike their unsigned
;; counterparts. These latter instructions use the unsigned min/max
;; (PMINU*/PMAXU*) and negate the result (PXOR with all 1s).
(rule (lower (icmp (IntCC.SignedGreaterThan) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.SignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))
(x64_pcmpgt ty a b))
(rule (lower (icmp (IntCC.SignedLessThan) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.SignedLessThan) a @ (value_type (ty_vec128 ty)) b))
(x64_pcmpgt ty b a))
(rule (lower (icmp (IntCC.UnsignedGreaterThan) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.UnsignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))
;; N.B.: we must manually prevent load coalescing of these operands; the
;; register allocator gets confused otherwise. TODO:
;; https://github.com/bytecodealliance/wasmtime/issues/3953.
@@ -1479,7 +1479,7 @@
(eq Xmm (x64_pcmpeq ty max xmm_b))
(all_ones Xmm (vector_all_ones ty)))
(x64_pxor eq all_ones)))
(rule (lower (icmp (IntCC.UnsignedLessThan) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.UnsignedLessThan) a @ (value_type (ty_vec128 ty)) b))
;; N.B.: see note above.
(let ((xmm_a Xmm (put_in_xmm a))
(xmm_b Xmm (put_in_xmm b))
@@ -1490,16 +1490,16 @@
;; To lower signed and unsigned *-or-equals comparisons, we find the minimum
;; number (PMIN[U|S]*) and compare that to one of the terms (PCMPEQ*). Note that
;; there is no 64x2 version of this lowering (see below).
(rule (lower (icmp (IntCC.SignedGreaterThanOrEqual) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.SignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(let ((max Xmm (x64_pmaxs ty a b)))
(x64_pcmpeq ty a max)))
(rule (lower (icmp (IntCC.SignedLessThanOrEqual) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.SignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(let ((min Xmm (x64_pmins ty a b)))
(x64_pcmpeq ty a min)))
(rule (lower (icmp (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(let ((max Xmm (x64_pmaxu ty a b)))
(x64_pcmpeq ty a max)))
(rule (lower (icmp (IntCC.UnsignedLessThanOrEqual) a @ (value_type (vec128 ty)) b))
(rule (lower (icmp (IntCC.UnsignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(let ((min Xmm (x64_pminu ty a b)))
(x64_pcmpeq ty a min)))
;; The PMIN[S|U]Q instruction is only available in AVX512VL/F so we must instead
@@ -1550,6 +1550,115 @@
(cmp Reg (x64_or $I64 cmp_lo cmp_hi)))
(with_flags (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) cmp) (x64_setcc (CC.NZ)))))
;;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; CLIF's `fcmp` instruction always operates on XMM registers--both scalar and
;; vector. For the scalar versions, we use the flag-setting behavior of the
;; `UCOMIS*` instruction to `SETcc` a 0 or 1 in a GPR register. Note that CLIF's
;; `select` uses the same kind of flag-setting behavior but chooses values other
;; than 0 or 1.
;;
;; Checking the result of `UCOMIS*` is unfortunately difficult in some cases
;; because we do not have `SETcc` instructions that explicitly check
;; simultaneously for the condition (i.e., `eq`, `le`, `gt`, etc.) *and*
;; orderedness. Instead, we must check the flags multiple times. The UCOMIS*
;; documentation (see Intel's Software Developer's Manual, volume 2, chapter 4)
;; is helpful:
;; - unordered assigns Z = 1, P = 1, C = 1
;; - greater than assigns Z = 0, P = 0, C = 0
;; - less than assigns Z = 0, P = 0, C = 1
;; - equal assigns Z = 1, P = 0, C = 0
(rule (lower (fcmp (FloatCC.Equal) a @ (value_type (ty_scalar_float ty)) b))
(let ((maybe ValueRegs (with_flags (x64_ucomis b a)
(consumes_flags_concat
(x64_setcc (CC.NP))
(x64_setcc (CC.Z)))))
(maybe_np Gpr (value_regs_get_gpr maybe 0))
(maybe_z Gpr (value_regs_get_gpr maybe 1)))
(x64_and $I32 maybe_np maybe_z)))
(rule (lower (fcmp (FloatCC.NotEqual) a @ (value_type (ty_scalar_float ty)) b))
(let ((maybe ValueRegs (with_flags (x64_ucomis b a)
(consumes_flags_concat
(x64_setcc (CC.P))
(x64_setcc (CC.NZ)))))
(maybe_p Gpr (value_regs_get_gpr maybe 0))
(maybe_nz Gpr (value_regs_get_gpr maybe 1)))
(x64_or $I32 maybe_p maybe_nz)))
;; Some scalar lowerings correspond to one condition code.
(rule (lower (fcmp (FloatCC.Ordered) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.NP))))
(rule (lower (fcmp (FloatCC.Unordered) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.P))))
(rule (lower (fcmp (FloatCC.OrderedNotEqual) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.NZ))))
(rule (lower (fcmp (FloatCC.UnorderedOrEqual) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.Z))))
(rule (lower (fcmp (FloatCC.GreaterThan) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.NBE))))
(rule (lower (fcmp (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.NB))))
(rule (lower (fcmp (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.B))))
(rule (lower (fcmp (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b))
(with_flags (x64_ucomis b a) (x64_setcc (CC.BE))))
;; Other scalar lowerings are made possible by flipping the operands and
;; reversing the condition code.
(rule (lower (fcmp (FloatCC.LessThan) a @ (value_type (ty_scalar_float ty)) b))
;; Same flags as `GreaterThan`.
(with_flags (x64_ucomis a b) (x64_setcc (CC.NBE))))
(rule (lower (fcmp (FloatCC.LessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b))
;; Same flags as `GreaterThanOrEqual`.
(with_flags (x64_ucomis a b) (x64_setcc (CC.NB))))
(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_scalar_float ty)) b))
;; Same flags as `UnorderedOrLessThan`.
(with_flags (x64_ucomis a b) (x64_setcc (CC.B))))
(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b))
;; Same flags as `UnorderedOrLessThanOrEqual`.
(with_flags (x64_ucomis a b) (x64_setcc (CC.BE))))
;; For vector lowerings, we use `CMPP*` instructions with a 3-bit operand that
;; determines the comparison to make. Note that comparisons that succeed will
;; fill the lane with 1s; comparisons that do not will fill the lane with 0s.
(rule (lower (fcmp (FloatCC.Equal) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.Equal)))
(rule (lower (fcmp (FloatCC.NotEqual) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.NotEqual)))
(rule (lower (fcmp (FloatCC.LessThan) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.LessThan)))
(rule (lower (fcmp (FloatCC.LessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.LessThanOrEqual)))
(rule (lower (fcmp (FloatCC.Ordered) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.Ordered)))
(rule (lower (fcmp (FloatCC.Unordered) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.Unordered)))
(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.UnorderedOrGreaterThan)))
(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty a b (FcmpImm.UnorderedOrGreaterThanOrEqual)))
;; Some vector lowerings rely on flipping the operands and using a reversed
;; comparison code.
(rule (lower (fcmp (FloatCC.GreaterThan) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty b a (FcmpImm.LessThan)))
(rule (lower (fcmp (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty b a (FcmpImm.LessThanOrEqual)))
(rule (lower (fcmp (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty b a (FcmpImm.UnorderedOrGreaterThan)))
(rule (lower (fcmp (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))
(x64_cmpp ty b a (FcmpImm.UnorderedOrGreaterThanOrEqual)))
;; Some vector lowerings are simply not supported for certain codes:
;; - FloatCC::OrderedNotEqual
;; - FloatCC::UnorderedOrEqual
;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; CLIF `select` instructions receive a testable argument (i.e. boolean or

99
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -930,104 +930,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}
Opcode::Fcmp => {
let cond_code = ctx.data(insn).fp_cond_code().unwrap();
let input_ty = ctx.input_ty(insn, 0);
if !input_ty.is_vector() {
// Unordered is returned by setting ZF, PF, CF <- 111
// Greater than by ZF, PF, CF <- 000
// Less than by ZF, PF, CF <- 001
// Equal by ZF, PF, CF <- 100
//
// Checking the result of comiss is somewhat annoying because you don't have setcc
// instructions that explicitly check simultaneously for the condition (i.e. eq, le,
// gt, etc) *and* orderedness.
//
// So that might mean we need more than one setcc check and then a logical "and" or
// "or" to determine both, in some cases. However knowing that if the parity bit is
// set, then the result was considered unordered and knowing that if the parity bit is
// set, then both the ZF and CF flag bits must also be set we can get away with using
// one setcc for most condition codes.
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
match emit_fcmp(ctx, insn, cond_code, FcmpSpec::Normal) {
FcmpCondResult::Condition(cc) => {
ctx.emit(Inst::setcc(cc, dst));
}
FcmpCondResult::AndConditions(cc1, cc2) => {
let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap();
ctx.emit(Inst::setcc(cc1, tmp));
ctx.emit(Inst::setcc(cc2, dst));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size32,
AluRmiROpcode::And,
RegMemImm::reg(tmp.to_reg()),
dst,
));
}
FcmpCondResult::OrConditions(cc1, cc2) => {
let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap();
ctx.emit(Inst::setcc(cc1, tmp));
ctx.emit(Inst::setcc(cc2, dst));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size32,
AluRmiROpcode::Or,
RegMemImm::reg(tmp.to_reg()),
dst,
));
}
FcmpCondResult::InvertedEqualOrConditions(_, _) => unreachable!(),
}
} else {
let op = match input_ty {
types::F32X4 => SseOpcode::Cmpps,
types::F64X2 => SseOpcode::Cmppd,
_ => panic!("Bad input type to fcmp: {}", input_ty),
};
// Since some packed comparisons are not available, some of the condition codes
// must be inverted, with a corresponding `flip` of the operands.
let (imm, flip) = match cond_code {
FloatCC::GreaterThan => (FcmpImm::LessThan, true),
FloatCC::GreaterThanOrEqual => (FcmpImm::LessThanOrEqual, true),
FloatCC::UnorderedOrLessThan => (FcmpImm::UnorderedOrGreaterThan, true),
FloatCC::UnorderedOrLessThanOrEqual => {
(FcmpImm::UnorderedOrGreaterThanOrEqual, true)
}
FloatCC::OrderedNotEqual | FloatCC::UnorderedOrEqual => {
panic!("unsupported float condition code: {}", cond_code)
}
_ => (FcmpImm::from(cond_code), false),
};
// Determine the operands of the comparison, possibly by flipping them.
let (lhs, rhs) = if flip {
(
put_input_in_reg(ctx, inputs[1]),
input_to_reg_mem(ctx, inputs[0]),
)
} else {
(
put_input_in_reg(ctx, inputs[0]),
input_to_reg_mem(ctx, inputs[1]),
)
};
// Move the `lhs` to the same register as `dst`; this may not emit an actual move
// but ensures that the registers are the same to match x86's read-write operand
// encoding.
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
ctx.emit(Inst::gen_move(dst, lhs, input_ty));
// Emit the comparison.
ctx.emit(Inst::xmm_rm_r_imm(
op,
rhs,
dst,
imm.encode(),
OperandSize::Size32,
));
}
implemented_in_isle(ctx);
}
Opcode::FallthroughReturn | Opcode::Return => {

6
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle b2bc986bcbbbb77
src/isa/x64/inst.isle bfb0fb7143d8dc34
src/isa/x64/lower.isle ccaee2b83bdf73e1
src/prelude.isle 74d9514ac948e163
src/isa/x64/inst.isle a002d62dcfce285
src/isa/x64/lower.isle d8facef52a4e2ac6

759
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

11
cranelift/codegen/src/machinst/isle.rs
View File

@@ -253,7 +253,16 @@ macro_rules! isle_prelude_methods {
}
}
fn vec128(&mut self, ty: Type) -> Option<Type> {
#[inline]
fn ty_scalar_float(&mut self, ty: Type) -> Option<Type> {
match ty {
F32 | F64 => Some(ty),
_ => None,
}
}
#[inline]
fn ty_vec128(&mut self, ty: Type) -> Option<Type> {
if ty.is_vector() && ty.bits() == 128 {
Some(ty)
} else {

8
cranelift/codegen/src/prelude.isle
View File

@@ -269,9 +269,13 @@
(decl ty_int_bool_128 (Type) Type)
(extern extractor ty_int_bool_128 ty_int_bool_128)
;; An extractor that only matches scalar floating-point types--F32 or F64.
(decl ty_scalar_float (Type) Type)
(extern extractor ty_scalar_float ty_scalar_float)
;; An extractor that only matches 128-bit vector types.
(decl vec128 (Type) Type)
(extern extractor vec128 vec128)
(decl ty_vec128 (Type) Type)
(extern extractor ty_vec128 ty_vec128)
;; An extractor that matches everything except i64x2
(decl not_i64x2 () Type)

4
cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif
View File

@@ -48,8 +48,8 @@ block0(v0: f64, v1: i64):
; Inst 1: movq %rsp, %rbp
; Inst 2: movsd 0(%rdi), %xmm1
; Inst 3: ucomisd %xmm1, %xmm0
; Inst 4: setnp %dil
; Inst 5: setz %sil
; Inst 4: setnp %sil
; Inst 5: setz %dil
; Inst 6: andl %edi, %esi
; Inst 7: andq $1, %rsi
; Inst 8: ucomisd %xmm0, %xmm1