diff --git a/build.rs b/build.rs index 889962724c..596f39caf6 100644 --- a/build.rs +++ b/build.rs @@ -192,7 +192,6 @@ fn x64_should_panic(testsuite: &str, testname: &str, strategy: &str) -> bool { match (testsuite, testname) { ("simd", "simd_i16x8_extadd_pairwise_i8x16") => return true, ("simd", "simd_i32x4_extadd_pairwise_i16x8") => return true, - ("simd", "simd_i32x4_trunc_sat_f64x2") => return true, ("simd", "simd_int_to_int_extend") => return true, ("simd", _) => return false, _ => {} diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index ebb80f0d3d..915ddfe347 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -497,6 +497,7 @@ pub enum SseOpcode { Cvtsi2sd, Cvtss2si, Cvtss2sd, + Cvttpd2dq, Cvttps2dq, Cvttss2si, Cvttsd2si, @@ -631,6 +632,7 @@ pub enum SseOpcode { Roundss, Roundsd, Rsqrtss, + Shufps, Sqrtps, Sqrtpd, Sqrtss, @@ -677,6 +679,7 @@ impl SseOpcode { | SseOpcode::Orps | SseOpcode::Rcpss | SseOpcode::Rsqrtss + | SseOpcode::Shufps | SseOpcode::Sqrtps | SseOpcode::Sqrtss | SseOpcode::Subps @@ -700,6 +703,7 @@ impl SseOpcode { | SseOpcode::Cvtsd2si | SseOpcode::Cvtsi2sd | SseOpcode::Cvtss2sd + | SseOpcode::Cvttpd2dq | SseOpcode::Cvttps2dq | SseOpcode::Cvttsd2si | SseOpcode::Divpd @@ -869,6 +873,7 @@ impl fmt::Debug for SseOpcode { SseOpcode::Cvtsi2sd => "cvtsi2sd", SseOpcode::Cvtss2si => "cvtss2si", SseOpcode::Cvtss2sd => "cvtss2sd", + SseOpcode::Cvttpd2dq => "cvttpd2dq", SseOpcode::Cvttps2dq => "cvttps2dq", SseOpcode::Cvttss2si => "cvttss2si", SseOpcode::Cvttsd2si => "cvttsd2si", @@ -1003,6 +1008,7 @@ impl fmt::Debug for SseOpcode { SseOpcode::Roundss => "roundss", SseOpcode::Roundsd => "roundsd", SseOpcode::Rsqrtss => "rsqrtss", + SseOpcode::Shufps => "shufps", SseOpcode::Sqrtps => "sqrtps", SseOpcode::Sqrtpd => "sqrtpd", SseOpcode::Sqrtss => "sqrtss", diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 826a601add..954fd98a5b 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1448,6 +1448,7 @@ pub(crate) fn emit( SseOpcode::Andnpd => (LegacyPrefixes::_66, 0x0F55, 2), SseOpcode::Blendvps => (LegacyPrefixes::_66, 0x0F3814, 3), SseOpcode::Blendvpd => (LegacyPrefixes::_66, 0x0F3815, 3), + SseOpcode::Cvttpd2dq => (LegacyPrefixes::_66, 0x0FE6, 2), SseOpcode::Cvttps2dq => (LegacyPrefixes::_F3, 0x0F5B, 2), SseOpcode::Cvtdq2ps => (LegacyPrefixes::None, 0x0F5B, 2), SseOpcode::Divps => (LegacyPrefixes::None, 0x0F5E, 2), @@ -1699,6 +1700,7 @@ pub(crate) fn emit( SseOpcode::Roundss => (LegacyPrefixes::_66, 0x0F3A0A, 3), SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3), SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3), + SseOpcode::Shufps => (LegacyPrefixes::None, 0x0FC6, 2), _ => unimplemented!("Opcode {:?} not implemented", op), }; let rex = RexFlags::from(*size); diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 26c97af019..c02593eab2 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3761,6 +3761,12 @@ fn test_x64_emit() { "cvtdq2ps %xmm1, %xmm8", )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Cvttpd2dq, RegMem::reg(xmm15), w_xmm7), + "66410FE6FF", + "cvttpd2dq %xmm15, %xmm7", + )); + insns.push(( Inst::xmm_rm_r(SseOpcode::Cvttps2dq, RegMem::reg(xmm9), w_xmm8), "F3450F5BC1", @@ -4125,6 +4131,18 @@ fn test_x64_emit() { "palignr $3, %xmm1, %xmm9", )); + insns.push(( + Inst::xmm_rm_r_imm( + SseOpcode::Shufps, + RegMem::reg(xmm1), + w_xmm10, + 136, + OperandSize::Size32, + ), + "440FC6D188", + "shufps $136, %xmm1, %xmm10", + )); + insns.push(( Inst::xmm_rm_r_imm( SseOpcode::Roundps, diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 2b2c450a65..121bad94b0 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -5014,28 +5014,91 @@ fn lower_insn_to_regs>( Opcode::Snarrow | Opcode::Unarrow => { let input_ty = ctx.input_ty(insn, 0); let output_ty = ctx.output_ty(insn, 0); - let src1 = put_input_in_reg(ctx, inputs[0]); - let src2 = put_input_in_reg(ctx, inputs[1]); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); if output_ty.is_vector() { match op { Opcode::Snarrow => match (input_ty, output_ty) { (types::I16X8, types::I8X16) => { + let src1 = put_input_in_reg(ctx, inputs[0]); + let src2 = put_input_in_reg(ctx, inputs[1]); ctx.emit(Inst::gen_move(dst, src1, input_ty)); ctx.emit(Inst::xmm_rm_r(SseOpcode::Packsswb, RegMem::reg(src2), dst)); } (types::I32X4, types::I16X8) => { + let src1 = put_input_in_reg(ctx, inputs[0]); + let src2 = put_input_in_reg(ctx, inputs[1]); ctx.emit(Inst::gen_move(dst, src1, input_ty)); ctx.emit(Inst::xmm_rm_r(SseOpcode::Packssdw, RegMem::reg(src2), dst)); } + // TODO: The type we are expecting as input as actually an F64X2 but the instruction is only defined + // for integers so here we use I64X2. This is a separate issue that needs to be fixed in instruction.rs. + (types::I64X2, types::I32X4) => { + if let Some(fcvt_inst) = + matches_input(ctx, inputs[0], Opcode::FcvtToSintSat) + { + //y = i32x4.trunc_sat_f64x2_s_zero(x) is lowered to: + //MOVE xmm_tmp, xmm_x + //CMPEQPD xmm_tmp, xmm_x + //MOVE xmm_y, xmm_x + //ANDPS xmm_tmp, [wasm_f64x2_splat(2147483647.0)] + //MINPD xmm_y, xmm_tmp + //CVTTPD2DQ xmm_y, xmm_y + + let fcvt_input = InsnInput { + insn: fcvt_inst, + input: 0, + }; + let src = put_input_in_reg(ctx, fcvt_input); + ctx.emit(Inst::gen_move(dst, src, input_ty)); + let tmp1 = ctx.alloc_tmp(output_ty).only_reg().unwrap(); + ctx.emit(Inst::gen_move(tmp1, src, input_ty)); + let cond = FcmpImm::from(FloatCC::Equal); + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Cmppd, + RegMem::reg(src), + tmp1, + cond.encode(), + OperandSize::Size32, + )); + + // 2147483647.0 is equivalent to 0x41DFFFFFFFC00000 + static UMAX_MASK: [u8; 16] = [ + 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xDF, 0x41, 0x00, 0x00, + 0xC0, 0xFF, 0xFF, 0xFF, 0xDF, 0x41, + ]; + let umax_const = + ctx.use_constant(VCodeConstantData::WellKnown(&UMAX_MASK)); + let umax_mask = ctx.alloc_tmp(types::F64X2).only_reg().unwrap(); + ctx.emit(Inst::xmm_load_const(umax_const, umax_mask, types::F64X2)); + + //ANDPD xmm_y, [wasm_f64x2_splat(2147483647.0)] + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Andps, + RegMem::from(umax_mask), + tmp1, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Minpd, RegMem::from(tmp1), dst)); + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Cvttpd2dq, + RegMem::from(dst), + dst, + )); + } else { + unreachable!(); + } + } _ => unreachable!(), }, Opcode::Unarrow => match (input_ty, output_ty) { (types::I16X8, types::I8X16) => { + let src1 = put_input_in_reg(ctx, inputs[0]); + let src2 = put_input_in_reg(ctx, inputs[1]); ctx.emit(Inst::gen_move(dst, src1, input_ty)); ctx.emit(Inst::xmm_rm_r(SseOpcode::Packuswb, RegMem::reg(src2), dst)); } (types::I32X4, types::I16X8) => { + let src1 = put_input_in_reg(ctx, inputs[0]); + let src2 = put_input_in_reg(ctx, inputs[1]); ctx.emit(Inst::gen_move(dst, src1, input_ty)); ctx.emit(Inst::xmm_rm_r(SseOpcode::Packusdw, RegMem::reg(src2), dst)); } @@ -6442,6 +6505,84 @@ fn lower_insn_to_regs>( )); } + Opcode::Uunarrow => { + if let Some(fcvt_inst) = matches_input(ctx, inputs[0], Opcode::FcvtToUintSat) { + //y = i32x4.trunc_sat_f64x2_u_zero(x) is lowered to: + //MOVAPD xmm_y, xmm_x + //XORPD xmm_tmp, xmm_tmp + //MAXPD xmm_y, xmm_tmp + //MINPD xmm_y, [wasm_f64x2_splat(4294967295.0)] + //ROUNDPD xmm_y, xmm_y, 0x0B + //ADDPD xmm_y, [wasm_f64x2_splat(0x1.0p+52)] + //SHUFPS xmm_y, xmm_xmp, 0x88 + + let fcvt_input = InsnInput { + insn: fcvt_inst, + input: 0, + }; + let input_ty = ctx.input_ty(fcvt_inst, 0); + let output_ty = ctx.output_ty(insn, 0); + let src = put_input_in_reg(ctx, fcvt_input); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + + ctx.emit(Inst::gen_move(dst, src, input_ty)); + let tmp1 = ctx.alloc_tmp(output_ty).only_reg().unwrap(); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Xorpd, RegMem::from(tmp1), tmp1)); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Maxpd, RegMem::from(tmp1), dst)); + + // 4294967295.0 is equivalent to 0x41EFFFFFFFE00000 + static UMAX_MASK: [u8; 16] = [ + 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xEF, 0x41, 0x00, 0x00, 0xE0, 0xFF, 0xFF, + 0xFF, 0xEF, 0x41, + ]; + let umax_const = ctx.use_constant(VCodeConstantData::WellKnown(&UMAX_MASK)); + let umax_mask = ctx.alloc_tmp(types::F64X2).only_reg().unwrap(); + ctx.emit(Inst::xmm_load_const(umax_const, umax_mask, types::F64X2)); + + //MINPD xmm_y, [wasm_f64x2_splat(4294967295.0)] + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Minpd, + RegMem::from(umax_mask), + dst, + )); + //ROUNDPD xmm_y, xmm_y, 0x0B + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Roundpd, + RegMem::reg(dst.to_reg()), + dst, + RoundImm::RoundZero.encode(), + OperandSize::Size32, + )); + //ADDPD xmm_y, [wasm_f64x2_splat(0x1.0p+52)] + static UINT_MASK: [u8; 16] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x43, + ]; + let uint_mask_const = ctx.use_constant(VCodeConstantData::WellKnown(&UINT_MASK)); + let uint_mask = ctx.alloc_tmp(types::F64X2).only_reg().unwrap(); + ctx.emit(Inst::xmm_load_const( + uint_mask_const, + uint_mask, + types::F64X2, + )); + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Addpd, + RegMem::from(uint_mask), + dst, + )); + + //SHUFPS xmm_y, xmm_xmp, 0x88 + ctx.emit(Inst::xmm_rm_r_imm( + SseOpcode::Shufps, + RegMem::reg(tmp1.to_reg()), + dst, + 0x88, + OperandSize::Size32, + )); + } else { + println!("Did not match fcvt input!"); + } + } // Unimplemented opcodes below. These are not currently used by Wasm // lowering or other known embeddings, but should be either supported or // removed eventually. @@ -6472,10 +6613,6 @@ fn lower_insn_to_regs>( unimplemented!("Vector split/concat ops not implemented."); } - Opcode::Uunarrow => { - unimplemented!("unimplemented lowering for opcode {:?}", op); - } - // Opcodes that should be removed by legalization. These should // eventually be removed if/when we replace in-situ legalization with // something better.