diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 29bb6a9003..dc00a84cc4 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -848,6 +848,21 @@ (rule (punpcklwd src1 src2) (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2)) +;; Helper for creating `andnps` instructions. +(decl andnps (Reg RegMem) Reg) +(rule (andnps src1 src2) + (xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2)) + +;; Helper for creating `andnpd` instructions. +(decl andnpd (Reg RegMem) Reg) +(rule (andnpd src1 src2) + (xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2)) + +;; Helper for creating `pandn` instructions. +(decl pandn (Reg RegMem) Reg) +(rule (pandn src1 src2) + (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2)) + ;; Helper for creating `MInst.XmmRmRImm` instructions. (decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg) (rule (xmm_rm_r_imm op src1 src2 imm size) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 71a5e1c6c5..ee67b81f11 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -926,3 +926,22 @@ 0x50 (OperandSize.Size32)))) (value_reg (pmuludq x2 (RegMem.Reg y2))))) + +;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Note the flipping of operands below. CLIF specifies +;; +;; band_not(x, y) = and(x, not(y)) +;; +;; while x86 does +;; +;; pandn(x, y) = and(not(x), y) + +(rule (lower (has_type $F32X4 (band_not x y))) + (value_reg (andnps (put_in_reg y) (put_in_reg_mem x)))) + +(rule (lower (has_type $F64X2 (band_not x y))) + (value_reg (andnpd (put_in_reg y) (put_in_reg_mem x)))) + +(rule (lower (has_type (multi_lane _bits _lanes) (band_not x y))) + (value_reg (pandn (put_in_reg y) (put_in_reg_mem x)))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 1a70a043db..71500f7c7d 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1519,7 +1519,8 @@ fn lower_insn_to_regs>( | Opcode::Band | Opcode::Bor | Opcode::Bxor - | Opcode::Imul => { + | Opcode::Imul + | Opcode::BandNot => { unreachable!( "implemented in ISLE: inst = `{}`, type = `{:?}`", ctx.dfg().display_inst(insn), @@ -1527,23 +1528,6 @@ fn lower_insn_to_regs>( ); } - Opcode::BandNot => { - let ty = ty.unwrap(); - debug_assert!(ty.is_vector() && ty.bytes() == 16); - let lhs = input_to_reg_mem(ctx, inputs[0]); - let rhs = put_input_in_reg(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let sse_op = match ty { - types::F32X4 => SseOpcode::Andnps, - types::F64X2 => SseOpcode::Andnpd, - _ => SseOpcode::Pandn, - }; - // Note the flipping of operands: the `rhs` operand is used as the destination instead - // of the `lhs` as in the other bit operations above (e.g. `band`). - ctx.emit(Inst::gen_move(dst, rhs, ty)); - ctx.emit(Inst::xmm_rm_r(sse_op, lhs, dst)); - } - Opcode::Iabs => { let src = input_to_reg_mem(ctx, inputs[0]); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index adeab6f063..da0932027e 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -1198,6 +1198,39 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) return Some(expr2_0); } +// Generated as internal constructor for term andnps. +pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 853. + let expr0_0: Type = F32X4; + let expr1_0 = SseOpcode::Andnps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term andnpd. +pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 858. + let expr0_0: Type = F64X2; + let expr1_0 = SseOpcode::Andnpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pandn. +pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 863. + let expr0_0: Type = F64X2; + let expr1_0 = SseOpcode::Pandn; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + // Generated as internal constructor for term xmm_rm_r_imm. pub fn constructor_xmm_rm_r_imm( ctx: &mut C, @@ -1212,7 +1245,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 853. + // Rule at src/isa/x64/inst.isle line 868. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1240,7 +1273,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 865. + // Rule at src/isa/x64/inst.isle line 880. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1258,7 +1291,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 889. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1283,7 +1316,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 887. + // Rule at src/isa/x64/inst.isle line 902. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1299,7 +1332,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 909. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1308,7 +1341,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 914. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1324,7 +1357,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 919. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -1342,7 +1375,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 916. + // Rule at src/isa/x64/inst.isle line 931. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1358,7 +1391,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 923. + // Rule at src/isa/x64/inst.isle line 938. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -1376,7 +1409,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 933. + // Rule at src/isa/x64/inst.isle line 948. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1386,7 +1419,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 938. + // Rule at src/isa/x64/inst.isle line 953. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1404,7 +1437,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 945. + // Rule at src/isa/x64/inst.isle line 960. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); @@ -1433,7 +1466,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 961. + // Rule at src/isa/x64/inst.isle line 976. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1800,6 +1833,15 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 940. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); + let expr2_0 = constructor_andnps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } _ => {} } } @@ -1839,6 +1881,15 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 943. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); + let expr2_0 = constructor_andnpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } _ => {} } } @@ -3010,6 +3061,15 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 946. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0); + let expr2_0 = constructor_pandn(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } _ => {} } }