diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 93676d7e2b..c523f35dcf 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -1861,6 +1861,86 @@ (rule (pandn src1 src2) (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2)) +;; Helper for creating `addss` instructions. +(decl addss (Xmm XmmMem) Xmm) +(rule (addss src1 src2) + (xmm_rm_r $F32 (SseOpcode.Addss) src1 src2)) + +;; Helper for creating `addsd` instructions. +(decl addsd (Xmm XmmMem) Xmm) +(rule (addsd src1 src2) + (xmm_rm_r $F64 (SseOpcode.Addsd) src1 src2)) + +;; Helper for creating `addps` instructions. +(decl addps (Xmm XmmMem) Xmm) +(rule (addps src1 src2) + (xmm_rm_r $F32 (SseOpcode.Addps) src1 src2)) + +;; Helper for creating `addpd` instructions. +(decl addpd (Xmm XmmMem) Xmm) +(rule (addpd src1 src2) + (xmm_rm_r $F32 (SseOpcode.Addpd) src1 src2)) + +;; Helper for creating `subss` instructions. +(decl subss (Xmm XmmMem) Xmm) +(rule (subss src1 src2) + (xmm_rm_r $F32 (SseOpcode.Subss) src1 src2)) + +;; Helper for creating `subsd` instructions. +(decl subsd (Xmm XmmMem) Xmm) +(rule (subsd src1 src2) + (xmm_rm_r $F64 (SseOpcode.Subsd) src1 src2)) + +;; Helper for creating `subps` instructions. +(decl subps (Xmm XmmMem) Xmm) +(rule (subps src1 src2) + (xmm_rm_r $F32 (SseOpcode.Subps) src1 src2)) + +;; Helper for creating `subpd` instructions. +(decl subpd (Xmm XmmMem) Xmm) +(rule (subpd src1 src2) + (xmm_rm_r $F32 (SseOpcode.Subpd) src1 src2)) + +;; Helper for creating `mulss` instructions. +(decl mulss (Xmm XmmMem) Xmm) +(rule (mulss src1 src2) + (xmm_rm_r $F32 (SseOpcode.Mulss) src1 src2)) + +;; Helper for creating `mulsd` instructions. +(decl mulsd (Xmm XmmMem) Xmm) +(rule (mulsd src1 src2) + (xmm_rm_r $F64 (SseOpcode.Mulsd) src1 src2)) + +;; Helper for creating `mulps` instructions. +(decl mulps (Xmm XmmMem) Xmm) +(rule (mulps src1 src2) + (xmm_rm_r $F32 (SseOpcode.Mulps) src1 src2)) + +;; Helper for creating `mulpd` instructions. +(decl mulpd (Xmm XmmMem) Xmm) +(rule (mulpd src1 src2) + (xmm_rm_r $F32 (SseOpcode.Mulpd) src1 src2)) + +;; Helper for creating `divss` instructions. +(decl divss (Xmm XmmMem) Xmm) +(rule (divss src1 src2) + (xmm_rm_r $F32 (SseOpcode.Divss) src1 src2)) + +;; Helper for creating `divsd` instructions. +(decl divsd (Xmm XmmMem) Xmm) +(rule (divsd src1 src2) + (xmm_rm_r $F64 (SseOpcode.Divsd) src1 src2)) + +;; Helper for creating `divps` instructions. +(decl divps (Xmm XmmMem) Xmm) +(rule (divps src1 src2) + (xmm_rm_r $F32 (SseOpcode.Divps) src1 src2)) + +;; Helper for creating `divpd` instructions. +(decl divpd (Xmm XmmMem) Xmm) +(rule (divpd src1 src2) + (xmm_rm_r $F32 (SseOpcode.Divpd) src1 src2)) + (decl sse_blend_op (Type) SseOpcode) (rule (sse_blend_op $F32X4) (SseOpcode.Blendvps)) (rule (sse_blend_op $F64X2) (SseOpcode.Blendvpd)) @@ -2041,6 +2121,16 @@ lane size)) +;; Helper for creating `pmaddwd` instructions. +(decl pmaddwd (Xmm XmmMem) Xmm) +(rule (pmaddwd src1 src2) + (let ((dst WritableXmm (temp_writable_xmm)) + (_ Unit (emit (MInst.XmmRmR (SseOpcode.Pmaddwd) + src1 + src2 + dst)))) + dst)) + ;; Helper for creating `insertps` instructions. (decl insertps (Xmm XmmMem u8) Xmm) (rule (insertps src1 src2 lane) @@ -2271,6 +2361,11 @@ (rule (ud2 code) (SideEffectNoResult.Inst (MInst.Ud2 code))) +;; Helper for creating `hlt` instructions. +(decl hlt () SideEffectNoResult) +(rule (hlt) + (SideEffectNoResult.Inst (MInst.Hlt))) + ;; Helper for creating `lzcnt` instructions. (decl lzcnt (Type Gpr) Gpr) (rule (lzcnt ty src) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 83679d5854..6c60b52597 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1986,3 +1986,76 @@ (rule (lower (has_type (fits_in_64 ty) (breduce src))) (value_regs_get_gpr src 0)) + +;; Rules for `bint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Booleans are stored as all-zeroes (0) or all-ones (-1). We AND out +;; the LSB to give a 0 / 1-valued integer result. + +(rule (lower (has_type (fits_in_64 ty) + (bint src))) + (x64_and ty src (RegMemImm.Imm 1))) +(rule (lower (has_type $I128 + (bint src))) + (value_regs + (x64_and $I64 src (RegMemImm.Imm 1)) + (imm $I64 0))) + +;; Rules for `debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (debugtrap)) + (side_effect (hlt))) + +;; Rules for `widening_pairwise_dot_product_s` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type $I32X4 + (widening_pairwise_dot_product_s x y))) + (pmaddwd x y)) + +;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; N.B.: there are no load-op merging rules here. We can't guarantee +;; the RHS (if a load) is 128-bit aligned, so we must avoid merging a +;; load. Likewise for other ops below. + +(rule (lower (has_type $F32 (fadd x y))) + (addss x y)) +(rule (lower (has_type $F64 (fadd x y))) + (addsd x y)) +(rule (lower (has_type $F32X4 (fadd x y))) + (addps x y)) +(rule (lower (has_type $F64X2 (fadd x y))) + (addpd x y)) + +;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type $F32 (fsub x y))) + (subss x y)) +(rule (lower (has_type $F64 (fsub x y))) + (subsd x y)) +(rule (lower (has_type $F32X4 (fsub x y))) + (subps x y)) +(rule (lower (has_type $F64X2 (fsub x y))) + (subpd x y)) + +;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type $F32 (fmul x y))) + (mulss x y)) +(rule (lower (has_type $F64 (fmul x y))) + (mulsd x y)) +(rule (lower (has_type $F32X4 (fmul x y))) + (mulps x y)) +(rule (lower (has_type $F64X2 (fmul x y))) + (mulpd x y)) + +;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type $F32 (fdiv x y))) + (divss x y)) +(rule (lower (has_type $F64 (fdiv x y))) + (divsd x y)) +(rule (lower (has_type $F32X4 (fdiv x y))) + (divps x y)) +(rule (lower (has_type $F64X2 (fdiv x y))) + (divpd x y)) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 71aa48fff0..0095ed0dc1 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -903,33 +903,14 @@ fn lower_insn_to_regs>( | Opcode::Sextend | Opcode::Breduce | Opcode::Bextend - | Opcode::Ireduce => implemented_in_isle(ctx), - - Opcode::Bint => { - // Booleans are stored as all-zeroes (0) or all-ones (-1). We AND - // out the LSB to give a 0 / 1-valued integer result. - let rn = put_input_in_reg(ctx, inputs[0]); - let rd = get_output_reg(ctx, outputs[0]); - let ty = ctx.output_ty(insn, 0); - - ctx.emit(Inst::gen_move(rd.regs()[0], rn, types::I64)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::imm(1), - rd.regs()[0], - )); - - if ty == types::I128 { - let upper = rd.regs()[1]; - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(upper.to_reg()), - upper, - )); - } - } + | Opcode::Ireduce + | Opcode::Bint + | Opcode::Debugtrap + | Opcode::WideningPairwiseDotProductS + | Opcode::Fadd + | Opcode::Fsub + | Opcode::Fmul + | Opcode::Fdiv => implemented_in_isle(ctx), Opcode::Icmp => { let condcode = ctx.data(insn).cond_code().unwrap(); @@ -1240,10 +1221,6 @@ fn lower_insn_to_regs>( abi.emit_stack_post_adjust(ctx); } - Opcode::Debugtrap => { - ctx.emit(Inst::Hlt); - } - Opcode::Trapif | Opcode::Trapff => { let trap_code = ctx.data(insn).trap_code().unwrap(); @@ -1301,77 +1278,6 @@ fn lower_insn_to_regs>( }; } - Opcode::WideningPairwiseDotProductS => { - let lhs = put_input_in_reg(ctx, inputs[0]); - let rhs = input_to_reg_mem(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let ty = ty.unwrap(); - - ctx.emit(Inst::gen_move(dst, lhs, ty)); - - if ty == types::I32X4 { - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmaddwd, rhs, dst)); - } else { - panic!( - "Opcode::WideningPairwiseDotProductS: unsupported laneage: {:?}", - ty - ); - } - } - - Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => { - let lhs = put_input_in_reg(ctx, inputs[0]); - // We can't guarantee the RHS (if a load) is 128-bit aligned, so we - // must avoid merging a load here. - let rhs = RegMem::reg(put_input_in_reg(ctx, inputs[1])); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let ty = ty.unwrap(); - - // Move the `lhs` to the same register as `dst`; this may not emit an actual move - // but ensures that the registers are the same to match x86's read-write operand - // encoding. - ctx.emit(Inst::gen_move(dst, lhs, ty)); - - // Note: min and max can't be handled here, because of the way Cranelift defines them: - // if any operand is a NaN, they must return the NaN operand, while the x86 machine - // instruction will return the second operand if either operand is a NaN. - let sse_op = match ty { - types::F32 => match op { - Opcode::Fadd => SseOpcode::Addss, - Opcode::Fsub => SseOpcode::Subss, - Opcode::Fmul => SseOpcode::Mulss, - Opcode::Fdiv => SseOpcode::Divss, - _ => unreachable!(), - }, - types::F64 => match op { - Opcode::Fadd => SseOpcode::Addsd, - Opcode::Fsub => SseOpcode::Subsd, - Opcode::Fmul => SseOpcode::Mulsd, - Opcode::Fdiv => SseOpcode::Divsd, - _ => unreachable!(), - }, - types::F32X4 => match op { - Opcode::Fadd => SseOpcode::Addps, - Opcode::Fsub => SseOpcode::Subps, - Opcode::Fmul => SseOpcode::Mulps, - Opcode::Fdiv => SseOpcode::Divps, - _ => unreachable!(), - }, - types::F64X2 => match op { - Opcode::Fadd => SseOpcode::Addpd, - Opcode::Fsub => SseOpcode::Subpd, - Opcode::Fmul => SseOpcode::Mulpd, - Opcode::Fdiv => SseOpcode::Divpd, - _ => unreachable!(), - }, - _ => panic!( - "invalid type: expected one of [F32, F64, F32X4, F64X2], found {}", - ty - ), - }; - ctx.emit(Inst::xmm_rm_r(sse_op, rhs, dst)); - } - Opcode::Fmin | Opcode::Fmax => { let lhs = put_input_in_reg(ctx, inputs[0]); let rhs = put_input_in_reg(ctx, inputs[1]); diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 6fc00080cf..42e4b978a1 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 src/prelude.isle b2bc986bcbbbb77 -src/isa/x64/inst.isle 67eb719e568c2a81 -src/isa/x64/lower.isle 2d06b233fb3a1e1c +src/isa/x64/inst.isle 9a8a3babd8257100 +src/isa/x64/lower.isle f0f4af691241209e diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index d2bfa465b2..8132cea9ea 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -2698,21 +2698,197 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O return Some(expr2_0); } +// Generated as internal constructor for term addss. +pub fn constructor_addss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1866. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Addss; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term addsd. +pub fn constructor_addsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1871. + let expr0_0: Type = F64; + let expr1_0 = SseOpcode::Addsd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term addps. +pub fn constructor_addps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1876. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Addps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term addpd. +pub fn constructor_addpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1881. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Addpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term subss. +pub fn constructor_subss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1886. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Subss; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term subsd. +pub fn constructor_subsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1891. + let expr0_0: Type = F64; + let expr1_0 = SseOpcode::Subsd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term subps. +pub fn constructor_subps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1896. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Subps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term subpd. +pub fn constructor_subpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1901. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Subpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term mulss. +pub fn constructor_mulss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1906. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Mulss; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term mulsd. +pub fn constructor_mulsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1911. + let expr0_0: Type = F64; + let expr1_0 = SseOpcode::Mulsd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term mulps. +pub fn constructor_mulps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1916. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Mulps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term mulpd. +pub fn constructor_mulpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1921. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Mulpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term divss. +pub fn constructor_divss(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1926. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Divss; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term divsd. +pub fn constructor_divsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1931. + let expr0_0: Type = F64; + let expr1_0 = SseOpcode::Divsd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term divps. +pub fn constructor_divps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1936. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Divps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term divpd. +pub fn constructor_divpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1941. + let expr0_0: Type = F32; + let expr1_0 = SseOpcode::Divpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + // Generated as internal constructor for term sse_blend_op. pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1865. + // Rule at src/isa/x64/inst.isle line 1945. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1866. + // Rule at src/isa/x64/inst.isle line 1946. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1867. + // Rule at src/isa/x64/inst.isle line 1947. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -2723,17 +2899,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1870. + // Rule at src/isa/x64/inst.isle line 1950. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1871. + // Rule at src/isa/x64/inst.isle line 1951. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1872. + // Rule at src/isa/x64/inst.isle line 1952. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -2752,7 +2928,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1876. + // Rule at src/isa/x64/inst.isle line 1956. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -2776,7 +2952,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1890. + // Rule at src/isa/x64/inst.isle line 1970. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); @@ -2796,7 +2972,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1904. + // Rule at src/isa/x64/inst.isle line 1984. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2807,7 +2983,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1909. + // Rule at src/isa/x64/inst.isle line 1989. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2818,7 +2994,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1914. + // Rule at src/isa/x64/inst.isle line 1994. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2829,7 +3005,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1919. + // Rule at src/isa/x64/inst.isle line 1999. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2840,7 +3016,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1924. + // Rule at src/isa/x64/inst.isle line 2004. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2851,7 +3027,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1929. + // Rule at src/isa/x64/inst.isle line 2009. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2862,7 +3038,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1934. + // Rule at src/isa/x64/inst.isle line 2014. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2873,7 +3049,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1939. + // Rule at src/isa/x64/inst.isle line 2019. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2884,7 +3060,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1944. + // Rule at src/isa/x64/inst.isle line 2024. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2895,7 +3071,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1949. + // Rule at src/isa/x64/inst.isle line 2029. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2906,7 +3082,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1954. + // Rule at src/isa/x64/inst.isle line 2034. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2917,7 +3093,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1959. + // Rule at src/isa/x64/inst.isle line 2039. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2928,7 +3104,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1964. + // Rule at src/isa/x64/inst.isle line 2044. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2939,7 +3115,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1969. + // Rule at src/isa/x64/inst.isle line 2049. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2950,7 +3126,7 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1974. + // Rule at src/isa/x64/inst.isle line 2054. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2961,7 +3137,7 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1979. + // Rule at src/isa/x64/inst.isle line 2059. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2972,7 +3148,7 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1984. + // Rule at src/isa/x64/inst.isle line 2064. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2993,7 +3169,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1989. + // Rule at src/isa/x64/inst.isle line 2069. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -3021,7 +3197,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2001. + // Rule at src/isa/x64/inst.isle line 2081. let expr0_0 = SseOpcode::Palignr; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3040,7 +3216,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2010. + // Rule at src/isa/x64/inst.isle line 2090. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3060,7 +3236,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2019. + // Rule at src/isa/x64/inst.isle line 2099. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3079,7 +3255,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2028. + // Rule at src/isa/x64/inst.isle line 2108. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3100,7 +3276,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2037. + // Rule at src/isa/x64/inst.isle line 2117. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -3109,6 +3285,24 @@ pub fn constructor_pinsrd( return Some(expr3_0); } +// Generated as internal constructor for term pmaddwd. +pub fn constructor_pmaddwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 2126. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = SseOpcode::Pmaddwd; + let expr2_0 = MInst::XmmRmR { + op: expr1_0, + src1: pattern0_0, + src2: pattern1_0.clone(), + dst: expr0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr4_0); +} + // Generated as internal constructor for term insertps. pub fn constructor_insertps( ctx: &mut C, @@ -3119,7 +3313,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2046. + // Rule at src/isa/x64/inst.isle line 2136. let expr0_0 = SseOpcode::Insertps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3138,7 +3332,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2055. + // Rule at src/isa/x64/inst.isle line 2145. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pshufd; let expr2_0 = constructor_writable_xmm_to_r_reg(ctx, expr0_0)?; @@ -3161,7 +3355,7 @@ pub fn constructor_pshufd( pub fn constructor_pshufb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2067. + // Rule at src/isa/x64/inst.isle line 2157. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pshufb; let expr2_0 = MInst::XmmRmR { @@ -3183,7 +3377,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2077. + // Rule at src/isa/x64/inst.isle line 2167. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), @@ -3198,7 +3392,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2084. + // Rule at src/isa/x64/inst.isle line 2174. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3207,7 +3401,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2089. + // Rule at src/isa/x64/inst.isle line 2179. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3216,7 +3410,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2094. + // Rule at src/isa/x64/inst.isle line 2184. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3225,7 +3419,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2099. + // Rule at src/isa/x64/inst.isle line 2189. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3234,7 +3428,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2104. + // Rule at src/isa/x64/inst.isle line 2194. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3248,7 +3442,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2109. + // Rule at src/isa/x64/inst.isle line 2199. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), @@ -3263,7 +3457,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2116. + // Rule at src/isa/x64/inst.isle line 2206. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3272,7 +3466,7 @@ pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option // Generated as internal constructor for term vpopcntb. pub fn constructor_vpopcntb(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2121. + // Rule at src/isa/x64/inst.isle line 2211. let expr0_0 = Avx512Opcode::Vpopcntb; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3288,7 +3482,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2126. + // Rule at src/isa/x64/inst.isle line 2216. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), @@ -3305,7 +3499,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2138. + // Rule at src/isa/x64/inst.isle line 2228. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3323,7 +3517,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2147. + // Rule at src/isa/x64/inst.isle line 2237. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::temp_writable_gpr(ctx); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -3352,7 +3546,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2162. + // Rule at src/isa/x64/inst.isle line 2252. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3368,7 +3562,7 @@ pub fn constructor_xmm_rmi_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2167. + // Rule at src/isa/x64/inst.isle line 2257. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmiReg { opcode: pattern0_0.clone(), @@ -3385,7 +3579,7 @@ pub fn constructor_xmm_rmi_xmm( pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2177. + // Rule at src/isa/x64/inst.isle line 2267. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3395,7 +3589,7 @@ pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2182. + // Rule at src/isa/x64/inst.isle line 2272. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3405,7 +3599,7 @@ pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2187. + // Rule at src/isa/x64/inst.isle line 2277. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3415,7 +3609,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2192. + // Rule at src/isa/x64/inst.isle line 2282. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3425,7 +3619,7 @@ pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2197. + // Rule at src/isa/x64/inst.isle line 2287. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3435,7 +3629,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2202. + // Rule at src/isa/x64/inst.isle line 2292. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3445,7 +3639,7 @@ pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2207. + // Rule at src/isa/x64/inst.isle line 2297. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3455,7 +3649,7 @@ pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2212. + // Rule at src/isa/x64/inst.isle line 2302. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3466,7 +3660,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Xmm, arg2: let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2217. + // Rule at src/isa/x64/inst.isle line 2307. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = SseOpcode::Pextrd; let expr2_0 = constructor_writable_gpr_to_r_reg(ctx, expr0_0)?; @@ -3499,7 +3693,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2233. + // Rule at src/isa/x64/inst.isle line 2323. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3519,7 +3713,7 @@ pub fn constructor_gpr_to_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2242. + // Rule at src/isa/x64/inst.isle line 2332. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::GprToXmm { op: pattern0_0.clone(), @@ -3536,7 +3730,7 @@ pub fn constructor_gpr_to_xmm( pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2249. + // Rule at src/isa/x64/inst.isle line 2339. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -3553,7 +3747,7 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2257. + // Rule at src/isa/x64/inst.isle line 2347. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { @@ -3569,7 +3763,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option // Generated as internal constructor for term lea. pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2264. + // Rule at src/isa/x64/inst.isle line 2354. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), @@ -3583,7 +3777,7 @@ pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option // Generated as internal constructor for term ud2. pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2271. + // Rule at src/isa/x64/inst.isle line 2361. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -3591,11 +3785,19 @@ pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option(ctx: &mut C) -> Option { + // Rule at src/isa/x64/inst.isle line 2366. + let expr0_0 = MInst::Hlt; + let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 }; + return Some(expr1_0); +} + // Generated as internal constructor for term lzcnt. pub fn constructor_lzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2276. + // Rule at src/isa/x64/inst.isle line 2371. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Lzcnt; @@ -3615,7 +3817,7 @@ pub fn constructor_lzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Opti pub fn constructor_tzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2284. + // Rule at src/isa/x64/inst.isle line 2379. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Tzcnt; @@ -3635,7 +3837,7 @@ pub fn constructor_tzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Opti pub fn constructor_bsr(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2292. + // Rule at src/isa/x64/inst.isle line 2387. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Bsr; @@ -3664,7 +3866,7 @@ pub fn constructor_bsr_or_else( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2301. + // Rule at src/isa/x64/inst.isle line 2396. let expr0_0 = constructor_bsr(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_produces_flags_get_reg(ctx, &expr0_0)?; let expr2_0 = C::gpr_new(ctx, expr1_0); @@ -3681,7 +3883,7 @@ pub fn constructor_bsr_or_else( pub fn constructor_bsf(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2312. + // Rule at src/isa/x64/inst.isle line 2407. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Bsf; @@ -3710,7 +3912,7 @@ pub fn constructor_bsf_or_else( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2321. + // Rule at src/isa/x64/inst.isle line 2416. let expr0_0 = constructor_bsf(ctx, pattern0_0, pattern1_0)?; let expr1_0 = constructor_produces_flags_get_reg(ctx, &expr0_0)?; let expr2_0 = C::gpr_new(ctx, expr1_0); @@ -3727,7 +3929,7 @@ pub fn constructor_bsf_or_else( pub fn constructor_x64_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2332. + // Rule at src/isa/x64/inst.isle line 2427. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = UnaryRmROpcode::Popcnt; @@ -3746,7 +3948,7 @@ pub fn constructor_x64_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> // Generated as internal constructor for term reg_to_xmm_mem. pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2388. + // Rule at src/isa/x64/inst.isle line 2483. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -3755,7 +3957,7 @@ pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option< // Generated as internal constructor for term xmm_to_reg_mem. pub fn constructor_xmm_to_reg_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2391. + // Rule at src/isa/x64/inst.isle line 2486. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_reg(ctx, expr0_0); let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -3769,7 +3971,7 @@ pub fn constructor_writable_gpr_to_r_reg( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2395. + // Rule at src/isa/x64/inst.isle line 2490. let expr0_0 = C::writable_gpr_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -3781,7 +3983,7 @@ pub fn constructor_writable_gpr_to_gpr_mem( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2398. + // Rule at src/isa/x64/inst.isle line 2493. let expr0_0 = C::writable_gpr_to_gpr(ctx, pattern0_0); let expr1_0 = C::gpr_to_gpr_mem(ctx, expr0_0); return Some(expr1_0); @@ -3793,7 +3995,7 @@ pub fn constructor_writable_gpr_to_value_regs( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2401. + // Rule at src/isa/x64/inst.isle line 2496. let expr0_0 = constructor_writable_gpr_to_r_reg(ctx, pattern0_0)?; let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); @@ -3805,7 +4007,7 @@ pub fn constructor_writable_xmm_to_r_reg( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2404. + // Rule at src/isa/x64/inst.isle line 2499. let expr0_0 = C::writable_xmm_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -3817,7 +4019,7 @@ pub fn constructor_writable_xmm_to_xmm_mem( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2407. + // Rule at src/isa/x64/inst.isle line 2502. let expr0_0 = C::writable_xmm_to_xmm(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -3829,7 +4031,7 @@ pub fn constructor_writable_xmm_to_value_regs( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2410. + // Rule at src/isa/x64/inst.isle line 2505. let expr0_0 = constructor_writable_xmm_to_r_reg(ctx, pattern0_0)?; let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); @@ -3841,7 +4043,7 @@ pub fn constructor_synthetic_amode_to_gpr_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2414. + // Rule at src/isa/x64/inst.isle line 2509. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_gpr_mem(ctx, &expr0_0); return Some(expr1_0); @@ -3853,7 +4055,7 @@ pub fn constructor_synthetic_amode_to_xmm_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2417. + // Rule at src/isa/x64/inst.isle line 2512. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); return Some(expr1_0); @@ -3994,6 +4196,16 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + if let &Opcode::Debugtrap = pattern2_0 { + // Rule at src/isa/x64/lower.isle line 2006. + let expr0_0 = constructor_hlt(ctx)?; + let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; + return Some(expr1_0); + } + } &InstructionData::UnaryIeee32 { opcode: ref pattern2_0, imm: pattern2_1, @@ -4745,6 +4957,22 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + // Rule at src/isa/x64/lower.isle line 1998. + let expr0_0: Type = I64; + let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; + let expr2_0: u32 = 1; + let expr3_0 = RegMemImm::Imm { simm32: expr2_0 }; + let expr4_0 = C::gpr_mem_imm_new(ctx, &expr3_0); + let expr5_0 = constructor_x64_and(ctx, expr0_0, expr1_0, &expr4_0)?; + let expr6_0 = C::gpr_to_reg(ctx, expr5_0); + let expr7_0: Type = I64; + let expr8_0: u64 = 0; + let expr9_0 = constructor_imm(ctx, expr7_0, expr8_0)?; + let expr10_0 = C::value_regs(ctx, expr6_0, expr9_0); + let expr11_0 = C::output(ctx, expr10_0); + return Some(expr11_0); + } &Opcode::Uextend => { let pattern7_0 = C::value_type(ctx, pattern5_1); if pattern7_0 == I64 { @@ -4808,6 +5036,102 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } } + if pattern2_0 == F32 { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match pattern5_0 { + &Opcode::Fadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2021. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_addss(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fsub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2032. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_subss(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fmul => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2043. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_mulss(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fdiv => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2054. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_divss(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + _ => {} + } + } + } + if pattern2_0 == F64 { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match pattern5_0 { + &Opcode::Fadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2023. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_addsd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fsub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2034. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_subsd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fmul => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2045. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_mulsd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fdiv => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2056. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_divsd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + _ => {} + } + } + } if pattern2_0 == I8X16 { let pattern4_0 = C::inst_data(ctx, pattern0_0); match &pattern4_0 { @@ -5151,6 +5475,15 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2011. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_pmaddwd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } _ => {} } } @@ -5275,48 +5608,140 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + match pattern5_0 { + &Opcode::Fadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2025. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_addps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fsub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2036. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_subps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fmul => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2047. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_mulps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fdiv => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2058. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_divps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + _ => {} + } } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + if let &Opcode::Fabs = pattern5_0 { + // Rule at src/isa/x64/lower.isle line 1238. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; + let expr1_0: Type = F32X4; + let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; + let expr3_0: u32 = 1; + let expr4_0 = RegMemImm::Imm { simm32: expr3_0 }; + let expr5_0 = constructor_mov_rmi_to_xmm(ctx, &expr4_0)?; + let expr6_0 = constructor_psrld(ctx, expr2_0, &expr5_0)?; + let expr7_0 = C::xmm_to_xmm_mem(ctx, expr6_0); + let expr8_0 = constructor_andps(ctx, expr0_0, &expr7_0)?; + let expr9_0 = constructor_output_xmm(ctx, expr8_0)?; + return Some(expr9_0); + } + } + _ => {} } } if pattern2_0 == F64X2 { let pattern4_0 = C::inst_data(ctx, pattern0_0); - if let &InstructionData::Unary { - opcode: ref pattern5_0, - arg: pattern5_1, - } = &pattern4_0 - { - if let &Opcode::Fabs = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1244. - let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; - let expr1_0: Type = F64X2; - let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; - let expr3_0: u32 = 1; - let expr4_0 = RegMemImm::Imm { simm32: expr3_0 }; - let expr5_0 = constructor_mov_rmi_to_xmm(ctx, &expr4_0)?; - let expr6_0 = constructor_psrlq(ctx, expr2_0, &expr5_0)?; - let expr7_0 = C::xmm_to_xmm_mem(ctx, expr6_0); - let expr8_0 = constructor_andpd(ctx, expr0_0, &expr7_0)?; - let expr9_0 = constructor_output_xmm(ctx, expr8_0)?; - return Some(expr9_0); + match &pattern4_0 { + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match pattern5_0 { + &Opcode::Fadd => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2027. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_addpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fsub => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2038. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_subpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fmul => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2049. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_mulpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Fdiv => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 2060. + let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr2_0 = constructor_divpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_xmm(ctx, expr2_0)?; + return Some(expr3_0); + } + _ => {} + } } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + if let &Opcode::Fabs = pattern5_0 { + // Rule at src/isa/x64/lower.isle line 1244. + let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; + let expr1_0: Type = F64X2; + let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?; + let expr3_0: u32 = 1; + let expr4_0 = RegMemImm::Imm { simm32: expr3_0 }; + let expr5_0 = constructor_mov_rmi_to_xmm(ctx, &expr4_0)?; + let expr6_0 = constructor_psrlq(ctx, expr2_0, &expr5_0)?; + let expr7_0 = C::xmm_to_xmm_mem(ctx, expr6_0); + let expr8_0 = constructor_andpd(ctx, expr0_0, &expr7_0)?; + let expr9_0 = constructor_output_xmm(ctx, expr8_0)?; + return Some(expr9_0); + } + } + _ => {} } } let pattern3_0 = C::inst_data(ctx, pattern0_0); @@ -7210,6 +7635,16 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + // Rule at src/isa/x64/lower.isle line 1995. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; + let expr1_0: u32 = 1; + let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; + let expr3_0 = C::gpr_mem_imm_new(ctx, &expr2_0); + let expr4_0 = constructor_x64_and(ctx, pattern3_0, expr0_0, &expr3_0)?; + let expr5_0 = constructor_output_gpr(ctx, expr4_0)?; + return Some(expr5_0); + } &Opcode::Ireduce => { // Rule at src/isa/x64/lower.isle line 1979. let expr0_0 = C::put_in_regs(ctx, pattern5_1); diff --git a/cranelift/filetests/filetests/isa/x64/fastcall.clif b/cranelift/filetests/filetests/isa/x64/fastcall.clif index 8a78d5cb38..521a142618 100644 --- a/cranelift/filetests/filetests/isa/x64/fastcall.clif +++ b/cranelift/filetests/filetests/isa/x64/fastcall.clif @@ -288,7 +288,7 @@ block0(v0: i64): ; Entry block: 0 ; Block 0: ; (original IR block: block0) -; (instruction range: 0 .. 87) +; (instruction range: 0 .. 85) ; Inst 0: pushq %rbp ; Inst 1: unwind PushFrameRegs { offset_upward_to_caller_sp: 16 } ; Inst 2: movq %rsp, %rbp @@ -314,13 +314,13 @@ block0(v0: i64): ; Inst 22: unwind SaveReg { clobber_offset: 128, reg: r14V } ; Inst 23: movdqu %xmm15, 208(%rsp) ; Inst 24: unwind SaveReg { clobber_offset: 144, reg: r15V } -; Inst 25: movsd 0(%rcx), %xmm4 -; Inst 26: movsd 8(%rcx), %xmm1 -; Inst 27: movsd 16(%rcx), %xmm0 -; Inst 28: movdqu %xmm0, rsp(32 + virtual offset) -; Inst 29: movsd 24(%rcx), %xmm3 -; Inst 30: movsd 32(%rcx), %xmm0 -; Inst 31: movdqu %xmm0, rsp(48 + virtual offset) +; Inst 25: movsd 0(%rcx), %xmm0 +; Inst 26: movdqu %xmm0, rsp(48 + virtual offset) +; Inst 27: movsd 8(%rcx), %xmm1 +; Inst 28: movsd 16(%rcx), %xmm0 +; Inst 29: movdqu %xmm0, rsp(32 + virtual offset) +; Inst 30: movsd 24(%rcx), %xmm3 +; Inst 31: movsd 32(%rcx), %xmm4 ; Inst 32: movsd 40(%rcx), %xmm5 ; Inst 33: movsd 48(%rcx), %xmm6 ; Inst 34: movsd 56(%rcx), %xmm7 @@ -332,49 +332,47 @@ block0(v0: i64): ; Inst 40: movsd 104(%rcx), %xmm13 ; Inst 41: movsd 112(%rcx), %xmm14 ; Inst 42: movsd 120(%rcx), %xmm15 -; Inst 43: movsd 128(%rcx), %xmm0 -; Inst 44: movdqu %xmm0, rsp(0 + virtual offset) -; Inst 45: movsd 136(%rcx), %xmm0 -; Inst 46: movsd 144(%rcx), %xmm2 -; Inst 47: movdqu %xmm2, rsp(16 + virtual offset) -; Inst 48: movsd 152(%rcx), %xmm2 -; Inst 49: addsd %xmm1, %xmm4 +; Inst 43: movsd 128(%rcx), %xmm2 +; Inst 44: movdqu %xmm2, rsp(0 + virtual offset) +; Inst 45: movsd 136(%rcx), %xmm2 +; Inst 46: movsd 144(%rcx), %xmm0 +; Inst 47: movdqu %xmm0, rsp(16 + virtual offset) +; Inst 48: movdqu rsp(48 + virtual offset), %xmm0 +; Inst 49: addsd %xmm1, %xmm0 ; Inst 50: movdqu rsp(32 + virtual offset), %xmm1 ; Inst 51: addsd %xmm3, %xmm1 -; Inst 52: movdqu rsp(48 + virtual offset), %xmm3 -; Inst 53: addsd %xmm5, %xmm3 -; Inst 54: addsd %xmm7, %xmm6 -; Inst 55: addsd %xmm9, %xmm8 -; Inst 56: addsd %xmm11, %xmm10 -; Inst 57: addsd %xmm13, %xmm12 -; Inst 58: addsd %xmm15, %xmm14 -; Inst 59: movdqu rsp(0 + virtual offset), %xmm5 -; Inst 60: addsd %xmm0, %xmm5 -; Inst 61: movdqu rsp(16 + virtual offset), %xmm0 -; Inst 62: addsd %xmm2, %xmm0 -; Inst 63: addsd %xmm1, %xmm4 -; Inst 64: addsd %xmm6, %xmm3 -; Inst 65: addsd %xmm10, %xmm8 -; Inst 66: addsd %xmm14, %xmm12 -; Inst 67: addsd %xmm0, %xmm5 -; Inst 68: addsd %xmm3, %xmm4 -; Inst 69: addsd %xmm12, %xmm8 -; Inst 70: addsd %xmm8, %xmm4 -; Inst 71: addsd %xmm5, %xmm4 -; Inst 72: movaps %xmm4, %xmm0 -; Inst 73: movdqu 64(%rsp), %xmm6 -; Inst 74: movdqu 80(%rsp), %xmm7 -; Inst 75: movdqu 96(%rsp), %xmm8 -; Inst 76: movdqu 112(%rsp), %xmm9 -; Inst 77: movdqu 128(%rsp), %xmm10 -; Inst 78: movdqu 144(%rsp), %xmm11 -; Inst 79: movdqu 160(%rsp), %xmm12 -; Inst 80: movdqu 176(%rsp), %xmm13 -; Inst 81: movdqu 192(%rsp), %xmm14 -; Inst 82: movdqu 208(%rsp), %xmm15 -; Inst 83: addq $224, %rsp -; Inst 84: movq %rbp, %rsp -; Inst 85: popq %rbp -; Inst 86: ret +; Inst 52: addsd %xmm5, %xmm4 +; Inst 53: addsd %xmm7, %xmm6 +; Inst 54: addsd %xmm9, %xmm8 +; Inst 55: addsd %xmm11, %xmm10 +; Inst 56: addsd %xmm13, %xmm12 +; Inst 57: addsd %xmm15, %xmm14 +; Inst 58: movdqu rsp(0 + virtual offset), %xmm3 +; Inst 59: addsd %xmm2, %xmm3 +; Inst 60: movdqu rsp(16 + virtual offset), %xmm2 +; Inst 61: addsd 152(%rcx), %xmm2 +; Inst 62: addsd %xmm1, %xmm0 +; Inst 63: addsd %xmm6, %xmm4 +; Inst 64: addsd %xmm10, %xmm8 +; Inst 65: addsd %xmm14, %xmm12 +; Inst 66: addsd %xmm2, %xmm3 +; Inst 67: addsd %xmm4, %xmm0 +; Inst 68: addsd %xmm12, %xmm8 +; Inst 69: addsd %xmm8, %xmm0 +; Inst 70: addsd %xmm3, %xmm0 +; Inst 71: movdqu 64(%rsp), %xmm6 +; Inst 72: movdqu 80(%rsp), %xmm7 +; Inst 73: movdqu 96(%rsp), %xmm8 +; Inst 74: movdqu 112(%rsp), %xmm9 +; Inst 75: movdqu 128(%rsp), %xmm10 +; Inst 76: movdqu 144(%rsp), %xmm11 +; Inst 77: movdqu 160(%rsp), %xmm12 +; Inst 78: movdqu 176(%rsp), %xmm13 +; Inst 79: movdqu 192(%rsp), %xmm14 +; Inst 80: movdqu 208(%rsp), %xmm15 +; Inst 81: addq $224, %rsp +; Inst 82: movq %rbp, %rsp +; Inst 83: popq %rbp +; Inst 84: ret ; }}