b78731839b
Instead of using `m_` like we used to, which was short for "mach inst" but not obvious or clear at all.
1436 lines
48 KiB
Common Lisp
1436 lines
48 KiB
Common Lisp
;; Extern type definitions and constructors for the x64 `MachInst` type.
|
|
|
|
;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(type MInst extern
|
|
(enum (Nop (len u8))
|
|
(AluRmiR (size OperandSize)
|
|
(op AluRmiROpcode)
|
|
(src1 Reg)
|
|
(src2 RegMemImm)
|
|
(dst WritableReg))
|
|
(MulHi (size OperandSize)
|
|
(signed bool)
|
|
(src1 Reg)
|
|
(src2 RegMem)
|
|
(dst_lo WritableReg)
|
|
(dst_hi WritableReg))
|
|
(XmmRmR (op SseOpcode)
|
|
(src1 Reg)
|
|
(src2 RegMem)
|
|
(dst WritableReg))
|
|
(XmmUnaryRmR (op SseOpcode)
|
|
(src RegMem)
|
|
(dst WritableReg))
|
|
(XmmUnaryRmREvex (op Avx512Opcode)
|
|
(src RegMem)
|
|
(dst WritableReg))
|
|
(XmmRmiReg (opcode SseOpcode)
|
|
(src1 Reg)
|
|
(src2 RegMemImm)
|
|
(dst WritableReg))
|
|
(XmmRmRImm (op SseOpcode)
|
|
(src1 Reg)
|
|
(src2 RegMem)
|
|
(dst WritableReg)
|
|
(imm u8)
|
|
(size OperandSize))
|
|
(XmmUninitializedValue (dst WritableReg))
|
|
(CmpRmiR (size OperandSize)
|
|
(opcode CmpOpcode)
|
|
(src RegMemImm)
|
|
(dst Reg))
|
|
(Imm (dst_size OperandSize)
|
|
(simm64 u64)
|
|
(dst WritableReg))
|
|
(ShiftR (size OperandSize)
|
|
(kind ShiftKind)
|
|
(src Reg)
|
|
(num_bits Imm8Reg)
|
|
(dst WritableReg))
|
|
(MovzxRmR (ext_mode ExtMode)
|
|
(src RegMem)
|
|
(dst WritableReg))
|
|
(MovsxRmR (ext_mode ExtMode)
|
|
(src RegMem)
|
|
(dst WritableReg))
|
|
(Mov64MR (src SyntheticAmode)
|
|
(dst WritableReg))
|
|
(Cmove (size OperandSize)
|
|
(cc CC)
|
|
(consequent RegMem)
|
|
(alternative Reg)
|
|
(dst WritableReg))
|
|
(XmmRmREvex (op Avx512Opcode)
|
|
(src1 RegMem)
|
|
(src2 Reg)
|
|
(dst WritableReg))
|
|
(GprToXmm (op SseOpcode)
|
|
(src RegMem)
|
|
(dst WritableReg)
|
|
(src_size OperandSize))
|
|
(Not (size OperandSize)
|
|
(src Reg)
|
|
(dst WritableReg))
|
|
(LoadEffectiveAddress (addr SyntheticAmode)
|
|
(dst WritableReg))))
|
|
|
|
(type OperandSize extern
|
|
(enum Size8
|
|
Size16
|
|
Size32
|
|
Size64))
|
|
|
|
;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits.
|
|
(decl operand_size_of_type_32_64 (Type) OperandSize)
|
|
(extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64)
|
|
|
|
;; Get the true `OperandSize` for a given `Type`, with no rounding.
|
|
(decl raw_operand_size_of_type (Type) OperandSize)
|
|
(extern constructor raw_operand_size_of_type raw_operand_size_of_type)
|
|
|
|
;; Get the bit width of an `OperandSize`.
|
|
(decl operand_size_bits (OperandSize) u16)
|
|
(rule (operand_size_bits (OperandSize.Size8)) 8)
|
|
(rule (operand_size_bits (OperandSize.Size16)) 16)
|
|
(rule (operand_size_bits (OperandSize.Size32)) 32)
|
|
(rule (operand_size_bits (OperandSize.Size64)) 64)
|
|
|
|
(type AluRmiROpcode extern
|
|
(enum Add
|
|
Adc
|
|
Sub
|
|
Sbb
|
|
And
|
|
Or
|
|
Xor
|
|
Mul
|
|
And8
|
|
Or8))
|
|
|
|
(type SseOpcode extern
|
|
(enum Addps
|
|
Addpd
|
|
Addss
|
|
Addsd
|
|
Andps
|
|
Andpd
|
|
Andnps
|
|
Andnpd
|
|
Blendvpd
|
|
Blendvps
|
|
Comiss
|
|
Comisd
|
|
Cmpps
|
|
Cmppd
|
|
Cmpss
|
|
Cmpsd
|
|
Cvtdq2ps
|
|
Cvtdq2pd
|
|
Cvtpd2ps
|
|
Cvtps2pd
|
|
Cvtsd2ss
|
|
Cvtsd2si
|
|
Cvtsi2ss
|
|
Cvtsi2sd
|
|
Cvtss2si
|
|
Cvtss2sd
|
|
Cvttpd2dq
|
|
Cvttps2dq
|
|
Cvttss2si
|
|
Cvttsd2si
|
|
Divps
|
|
Divpd
|
|
Divss
|
|
Divsd
|
|
Insertps
|
|
Maxps
|
|
Maxpd
|
|
Maxss
|
|
Maxsd
|
|
Minps
|
|
Minpd
|
|
Minss
|
|
Minsd
|
|
Movaps
|
|
Movapd
|
|
Movd
|
|
Movdqa
|
|
Movdqu
|
|
Movlhps
|
|
Movmskps
|
|
Movmskpd
|
|
Movq
|
|
Movss
|
|
Movsd
|
|
Movups
|
|
Movupd
|
|
Mulps
|
|
Mulpd
|
|
Mulss
|
|
Mulsd
|
|
Orps
|
|
Orpd
|
|
Pabsb
|
|
Pabsw
|
|
Pabsd
|
|
Packssdw
|
|
Packsswb
|
|
Packusdw
|
|
Packuswb
|
|
Paddb
|
|
Paddd
|
|
Paddq
|
|
Paddw
|
|
Paddsb
|
|
Paddsw
|
|
Paddusb
|
|
Paddusw
|
|
Palignr
|
|
Pand
|
|
Pandn
|
|
Pavgb
|
|
Pavgw
|
|
Pblendvb
|
|
Pcmpeqb
|
|
Pcmpeqw
|
|
Pcmpeqd
|
|
Pcmpeqq
|
|
Pcmpgtb
|
|
Pcmpgtw
|
|
Pcmpgtd
|
|
Pcmpgtq
|
|
Pextrb
|
|
Pextrw
|
|
Pextrd
|
|
Pinsrb
|
|
Pinsrw
|
|
Pinsrd
|
|
Pmaddubsw
|
|
Pmaddwd
|
|
Pmaxsb
|
|
Pmaxsw
|
|
Pmaxsd
|
|
Pmaxub
|
|
Pmaxuw
|
|
Pmaxud
|
|
Pminsb
|
|
Pminsw
|
|
Pminsd
|
|
Pminub
|
|
Pminuw
|
|
Pminud
|
|
Pmovmskb
|
|
Pmovsxbd
|
|
Pmovsxbw
|
|
Pmovsxbq
|
|
Pmovsxwd
|
|
Pmovsxwq
|
|
Pmovsxdq
|
|
Pmovzxbd
|
|
Pmovzxbw
|
|
Pmovzxbq
|
|
Pmovzxwd
|
|
Pmovzxwq
|
|
Pmovzxdq
|
|
Pmuldq
|
|
Pmulhw
|
|
Pmulhuw
|
|
Pmulhrsw
|
|
Pmulld
|
|
Pmullw
|
|
Pmuludq
|
|
Por
|
|
Pshufb
|
|
Pshufd
|
|
Psllw
|
|
Pslld
|
|
Psllq
|
|
Psraw
|
|
Psrad
|
|
Psrlw
|
|
Psrld
|
|
Psrlq
|
|
Psubb
|
|
Psubd
|
|
Psubq
|
|
Psubw
|
|
Psubsb
|
|
Psubsw
|
|
Psubusb
|
|
Psubusw
|
|
Ptest
|
|
Punpckhbw
|
|
Punpckhwd
|
|
Punpcklbw
|
|
Punpcklwd
|
|
Pxor
|
|
Rcpss
|
|
Roundps
|
|
Roundpd
|
|
Roundss
|
|
Roundsd
|
|
Rsqrtss
|
|
Shufps
|
|
Sqrtps
|
|
Sqrtpd
|
|
Sqrtss
|
|
Sqrtsd
|
|
Subps
|
|
Subpd
|
|
Subss
|
|
Subsd
|
|
Ucomiss
|
|
Ucomisd
|
|
Unpcklps
|
|
Xorps
|
|
Xorpd))
|
|
|
|
(type CmpOpcode extern
|
|
(enum Cmp
|
|
Test))
|
|
|
|
(type RegMemImm extern
|
|
(enum
|
|
(Reg (reg Reg))
|
|
(Mem (addr SyntheticAmode))
|
|
(Imm (simm32 u32))))
|
|
|
|
;; Put the given clif value into a `RegMemImm` operand.
|
|
;;
|
|
;; Asserts that the value fits into a single register, and doesn't require
|
|
;; multiple registers for its representation (like `i128` for example).
|
|
;;
|
|
;; As a side effect, this marks the value as used.
|
|
(decl put_in_reg_mem_imm (Value) RegMemImm)
|
|
(extern constructor put_in_reg_mem_imm put_in_reg_mem_imm)
|
|
|
|
(type RegMem extern
|
|
(enum
|
|
(Reg (reg Reg))
|
|
(Mem (addr SyntheticAmode))))
|
|
|
|
;; Put the given clif value into a `RegMem` operand.
|
|
;;
|
|
;; Asserts that the value fits into a single register, and doesn't require
|
|
;; multiple registers for its representation (like `i128` for example).
|
|
;;
|
|
;; As a side effect, this marks the value as used.
|
|
(decl put_in_reg_mem (Value) RegMem)
|
|
(extern constructor put_in_reg_mem put_in_reg_mem)
|
|
|
|
(type SyntheticAmode extern (enum))
|
|
|
|
(decl synthetic_amode_to_reg_mem (SyntheticAmode) RegMem)
|
|
(extern constructor synthetic_amode_to_reg_mem synthetic_amode_to_reg_mem)
|
|
|
|
(type Amode extern (enum))
|
|
|
|
(decl amode_imm_reg_reg_shift (u32 Reg Reg u8) Amode)
|
|
(extern constructor amode_imm_reg_reg_shift amode_imm_reg_reg_shift)
|
|
|
|
(decl amode_to_synthetic_amode (Amode) SyntheticAmode)
|
|
(extern constructor amode_to_synthetic_amode amode_to_synthetic_amode)
|
|
|
|
(type ShiftKind extern
|
|
(enum ShiftLeft
|
|
ShiftRightLogical
|
|
ShiftRightArithmetic
|
|
RotateLeft
|
|
RotateRight))
|
|
|
|
(type Imm8Reg extern
|
|
(enum (Imm8 (imm u8))
|
|
(Reg (reg Reg))))
|
|
|
|
;; Put the given clif value into a `Imm8Reg` operand, masked to the bit width of
|
|
;; the given type.
|
|
;;
|
|
;; Asserts that the value fits into a single register, and doesn't require
|
|
;; multiple registers for its representation (like `i128` for example).
|
|
;;
|
|
;; As a side effect, this marks the value as used.
|
|
;;
|
|
;; This is used when lowering various shifts and rotates.
|
|
(decl put_masked_in_imm8_reg (Value Type) Imm8Reg)
|
|
(extern constructor put_masked_in_imm8_reg put_masked_in_imm8_reg)
|
|
|
|
(type CC extern
|
|
(enum O
|
|
NO
|
|
B
|
|
NB
|
|
Z
|
|
NZ
|
|
BE
|
|
NBE
|
|
S
|
|
NS
|
|
L
|
|
NL
|
|
LE
|
|
NLE
|
|
P
|
|
NP))
|
|
|
|
(type Avx512Opcode extern
|
|
(enum Vcvtudq2ps
|
|
Vpabsq
|
|
Vpermi2b
|
|
Vpmullq
|
|
Vpopcntb))
|
|
|
|
(type FcmpImm extern
|
|
(enum Equal
|
|
LessThan
|
|
LessThanOrEqual
|
|
Unordered
|
|
NotEqual
|
|
UnorderedOrGreaterThanOrEqual
|
|
UnorderedOrGreaterThan
|
|
Ordered))
|
|
|
|
(decl encode_fcmp_imm (FcmpImm) u8)
|
|
(extern constructor encode_fcmp_imm encode_fcmp_imm)
|
|
|
|
;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;;
|
|
;; These should only be used for legalization purposes, when we can't otherwise
|
|
;; rely on something like `Inst::mov_mitosis` to put an operand into the
|
|
;; appropriate physical register for whatever reason.
|
|
|
|
(decl xmm0 () WritableReg)
|
|
(extern constructor xmm0 xmm0)
|
|
|
|
;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(decl avx512vl_enabled () Type)
|
|
(extern extractor avx512vl_enabled avx512vl_enabled)
|
|
|
|
(decl avx512dq_enabled () Type)
|
|
(extern extractor avx512dq_enabled avx512dq_enabled)
|
|
|
|
(decl avx512f_enabled () Type)
|
|
(extern extractor avx512f_enabled avx512f_enabled)
|
|
|
|
;;;; Helpers for Merging and Sinking Immediates/Loads ;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Extract a constant `Imm8Reg.Imm8` from a value operand.
|
|
(decl imm8_from_value (Imm8Reg) Value)
|
|
(extern extractor imm8_from_value imm8_from_value)
|
|
|
|
;; Mask a constant to the bit-width of the given type and package it into an
|
|
;; `Imm8Reg.Imm8`. This is used for shifts and rotates, so that we don't try and
|
|
;; shift/rotate more bits than the type has available, per Cranelift's
|
|
;; semantics.
|
|
(decl const_to_type_masked_imm8 (u64 Type) Imm8Reg)
|
|
(extern constructor const_to_type_masked_imm8 const_to_type_masked_imm8)
|
|
|
|
;; Extract a constant `RegMemImm.Imm` from a value operand.
|
|
(decl simm32_from_value (RegMemImm) Value)
|
|
(extern extractor simm32_from_value simm32_from_value)
|
|
|
|
;; Extract a constant `RegMemImm.Imm` from an `Imm64` immediate.
|
|
(decl simm32_from_imm64 (RegMemImm) Imm64)
|
|
(extern extractor simm32_from_imm64 simm32_from_imm64)
|
|
|
|
;; A load that can be sunk into another operation.
|
|
(type SinkableLoad extern (enum))
|
|
|
|
;; Extract a `SinkableLoad` that works with `RegMemImm.Mem` from a value
|
|
;; operand.
|
|
(decl sinkable_load (SinkableLoad) Value)
|
|
(extern extractor sinkable_load sinkable_load)
|
|
|
|
;; Sink a `SinkableLoad` into a `RegMemImm.Mem`.
|
|
;;
|
|
;; This is a side-effectful operation that notifies the context that the
|
|
;; instruction that produced the `SinkableImm` has been sunk into another
|
|
;; instruction, and no longer needs to be lowered.
|
|
(decl sink_load (SinkableLoad) RegMemImm)
|
|
(extern constructor sink_load sink_load)
|
|
|
|
;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(type ExtKind extern
|
|
(enum None
|
|
SignExtend
|
|
ZeroExtend))
|
|
|
|
(type ExtendKind (enum Sign Zero))
|
|
|
|
(type ExtMode extern (enum BL BQ WL WQ LQ))
|
|
|
|
;; `ExtMode::new`
|
|
(decl ext_mode (u16 u16) ExtMode)
|
|
(extern constructor ext_mode ext_mode)
|
|
|
|
;; Put the given value into a register, but extended as the given type.
|
|
(decl extend_to_reg (Value Type ExtendKind) Reg)
|
|
|
|
;; If the value is already of the requested type, no extending is necessary.
|
|
(rule (extend_to_reg (and val (value_type ty)) =ty _kind)
|
|
(put_in_reg val))
|
|
|
|
(rule (extend_to_reg (and val (value_type from_ty))
|
|
to_ty
|
|
kind)
|
|
(let ((from_bits u16 (ty_bits_u16 from_ty))
|
|
;; Use `operand_size_of_type` so that the we clamp the output to 32-
|
|
;; or 64-bit width types.
|
|
(to_bits u16 (operand_size_bits (operand_size_of_type_32_64 to_ty))))
|
|
(extend kind
|
|
to_ty
|
|
(ext_mode from_bits to_bits)
|
|
(put_in_reg_mem val))))
|
|
|
|
;; Do a sign or zero extension of the given `RegMem`.
|
|
(decl extend (ExtendKind Type ExtMode RegMem) Reg)
|
|
|
|
;; Zero extending uses `movzx`.
|
|
(rule (extend (ExtendKind.Zero) ty mode src)
|
|
(movzx ty mode src))
|
|
|
|
;; Sign extending uses `movsx`.
|
|
(rule (extend (ExtendKind.Sign) ty mode src)
|
|
(movsx ty mode src))
|
|
|
|
;;;; Helpers for Working SSE tidbits ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Determine the appropriate operation for xor-ing vectors of the specified type
|
|
(decl sse_xor_op (Type) SseOpcode)
|
|
(rule (sse_xor_op $F32X4) (SseOpcode.Xorps))
|
|
(rule (sse_xor_op $F64X2) (SseOpcode.Xorpd))
|
|
(rule (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor))
|
|
|
|
;; Performs an xor operation of the two operands specified
|
|
(decl sse_xor (Type Reg RegMem) Reg)
|
|
(rule (sse_xor ty x y) (xmm_rm_r ty (sse_xor_op ty) x y))
|
|
|
|
;; Determine the appropriate operation to compare two vectors of the specified
|
|
;; type.
|
|
(decl sse_cmp_op (Type) SseOpcode)
|
|
(rule (sse_cmp_op (multi_lane 8 16)) (SseOpcode.Pcmpeqb))
|
|
(rule (sse_cmp_op (multi_lane 16 8)) (SseOpcode.Pcmpeqw))
|
|
(rule (sse_cmp_op (multi_lane 32 4)) (SseOpcode.Pcmpeqd))
|
|
(rule (sse_cmp_op (multi_lane 64 2)) (SseOpcode.Pcmpeqq))
|
|
(rule (sse_cmp_op $F32X4) (SseOpcode.Cmpps))
|
|
(rule (sse_cmp_op $F64X2) (SseOpcode.Cmppd))
|
|
|
|
;; Generates a register value which has an all-ones pattern of the specified
|
|
;; type.
|
|
;;
|
|
;; Note that this is accomplished by comparing a fresh register with itself,
|
|
;; which for integers is always true. Also note that the comparison is always
|
|
;; done for integers, it doesn't actually take the input `ty` into account. This
|
|
;; is because we're comparing a fresh register to itself and we don't know the
|
|
;; previous contents of the register. If a floating-point comparison is used
|
|
;; then it runs the risk of comparing NaN against NaN and not actually producing
|
|
;; an all-ones mask. By using integer comparision operations we're guaranteeed
|
|
;; that everything is equal to itself.
|
|
(decl vector_all_ones (Type) Reg)
|
|
(rule (vector_all_ones ty)
|
|
(let ((wr WritableReg (temp_writable_reg ty))
|
|
(r Reg (writable_reg_to_reg wr))
|
|
(_ Unit (emit (MInst.XmmRmR (sse_cmp_op $I32X4)
|
|
r
|
|
(RegMem.Reg r)
|
|
wr))))
|
|
r))
|
|
|
|
;; Helper for creating an SSE register holding an `i64x2` from two `i64` values.
|
|
(decl make_i64x2_from_lanes (RegMem RegMem) Reg)
|
|
(rule (make_i64x2_from_lanes lo hi)
|
|
(let ((dst_w WritableReg (temp_writable_reg $I64X2))
|
|
(dst_r Reg (writable_reg_to_reg dst_w))
|
|
(_0 Unit (emit (MInst.XmmUninitializedValue dst_w)))
|
|
(_1 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd)
|
|
dst_r
|
|
lo
|
|
dst_w
|
|
0
|
|
(OperandSize.Size64))))
|
|
(_2 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd)
|
|
dst_r
|
|
hi
|
|
dst_w
|
|
1
|
|
(OperandSize.Size64)))))
|
|
dst_r))
|
|
|
|
;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary.
|
|
(decl reg_mem_imm_to_xmm (RegMemImm) RegMemImm)
|
|
(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Mem _)) rmi)
|
|
(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Imm _)) rmi)
|
|
(rule (reg_mem_imm_to_xmm (RegMemImm.Reg r))
|
|
(RegMemImm.Reg (gpr_to_xmm $I8X16
|
|
(SseOpcode.Movd)
|
|
(RegMem.Reg r)
|
|
(OperandSize.Size32))))
|
|
|
|
;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Load a value into a register.
|
|
(decl x64_load (Type SyntheticAmode ExtKind) Reg)
|
|
|
|
(rule (x64_load (fits_in_32 ty) addr (ExtKind.SignExtend))
|
|
(movsx ty
|
|
(ext_mode (ty_bytes ty) 8)
|
|
(synthetic_amode_to_reg_mem addr)))
|
|
|
|
(rule (x64_load $I64 addr _ext_kind)
|
|
(let ((dst WritableReg (temp_writable_reg $I64))
|
|
(_ Unit (emit (MInst.Mov64MR addr dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
(rule (x64_load $F32 addr _ext_kind)
|
|
(xmm_unary_rm_r (SseOpcode.Movss)
|
|
(synthetic_amode_to_reg_mem addr)))
|
|
|
|
(rule (x64_load $F64 addr _ext_kind)
|
|
(xmm_unary_rm_r (SseOpcode.Movsd)
|
|
(synthetic_amode_to_reg_mem addr)))
|
|
|
|
(rule (x64_load $F32X4 addr _ext_kind)
|
|
(xmm_unary_rm_r (SseOpcode.Movups)
|
|
(synthetic_amode_to_reg_mem addr)))
|
|
|
|
(rule (x64_load $F64X2 addr _ext_kind)
|
|
(xmm_unary_rm_r (SseOpcode.Movupd)
|
|
(synthetic_amode_to_reg_mem addr)))
|
|
|
|
(rule (x64_load (multi_lane _bits _lanes) addr _ext_kind)
|
|
(xmm_unary_rm_r (SseOpcode.Movdqu)
|
|
(synthetic_amode_to_reg_mem addr)))
|
|
|
|
;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;;
|
|
;; These constructors create SSA-style `MInst`s. It is their responsibility to
|
|
;; maintain the invariant that each temporary register they allocate and define
|
|
;; only gets defined the once.
|
|
|
|
;; Emit an instruction.
|
|
;;
|
|
;; This is low-level and side-effectful; it should only be used as an
|
|
;; implementation detail by helpers that preserve the SSA facade themselves.
|
|
(decl emit (MInst) Unit)
|
|
(extern constructor emit emit)
|
|
|
|
;; Helper for emitting `MInst.AluRmiR` instructions.
|
|
(decl alu_rmi_r (Type AluRmiROpcode Reg RegMemImm) Reg)
|
|
(rule (alu_rmi_r ty opcode src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(size OperandSize (operand_size_of_type_32_64 ty))
|
|
(_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for emitting `add` instructions.
|
|
(decl add (Type Reg RegMemImm) Reg)
|
|
(rule (add ty src1 src2)
|
|
(alu_rmi_r ty
|
|
(AluRmiROpcode.Add)
|
|
src1
|
|
src2))
|
|
|
|
;; Helper for creating `add` instructions whose flags are also used.
|
|
(decl add_with_flags (Type Reg RegMemImm) ProducesFlags)
|
|
(rule (add_with_flags ty src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg ty)))
|
|
(ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
|
|
(AluRmiROpcode.Add)
|
|
src1
|
|
src2
|
|
dst)
|
|
(writable_reg_to_reg dst))))
|
|
|
|
;; Helper for creating `adc` instructions.
|
|
(decl adc (Type Reg RegMemImm) ConsumesFlags)
|
|
(rule (adc ty src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg ty)))
|
|
(ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
|
|
(AluRmiROpcode.Adc)
|
|
src1
|
|
src2
|
|
dst)
|
|
(writable_reg_to_reg dst))))
|
|
|
|
;; Helper for emitting `sub` instructions.
|
|
(decl sub (Type Reg RegMemImm) Reg)
|
|
(rule (sub ty src1 src2)
|
|
(alu_rmi_r ty
|
|
(AluRmiROpcode.Sub)
|
|
src1
|
|
src2))
|
|
|
|
;; Helper for creating `sub` instructions whose flags are also used.
|
|
(decl sub_with_flags (Type Reg RegMemImm) ProducesFlags)
|
|
(rule (sub_with_flags ty src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg ty)))
|
|
(ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
|
|
(AluRmiROpcode.Sub)
|
|
src1
|
|
src2
|
|
dst)
|
|
(writable_reg_to_reg dst))))
|
|
|
|
;; Helper for creating `sbb` instructions.
|
|
(decl sbb (Type Reg RegMemImm) ConsumesFlags)
|
|
(rule (sbb ty src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg ty)))
|
|
(ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
|
|
(AluRmiROpcode.Sbb)
|
|
src1
|
|
src2
|
|
dst)
|
|
(writable_reg_to_reg dst))))
|
|
|
|
;; Helper for creating `mul` instructions.
|
|
(decl mul (Type Reg RegMemImm) Reg)
|
|
(rule (mul ty src1 src2)
|
|
(alu_rmi_r ty
|
|
(AluRmiROpcode.Mul)
|
|
src1
|
|
src2))
|
|
|
|
;; Helper for emitting `and` instructions.
|
|
;;
|
|
;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin
|
|
;; `and` operator.
|
|
(decl x64_and (Type Reg RegMemImm) Reg)
|
|
(rule (x64_and ty src1 src2)
|
|
(alu_rmi_r ty
|
|
(AluRmiROpcode.And)
|
|
src1
|
|
src2))
|
|
|
|
;; Helper for emitting `or` instructions.
|
|
(decl or (Type Reg RegMemImm) Reg)
|
|
(rule (or ty src1 src2)
|
|
(alu_rmi_r ty
|
|
(AluRmiROpcode.Or)
|
|
src1
|
|
src2))
|
|
|
|
;; Helper for emitting `xor` instructions.
|
|
(decl xor (Type Reg RegMemImm) Reg)
|
|
(rule (xor ty src1 src2)
|
|
(alu_rmi_r ty
|
|
(AluRmiROpcode.Xor)
|
|
src1
|
|
src2))
|
|
|
|
;; Helper for emitting immediates.
|
|
(decl imm (Type u64) Reg)
|
|
|
|
;; Integer immediates.
|
|
(rule (imm ty simm64)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(size OperandSize (operand_size_of_type_32_64 ty))
|
|
(_ Unit (emit (MInst.Imm size simm64 dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; `f32` immediates.
|
|
(rule (imm $F32 bits)
|
|
(gpr_to_xmm $F32 (SseOpcode.Movd) (RegMem.Reg (imm $I32 bits)) (OperandSize.Size32)))
|
|
|
|
;; `f64` immediates.
|
|
(rule (imm $F64 bits)
|
|
(gpr_to_xmm $F64 (SseOpcode.Movq) (RegMem.Reg (imm $I64 bits)) (OperandSize.Size64)))
|
|
|
|
(decl nonzero_u64_fits_in_u32 (u64) u64)
|
|
(extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32)
|
|
|
|
;; Special case for when a 64-bit immediate fits into 32-bits. We can use a
|
|
;; 32-bit move that zero-extends the value, which has a smaller encoding.
|
|
(rule (imm $I64 (nonzero_u64_fits_in_u32 x))
|
|
(let ((dst WritableReg (temp_writable_reg $I64))
|
|
(_ Unit (emit (MInst.Imm (OperandSize.Size32) x dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Special case for integer zero immediates: turn them into an `xor r, r`.
|
|
(rule (imm ty 0)
|
|
(let ((wr WritableReg (temp_writable_reg ty))
|
|
(r Reg (writable_reg_to_reg wr))
|
|
(size OperandSize (operand_size_of_type_32_64 ty))
|
|
(_ Unit (emit (MInst.AluRmiR size
|
|
(AluRmiROpcode.Xor)
|
|
r
|
|
(RegMemImm.Reg r)
|
|
wr))))
|
|
r))
|
|
|
|
;; Special case for zero immediates with vector types, they turn into an xor
|
|
;; specific to the vector type.
|
|
(rule (imm ty @ (multi_lane _bits _lanes) 0)
|
|
(let ((wr WritableReg (temp_writable_reg ty))
|
|
(r Reg (writable_reg_to_reg wr))
|
|
(_ Unit (emit (MInst.XmmRmR (sse_xor_op ty)
|
|
r
|
|
(RegMem.Reg r)
|
|
wr))))
|
|
r))
|
|
|
|
;; Special case for `f32` zero immediates to use `xorps`.
|
|
(rule (imm $F32 0)
|
|
(let ((wr WritableReg (temp_writable_reg $F32))
|
|
(r Reg (writable_reg_to_reg wr))
|
|
(_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorps)
|
|
r
|
|
(RegMem.Reg r)
|
|
wr))))
|
|
r))
|
|
|
|
;; TODO: use cmpeqps for all 1s
|
|
|
|
;; Special case for `f64` zero immediates to use `xorpd`.
|
|
(rule (imm $F64 0)
|
|
(let ((wr WritableReg (temp_writable_reg $F64))
|
|
(r Reg (writable_reg_to_reg wr))
|
|
(_ Unit (emit (MInst.XmmRmR (SseOpcode.Xorpd)
|
|
r
|
|
(RegMem.Reg r)
|
|
wr))))
|
|
r))
|
|
|
|
;; TODO: use cmpeqpd for all 1s
|
|
|
|
;; Helper for creating `MInst.ShifR` instructions.
|
|
(decl shift_r (Type ShiftKind Reg Imm8Reg) Reg)
|
|
(rule (shift_r ty kind src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
;; Use actual 8/16-bit instructions when appropriate: we
|
|
;; rely on their shift-amount-masking semantics.
|
|
(size OperandSize (raw_operand_size_of_type ty))
|
|
(_ Unit (emit (MInst.ShiftR size kind src1 src2 dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach
|
|
;; inst", to disambiguate this from clif's `rotl`).
|
|
(decl x64_rotl (Type Reg Imm8Reg) Reg)
|
|
(rule (x64_rotl ty src1 src2)
|
|
(shift_r ty (ShiftKind.RotateLeft) src1 src2))
|
|
|
|
;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach
|
|
;; inst", to disambiguate this from clif's `rotr`).
|
|
(decl x64_rotr (Type Reg Imm8Reg) Reg)
|
|
(rule (x64_rotr ty src1 src2)
|
|
(shift_r ty (ShiftKind.RotateRight) src1 src2))
|
|
|
|
;; Helper for creating `shl` instructions.
|
|
(decl shl (Type Reg Imm8Reg) Reg)
|
|
(rule (shl ty src1 src2)
|
|
(shift_r ty (ShiftKind.ShiftLeft) src1 src2))
|
|
|
|
;; Helper for creating logical shift-right instructions.
|
|
(decl shr (Type Reg Imm8Reg) Reg)
|
|
(rule (shr ty src1 src2)
|
|
(shift_r ty (ShiftKind.ShiftRightLogical) src1 src2))
|
|
|
|
;; Helper for creating arithmetic shift-right instructions.
|
|
(decl sar (Type Reg Imm8Reg) Reg)
|
|
(rule (sar ty src1 src2)
|
|
(shift_r ty (ShiftKind.ShiftRightArithmetic) src1 src2))
|
|
|
|
;; Helper for creating `MInst.CmpRmiR` instructions.
|
|
(decl cmp_rmi_r (OperandSize CmpOpcode RegMemImm Reg) ProducesFlags)
|
|
(rule (cmp_rmi_r size opcode src1 src2)
|
|
(ProducesFlags.ProducesFlags (MInst.CmpRmiR size
|
|
opcode
|
|
src1
|
|
src2)
|
|
(invalid_reg)))
|
|
|
|
;; Helper for creating `cmp` instructions.
|
|
(decl cmp (OperandSize RegMemImm Reg) ProducesFlags)
|
|
(rule (cmp size src1 src2)
|
|
(cmp_rmi_r size (CmpOpcode.Cmp) src1 src2))
|
|
|
|
;; Helper for creating `test` instructions.
|
|
(decl test (OperandSize RegMemImm Reg) ProducesFlags)
|
|
(rule (test size src1 src2)
|
|
(cmp_rmi_r size (CmpOpcode.Test) src1 src2))
|
|
|
|
;; Helper for creating `MInst.Cmove` instructions.
|
|
(decl cmove (Type CC RegMem Reg) ConsumesFlags)
|
|
(rule (cmove ty cc consequent alternative)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(size OperandSize (operand_size_of_type_32_64 ty)))
|
|
(ConsumesFlags.ConsumesFlags (MInst.Cmove size cc consequent alternative dst)
|
|
(writable_reg_to_reg dst))))
|
|
|
|
;; Helper for creating `MInst.MovzxRmR` instructions.
|
|
(decl movzx (Type ExtMode RegMem) Reg)
|
|
(rule (movzx ty mode src)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(_ Unit (emit (MInst.MovzxRmR mode src dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `MInst.MovsxRmR` instructions.
|
|
(decl movsx (Type ExtMode RegMem) Reg)
|
|
(rule (movsx ty mode src)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(_ Unit (emit (MInst.MovsxRmR mode src dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `MInst.XmmRmR` instructions.
|
|
(decl xmm_rm_r (Type SseOpcode Reg RegMem) Reg)
|
|
(rule (xmm_rm_r ty op src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(_ Unit (emit (MInst.XmmRmR op src1 src2 dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `paddb` instructions.
|
|
(decl paddb (Reg RegMem) Reg)
|
|
(rule (paddb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2))
|
|
|
|
;; Helper for creating `paddw` instructions.
|
|
(decl paddw (Reg RegMem) Reg)
|
|
(rule (paddw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2))
|
|
|
|
;; Helper for creating `paddd` instructions.
|
|
(decl paddd (Reg RegMem) Reg)
|
|
(rule (paddd src1 src2)
|
|
(xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2))
|
|
|
|
;; Helper for creating `paddq` instructions.
|
|
(decl paddq (Reg RegMem) Reg)
|
|
(rule (paddq src1 src2)
|
|
(xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2))
|
|
|
|
;; Helper for creating `paddsb` instructions.
|
|
(decl paddsb (Reg RegMem) Reg)
|
|
(rule (paddsb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2))
|
|
|
|
;; Helper for creating `paddsw` instructions.
|
|
(decl paddsw (Reg RegMem) Reg)
|
|
(rule (paddsw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2))
|
|
|
|
;; Helper for creating `paddusb` instructions.
|
|
(decl paddusb (Reg RegMem) Reg)
|
|
(rule (paddusb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2))
|
|
|
|
;; Helper for creating `paddusw` instructions.
|
|
(decl paddusw (Reg RegMem) Reg)
|
|
(rule (paddusw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2))
|
|
|
|
;; Helper for creating `psubb` instructions.
|
|
(decl psubb (Reg RegMem) Reg)
|
|
(rule (psubb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2))
|
|
|
|
;; Helper for creating `psubw` instructions.
|
|
(decl psubw (Reg RegMem) Reg)
|
|
(rule (psubw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2))
|
|
|
|
;; Helper for creating `psubd` instructions.
|
|
(decl psubd (Reg RegMem) Reg)
|
|
(rule (psubd src1 src2)
|
|
(xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2))
|
|
|
|
;; Helper for creating `psubq` instructions.
|
|
(decl psubq (Reg RegMem) Reg)
|
|
(rule (psubq src1 src2)
|
|
(xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2))
|
|
|
|
;; Helper for creating `psubsb` instructions.
|
|
(decl psubsb (Reg RegMem) Reg)
|
|
(rule (psubsb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2))
|
|
|
|
;; Helper for creating `psubsw` instructions.
|
|
(decl psubsw (Reg RegMem) Reg)
|
|
(rule (psubsw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2))
|
|
|
|
;; Helper for creating `psubusb` instructions.
|
|
(decl psubusb (Reg RegMem) Reg)
|
|
(rule (psubusb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2))
|
|
|
|
;; Helper for creating `psubusw` instructions.
|
|
(decl psubusw (Reg RegMem) Reg)
|
|
(rule (psubusw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2))
|
|
|
|
;; Helper for creating `pavgb` instructions.
|
|
(decl pavgb (Reg RegMem) Reg)
|
|
(rule (pavgb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2))
|
|
|
|
;; Helper for creating `pavgw` instructions.
|
|
(decl pavgw (Reg RegMem) Reg)
|
|
(rule (pavgw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2))
|
|
|
|
;; Helper for creating `pand` instructions.
|
|
(decl pand (Reg RegMem) Reg)
|
|
(rule (pand src1 src2)
|
|
(xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2))
|
|
|
|
;; Helper for creating `andps` instructions.
|
|
(decl andps (Reg RegMem) Reg)
|
|
(rule (andps src1 src2)
|
|
(xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2))
|
|
|
|
;; Helper for creating `andpd` instructions.
|
|
(decl andpd (Reg RegMem) Reg)
|
|
(rule (andpd src1 src2)
|
|
(xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2))
|
|
|
|
;; Helper for creating `por` instructions.
|
|
(decl por (Reg RegMem) Reg)
|
|
(rule (por src1 src2)
|
|
(xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2))
|
|
|
|
;; Helper for creating `orps` instructions.
|
|
(decl orps (Reg RegMem) Reg)
|
|
(rule (orps src1 src2)
|
|
(xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2))
|
|
|
|
;; Helper for creating `orpd` instructions.
|
|
(decl orpd (Reg RegMem) Reg)
|
|
(rule (orpd src1 src2)
|
|
(xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2))
|
|
|
|
;; Helper for creating `pxor` instructions.
|
|
(decl pxor (Reg RegMem) Reg)
|
|
(rule (pxor src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2))
|
|
|
|
;; Helper for creating `xorps` instructions.
|
|
(decl xorps (Reg RegMem) Reg)
|
|
(rule (xorps src1 src2)
|
|
(xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2))
|
|
|
|
;; Helper for creating `xorpd` instructions.
|
|
(decl xorpd (Reg RegMem) Reg)
|
|
(rule (xorpd src1 src2)
|
|
(xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2))
|
|
|
|
;; Helper for creating `pmullw` instructions.
|
|
(decl pmullw (Reg RegMem) Reg)
|
|
(rule (pmullw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2))
|
|
|
|
;; Helper for creating `pmulld` instructions.
|
|
(decl pmulld (Reg RegMem) Reg)
|
|
(rule (pmulld src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2))
|
|
|
|
;; Helper for creating `pmulhw` instructions.
|
|
(decl pmulhw (Reg RegMem) Reg)
|
|
(rule (pmulhw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2))
|
|
|
|
;; Helper for creating `pmulhuw` instructions.
|
|
(decl pmulhuw (Reg RegMem) Reg)
|
|
(rule (pmulhuw src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2))
|
|
|
|
;; Helper for creating `pmuldq` instructions.
|
|
(decl pmuldq (Reg RegMem) Reg)
|
|
(rule (pmuldq src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2))
|
|
|
|
;; Helper for creating `pmuludq` instructions.
|
|
(decl pmuludq (Reg RegMem) Reg)
|
|
(rule (pmuludq src1 src2)
|
|
(xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2))
|
|
|
|
;; Helper for creating `punpckhwd` instructions.
|
|
(decl punpckhwd (Reg RegMem) Reg)
|
|
(rule (punpckhwd src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2))
|
|
|
|
;; Helper for creating `punpcklwd` instructions.
|
|
(decl punpcklwd (Reg RegMem) Reg)
|
|
(rule (punpcklwd src1 src2)
|
|
(xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2))
|
|
|
|
;; Helper for creating `andnps` instructions.
|
|
(decl andnps (Reg RegMem) Reg)
|
|
(rule (andnps src1 src2)
|
|
(xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2))
|
|
|
|
;; Helper for creating `andnpd` instructions.
|
|
(decl andnpd (Reg RegMem) Reg)
|
|
(rule (andnpd src1 src2)
|
|
(xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2))
|
|
|
|
;; Helper for creating `pandn` instructions.
|
|
(decl pandn (Reg RegMem) Reg)
|
|
(rule (pandn src1 src2)
|
|
(xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2))
|
|
|
|
(decl sse_blend_op (Type) SseOpcode)
|
|
(rule (sse_blend_op $F32X4) (SseOpcode.Blendvps))
|
|
(rule (sse_blend_op $F64X2) (SseOpcode.Blendvpd))
|
|
(rule (sse_blend_op (multi_lane _bits _lanes)) (SseOpcode.Pblendvb))
|
|
|
|
(decl sse_mov_op (Type) SseOpcode)
|
|
(rule (sse_mov_op $F32X4) (SseOpcode.Movaps))
|
|
(rule (sse_mov_op $F64X2) (SseOpcode.Movapd))
|
|
(rule (sse_mov_op (multi_lane _bits _lanes)) (SseOpcode.Movdqa))
|
|
|
|
;; Helper for creating `blendvp{d,s}` and `pblendvb` instructions.
|
|
(decl sse_blend (Type RegMem RegMem Reg) Reg)
|
|
(rule (sse_blend ty mask src1 src2)
|
|
;; Move the mask into `xmm0`, as blend instructions implicitly operate on
|
|
;; that register. (This kind of thing would normally happen inside of
|
|
;; `Inst::mov_mitosis`, but has to happen here, where we still have the
|
|
;; mask register, because the mask is implicit and doesn't appear in the
|
|
;; `Inst` itself.)
|
|
(let ((mask2 WritableReg (xmm0))
|
|
(_ Unit (emit (MInst.XmmUnaryRmR (sse_mov_op ty) mask mask2))))
|
|
(xmm_rm_r ty (sse_blend_op ty) src2 src1)))
|
|
|
|
;; Helper for creating `blendvpd` instructions.
|
|
(decl blendvpd (Reg RegMem Reg) Reg)
|
|
(rule (blendvpd src1 src2 mask)
|
|
;; Move the mask into `xmm0`, as `blendvpd` implicitly operates on that
|
|
;; register. (This kind of thing would normally happen inside of
|
|
;; `Inst::mov_mitosis`, but has to happen here, where we still have the
|
|
;; mask register, because the mask is implicit and doesn't appear in the
|
|
;; `Inst` itself.)
|
|
(let ((mask2 WritableReg (xmm0))
|
|
(_ Unit (emit (MInst.XmmUnaryRmR (SseOpcode.Movapd) (RegMem.Reg mask) mask2))))
|
|
(xmm_rm_r $F64X2 (SseOpcode.Blendvpd) src1 src2)))
|
|
|
|
;; Helper for creating `movsd` instructions.
|
|
(decl movsd (Reg RegMem) Reg)
|
|
(rule (movsd src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Movsd) src1 src2))
|
|
|
|
;; Helper for creating `movlhps` instructions.
|
|
(decl movlhps (Reg RegMem) Reg)
|
|
(rule (movlhps src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Movlhps) src1 src2))
|
|
|
|
;; Helper for creating `pmaxsb` instructions.
|
|
(decl pmaxsb (Reg RegMem) Reg)
|
|
(rule (pmaxsb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pmaxsb) src1 src2))
|
|
|
|
;; Helper for creating `pmaxsw` instructions.
|
|
(decl pmaxsw (Reg RegMem) Reg)
|
|
(rule (pmaxsw src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pmaxsw) src1 src2))
|
|
|
|
;; Helper for creating `pmaxsd` instructions.
|
|
(decl pmaxsd (Reg RegMem) Reg)
|
|
(rule (pmaxsd src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pmaxsd) src1 src2))
|
|
|
|
;; Helper for creating `pminsb` instructions.
|
|
(decl pminsb (Reg RegMem) Reg)
|
|
(rule (pminsb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pminsb) src1 src2))
|
|
|
|
;; Helper for creating `pminsw` instructions.
|
|
(decl pminsw (Reg RegMem) Reg)
|
|
(rule (pminsw src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pminsw) src1 src2))
|
|
|
|
;; Helper for creating `pminsd` instructions.
|
|
(decl pminsd (Reg RegMem) Reg)
|
|
(rule (pminsd src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pminsd) src1 src2))
|
|
|
|
;; Helper for creating `pmaxub` instructions.
|
|
(decl pmaxub (Reg RegMem) Reg)
|
|
(rule (pmaxub src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pmaxub) src1 src2))
|
|
|
|
;; Helper for creating `pmaxuw` instructions.
|
|
(decl pmaxuw (Reg RegMem) Reg)
|
|
(rule (pmaxuw src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pmaxuw) src1 src2))
|
|
|
|
;; Helper for creating `pmaxud` instructions.
|
|
(decl pmaxud (Reg RegMem) Reg)
|
|
(rule (pmaxud src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pmaxud) src1 src2))
|
|
|
|
;; Helper for creating `pminub` instructions.
|
|
(decl pminub (Reg RegMem) Reg)
|
|
(rule (pminub src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pminub) src1 src2))
|
|
|
|
;; Helper for creating `pminuw` instructions.
|
|
(decl pminuw (Reg RegMem) Reg)
|
|
(rule (pminuw src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pminuw) src1 src2))
|
|
|
|
;; Helper for creating `pminud` instructions.
|
|
(decl pminud (Reg RegMem) Reg)
|
|
(rule (pminud src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Pminud) src1 src2))
|
|
|
|
;; Helper for creating `punpcklbw` instructions.
|
|
(decl punpcklbw (Reg RegMem) Reg)
|
|
(rule (punpcklbw src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Punpcklbw) src1 src2))
|
|
|
|
;; Helper for creating `punpckhbw` instructions.
|
|
(decl punpckhbw (Reg RegMem) Reg)
|
|
(rule (punpckhbw src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Punpckhbw) src1 src2))
|
|
|
|
;; Helper for creating `packsswb` instructions.
|
|
(decl packsswb (Reg RegMem) Reg)
|
|
(rule (packsswb src1 src2)
|
|
(xmm_rm_r $I8X16 (SseOpcode.Packsswb) src1 src2))
|
|
|
|
;; Helper for creating `MInst.XmmRmRImm` instructions.
|
|
(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg)
|
|
(rule (xmm_rm_r_imm op src1 src2 imm size)
|
|
(let ((dst WritableReg (temp_writable_reg $I8X16))
|
|
(_ Unit (emit (MInst.XmmRmRImm op
|
|
src1
|
|
src2
|
|
dst
|
|
imm
|
|
size))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `palignr` instructions.
|
|
(decl palignr (Reg RegMem u8 OperandSize) Reg)
|
|
(rule (palignr src1 src2 imm size)
|
|
(xmm_rm_r_imm (SseOpcode.Palignr)
|
|
src1
|
|
src2
|
|
imm
|
|
size))
|
|
|
|
;; Helper for creating `pshufd` instructions.
|
|
(decl pshufd (RegMem u8 OperandSize) Reg)
|
|
(rule (pshufd src imm size)
|
|
(let ((w_dst WritableReg (temp_writable_reg $I8X16))
|
|
(dst Reg (writable_reg_to_reg w_dst))
|
|
(_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd)
|
|
dst
|
|
src
|
|
w_dst
|
|
imm
|
|
size))))
|
|
dst))
|
|
|
|
;; Helper for creating `MInst.XmmUnaryRmR` instructions.
|
|
(decl xmm_unary_rm_r (SseOpcode RegMem) Reg)
|
|
(rule (xmm_unary_rm_r op src)
|
|
(let ((dst WritableReg (temp_writable_reg $I8X16))
|
|
(_ Unit (emit (MInst.XmmUnaryRmR op src dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `pmovsxbw` instructions.
|
|
(decl pmovsxbw (RegMem) Reg)
|
|
(rule (pmovsxbw src)
|
|
(xmm_unary_rm_r (SseOpcode.Pmovsxbw) src))
|
|
|
|
;; Helper for creating `pmovzxbw` instructions.
|
|
(decl pmovzxbw (RegMem) Reg)
|
|
(rule (pmovzxbw src)
|
|
(xmm_unary_rm_r (SseOpcode.Pmovzxbw) src))
|
|
|
|
;; Helper for creating `pabsb` instructions.
|
|
(decl pabsb (RegMem) Reg)
|
|
(rule (pabsb src)
|
|
(xmm_unary_rm_r (SseOpcode.Pabsb) src))
|
|
|
|
;; Helper for creating `pabsw` instructions.
|
|
(decl pabsw (RegMem) Reg)
|
|
(rule (pabsw src)
|
|
(xmm_unary_rm_r (SseOpcode.Pabsw) src))
|
|
|
|
;; Helper for creating `pabsd` instructions.
|
|
(decl pabsd (RegMem) Reg)
|
|
(rule (pabsd src)
|
|
(xmm_unary_rm_r (SseOpcode.Pabsd) src))
|
|
|
|
;; Helper for creating `MInst.XmmUnaryRmREvex` instructions.
|
|
(decl xmm_unary_rm_r_evex (Avx512Opcode RegMem) Reg)
|
|
(rule (xmm_unary_rm_r_evex op src)
|
|
(let ((dst WritableReg (temp_writable_reg $I8X16))
|
|
(_ Unit (emit (MInst.XmmUnaryRmREvex op src dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `vpabsq` instructions.
|
|
(decl vpabsq (RegMem) Reg)
|
|
(rule (vpabsq src)
|
|
(xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src))
|
|
|
|
;; Helper for creating `MInst.XmmRmREvex` instructions.
|
|
(decl xmm_rm_r_evex (Avx512Opcode RegMem Reg) Reg)
|
|
(rule (xmm_rm_r_evex op src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg $I8X16))
|
|
(_ Unit (emit (MInst.XmmRmREvex op
|
|
src1
|
|
src2
|
|
dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `vpmullq` instructions.
|
|
;;
|
|
;; Requires AVX-512 vl and dq.
|
|
(decl vpmullq (RegMem Reg) Reg)
|
|
(rule (vpmullq src1 src2)
|
|
(xmm_rm_r_evex (Avx512Opcode.Vpmullq)
|
|
src1
|
|
src2))
|
|
|
|
;; Helper for creating `MInst.XmmRmiReg` instructions.
|
|
(decl xmm_rmi_reg (SseOpcode Reg RegMemImm) Reg)
|
|
(rule (xmm_rmi_reg op src1 src2)
|
|
(let ((dst WritableReg (temp_writable_reg $I8X16))
|
|
(_ Unit (emit (MInst.XmmRmiReg op
|
|
src1
|
|
src2
|
|
dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `psllw` instructions.
|
|
(decl psllw (Reg RegMemImm) Reg)
|
|
(rule (psllw src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Psllw) src1 src2))
|
|
|
|
;; Helper for creating `pslld` instructions.
|
|
(decl pslld (Reg RegMemImm) Reg)
|
|
(rule (pslld src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Pslld) src1 src2))
|
|
|
|
;; Helper for creating `psllq` instructions.
|
|
(decl psllq (Reg RegMemImm) Reg)
|
|
(rule (psllq src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Psllq) src1 src2))
|
|
|
|
;; Helper for creating `psrlw` instructions.
|
|
(decl psrlw (Reg RegMemImm) Reg)
|
|
(rule (psrlw src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Psrlw) src1 src2))
|
|
|
|
;; Helper for creating `psrld` instructions.
|
|
(decl psrld (Reg RegMemImm) Reg)
|
|
(rule (psrld src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Psrld) src1 src2))
|
|
|
|
;; Helper for creating `psrlq` instructions.
|
|
(decl psrlq (Reg RegMemImm) Reg)
|
|
(rule (psrlq src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Psrlq) src1 src2))
|
|
|
|
;; Helper for creating `psraw` instructions.
|
|
(decl psraw (Reg RegMemImm) Reg)
|
|
(rule (psraw src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Psraw) src1 src2))
|
|
|
|
;; Helper for creating `psrad` instructions.
|
|
(decl psrad (Reg RegMemImm) Reg)
|
|
(rule (psrad src1 src2)
|
|
(xmm_rmi_reg (SseOpcode.Psrad) src1 src2))
|
|
|
|
;; Helper for creating `MInst.MulHi` instructions.
|
|
;;
|
|
;; Returns the (lo, hi) register halves of the multiplication.
|
|
(decl mul_hi (Type bool Reg RegMem) ValueRegs)
|
|
(rule (mul_hi ty signed src1 src2)
|
|
(let ((dst_lo WritableReg (temp_writable_reg ty))
|
|
(dst_hi WritableReg (temp_writable_reg ty))
|
|
(size OperandSize (operand_size_of_type_32_64 ty))
|
|
(_ Unit (emit (MInst.MulHi size
|
|
signed
|
|
src1
|
|
src2
|
|
dst_lo
|
|
dst_hi))))
|
|
(value_regs (writable_reg_to_reg dst_lo)
|
|
(writable_reg_to_reg dst_hi))))
|
|
|
|
;; Helper for creating `mul` instructions that return both the lower and
|
|
;; (unsigned) higher halves of the result.
|
|
(decl mulhi_u (Type Reg RegMem) ValueRegs)
|
|
(rule (mulhi_u ty src1 src2)
|
|
(mul_hi ty $false src1 src2))
|
|
|
|
;; Helper for creating `cmpps` instructions.
|
|
(decl cmpps (Reg RegMem FcmpImm) Reg)
|
|
(rule (cmpps src1 src2 imm)
|
|
(xmm_rm_r_imm (SseOpcode.Cmpps)
|
|
src1
|
|
src2
|
|
(encode_fcmp_imm imm)
|
|
(OperandSize.Size32)))
|
|
|
|
;; Helper for creating `cmppd` instructions.
|
|
;;
|
|
;; Note that `Size32` is intentional despite this being used for 64-bit
|
|
;; operations, since this presumably induces the correct encoding of the
|
|
;; instruction.
|
|
(decl cmppd (Reg RegMem FcmpImm) Reg)
|
|
(rule (cmppd src1 src2 imm)
|
|
(xmm_rm_r_imm (SseOpcode.Cmppd)
|
|
src1
|
|
src2
|
|
(encode_fcmp_imm imm)
|
|
(OperandSize.Size32)))
|
|
|
|
;; Helper for creating `MInst.GprToXmm` instructions.
|
|
(decl gpr_to_xmm (Type SseOpcode RegMem OperandSize) Reg)
|
|
(rule (gpr_to_xmm ty op src size)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(_ Unit (emit (MInst.GprToXmm op src dst size))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
;; Helper for creating `pinsrb` instructions.
|
|
(decl pinsrb (Reg RegMem u8) Reg)
|
|
(rule (pinsrb src1 src2 lane)
|
|
(xmm_rm_r_imm (SseOpcode.Pinsrb) src1 src2 lane (OperandSize.Size32)))
|
|
|
|
;; Helper for creating `pinsrw` instructions.
|
|
(decl pinsrw (Reg RegMem u8) Reg)
|
|
(rule (pinsrw src1 src2 lane)
|
|
(xmm_rm_r_imm (SseOpcode.Pinsrw) src1 src2 lane (OperandSize.Size32)))
|
|
|
|
;; Helper for creating `pinsrd` instructions.
|
|
(decl pinsrd (Reg RegMem u8 OperandSize) Reg)
|
|
(rule (pinsrd src1 src2 lane size)
|
|
(xmm_rm_r_imm (SseOpcode.Pinsrd) src1 src2 lane size))
|
|
|
|
;; Helper for creating `insertps` instructions.
|
|
(decl insertps (Reg RegMem u8) Reg)
|
|
(rule (insertps src1 src2 lane)
|
|
(xmm_rm_r_imm (SseOpcode.Insertps) src1 src2 lane (OperandSize.Size32)))
|
|
|
|
;; Helper for creating `pextrd` instructions.
|
|
(decl pextrd (Type Reg u8) Reg)
|
|
(rule (pextrd ty src lane)
|
|
(let ((w_dst WritableReg (temp_writable_reg ty))
|
|
(r_dst Reg (writable_reg_to_reg w_dst))
|
|
(_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd)
|
|
r_dst
|
|
(RegMem.Reg src)
|
|
w_dst
|
|
lane
|
|
(operand_size_of_type_32_64 (lane_type ty))))))
|
|
r_dst))
|
|
|
|
;; Helper for creating `not` instructions.
|
|
(decl not (Type Reg) Reg)
|
|
(rule (not ty src)
|
|
(let ((dst WritableReg (temp_writable_reg ty))
|
|
(size OperandSize (operand_size_of_type_32_64 ty))
|
|
(_ Unit (emit (MInst.Not size src dst))))
|
|
(writable_reg_to_reg dst)))
|
|
|
|
(decl lea (SyntheticAmode) Reg)
|
|
(rule (lea addr)
|
|
(let ((dst WritableReg (temp_writable_reg $I64))
|
|
(_ Unit (emit (MInst.LoadEffectiveAddress addr dst))))
|
|
(writable_reg_to_reg dst)))
|