T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cranelift: Use GPR newtypes extensively in x64 lowering (#3798)

Browse Source 
We already defined the `Gpr` newtype and used it in a few places, and we already
defined the `Xmm` newtype and used it extensively. This finishes the transition
to using the newtypes extensively in lowering by making use of `Gpr` in more
places.

Fixes #3685
This commit is contained in:
Nick Fitzgerald
2022-02-14 12:54:41 -08:00
committed by GitHub
parent 84b9c7bb8a
commit dc86e7a6dc
9 changed files with 1804 additions and 1482 deletions
Download Patch File Download Diff File Expand all files Collapse all files

291
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -16,9 +16,9 @@
;; Integer arithmetic/bit-twiddling.
(AluRmiR (size OperandSize) ;; 4 or 8
(op AluRmiROpcode)
(src1 Reg)
(src2 RegMemImm)
(dst WritableReg))
(src1 Gpr)
(src2 GprMemImm)
(dst WritableGpr))
;; Instructions on general-purpose registers that only read src and
;; defines dst (dst is not modified). `bsr`, etc.
@@ -40,19 +40,19 @@
;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
(Div (size OperandSize) ;; 1, 2, 4, or 8
(signed bool)
(divisor RegMem)
(dividend Reg)
(dst_quotient WritableReg)
(dst_remainder WritableReg))
(divisor GprMem)
(dividend Gpr)
(dst_quotient WritableGpr)
(dst_remainder WritableGpr))
;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX *
;; rhs`.
(MulHi (size OperandSize)
(signed bool)
(src1 Reg)
(src2 RegMem)
(dst_lo WritableReg)
(dst_hi WritableReg))
(src1 Gpr)
(src2 GprMem)
(dst_lo WritableGpr)
(dst_hi WritableGpr))
;; A synthetic sequence to implement the right inline checks for
;; remainder and division, assuming the dividend is in %rax.
@@ -69,32 +69,32 @@
;; regalloc failures where %rdx is live before its first def!
(CheckedDivOrRemSeq (kind DivOrRemKind)
(size OperandSize)
(dividend Reg)
(dividend Gpr)
;; The divisor operand. Note it's marked as modified
;; so that it gets assigned a register different from
;; the temporary.
(divisor WritableReg)
(dst_quotient WritableReg)
(dst_remainder WritableReg)
(tmp OptionWritableReg))
(divisor WritableGpr)
(dst_quotient WritableGpr)
(dst_remainder WritableGpr)
(tmp OptionWritableGpr))
;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd
;; cdq cqo) or al into ah: (cbw)
(SignExtendData (size OperandSize) ;; 1, 2, 4, or 8
(src Reg)
(dst WritableReg))
(src Gpr)
(dst WritableGpr))
;; Constant materialization: (imm32 imm64) reg.
;;
;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32.
(Imm (dst_size OperandSize) ;; 4 or 8
(simm64 u64)
(dst WritableReg))
(dst WritableGpr))
;; GPR to GPR move: mov (64 32) reg reg.
(MovRR (size OperandSize) ;; 4 or 8
(src Reg)
(dst WritableReg))
(src Gpr)
(dst WritableGpr))
;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr
;; reg.
@@ -103,12 +103,12 @@
;; zero-extend rule makes it unnecessary. For that case we emit the
;; equivalent "movl AM, reg32".
(MovzxRmR (ext_mode ExtMode)
(src RegMem)
(dst WritableReg))
(src GprMem)
(dst WritableGpr))
;; A plain 64-bit integer load, since MovZX_RM_R can't represent that.
(Mov64MR (src SyntheticAmode)
(dst WritableReg))
(dst WritableGpr))
;; Loads the memory address of addr into dst.
(LoadEffectiveAddress (addr SyntheticAmode)
@@ -116,22 +116,22 @@
;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg.
(MovsxRmR (ext_mode ExtMode)
(src RegMem)
(dst WritableReg))
(src GprMem)
(dst WritableGpr))
;; Integer stores: mov (b w l q) reg addr.
(MovRM (size OperandSize) ;; 1, 2, 4, or 8
(src Reg)
(src Gpr)
(dst SyntheticAmode))
;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
(ShiftR (size OperandSize) ;; 1, 2, 4, or 8
(kind ShiftKind)
(src Reg)
;; shift count: `Imm8Reg::Imm8(0 .. #bits-in-type - 1)` or
;; `Imm8Reg::Reg(r)` where `r` get's move mitosis'd into `%cl`.
(num_bits Imm8Reg)
(dst WritableReg))
(src Gpr)
;; shift count: `Imm8Gpr::Imm8(0 .. #bits-in-type - 1)` or
;; `Imm8Reg::Gpr(r)` where `r` get's move mitosis'd into `%cl`.
(num_bits Imm8Gpr)
(dst WritableGpr))
;; Arithmetic SIMD shifts.
(XmmRmiReg (opcode SseOpcode)
@@ -142,30 +142,30 @@
;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
(CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8
(opcode CmpOpcode)
(src RegMemImm)
(dst Reg))
(src GprMemImm)
(dst Gpr))
;; Materializes the requested condition code in the destinaton reg.
(Setcc (cc CC)
(dst WritableReg))
(dst WritableGpr))
;; Integer conditional move.
;;
;; Overwrites the destination register.
(Cmove (size OperandSize)
(cc CC)
(consequent RegMem)
(alternative Reg)
(dst WritableReg))
(consequent GprMem)
(alternative Gpr)
(dst WritableGpr))
;; =========================================
;; Stack manipulation.
;; pushq (reg addr imm)
(Push64 (src RegMemImm))
(Push64 (src GprMemImm))
;; popq reg
(Pop64 (dst WritableReg))
(Pop64 (dst WritableGpr))
;; =========================================
;; Floating-point operations.
@@ -221,7 +221,7 @@
;; XMM (scalar) unary op (from integer to float reg): movd, movq,
;; cvtsi2s{s,d}
(GprToXmm (op SseOpcode)
(src RegMem)
(src GprMem)
(dst WritableXmm)
(src_size OperandSize))
@@ -272,21 +272,21 @@
;; registers.
(XmmMinMaxSeq (size OperandSize)
(is_min bool)
(lhs Reg)
(rhs_dst WritableReg))
(lhs Xmm)
(rhs_dst WritableXmm))
;; XMM (scalar) conditional move.
;;
;; Overwrites the destination register if cc is set.
(XmmCmove (size OperandSize)
(cc CC)
(src RegMem)
(dst WritableReg))
(src XmmMem)
(dst WritableXmm))
;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
(XmmCmpRmR (op SseOpcode)
(src RegMem)
(dst Reg))
(src XmmMem)
(dst Xmm))
;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm
;; (reg addr) reg
@@ -780,8 +780,8 @@
;; As a side effect, this marks the value as used.
;;
;; This is used when lowering various shifts and rotates.
(decl put_masked_in_imm8_reg (Value Type) Imm8Reg)
(extern constructor put_masked_in_imm8_reg put_masked_in_imm8_reg)
(decl put_masked_in_imm8_gpr (Value Type) Imm8Gpr)
(extern constructor put_masked_in_imm8_gpr put_masked_in_imm8_gpr)
(type CC extern
(enum O
@@ -825,14 +825,21 @@
(type Gpr (primitive Gpr))
(type WritableGpr (primitive WritableGpr))
(type OptionWritableGpr (primitive OptionWritableGpr))
(type GprMem extern (enum))
(type GprMemImm extern (enum))
(type Imm8Gpr extern (enum))
(type Xmm (primitive Xmm))
(type WritableXmm (primitive WritableXmm))
(type OptionWritableXmm (primitive OptionWritableXmm))
(type XmmMem extern (enum))
(type XmmMemImm extern (enum))
;; Convert an `Imm8Reg` into an `Imm8Gpr`.
(decl imm8_reg_to_imm8_gpr (Imm8Reg) Imm8Gpr)
(extern constructor imm8_reg_to_imm8_gpr imm8_reg_to_imm8_gpr)
;; Convert a `WritableGpr` to a `WritableReg`.
(decl writable_gpr_to_reg (WritableGpr) WritableReg)
(extern constructor writable_gpr_to_reg writable_gpr_to_reg)
@@ -857,6 +864,14 @@
(decl gpr_to_reg (Gpr) Reg)
(extern constructor gpr_to_reg gpr_to_reg)
;; Convert an `Gpr` to a `GprMem`.
(decl gpr_to_gpr_mem (Gpr) GprMem)
(extern constructor gpr_to_gpr_mem gpr_to_gpr_mem)
;; Convert an `Gpr` to a `GprMemImm`.
(decl gpr_to_gpr_mem_imm (Gpr) GprMemImm)
(extern constructor gpr_to_gpr_mem_imm gpr_to_gpr_mem_imm)
;; Convert an `Xmm` to a `Reg`.
(decl xmm_to_reg (Xmm) Reg)
(extern constructor xmm_to_reg xmm_to_reg)
@@ -984,6 +999,25 @@
(rule (value_xmm x)
(value_reg (xmm_to_reg x)))
;; Get the `n`th reg in a `ValueRegs` and construct a GPR from it.
;;
;; Asserts that the register is a GPR.
(decl value_regs_get_gpr (ValueRegs usize) Gpr)
(rule (value_regs_get_gpr regs n)
(gpr_new (value_regs_get regs n)))
;; Convert a `Gpr` to an `Imm8Gpr`.
(decl gpr_to_imm8_gpr (Gpr) Imm8Gpr)
(extern constructor gpr_to_imm8_gpr gpr_to_imm8_gpr)
;; Convert an 8-bit immediate into an `Imm8Gpr`.
(decl imm8_to_imm8_gpr (u8) Imm8Gpr)
(extern constructor imm8_to_imm8_gpr imm8_to_imm8_gpr)
;; Get the low half of the given `Value` as a GPR.
(decl lo_gpr (Value) Gpr)
(rule (lo_gpr regs) (gpr_new (lo_reg regs)))
;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; These should only be used for legalization purposes, when we can't otherwise
@@ -1014,15 +1048,15 @@
;; `Imm8Reg.Imm8`. This is used for shifts and rotates, so that we don't try and
;; shift/rotate more bits than the type has available, per Cranelift's
;; semantics.
(decl const_to_type_masked_imm8 (u64 Type) Imm8Reg)
(decl const_to_type_masked_imm8 (u64 Type) Imm8Gpr)
(extern constructor const_to_type_masked_imm8 const_to_type_masked_imm8)
;; Extract a constant `RegMemImm.Imm` from a value operand.
(decl simm32_from_value (RegMemImm) Value)
;; Extract a constant `GprMemImm.Imm` from a value operand.
(decl simm32_from_value (GprMemImm) Value)
(extern extractor simm32_from_value simm32_from_value)
;; Extract a constant `RegMemImm.Imm` from an `Imm64` immediate.
(decl simm32_from_imm64 (RegMemImm) Imm64)
(decl simm32_from_imm64 (GprMemImm) Imm64)
(extern extractor simm32_from_imm64 simm32_from_imm64)
;; A load that can be sunk into another operation.
@@ -1041,6 +1075,10 @@
(decl sink_load (SinkableLoad) RegMemImm)
(extern constructor sink_load sink_load)
(decl sink_load_to_gpr_mem_imm (SinkableLoad) GprMemImm)
(rule (sink_load_to_gpr_mem_imm load)
(gpr_mem_imm_new (sink_load load)))
;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(type ExtKind extern
@@ -1057,13 +1095,13 @@
(extern constructor ext_mode ext_mode)
;; Put the given value into a register, but extended as the given type.
(decl extend_to_reg (Value Type ExtendKind) Reg)
(decl extend_to_gpr (Value Type ExtendKind) Gpr)
;; If the value is already of the requested type, no extending is necessary.
(rule (extend_to_reg (and val (value_type ty)) =ty _kind)
(put_in_reg val))
(rule (extend_to_gpr (and val (value_type ty)) =ty _kind)
(put_in_gpr val))
(rule (extend_to_reg (and val (value_type from_ty))
(rule (extend_to_gpr (and val (value_type from_ty))
to_ty
kind)
(let ((from_bits u16 (ty_bits_u16 from_ty))
@@ -1073,10 +1111,10 @@
(extend kind
to_ty
(ext_mode from_bits to_bits)
(put_in_reg_mem val))))
(put_in_gpr_mem val))))
;; Do a sign or zero extension of the given `RegMem`.
(decl extend (ExtendKind Type ExtMode RegMem) Reg)
;; Do a sign or zero extension of the given `GprMem`.
(decl extend (ExtendKind Type ExtMode GprMem) Gpr)
;; Zero extending uses `movzx`.
(rule (extend (ExtendKind.Zero) ty mode src)
@@ -1166,14 +1204,14 @@
(decl x64_load (Type SyntheticAmode ExtKind) Reg)
(rule (x64_load (fits_in_32 ty) addr (ExtKind.SignExtend))
(movsx ty
(gpr_to_reg (movsx ty
(ext_mode (ty_bytes ty) 8)
(synthetic_amode_to_reg_mem addr)))
(reg_mem_to_gpr_mem (synthetic_amode_to_reg_mem addr)))))
(rule (x64_load $I64 addr _ext_kind)
(let ((dst WritableReg (temp_writable_reg $I64))
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.Mov64MR addr dst))))
(writable_reg_to_reg dst)))
(gpr_to_reg (writable_gpr_to_gpr dst))))
(rule (x64_load $F32 addr _ext_kind)
(xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss)
@@ -1202,15 +1240,15 @@
;; only gets defined the once.
;; Helper for emitting `MInst.AluRmiR` instructions.
(decl alu_rmi_r (Type AluRmiROpcode Reg RegMemImm) Reg)
(decl alu_rmi_r (Type AluRmiROpcode Gpr GprMemImm) Gpr)
(rule (alu_rmi_r ty opcode src1 src2)
(let ((dst WritableReg (temp_writable_reg ty))
(let ((dst WritableGpr (temp_writable_gpr))
(size OperandSize (operand_size_of_type_32_64 ty))
(_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst))))
(writable_reg_to_reg dst)))
(writable_gpr_to_gpr dst)))
;; Helper for emitting `add` instructions.
(decl add (Type Reg RegMemImm) Reg)
(decl add (Type Gpr GprMemImm) Gpr)
(rule (add ty src1 src2)
(alu_rmi_r ty
(AluRmiROpcode.Add)
@@ -1218,29 +1256,29 @@
src2))
;; Helper for creating `add` instructions whose flags are also used.
(decl add_with_flags (Type Reg RegMemImm) ProducesFlags)
(decl add_with_flags (Type Gpr GprMemImm) ProducesFlags)
(rule (add_with_flags ty src1 src2)
(let ((dst WritableReg (temp_writable_reg ty)))
(let ((dst WritableGpr (temp_writable_gpr)))
(ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
(AluRmiROpcode.Add)
src1
src2
dst)
(writable_reg_to_reg dst))))
(gpr_to_reg (writable_gpr_to_gpr dst)))))
;; Helper for creating `adc` instructions.
(decl adc (Type Reg RegMemImm) ConsumesFlags)
(decl adc (Type Gpr GprMemImm) ConsumesFlags)
(rule (adc ty src1 src2)
(let ((dst WritableReg (temp_writable_reg ty)))
(let ((dst WritableGpr (temp_writable_gpr)))
(ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
(AluRmiROpcode.Adc)
src1
src2
dst)
(writable_reg_to_reg dst))))
(gpr_to_reg (writable_gpr_to_gpr dst)))))
;; Helper for emitting `sub` instructions.
(decl sub (Type Reg RegMemImm) Reg)
(decl sub (Type Gpr GprMemImm) Gpr)
(rule (sub ty src1 src2)
(alu_rmi_r ty
(AluRmiROpcode.Sub)
@@ -1248,29 +1286,29 @@
src2))
;; Helper for creating `sub` instructions whose flags are also used.
(decl sub_with_flags (Type Reg RegMemImm) ProducesFlags)
(decl sub_with_flags (Type Gpr GprMemImm) ProducesFlags)
(rule (sub_with_flags ty src1 src2)
(let ((dst WritableReg (temp_writable_reg ty)))
(let ((dst WritableGpr (temp_writable_gpr)))
(ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
(AluRmiROpcode.Sub)
src1
src2
dst)
(writable_reg_to_reg dst))))
(gpr_to_reg (writable_gpr_to_gpr dst)))))
;; Helper for creating `sbb` instructions.
(decl sbb (Type Reg RegMemImm) ConsumesFlags)
(decl sbb (Type Gpr GprMemImm) ConsumesFlags)
(rule (sbb ty src1 src2)
(let ((dst WritableReg (temp_writable_reg ty)))
(let ((dst WritableGpr (temp_writable_gpr)))
(ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty)
(AluRmiROpcode.Sbb)
src1
src2
dst)
(writable_reg_to_reg dst))))
(gpr_to_reg (writable_gpr_to_gpr dst)))))
;; Helper for creating `mul` instructions.
(decl mul (Type Reg RegMemImm) Reg)
(decl mul (Type Gpr GprMemImm) Gpr)
(rule (mul ty src1 src2)
(alu_rmi_r ty
(AluRmiROpcode.Mul)
@@ -1278,10 +1316,7 @@
src2))
;; Helper for emitting `and` instructions.
;;
;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin
;; `and` operator.
(decl x64_and (Type Reg RegMemImm) Reg)
(decl x64_and (Type Gpr GprMemImm) Gpr)
(rule (x64_and ty src1 src2)
(alu_rmi_r ty
(AluRmiROpcode.And)
@@ -1289,7 +1324,7 @@
src2))
;; Helper for emitting `or` instructions.
(decl or (Type Reg RegMemImm) Reg)
(decl or (Type Gpr GprMemImm) Gpr)
(rule (or ty src1 src2)
(alu_rmi_r ty
(AluRmiROpcode.Or)
@@ -1297,7 +1332,7 @@
src2))
;; Helper for emitting `xor` instructions.
(decl xor (Type Reg RegMemImm) Reg)
(decl xor (Type Gpr GprMemImm) Gpr)
(rule (xor ty src1 src2)
(alu_rmi_r ty
(AluRmiROpcode.Xor)
@@ -1309,10 +1344,10 @@
;; Integer immediates.
(rule (imm (fits_in_64 ty) simm64)
(let ((dst WritableReg (temp_writable_reg ty))
(let ((dst WritableGpr (temp_writable_gpr))
(size OperandSize (operand_size_of_type_32_64 ty))
(_ Unit (emit (MInst.Imm size simm64 dst))))
(writable_reg_to_reg dst)))
(gpr_to_reg (writable_gpr_to_gpr dst))))
;; `f32` immediates.
(rule (imm $F32 bits)
@@ -1332,21 +1367,21 @@
;; Special case for when a 64-bit immediate fits into 32-bits. We can use a
;; 32-bit move that zero-extends the value, which has a smaller encoding.
(rule (imm $I64 (nonzero_u64_fits_in_u32 x))
(let ((dst WritableReg (temp_writable_reg $I64))
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.Imm (OperandSize.Size32) x dst))))
(writable_reg_to_reg dst)))
(gpr_to_reg (writable_gpr_to_gpr dst))))
;; Special case for integer zero immediates: turn them into an `xor r, r`.
(rule (imm (fits_in_64 ty) 0)
(let ((wr WritableReg (temp_writable_reg ty))
(r Reg (writable_reg_to_reg wr))
(let ((wgpr WritableGpr (temp_writable_gpr))
(g Gpr (writable_gpr_to_gpr wgpr))
(size OperandSize (operand_size_of_type_32_64 ty))
(_ Unit (emit (MInst.AluRmiR size
(AluRmiROpcode.Xor)
r
(RegMemImm.Reg r)
wr))))
r))
g
(gpr_to_gpr_mem_imm g)
wgpr))))
(gpr_to_reg g)))
;; Special case for zero immediates with vector types, they turn into an xor
;; specific to the vector type.
@@ -1384,44 +1419,42 @@
;; TODO: use cmpeqpd for all 1s
;; Helper for creating `MInst.ShifR` instructions.
(decl shift_r (Type ShiftKind Reg Imm8Reg) Reg)
(decl shift_r (Type ShiftKind Gpr Imm8Gpr) Gpr)
(rule (shift_r ty kind src1 src2)
(let ((dst WritableReg (temp_writable_reg ty))
(let ((dst WritableGpr (temp_writable_gpr))
;; Use actual 8/16-bit instructions when appropriate: we
;; rely on their shift-amount-masking semantics.
(size OperandSize (raw_operand_size_of_type ty))
(_ Unit (emit (MInst.ShiftR size kind src1 src2 dst))))
(writable_reg_to_reg dst)))
(writable_gpr_to_gpr dst)))
;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach
;; inst", to disambiguate this from clif's `rotl`).
(decl x64_rotl (Type Reg Imm8Reg) Reg)
;; Helper for creating `rotl` instructions.
(decl x64_rotl (Type Gpr Imm8Gpr) Gpr)
(rule (x64_rotl ty src1 src2)
(shift_r ty (ShiftKind.RotateLeft) src1 src2))
;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach
;; inst", to disambiguate this from clif's `rotr`).
(decl x64_rotr (Type Reg Imm8Reg) Reg)
;; Helper for creating `rotr` instructions.
(decl x64_rotr (Type Gpr Imm8Gpr) Gpr)
(rule (x64_rotr ty src1 src2)
(shift_r ty (ShiftKind.RotateRight) src1 src2))
;; Helper for creating `shl` instructions.
(decl shl (Type Reg Imm8Reg) Reg)
(decl shl (Type Gpr Imm8Gpr) Gpr)
(rule (shl ty src1 src2)
(shift_r ty (ShiftKind.ShiftLeft) src1 src2))
;; Helper for creating logical shift-right instructions.
(decl shr (Type Reg Imm8Reg) Reg)
(decl shr (Type Gpr Imm8Gpr) Gpr)
(rule (shr ty src1 src2)
(shift_r ty (ShiftKind.ShiftRightLogical) src1 src2))
;; Helper for creating arithmetic shift-right instructions.
(decl sar (Type Reg Imm8Reg) Reg)
(decl sar (Type Gpr Imm8Gpr) Gpr)
(rule (sar ty src1 src2)
(shift_r ty (ShiftKind.ShiftRightArithmetic) src1 src2))
;; Helper for creating `MInst.CmpRmiR` instructions.
(decl cmp_rmi_r (OperandSize CmpOpcode RegMemImm Reg) ProducesFlags)
(decl cmp_rmi_r (OperandSize CmpOpcode GprMemImm Gpr) ProducesFlags)
(rule (cmp_rmi_r size opcode src1 src2)
(ProducesFlags.ProducesFlags (MInst.CmpRmiR size
opcode
@@ -1430,36 +1463,36 @@
(invalid_reg)))
;; Helper for creating `cmp` instructions.
(decl cmp (OperandSize RegMemImm Reg) ProducesFlags)
(decl cmp (OperandSize GprMemImm Gpr) ProducesFlags)
(rule (cmp size src1 src2)
(cmp_rmi_r size (CmpOpcode.Cmp) src1 src2))
;; Helper for creating `test` instructions.
(decl test (OperandSize RegMemImm Reg) ProducesFlags)
(decl test (OperandSize GprMemImm Gpr) ProducesFlags)
(rule (test size src1 src2)
(cmp_rmi_r size (CmpOpcode.Test) src1 src2))
;; Helper for creating `MInst.Cmove` instructions.
(decl cmove (Type CC RegMem Reg) ConsumesFlags)
(decl cmove (Type CC GprMem Gpr) ConsumesFlags)
(rule (cmove ty cc consequent alternative)
(let ((dst WritableReg (temp_writable_reg ty))
(let ((dst WritableGpr (temp_writable_gpr))
(size OperandSize (operand_size_of_type_32_64 ty)))
(ConsumesFlags.ConsumesFlags (MInst.Cmove size cc consequent alternative dst)
(writable_reg_to_reg dst))))
(gpr_to_reg (writable_gpr_to_gpr dst)))))
;; Helper for creating `MInst.MovzxRmR` instructions.
(decl movzx (Type ExtMode RegMem) Reg)
(decl movzx (Type ExtMode GprMem) Gpr)
(rule (movzx ty mode src)
(let ((dst WritableReg (temp_writable_reg ty))
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.MovzxRmR mode src dst))))
(writable_reg_to_reg dst)))
(writable_gpr_to_gpr dst)))
;; Helper for creating `MInst.MovsxRmR` instructions.
(decl movsx (Type ExtMode RegMem) Reg)
(decl movsx (Type ExtMode GprMem) Gpr)
(rule (movsx ty mode src)
(let ((dst WritableReg (temp_writable_reg ty))
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.MovsxRmR mode src dst))))
(writable_reg_to_reg dst)))
(writable_gpr_to_gpr dst)))
;; Helper for creating `MInst.XmmRmR` instructions.
(decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm)
@@ -1926,10 +1959,10 @@
;; Helper for creating `MInst.MulHi` instructions.
;;
;; Returns the (lo, hi) register halves of the multiplication.
(decl mul_hi (Type bool Reg RegMem) ValueRegs)
(decl mul_hi (Type bool Gpr GprMem) ValueRegs)
(rule (mul_hi ty signed src1 src2)
(let ((dst_lo WritableReg (temp_writable_reg ty))
(dst_hi WritableReg (temp_writable_reg ty))
(let ((dst_lo WritableGpr (temp_writable_gpr))
(dst_hi WritableGpr (temp_writable_gpr))
(size OperandSize (operand_size_of_type_32_64 ty))
(_ Unit (emit (MInst.MulHi size
signed
@@ -1937,12 +1970,12 @@
src2
dst_lo
dst_hi))))
(value_regs (writable_reg_to_reg dst_lo)
(writable_reg_to_reg dst_hi))))
(value_gprs (writable_gpr_to_gpr dst_lo)
(writable_gpr_to_gpr dst_hi))))
;; Helper for creating `mul` instructions that return both the lower and
;; (unsigned) higher halves of the result.
(decl mulhi_u (Type Reg RegMem) ValueRegs)
(decl mulhi_u (Type Gpr GprMem) ValueRegs)
(rule (mulhi_u ty src1 src2)
(mul_hi ty $false src1 src2))
@@ -2026,7 +2059,7 @@
(decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm)
(rule (gpr_to_xmm op src size)
(let ((dst WritableXmm (temp_writable_xmm))
(_ Unit (emit (MInst.GprToXmm op (gpr_mem_to_reg_mem src) dst size))))
(_ Unit (emit (MInst.GprToXmm op src dst size))))
(writable_xmm_to_xmm dst)))
;; Helper for creating `not` instructions.

69
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -48,8 +48,10 @@ macro_rules! newtype_of_reg {
(
$newtype_reg:ident,
$newtype_writable_reg:ident,
$newtype_option_writable_reg:ident,
$newtype_reg_mem:ident,
$newtype_reg_mem_imm:ident,
$newtype_imm8_reg:ident,
|$check_reg:ident| $check:expr
) => {
/// A newtype wrapper around `Reg`.
@@ -122,6 +124,9 @@ macro_rules! newtype_of_reg {
pub type $newtype_writable_reg = Writable<$newtype_reg>;
#[allow(dead_code)] // Used by some newtypes and not others.
pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
impl ToWritableReg for $newtype_writable_reg {
fn to_writable_reg(&self) -> Writable<Reg> {
Writable::from_reg(self.to_reg().to_reg())
@@ -218,6 +223,11 @@ macro_rules! newtype_of_reg {
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
self.0.get_regs_as_uses(collector);
}
}
impl PrettyPrint for $newtype_reg_mem {
@@ -290,6 +300,11 @@ macro_rules! newtype_of_reg {
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
self.0.get_regs_as_uses(collector);
}
}
impl PrettyPrint for $newtype_reg_mem_imm {
@@ -303,18 +318,60 @@ macro_rules! newtype_of_reg {
self.0.show_rru_sized(mb_rru, size)
}
}
/// A newtype wrapper around `Imm8Reg`.
#[derive(Clone, Debug)]
#[allow(dead_code)] // Used by some newtypes and not others.
pub struct $newtype_imm8_reg(Imm8Reg);
impl From<$newtype_reg> for $newtype_imm8_reg {
fn from(r: $newtype_reg) -> Self {
Self(Imm8Reg::Reg { reg: r.to_reg() })
}
}
impl $newtype_imm8_reg {
/// Construct this newtype from the given `Imm8Reg`, or return
/// `None` if the `Imm8Reg` is not a valid instance of this newtype.
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
match imm8_reg {
Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
Imm8Reg::Reg { reg: $check_reg } if $check => Some(Self(imm8_reg)),
Imm8Reg::Reg { reg: _ } => None,
}
}
/// Convert this newtype into its underlying `Imm8Reg`.
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn to_imm8_reg(self) -> Imm8Reg {
self.0
}
}
};
}
// Define a newtype of `Reg` for general-purpose registers.
newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| {
reg.get_class() == RegClass::I64
});
newtype_of_reg!(
Gpr,
WritableGpr,
OptionWritableGpr,
GprMem,
GprMemImm,
Imm8Gpr,
|reg| reg.get_class() == RegClass::I64
);
// Define a newtype of `Reg` for XMM registers.
newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| {
reg.get_class() == RegClass::V128
});
newtype_of_reg!(
Xmm,
WritableXmm,
OptionWritableXmm,
XmmMem,
XmmMemImm,
Imm8Xmm,
|reg| reg.get_class() == RegClass::V128
);
/// A possible addressing mode (amode) that can be used in instructions.
/// These denote a 64-bit value only.

196
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -156,15 +156,15 @@ pub(crate) fn emit(
if *op == AluRmiROpcode::Mul {
// We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so
// we have to special-case it.
match src2 {
match src2.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg: reg_e } => {
emit_std_reg_reg(
sink,
LegacyPrefixes::None,
0x0FAF,
2,
reg_g.to_reg(),
*reg_e,
reg_g.to_reg().to_reg(),
reg_e,
rex,
);
}
@@ -178,14 +178,14 @@ pub(crate) fn emit(
LegacyPrefixes::None,
0x0FAF,
2,
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
&amode,
rex,
);
}
RegMemImm::Imm { simm32 } => {
let use_imm8 = low8_will_sign_extend_to_32(*simm32);
let use_imm8 = low8_will_sign_extend_to_32(simm32);
let opcode = if use_imm8 { 0x6B } else { 0x69 };
// Yes, really, reg_g twice.
emit_std_reg_reg(
@@ -193,11 +193,11 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcode,
1,
reg_g.to_reg(),
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
reg_g.to_reg().to_reg(),
rex,
);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32);
}
}
} else {
@@ -215,11 +215,11 @@ pub(crate) fn emit(
};
assert!(!(is_8bit && *size == OperandSize::Size64));
match src2 {
match src2.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg: reg_e } => {
if is_8bit {
rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(reg_g.to_reg());
rex.always_emit_if_8bit_needed(reg_e);
rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg());
}
// GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R
// duality). Do this too, so as to be able to compare generated machine
@@ -229,15 +229,15 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcode_r,
1,
*reg_e,
reg_g.to_reg(),
reg_e,
reg_g.to_reg().to_reg(),
rex,
);
}
RegMemImm::Mem { addr } => {
if is_8bit {
rex.always_emit_if_8bit_needed(reg_g.to_reg());
rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg());
}
// Here we revert to the "normal" G-E ordering.
let amode = addr.finalize(state, sink);
@@ -248,7 +248,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcode_m,
1,
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
&amode,
rex,
);
@@ -256,10 +256,10 @@ pub(crate) fn emit(
RegMemImm::Imm { simm32 } => {
assert!(!is_8bit);
let use_imm8 = low8_will_sign_extend_to_32(*simm32);
let use_imm8 = low8_will_sign_extend_to_32(simm32);
let opcode = if use_imm8 { 0x83 } else { 0x81 };
// And also here we use the "normal" G-E ordering.
let enc_g = int_reg_enc(reg_g.to_reg());
let enc_g = int_reg_enc(reg_g.to_reg().to_reg());
emit_std_enc_enc(
sink,
LegacyPrefixes::None,
@@ -269,7 +269,7 @@ pub(crate) fn emit(
enc_g,
rex,
);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32);
}
}
}
@@ -377,9 +377,9 @@ pub(crate) fn emit(
sink.add_trap(loc, TrapCode::IntegerDivisionByZero);
let subopcode = if *signed { 7 } else { 6 };
match divisor {
match divisor.clone().to_reg_mem() {
RegMem::Reg { reg } => {
let src = int_reg_enc(*reg);
let src = int_reg_enc(reg);
emit_std_enc_enc(
sink,
prefix,
@@ -387,7 +387,7 @@ pub(crate) fn emit(
1,
subopcode,
src,
RexFlags::from((*size, *reg)),
RexFlags::from((*size, reg)),
)
}
RegMem::Mem { addr: src } => {
@@ -428,9 +428,9 @@ pub(crate) fn emit(
};
let subopcode = if *signed { 5 } else { 4 };
match src2 {
match src2.clone().to_reg_mem() {
RegMem::Reg { reg } => {
let src = int_reg_enc(*reg);
let src = int_reg_enc(reg);
emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags)
}
RegMem::Mem { addr: src } => {
@@ -504,7 +504,7 @@ pub(crate) fn emit(
// $done:
// Check if the divisor is zero, first.
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg());
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg().to_reg());
inst.emit(sink, info, state);
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero);
@@ -512,7 +512,8 @@ pub(crate) fn emit(
let (do_op, done_label) = if kind.is_signed() {
// Now check if the divisor is -1.
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg());
let inst =
Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg().to_reg());
inst.emit(sink, info, state);
let do_op = sink.get_label();
@@ -537,12 +538,16 @@ pub(crate) fn emit(
if *size == OperandSize::Size64 {
let tmp = tmp.expect("temporary for i64 sdiv");
let inst = Inst::imm(OperandSize::Size64, 0x8000000000000000, tmp);
let inst = Inst::imm(
OperandSize::Size64,
0x8000000000000000,
tmp.to_writable_reg(),
);
inst.emit(sink, info, state);
let inst = Inst::cmp_rmi_r(
OperandSize::Size64,
RegMemImm::reg(tmp.to_reg()),
RegMemImm::reg(tmp.to_reg().to_reg()),
regs::rax(),
);
inst.emit(sink, info, state);
@@ -576,7 +581,11 @@ pub(crate) fn emit(
inst.emit(sink, info, state);
}
let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg()));
let inst = Inst::div(
*size,
kind.is_signed(),
RegMem::reg(divisor.to_reg().to_reg()),
);
inst.emit(sink, info, state);
// Lowering takes care of moving the result back into the right register, see comment
@@ -626,8 +635,8 @@ pub(crate) fn emit(
LegacyPrefixes::None,
0x89,
1,
*src,
dst.to_reg(),
src.to_reg(),
dst.to_reg().to_reg(),
RexFlags::from(*size),
);
}
@@ -664,12 +673,12 @@ pub(crate) fn emit(
}
};
match src {
match src.clone().to_reg_mem() {
RegMem::Reg { reg: src } => {
match ext_mode {
ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs.
rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(src);
}
_ => {}
}
@@ -678,8 +687,8 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
*src,
dst.to_reg().to_reg(),
src,
rex_flags,
)
}
@@ -694,7 +703,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
dst.to_reg().to_reg(),
src,
rex_flags,
)
@@ -712,7 +721,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
0x8B,
1,
dst.to_reg(),
dst.to_reg().to_reg(),
src,
RexFlags::set_w(),
)
@@ -758,12 +767,12 @@ pub(crate) fn emit(
}
};
match src {
match src.clone().to_reg_mem() {
RegMem::Reg { reg: src } => {
match ext_mode {
ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs.
rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(src);
}
_ => {}
}
@@ -772,8 +781,8 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
*src,
dst.to_reg().to_reg(),
src,
rex_flags,
)
}
@@ -788,7 +797,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
dst.to_reg().to_reg(),
src,
rex_flags,
)
@@ -812,13 +821,13 @@ pub(crate) fn emit(
// This is one of the few places where the presence of a
// redundant REX prefix changes the meaning of the
// instruction.
let rex = RexFlags::from((*size, *src));
let rex = RexFlags::from((*size, src.to_reg()));
// 8-bit: MOV r8, r/m8 is (REX.W==0) 88 /r
// 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
// 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
// 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *src, dst, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src.to_reg(), dst, rex);
}
Inst::ShiftR {
@@ -837,10 +846,10 @@ pub(crate) fn emit(
ShiftKind::ShiftRightArithmetic => 7,
};
let enc_dst = int_reg_enc(dst.to_reg());
let rex_flags = RexFlags::from((*size, dst.to_reg()));
match num_bits {
let rex_flags = RexFlags::from((*size, dst.to_reg().to_reg()));
match num_bits.clone().to_imm8_reg() {
Imm8Reg::Reg { reg } => {
debug_assert_eq!(*reg, regs::rcx());
debug_assert_eq!(reg, regs::rcx());
let (opcode, prefix) = match size {
OperandSize::Size8 => (0xD2, LegacyPrefixes::None),
OperandSize::Size16 => (0xD3, LegacyPrefixes::_66),
@@ -870,7 +879,7 @@ pub(crate) fn emit(
// When the shift amount is 1, there's an even shorter encoding, but we don't
// bother with that nicety here.
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags);
sink.put1(*num_bits);
sink.put1(num_bits);
}
}
}
@@ -963,13 +972,13 @@ pub(crate) fn emit(
prefix = LegacyPrefixes::_66;
}
// A redundant REX prefix can change the meaning of this instruction.
let mut rex = RexFlags::from((*size, *reg_g));
let mut rex = RexFlags::from((*size, reg_g.to_reg()));
match src_e {
match src_e.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg: reg_e } => {
if *size == OperandSize::Size8 {
// Check whether the E register forces the use of a redundant REX.
rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(reg_e);
}
// Use the swapped operands encoding for CMP, to stay consistent with the output of
@@ -980,7 +989,7 @@ pub(crate) fn emit(
(OperandSize::Size8, false) => 0x84,
(_, false) => 0x85,
};
emit_std_reg_reg(sink, prefix, opcode, 1, *reg_e, *reg_g, rex);
emit_std_reg_reg(sink, prefix, opcode, 1, reg_e, reg_g.to_reg(), rex);
}
RegMemImm::Mem { addr } => {
@@ -992,13 +1001,23 @@ pub(crate) fn emit(
(OperandSize::Size8, false) => 0x84,
(_, false) => 0x85,
};
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *reg_g, addr, rex);
emit_std_reg_mem(
sink,
state,
info,
prefix,
opcode,
1,
reg_g.to_reg(),
addr,
rex,
);
}
RegMemImm::Imm { simm32 } => {
// FIXME JRS 2020Feb11: there are shorter encodings for
// cmp $imm, rax/eax/ax/al.
let use_imm8 = is_cmp && low8_will_sign_extend_to_32(*simm32);
let use_imm8 = is_cmp && low8_will_sign_extend_to_32(simm32);
// And also here we use the "normal" G-E ordering.
let opcode = if is_cmp {
@@ -1018,9 +1037,9 @@ pub(crate) fn emit(
};
let subopcode = if is_cmp { 7 } else { 0 };
let enc_g = int_reg_enc(*reg_g);
let enc_g = int_reg_enc(reg_g.to_reg());
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_g, rex);
emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, *simm32);
emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, simm32);
}
}
}
@@ -1056,9 +1075,17 @@ pub(crate) fn emit(
_ => unreachable!("invalid size spec for cmove"),
};
let opcode = 0x0F40 + cc.get_enc() as u32;
match consequent {
match consequent.clone().to_reg_mem() {
RegMem::Reg { reg: reg_e } => {
emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex_flags);
emit_std_reg_reg(
sink,
prefix,
opcode,
2,
reg_g.to_reg().to_reg(),
reg_e,
rex_flags,
);
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
@@ -1069,7 +1096,7 @@ pub(crate) fn emit(
prefix,
opcode,
2,
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
addr,
rex_flags,
);
@@ -1090,7 +1117,7 @@ pub(crate) fn emit(
} else {
SseOpcode::Movss
};
let inst = Inst::xmm_unary_rm_r(op, src.clone(), *dst);
let inst = Inst::xmm_unary_rm_r(op, src.clone().to_reg_mem(), dst.to_writable_reg());
inst.emit(sink, info, state);
sink.bind_label(next);
@@ -1101,9 +1128,9 @@ pub(crate) fn emit(
sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow);
}
match src {
match src.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg } => {
let enc_reg = int_reg_enc(*reg);
let enc_reg = int_reg_enc(reg);
let rex = 0x40 | ((enc_reg >> 3) & 1);
if rex != 0x40 {
sink.put1(rex);
@@ -1127,12 +1154,12 @@ pub(crate) fn emit(
}
RegMemImm::Imm { simm32 } => {
if low8_will_sign_extend_to_64(*simm32) {
if low8_will_sign_extend_to_64(simm32) {
sink.put1(0x6A);
sink.put1(*simm32 as u8);
sink.put1(simm32 as u8);
} else {
sink.put1(0x68);
sink.put4(*simm32);
sink.put4(simm32);
}
}
}
@@ -1711,7 +1738,8 @@ pub(crate) fn emit(
_ => unreachable!(),
};
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(*lhs), rhs_dst.to_reg());
let inst =
Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_reg().to_reg());
inst.emit(sink, info, state);
one_way_jmp(sink, CC::NZ, do_min_max);
@@ -1721,7 +1749,7 @@ pub(crate) fn emit(
// and negative zero. These instructions merge the sign bits in that
// case, and are no-ops otherwise.
let op = if *is_min { or_op } else { and_op };
let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst);
let inst = Inst::xmm_rm_r(op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg());
inst.emit(sink, info, state);
let inst = Inst::jmp_known(done);
@@ -1731,13 +1759,17 @@ pub(crate) fn emit(
// read-only operand: perform an addition between the two operands, which has the
// desired NaN propagation effects.
sink.bind_label(propagate_nan);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg());
inst.emit(sink, info, state);
one_way_jmp(sink, CC::P, done);
sink.bind_label(do_min_max);
let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst);
let inst = Inst::xmm_rm_r(
min_max_op,
RegMem::reg(lhs.to_reg()),
rhs_dst.to_writable_reg(),
);
inst.emit(sink, info, state);
sink.bind_label(done);
@@ -1890,17 +1922,9 @@ pub(crate) fn emit(
_ => panic!("unexpected opcode {:?}", op),
};
let rex = RexFlags::from(*src_size);
match src_e {
match src_e.clone().to_reg_mem() {
RegMem::Reg { reg: reg_e } => {
emit_std_reg_reg(
sink,
prefix,
opcode,
2,
reg_g.to_reg().to_reg(),
*reg_e,
rex,
);
emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg().to_reg(), reg_e, rex);
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
@@ -1928,13 +1952,23 @@ pub(crate) fn emit(
_ => unimplemented!("Emit xmm cmp rm r"),
};
match src {
match src.clone().to_reg_mem() {
RegMem::Reg { reg } => {
emit_std_reg_reg(sink, prefix, opcode, len, *dst, *reg, rex);
emit_std_reg_reg(sink, prefix, opcode, len, dst.to_reg(), reg, rex);
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, *dst, addr, rex);
emit_std_reg_mem(
sink,
state,
info,
prefix,
opcode,
len,
dst.to_reg(),
addr,
rex,
);
}
}
}

304
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -133,9 +133,9 @@ impl Inst {
Self::AluRmiR {
size,
op,
src1: dst.to_reg(),
src2: src,
dst,
src1: Gpr::new(dst.to_reg()).unwrap(),
src2: GprMemImm::new(src).unwrap(),
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -174,10 +174,10 @@ impl Inst {
Inst::Div {
size,
signed,
divisor,
dividend: regs::rax(),
dst_quotient: Writable::from_reg(regs::rax()),
dst_remainder: Writable::from_reg(regs::rdx()),
divisor: GprMem::new(divisor).unwrap(),
dividend: Gpr::new(regs::rax()).unwrap(),
dst_quotient: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
}
}
@@ -191,10 +191,10 @@ impl Inst {
Inst::MulHi {
size,
signed,
src1: regs::rax(),
src2: rhs,
dst_lo: Writable::from_reg(regs::rax()),
dst_hi: Writable::from_reg(regs::rdx()),
src1: Gpr::new(regs::rax()).unwrap(),
src2: GprMem::new(rhs).unwrap(),
dst_lo: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_hi: WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
}
}
@@ -211,19 +211,19 @@ impl Inst {
Inst::CheckedDivOrRemSeq {
kind,
size,
divisor,
dividend: regs::rax(),
dst_quotient: Writable::from_reg(regs::rax()),
dst_remainder: Writable::from_reg(regs::rdx()),
tmp,
divisor: WritableGpr::from_writable_reg(divisor).unwrap(),
dividend: Gpr::new(regs::rax()).unwrap(),
dst_quotient: Writable::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
tmp: tmp.map(|tmp| WritableGpr::from_writable_reg(tmp).unwrap()),
}
}
pub(crate) fn sign_extend_data(size: OperandSize) -> Inst {
Inst::SignExtendData {
size,
src: regs::rax(),
dst: Writable::from_reg(regs::rdx()),
src: Gpr::new(regs::rax()).unwrap(),
dst: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
}
}
@@ -239,7 +239,7 @@ impl Inst {
Inst::Imm {
dst_size,
simm64,
dst,
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -247,6 +247,8 @@ impl Inst {
debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
debug_assert!(src.get_class() == RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let src = Gpr::new(src).unwrap();
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::MovRR { size, src, dst }
}
@@ -360,7 +362,7 @@ impl Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::GprToXmm {
op,
src,
src: GprMem::new(src).unwrap(),
dst: WritableXmm::from_writable_reg(dst).unwrap(),
src_size,
}
@@ -369,6 +371,8 @@ impl Inst {
pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst {
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.get_class() == RegClass::V128);
let src = XmmMem::new(src).unwrap();
let dst = Xmm::new(dst).unwrap();
Inst::XmmCmpRmR { op, src, dst }
}
@@ -457,8 +461,8 @@ impl Inst {
Inst::XmmMinMaxSeq {
size,
is_min,
lhs,
rhs_dst,
lhs: Xmm::new(lhs).unwrap(),
rhs_dst: WritableXmm::from_writable_reg(rhs_dst).unwrap(),
}
}
@@ -483,6 +487,8 @@ impl Inst {
pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let src = GprMem::new(src).unwrap();
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::MovzxRmR { ext_mode, src, dst }
}
@@ -500,6 +506,8 @@ impl Inst {
pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let src = GprMem::new(src).unwrap();
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::MovsxRmR { ext_mode, src, dst }
}
@@ -507,7 +515,7 @@ impl Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::Mov64MR {
src: src.into(),
dst,
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -524,7 +532,7 @@ impl Inst {
debug_assert!(src.get_class() == RegClass::I64);
Inst::MovRM {
size,
src,
src: Gpr::new(src).unwrap(),
dst: dst.into(),
}
}
@@ -552,12 +560,13 @@ impl Inst {
Inst::ShiftR {
size,
kind,
src: dst.to_reg(),
num_bits: match num_bits {
src: Gpr::new(dst.to_reg()).unwrap(),
num_bits: Imm8Gpr::new(match num_bits {
Some(imm) => Imm8Reg::Imm8 { imm },
None => Imm8Reg::Reg { reg: regs::rcx() },
},
dst,
})
.unwrap(),
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -568,8 +577,8 @@ impl Inst {
debug_assert_eq!(dst.get_class(), RegClass::I64);
Inst::CmpRmiR {
size,
src,
dst,
src: GprMemImm::new(src).unwrap(),
dst: Gpr::new(dst).unwrap(),
opcode: CmpOpcode::Cmp,
}
}
@@ -580,8 +589,8 @@ impl Inst {
debug_assert_eq!(dst.get_class(), RegClass::I64);
Inst::CmpRmiR {
size,
src,
dst,
src: GprMemImm::new(src).unwrap(),
dst: Gpr::new(dst).unwrap(),
opcode: CmpOpcode::Test,
}
}
@@ -594,6 +603,7 @@ impl Inst {
pub(crate) fn setcc(cc: CC, dst: Writable<Reg>) -> Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::Setcc { cc, dst }
}
@@ -607,9 +617,9 @@ impl Inst {
Inst::Cmove {
size,
cc,
consequent: src,
alternative: dst.to_reg(),
dst,
consequent: GprMem::new(src).unwrap(),
alternative: Gpr::new(dst.to_reg()).unwrap(),
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -617,16 +627,20 @@ impl Inst {
debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
let src = XmmMem::new(src).unwrap();
let dst = WritableXmm::from_writable_reg(dst).unwrap();
Inst::XmmCmove { size, cc, src, dst }
}
pub(crate) fn push64(src: RegMemImm) -> Inst {
src.assert_regclass_is(RegClass::I64);
let src = GprMemImm::new(src).unwrap();
Inst::Push64 { src }
}
pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::Pop64 { dst }
}
@@ -780,7 +794,7 @@ impl Inst {
fn produces_const(&self) -> bool {
match self {
Self::AluRmiR { op, src2, dst, .. } => {
src2.to_reg() == Some(dst.to_reg())
src2.clone().to_reg_mem_imm().to_reg() == Some(dst.to_reg().to_reg())
&& (*op == AluRmiROpcode::Xor || *op == AluRmiROpcode::Sub)
}
@@ -838,7 +852,11 @@ impl Inst {
Inst::AluRmiR { src1, dst, .. } => {
if *src1 != dst.to_reg() {
debug_assert!(src1.is_virtual());
insts.push(Self::gen_move(*dst, *src1, types::I64));
insts.push(Self::gen_move(
dst.to_writable_reg(),
src1.to_reg(),
types::I64,
));
*src1 = dst.to_reg();
}
insts.push(self);
@@ -883,7 +901,11 @@ impl Inst {
} => {
if *alternative != dst.to_reg() {
debug_assert!(alternative.is_virtual());
insts.push(Self::mov_r_r(*size, *alternative, *dst));
insts.push(Self::mov_r_r(
*size,
alternative.to_reg(),
dst.to_writable_reg(),
));
*alternative = dst.to_reg();
}
insts.push(self);
@@ -916,22 +938,30 @@ impl Inst {
debug_assert!(dividend.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rax()),
*dividend,
dividend.to_reg(),
types::I64,
));
*dividend = regs::rax();
*dividend = Gpr::new(regs::rax()).unwrap();
}
let mut quotient_mov = None;
if dst_quotient.to_reg() != regs::rax() {
debug_assert!(dst_quotient.to_reg().is_virtual());
quotient_mov = Some(Self::gen_move(*dst_quotient, regs::rax(), types::I64));
*dst_quotient = Writable::from_reg(regs::rax());
quotient_mov = Some(Self::gen_move(
dst_quotient.to_writable_reg(),
regs::rax(),
types::I64,
));
*dst_quotient = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
}
let mut remainder_mov = None;
if dst_remainder.to_reg() != regs::rdx() {
debug_assert!(dst_remainder.to_reg().is_virtual());
remainder_mov = Some(Self::gen_move(*dst_remainder, regs::rdx(), types::I64));
*dst_remainder = Writable::from_reg(regs::rdx());
remainder_mov = Some(Self::gen_move(
dst_remainder.to_writable_reg(),
regs::rdx(),
types::I64,
));
*dst_remainder = Writable::from_reg(Gpr::new(regs::rdx()).unwrap());
}
insts.push(self);
insts.extend(quotient_mov);
@@ -947,22 +977,30 @@ impl Inst {
debug_assert!(src1.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rax()),
*src1,
src1.to_reg(),
types::I64,
));
*src1 = regs::rax();
*src1 = Gpr::new(regs::rax()).unwrap();
}
let mut dst_lo_mov = None;
if dst_lo.to_reg() != regs::rax() {
debug_assert!(dst_lo.to_reg().is_virtual());
dst_lo_mov = Some(Self::gen_move(*dst_lo, regs::rax(), types::I64));
*dst_lo = Writable::from_reg(regs::rax());
dst_lo_mov = Some(Self::gen_move(
dst_lo.to_writable_reg(),
regs::rax(),
types::I64,
));
*dst_lo = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
}
let mut dst_hi_mov = None;
if dst_hi.to_reg() != regs::rdx() {
debug_assert!(dst_hi.to_reg().is_virtual());
dst_hi_mov = Some(Self::gen_move(*dst_hi, regs::rdx(), types::I64));
*dst_hi = Writable::from_reg(regs::rdx());
dst_hi_mov = Some(Self::gen_move(
dst_hi.to_writable_reg(),
regs::rdx(),
types::I64,
));
*dst_hi = Writable::from_reg(Gpr::new(regs::rdx()).unwrap());
}
insts.push(self);
insts.extend(dst_lo_mov);
@@ -973,16 +1011,20 @@ impl Inst {
debug_assert!(src.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rax()),
*src,
src.to_reg(),
types::I64,
));
*src = regs::rax();
*src = Gpr::new(regs::rax()).unwrap();
}
let mut dst_mov = None;
if dst.to_reg() != regs::rax() {
debug_assert!(dst.to_reg().is_virtual());
dst_mov = Some(Self::gen_move(*dst, dst.to_reg(), types::I64));
*dst = Writable::from_reg(regs::rax());
dst_mov = Some(Self::gen_move(
dst.to_writable_reg(),
dst.to_reg().to_reg(),
types::I64,
));
*dst = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
}
insts.push(self);
insts.extend(dst_mov);
@@ -992,18 +1034,22 @@ impl Inst {
} => {
if *src != dst.to_reg() {
debug_assert!(src.is_virtual());
insts.push(Self::gen_move(*dst, *src, types::I64));
insts.push(Self::gen_move(
dst.to_writable_reg(),
src.to_reg(),
types::I64,
));
*src = dst.to_reg();
}
if let Imm8Reg::Reg { reg } = num_bits {
if *reg != regs::rcx() {
if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() {
if reg != regs::rcx() {
debug_assert!(reg.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rcx()),
*reg,
reg,
types::I64,
));
*reg = regs::rcx();
*num_bits = Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap();
}
}
insts.push(self);
@@ -1146,7 +1192,7 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())),
src2.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())),
show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
),
Inst::UnaryRmR { src, dst, op, size } => format!(
@@ -1208,7 +1254,7 @@ impl PrettyPrint for Inst {
DivOrRemKind::SignedRem => "srem",
DivOrRemKind::UnsignedRem => "urem",
},
show_ireg_sized(divisor.to_reg(), mb_rru, size.to_bytes()),
show_ireg_sized(divisor.to_reg().to_reg(), mb_rru, size.to_bytes()),
),
Inst::SignExtendData { size, .. } => match size {
@@ -1276,8 +1322,8 @@ impl PrettyPrint for Inst {
},
format!("f{}", size.to_bits())
),
show_ireg_sized(*lhs, mb_rru, 8),
show_ireg_sized(rhs_dst.to_reg(), mb_rru, 8),
show_ireg_sized(lhs.to_reg(), mb_rru, 8),
show_ireg_sized(rhs_dst.to_reg().to_reg(), mb_rru, 8),
),
Inst::XmmRmRImm {
@@ -1342,7 +1388,7 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify(op.to_string()),
src.show_rru_sized(mb_rru, 8),
show_ireg_sized(*dst, mb_rru, 8),
show_ireg_sized(dst.to_reg(), mb_rru, 8),
),
Inst::CvtUint64ToFloatSeq {
@@ -1405,14 +1451,14 @@ impl PrettyPrint for Inst {
"{} ${}, {}",
ljustify("movabsq".to_string()),
*simm64 as i64,
show_ireg_sized(dst.to_reg(), mb_rru, 8)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8)
)
} else {
format!(
"{} ${}, {}",
ljustify("movl".to_string()),
(*simm64 as u32) as i32,
show_ireg_sized(dst.to_reg(), mb_rru, 4)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4)
)
}
}
@@ -1420,8 +1466,8 @@ impl PrettyPrint for Inst {
Inst::MovRR { size, src, dst } => format!(
"{} {}, {}",
ljustify2("mov".to_string(), suffix_lq(*size)),
show_ireg_sized(*src, mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
Inst::MovzxRmR {
@@ -1432,14 +1478,14 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify("movl".to_string()),
src.show_rru_sized(mb_rru, ext_mode.src_size()),
show_ireg_sized(dst.to_reg(), mb_rru, 4)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4)
)
} else {
format!(
"{} {}, {}",
ljustify2("movz".to_string(), ext_mode.to_string()),
src.show_rru_sized(mb_rru, ext_mode.src_size()),
show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size())
)
}
}
@@ -1464,13 +1510,13 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify2("movs".to_string(), ext_mode.to_string()),
src.show_rru_sized(mb_rru, ext_mode.src_size()),
show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size())
),
Inst::MovRM { size, src, dst, .. } => format!(
"{} {}, {}",
ljustify2("mov".to_string(), suffix_bwlq(*size)),
show_ireg_sized(*src, mb_rru, size.to_bytes()),
show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()),
dst.show_rru(mb_rru)
),
@@ -1480,19 +1526,19 @@ impl PrettyPrint for Inst {
num_bits,
dst,
..
} => match num_bits {
} => match num_bits.clone().to_imm8_reg() {
Imm8Reg::Reg { reg } => format!(
"{} {}, {}",
ljustify2(kind.to_string(), suffix_bwlq(*size)),
show_ireg_sized(*reg, mb_rru, 1),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(reg, mb_rru, 1),
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
Imm8Reg::Imm8 { imm: num_bits } => format!(
"{} ${}, {}",
ljustify2(kind.to_string(), suffix_bwlq(*size)),
num_bits,
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
},
@@ -1519,14 +1565,14 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify2(op.to_string(), suffix_bwlq(*size)),
src.show_rru_sized(mb_rru, size.to_bytes()),
show_ireg_sized(*dst, mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
)
}
Inst::Setcc { cc, dst } => format!(
"{} {}",
ljustify2("set".to_string(), cc.to_string()),
show_ireg_sized(dst.to_reg(), mb_rru, 1)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 1)
),
Inst::Cmove {
@@ -1539,7 +1585,7 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))),
src.show_rru_sized(mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
Inst::XmmCmove { size, cc, src, dst } => {
@@ -1552,7 +1598,7 @@ impl PrettyPrint for Inst {
"ss"
},
src.show_rru_sized(mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
)
}
@@ -1693,10 +1739,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
debug_assert_eq!(*src1, dst.to_reg());
if inst.produces_const() {
// No need to account for src2, since src2 == dst.
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
} else {
src2.get_regs_as_uses(collector);
collector.add_mod(*dst);
collector.add_mod(dst.to_writable_reg());
}
}
Inst::Not { src, dst, .. } => {
@@ -1760,9 +1806,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
// the rdx register *before* the instruction, which is not too bad.
collector.add_mod(Writable::from_reg(regs::rax()));
collector.add_mod(Writable::from_reg(regs::rdx()));
collector.add_mod(*divisor);
collector.add_mod(divisor.to_writable_reg());
if let Some(tmp) = tmp {
collector.add_def(*tmp);
collector.add_def(tmp.to_writable_reg());
}
}
Inst::SignExtendData { size, src, dst } => {
@@ -1852,8 +1898,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()),
Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst),
Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => {
collector.add_use(*lhs);
collector.add_mod(*rhs_dst);
collector.add_use(lhs.to_reg());
collector.add_mod(rhs_dst.to_writable_reg());
}
Inst::XmmRmiReg {
src1, src2, dst, ..
@@ -1868,14 +1914,14 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
}
Inst::XmmCmpRmR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_use(*dst);
collector.add_use(dst.to_reg());
}
Inst::Imm { dst, .. } => {
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::MovRR { src, dst, .. } => {
collector.add_use(*src);
collector.add_def(*dst);
collector.add_use(src.to_reg());
collector.add_def(dst.to_writable_reg());
}
Inst::XmmToGpr { src, dst, .. } => {
collector.add_use(src.to_reg());
@@ -1918,11 +1964,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
}
Inst::MovzxRmR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::Mov64MR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_def(*dst)
collector.add_def(dst.to_writable_reg())
}
Inst::LoadEffectiveAddress { addr: src, dst } => {
src.get_regs_as_uses(collector);
@@ -1930,41 +1976,44 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
}
Inst::MovsxRmR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::MovRM { src, dst, .. } => {
collector.add_use(*src);
collector.add_use(src.to_reg());
dst.get_regs_as_uses(collector);
}
Inst::ShiftR { num_bits, dst, .. } => {
if let Imm8Reg::Reg { reg } = num_bits {
debug_assert_eq!(*reg, regs::rcx());
if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() {
debug_assert_eq!(reg, regs::rcx());
collector.add_use(regs::rcx());
}
collector.add_mod(*dst);
collector.add_mod(dst.to_writable_reg());
}
Inst::CmpRmiR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_use(*dst); // yes, really `add_use`
collector.add_use(dst.to_reg()); // yes, really `add_use`
}
Inst::Setcc { dst, .. } => {
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::Cmove {
consequent: src,
dst,
..
}
| Inst::XmmCmove { src, dst, .. } => {
} => {
src.get_regs_as_uses(collector);
collector.add_mod(*dst);
collector.add_mod(dst.to_writable_reg());
}
Inst::XmmCmove { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_mod(dst.to_writable_reg());
}
Inst::Push64 { src } => {
src.get_regs_as_uses(collector);
collector.add_mod(Writable::from_reg(regs::rsp()));
}
Inst::Pop64 { dst } => {
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::CallKnown {
@@ -2149,11 +2198,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
debug_assert_eq!(*src1, dst.to_reg());
if produces_const {
src2.map_as_def(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
*src1 = dst.to_reg();
} else {
src2.map_uses(mapper);
mapper.map_mod(dst);
dst.map_mod(mapper);
*src1 = dst.to_reg();
}
}
@@ -2165,9 +2214,9 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
Inst::Div { divisor, .. } => divisor.map_uses(mapper),
Inst::MulHi { src2, .. } => src2.map_uses(mapper),
Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => {
mapper.map_mod(divisor);
divisor.map_mod(mapper);
if let Some(tmp) = tmp {
mapper.map_def(tmp)
tmp.map_def(mapper)
}
}
Inst::SignExtendData { .. } => {}
@@ -2275,8 +2324,8 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
ref mut rhs_dst,
..
} => {
mapper.map_use(lhs);
mapper.map_mod(rhs_dst);
lhs.map_use(mapper);
rhs_dst.map_mod(mapper);
}
Inst::XmmMovRM {
ref mut src,
@@ -2292,16 +2341,16 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_use(dst);
dst.map_use(mapper);
}
Inst::Imm { ref mut dst, .. } => mapper.map_def(dst),
Inst::Imm { ref mut dst, .. } => dst.map_def(mapper),
Inst::MovRR {
ref mut src,
ref mut dst,
..
} => {
mapper.map_use(src);
mapper.map_def(dst);
src.map_use(mapper);
dst.map_def(mapper);
}
Inst::XmmToGpr {
ref mut src,
@@ -2356,11 +2405,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::Mov64MR { src, dst, .. } => {
src.map_uses(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::LoadEffectiveAddress { addr: src, dst } => {
src.map_uses(mapper);
@@ -2372,14 +2421,14 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::MovRM {
ref mut src,
ref mut dst,
..
} => {
mapper.map_use(src);
src.map_use(mapper);
dst.map_uses(mapper);
}
Inst::ShiftR {
@@ -2388,7 +2437,7 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
debug_assert_eq!(*src, dst.to_reg());
mapper.map_mod(dst);
dst.map_mod(mapper);
*src = dst.to_reg();
}
Inst::CmpRmiR {
@@ -2397,9 +2446,9 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_use(dst);
dst.map_use(mapper);
}
Inst::Setcc { ref mut dst, .. } => mapper.map_def(dst),
Inst::Setcc { ref mut dst, .. } => dst.map_def(mapper),
Inst::Cmove {
consequent: ref mut src,
ref mut dst,
@@ -2407,7 +2456,7 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_mod(dst);
dst.map_mod(mapper);
*alternative = dst.to_reg();
}
Inst::XmmCmove {
@@ -2416,11 +2465,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_mod(dst);
dst.map_mod(mapper);
}
Inst::Push64 { ref mut src } => src.map_uses(mapper),
Inst::Pop64 { ref mut dst } => {
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::CallKnown {
@@ -2520,7 +2569,7 @@ impl MachInst for Inst {
// conceivably use `movl %reg, %reg` to zero out the top 32 bits of
// %reg.
Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => {
Some((*dst, *src))
Some((dst.to_writable_reg(), src.to_reg()))
}
// Note as well that MOVS[S|D] when used in the `XmmUnaryRmR` context are pure moves of
// scalar floating-point values (and annotate `dst` as `def`s to the register allocator)
@@ -2578,12 +2627,15 @@ impl MachInst for Inst {
size: OperandSize::Size8,
src,
dst: SyntheticAmode::NominalSPOffset { simm32 },
} => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)),
} => Some(MachInstStackOpInfo::StoreNomSPOff(
src.to_reg(),
*simm32 as i64,
)),
Self::Mov64MR {
src: SyntheticAmode::NominalSPOffset { simm32 },
dst,
} => Some(MachInstStackOpInfo::LoadNomSPOff(
dst.to_reg(),
dst.to_reg().to_reg(),
*simm32 as i64,
)),
_ => None,

507
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -63,33 +63,33 @@
;; Add two registers.
(rule (lower (has_type (fits_in_64 ty)
(iadd x y)))
(value_reg (add ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
(value_gpr (add ty
(put_in_gpr x)
(gpr_to_gpr_mem_imm (put_in_gpr y)))))
;; Add a register and an immediate.
(rule (lower (has_type (fits_in_64 ty)
(iadd x (simm32_from_value y))))
(value_reg (add ty (put_in_reg x) y)))
(value_gpr (add ty (put_in_gpr x) y)))
(rule (lower (has_type (fits_in_64 ty)
(iadd (simm32_from_value x) y)))
(value_reg (add ty (put_in_reg y) x)))
(value_gpr (add ty (put_in_gpr y) x)))
;; Add a register and memory.
(rule (lower (has_type (fits_in_64 ty)
(iadd x (sinkable_load y))))
(value_reg (add ty
(put_in_reg x)
(sink_load y))))
(value_gpr (add ty
(put_in_gpr x)
(sink_load_to_gpr_mem_imm y))))
(rule (lower (has_type (fits_in_64 ty)
(iadd (sinkable_load x) y)))
(value_reg (add ty
(put_in_reg y)
(sink_load x))))
(value_gpr (add ty
(put_in_gpr y)
(sink_load_to_gpr_mem_imm x))))
;; SSE.
@@ -117,15 +117,15 @@
(rule (lower (has_type $I128 (iadd x y)))
;; Get the high/low registers for `x`.
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1)))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1)))
;; Get the high/low registers for `y`.
(let ((y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1)))
(y_lo Gpr (value_regs_get_gpr y_regs 0))
(y_hi Gpr (value_regs_get_gpr y_regs 1)))
;; Do an add followed by an add-with-carry.
(with_flags (add_with_flags $I64 x_lo (RegMemImm.Reg y_lo))
(adc $I64 x_hi (RegMemImm.Reg y_hi))))))
(with_flags (add_with_flags $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
(adc $I64 x_hi (gpr_to_gpr_mem_imm y_hi))))))
;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -166,42 +166,42 @@
;; Add two registers.
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout x y)))
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))
(let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
(value_gprs (add ty
(put_in_gpr x)
(put_in_gpr_mem_imm y))
unused_iflags)))
;; Add a register and an immediate.
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout x (simm32_from_value y))))
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty (put_in_reg x) y)
(let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
(value_gprs (add ty (put_in_gpr x) y)
unused_iflags)))
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout (simm32_from_value x) y)))
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty (put_in_reg y) x)
(let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
(value_gprs (add ty (put_in_gpr y) x)
unused_iflags)))
;; Add a register and memory.
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout x (sinkable_load y))))
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty
(put_in_reg x)
(sink_load y))
(let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
(value_gprs (add ty
(put_in_gpr x)
(sink_load_to_gpr_mem_imm y))
unused_iflags)))
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout (sinkable_load x) y)))
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty
(put_in_reg y)
(sink_load x))
(let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr))))
(value_gprs (add ty
(put_in_gpr y)
(sink_load_to_gpr_mem_imm x))
unused_iflags)))
;; (No `iadd_ifcout` for `i128`.)
@@ -212,30 +212,30 @@
;; When the immediate fits in a `RegMemImm.Imm`, use that.
(rule (lower (has_type (fits_in_64 ty) (iadd_imm y (simm32_from_imm64 x))))
(value_reg (add ty (put_in_reg y) x)))
(value_gpr (add ty (put_in_gpr y) x)))
;; Otherwise, put the immediate into a register.
(rule (lower (has_type (fits_in_64 ty) (iadd_imm y (u64_from_imm64 x))))
(value_reg (add ty (put_in_reg y) (RegMemImm.Reg (imm ty x)))))
(value_gpr (add ty (put_in_gpr y) (gpr_to_gpr_mem_imm (gpr_new (imm ty x))))))
;; `i128`
;; When the immediate fits in a `RegMemImm.Imm`, use that.
(rule (lower (has_type $I128 (iadd_imm y (simm32_from_imm64 x))))
(let ((y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1)))
(y_lo Gpr (value_regs_get_gpr y_regs 0))
(y_hi Gpr (value_regs_get_gpr y_regs 1)))
(with_flags (add_with_flags $I64 y_lo x)
(adc $I64 y_hi (RegMemImm.Imm 0)))))
(adc $I64 y_hi (gpr_mem_imm_new (RegMemImm.Imm 0))))))
;; Otherwise, put the immediate into a register.
(rule (lower (has_type $I128 (iadd_imm y (u64_from_imm64 x))))
(let ((y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1))
(x_lo Reg (imm $I64 x)))
(with_flags (add_with_flags $I64 y_lo (RegMemImm.Reg x_lo))
(adc $I64 y_hi (RegMemImm.Imm 0)))))
(y_lo Gpr (value_regs_get_gpr y_regs 0))
(y_hi Gpr (value_regs_get_gpr y_regs 1))
(x_lo Gpr (gpr_new (imm $I64 x))))
(with_flags (add_with_flags $I64 y_lo (gpr_to_gpr_mem_imm x_lo))
(adc $I64 y_hi (gpr_mem_imm_new (RegMemImm.Imm 0))))))
;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -244,21 +244,21 @@
;; Sub two registers.
(rule (lower (has_type (fits_in_64 ty)
(isub x y)))
(value_reg (sub ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
(value_gpr (sub ty
(put_in_gpr x)
(put_in_gpr_mem_imm y))))
;; Sub a register and an immediate.
(rule (lower (has_type (fits_in_64 ty)
(isub x (simm32_from_value y))))
(value_reg (sub ty (put_in_reg x) y)))
(value_gpr (sub ty (put_in_gpr x) y)))
;; Sub a register and memory.
(rule (lower (has_type (fits_in_64 ty)
(isub x (sinkable_load y))))
(value_reg (sub ty
(put_in_reg x)
(sink_load y))))
(value_gpr (sub ty
(put_in_gpr x)
(sink_load_to_gpr_mem_imm y))))
;; SSE.
@@ -286,15 +286,15 @@
(rule (lower (has_type $I128 (isub x y)))
;; Get the high/low registers for `x`.
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1)))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1)))
;; Get the high/low registers for `y`.
(let ((y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1)))
(y_lo Gpr (value_regs_get_gpr y_regs 0))
(y_hi Gpr (value_regs_get_gpr y_regs 1)))
;; Do a sub followed by an sub-with-borrow.
(with_flags (sub_with_flags $I64 x_lo (RegMemImm.Reg y_lo))
(sbb $I64 x_hi (RegMemImm.Reg y_hi))))))
(with_flags (sub_with_flags $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
(sbb $I64 x_hi (gpr_to_gpr_mem_imm y_hi))))))
;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -326,36 +326,36 @@
;; And two registers.
(rule (lower (has_type (fits_in_64 ty) (band x y)))
(value_reg (x64_and ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
(value_gpr (x64_and ty
(put_in_gpr x)
(put_in_gpr_mem_imm y))))
;; And with a memory operand.
(rule (lower (has_type (fits_in_64 ty)
(band x (sinkable_load y))))
(value_reg (x64_and ty
(put_in_reg x)
(sink_load y))))
(value_gpr (x64_and ty
(put_in_gpr x)
(sink_load_to_gpr_mem_imm y))))
(rule (lower (has_type (fits_in_64 ty)
(band (sinkable_load x) y)))
(value_reg (x64_and ty
(put_in_reg y)
(sink_load x))))
(value_gpr (x64_and ty
(put_in_gpr y)
(sink_load_to_gpr_mem_imm x))))
;; And with an immediate.
(rule (lower (has_type (fits_in_64 ty)
(band x (simm32_from_value y))))
(value_reg (x64_and ty
(put_in_reg x)
(value_gpr (x64_and ty
(put_in_gpr x)
y)))
(rule (lower (has_type (fits_in_64 ty)
(band (simm32_from_value x) y)))
(value_reg (x64_and ty
(put_in_reg y)
(value_gpr (x64_and ty
(put_in_gpr y)
x)))
;; SSE.
@@ -375,23 +375,23 @@
(rule (lower (has_type $I128 (band x y)))
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1)))
(value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo))
(x64_and $I64 x_hi (RegMemImm.Reg y_hi)))))
(y_lo Gpr (value_regs_get_gpr y_regs 0))
(y_hi Gpr (value_regs_get_gpr y_regs 1)))
(value_gprs (x64_and $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
(x64_and $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))
(rule (lower (has_type $B128 (band x y)))
;; Booleans are always `0` or `1`, so we only need to do the `and` on the
;; low half. The high half is always zero but, rather than generate a new
;; zero, we just reuse `x`'s high half which is already zero.
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(y_lo Reg (lo_reg y)))
(value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_lo Gpr (lo_gpr y)))
(value_gprs (x64_and $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
x_hi)))
;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -400,36 +400,36 @@
;; Or two registers.
(rule (lower (has_type (fits_in_64 ty) (bor x y)))
(value_reg (or ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
(value_gpr (or ty
(put_in_gpr x)
(put_in_gpr_mem_imm y))))
;; Or with a memory operand.
(rule (lower (has_type (fits_in_64 ty)
(bor x (sinkable_load y))))
(value_reg (or ty
(put_in_reg x)
(sink_load y))))
(value_gpr (or ty
(put_in_gpr x)
(sink_load_to_gpr_mem_imm y))))
(rule (lower (has_type (fits_in_64 ty)
(bor (sinkable_load x) y)))
(value_reg (or ty
(put_in_reg y)
(sink_load x))))
(value_gpr (or ty
(put_in_gpr y)
(sink_load_to_gpr_mem_imm x))))
;; Or with an immediate.
(rule (lower (has_type (fits_in_64 ty)
(bor x (simm32_from_value y))))
(value_reg (or ty
(put_in_reg x)
(value_gpr (or ty
(put_in_gpr x)
y)))
(rule (lower (has_type (fits_in_64 ty)
(bor (simm32_from_value x) y)))
(value_reg (or ty
(put_in_reg y)
(value_gpr (or ty
(put_in_gpr y)
x)))
;; SSE.
@@ -449,12 +449,12 @@
(decl or_i128 (ValueRegs ValueRegs) ValueRegs)
(rule (or_i128 x y)
(let ((x_lo Reg (value_regs_get x 0))
(x_hi Reg (value_regs_get x 1))
(y_lo Reg (value_regs_get y 0))
(y_hi Reg (value_regs_get y 1)))
(value_regs (or $I64 x_lo (RegMemImm.Reg y_lo))
(or $I64 x_hi (RegMemImm.Reg y_hi)))))
(let ((x_lo Gpr (value_regs_get_gpr x 0))
(x_hi Gpr (value_regs_get_gpr x 1))
(y_lo Gpr (value_regs_get_gpr y 0))
(y_hi Gpr (value_regs_get_gpr y 1)))
(value_gprs (or $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
(or $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))
(rule (lower (has_type $I128 (bor x y)))
(or_i128 (put_in_regs x) (put_in_regs y)))
@@ -464,10 +464,10 @@
;; low half. The high half is always zero but, rather than generate a new
;; zero, we just reuse `x`'s high half which is already zero.
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(y_lo Reg (lo_reg y)))
(value_regs (or $I64 x_lo (RegMemImm.Reg y_lo))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_lo Gpr (lo_gpr y)))
(value_gprs (or $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
x_hi)))
;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -476,36 +476,36 @@
;; Xor two registers.
(rule (lower (has_type (fits_in_64 ty) (bxor x y)))
(value_reg (xor ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
(value_gpr (xor ty
(put_in_gpr x)
(put_in_gpr_mem_imm y))))
;; Xor with a memory operand.
(rule (lower (has_type (fits_in_64 ty)
(bxor x (sinkable_load y))))
(value_reg (xor ty
(put_in_reg x)
(sink_load y))))
(value_gpr (xor ty
(put_in_gpr x)
(sink_load_to_gpr_mem_imm y))))
(rule (lower (has_type (fits_in_64 ty)
(bxor (sinkable_load x) y)))
(value_reg (xor ty
(put_in_reg y)
(sink_load x))))
(value_gpr (xor ty
(put_in_gpr y)
(sink_load_to_gpr_mem_imm x))))
;; Xor with an immediate.
(rule (lower (has_type (fits_in_64 ty)
(bxor x (simm32_from_value y))))
(value_reg (xor ty
(put_in_reg x)
(value_gpr (xor ty
(put_in_gpr x)
y)))
(rule (lower (has_type (fits_in_64 ty)
(bxor (simm32_from_value x) y)))
(value_reg (xor ty
(put_in_reg y)
(value_gpr (xor ty
(put_in_gpr y)
x)))
;; SSE.
@@ -517,23 +517,23 @@
(rule (lower (has_type $I128 (bxor x y)))
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1)))
(value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo))
(xor $I64 x_hi (RegMemImm.Reg y_hi)))))
(y_lo Gpr (value_regs_get_gpr y_regs 0))
(y_hi Gpr (value_regs_get_gpr y_regs 1)))
(value_gprs (xor $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
(xor $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))
(rule (lower (has_type $B128 (bxor x y)))
;; Booleans are always `0` or `1`, so we only need to do the `xor` on the
;; low half. The high half is always zero but, rather than generate a new
;; zero, we just reuse `x`'s high half which is already zero.
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(y_lo Reg (lo_reg y)))
(value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
(y_lo Gpr (lo_gpr y)))
(value_gprs (xor $I64 x_lo (gpr_to_gpr_mem_imm y_lo))
x_hi)))
;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -541,38 +541,49 @@
;; `i64` and smaller.
(rule (lower (has_type (fits_in_64 ty) (ishl src amt)))
(value_reg (shl ty (put_in_reg src) (put_masked_in_imm8_reg amt ty))))
(value_gpr (shl ty (put_in_gpr src) (put_masked_in_imm8_gpr amt ty))))
;; `i128`.
(decl shl_i128 (ValueRegs Reg) ValueRegs)
(decl shl_i128 (ValueRegs Gpr) ValueRegs)
(rule (shl_i128 src amt)
;; Unpack the registers that make up the 128-bit value being shifted.
(let ((src_lo Reg (value_regs_get src 0))
(src_hi Reg (value_regs_get src 1))
(let ((src_lo Gpr (value_regs_get_gpr src 0))
(src_hi Gpr (value_regs_get_gpr src 1))
;; Do two 64-bit shifts.
(lo_shifted Reg (shl $I64 src_lo (Imm8Reg.Reg amt)))
(hi_shifted Reg (shl $I64 src_hi (Imm8Reg.Reg amt)))
(lo_shifted Gpr (shl $I64 src_lo (gpr_to_imm8_gpr amt)))
(hi_shifted Gpr (shl $I64 src_hi (gpr_to_imm8_gpr amt)))
;; `src_lo >> (64 - amt)` are the bits to carry over from the lo
;; into the hi.
(carry Reg (shr $I64 src_lo (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
(zero Reg (imm $I64 0))
(carry Gpr (shr $I64
src_lo
(gpr_to_imm8_gpr (sub $I64
(gpr_new (imm $I64 64))
(gpr_to_gpr_mem_imm amt)))))
(zero Gpr (gpr_new (imm $I64 0)))
;; Nullify the carry if we are shifting in by a multiple of 128.
(carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
(cmove $I64 (CC.Z) (RegMem.Reg zero) carry)))
(carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64)
(gpr_mem_imm_new (RegMemImm.Imm 127))
amt)
(cmove $I64
(CC.Z)
(gpr_to_gpr_mem zero)
carry))))
;; Add the carry into the high half.
(hi_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg hi_shifted))))
(hi_shifted_ Gpr (or $I64 carry_ (gpr_to_gpr_mem_imm hi_shifted))))
;; Combine the two shifted halves. However, if we are shifting by >= 64
;; (modulo 128), then the low bits are zero and the high bits are our
;; low bits.
(with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
(cmove $I64 (CC.Z) (RegMem.Reg lo_shifted) zero)
(cmove $I64 (CC.Z) (RegMem.Reg hi_shifted_) lo_shifted))))
(with_flags_2 (test (OperandSize.Size64)
(gpr_mem_imm_new (RegMemImm.Imm 64))
amt)
(cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted) zero)
(cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted_) lo_shifted))))
(rule (lower (has_type $I128 (ishl src amt)))
;; NB: Only the low bits of `amt` matter since we logically mask the shift
;; amount to the value's bit width.
(let ((amt_ Reg (lo_reg amt)))
(let ((amt_ Gpr (lo_gpr amt)))
(shl_i128 (put_in_regs src) amt_)))
;; SSE.
@@ -613,10 +624,12 @@
(rule (ishl_i8x16_mask (RegMemImm.Reg amt))
(let ((mask_table SyntheticAmode (ishl_i8x16_mask_table))
(base_mask_addr Gpr (lea mask_table))
(mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4))))
(mask_offset Gpr (shl $I64
(gpr_new amt)
(imm8_to_imm8_gpr 4))))
(amode_to_synthetic_amode (amode_imm_reg_reg_shift 0
base_mask_addr
(gpr_new mask_offset)
mask_offset
0))))
(rule (ishl_i8x16_mask (RegMemImm.Mem amt))
(ishl_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None)))))
@@ -640,38 +653,49 @@
;; `i64` and smaller.
(rule (lower (has_type (fits_in_64 ty) (ushr src amt)))
(let ((src_ Reg (extend_to_reg src ty (ExtendKind.Zero))))
(value_reg (shr ty src_ (put_masked_in_imm8_reg amt ty)))))
(let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Zero))))
(value_gpr (shr ty src_ (put_masked_in_imm8_gpr amt ty)))))
;; `i128`.
(decl shr_i128 (ValueRegs Reg) ValueRegs)
(decl shr_i128 (ValueRegs Gpr) ValueRegs)
(rule (shr_i128 src amt)
;; Unpack the lo/hi halves of `src`.
(let ((src_lo Reg (value_regs_get src 0))
(src_hi Reg (value_regs_get src 1))
(let ((src_lo Gpr (value_regs_get_gpr src 0))
(src_hi Gpr (value_regs_get_gpr src 1))
;; Do a shift on each half.
(lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt)))
(hi_shifted Reg (shr $I64 src_hi (Imm8Reg.Reg amt)))
(lo_shifted Gpr (shr $I64 src_lo (gpr_to_imm8_gpr amt)))
(hi_shifted Gpr (shr $I64 src_hi (gpr_to_imm8_gpr amt)))
;; `src_hi << (64 - amt)` are the bits to carry over from the hi
;; into the lo.
(carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
(carry Gpr (shl $I64
src_hi
(gpr_to_imm8_gpr (sub $I64
(gpr_new (imm $I64 64))
(gpr_to_gpr_mem_imm amt)))))
;; Nullify the carry if we are shifting by a multiple of 128.
(carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
(cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry)))
(carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64)
(gpr_mem_imm_new (RegMemImm.Imm 127))
amt)
(cmove $I64
(CC.Z)
(gpr_to_gpr_mem (gpr_new (imm $I64 0)))
carry))))
;; Add the carry bits into the lo.
(lo_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg lo_shifted))))
(lo_shifted_ Gpr (or $I64 carry_ (gpr_to_gpr_mem_imm lo_shifted))))
;; Combine the two shifted halves. However, if we are shifting by >= 64
;; (modulo 128), then the hi bits are zero and the lo bits are what
;; would otherwise be our hi bits.
(with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
(cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted)
(cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) (imm $I64 0)))))
(with_flags_2 (test (OperandSize.Size64)
(gpr_mem_imm_new (RegMemImm.Imm 64))
amt)
(cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted_) hi_shifted)
(cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted) (gpr_new (imm $I64 0))))))
(rule (lower (has_type $I128 (ushr src amt)))
;; NB: Only the low bits of `amt` matter since we logically mask the shift
;; amount to the value's bit width.
(let ((amt_ Reg (lo_reg amt)))
(let ((amt_ Gpr (lo_gpr amt)))
(shr_i128 (put_in_regs src) amt_)))
;; SSE.
@@ -712,10 +736,12 @@
(rule (ushr_i8x16_mask (RegMemImm.Reg amt))
(let ((mask_table SyntheticAmode (ushr_i8x16_mask_table))
(base_mask_addr Gpr (lea mask_table))
(mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4))))
(mask_offset Gpr (shl $I64
(gpr_new amt)
(imm8_to_imm8_gpr 4))))
(amode_to_synthetic_amode (amode_imm_reg_reg_shift 0
base_mask_addr
(gpr_new mask_offset)
mask_offset
0))))
(rule (ushr_i8x16_mask (RegMemImm.Mem amt))
(ushr_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None)))))
@@ -739,41 +765,52 @@
;; `i64` and smaller.
(rule (lower (has_type (fits_in_64 ty) (sshr src amt)))
(let ((src_ Reg (extend_to_reg src ty (ExtendKind.Sign))))
(value_reg (sar ty src_ (put_masked_in_imm8_reg amt ty)))))
(let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Sign))))
(value_gpr (sar ty src_ (put_masked_in_imm8_gpr amt ty)))))
;; `i128`.
(decl sar_i128 (ValueRegs Reg) ValueRegs)
(decl sar_i128 (ValueRegs Gpr) ValueRegs)
(rule (sar_i128 src amt)
;; Unpack the low/high halves of `src`.
(let ((src_lo Reg (value_regs_get src 0))
(src_hi Reg (value_regs_get src 1))
(let ((src_lo Gpr (value_regs_get_gpr src 0))
(src_hi Gpr (value_regs_get_gpr src 1))
;; Do a shift of each half. NB: the low half uses an unsigned shift
;; because its MSB is not a sign bit.
(lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt)))
(hi_shifted Reg (sar $I64 src_hi (Imm8Reg.Reg amt)))
(lo_shifted Gpr (shr $I64 src_lo (gpr_to_imm8_gpr amt)))
(hi_shifted Gpr (sar $I64 src_hi (gpr_to_imm8_gpr amt)))
;; `src_hi << (64 - amt)` are the bits to carry over from the low
;; half to the high half.
(carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
(carry Gpr (shl $I64
src_hi
(gpr_to_imm8_gpr (sub $I64
(gpr_new (imm $I64 64))
(gpr_to_gpr_mem_imm amt)))))
;; Nullify the carry if we are shifting by a multiple of 128.
(carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
(cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry)))
(carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64)
(gpr_mem_imm_new (RegMemImm.Imm 127))
amt)
(cmove $I64
(CC.Z)
(gpr_to_gpr_mem (gpr_new (imm $I64 0)))
carry))))
;; Add the carry into the low half.
(lo_shifted_ Reg (or $I64 lo_shifted (RegMemImm.Reg carry_)))
(lo_shifted_ Gpr (or $I64 lo_shifted (gpr_to_gpr_mem_imm carry_)))
;; Get all sign bits.
(sign_bits Reg (sar $I64 src_hi (Imm8Reg.Imm8 63))))
(sign_bits Gpr (sar $I64 src_hi (imm8_to_imm8_gpr 63))))
;; Combine the two shifted halves. However, if we are shifting by >= 64
;; (modulo 128), then the hi bits are all sign bits and the lo bits are
;; what would otherwise be our hi bits.
(with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
(cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted)
(cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) sign_bits))))
(with_flags_2 (test (OperandSize.Size64)
(gpr_mem_imm_new (RegMemImm.Imm 64))
amt)
(cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted_) hi_shifted)
(cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted) sign_bits))))
(rule (lower (has_type $I128 (sshr src amt)))
;; NB: Only the low bits of `amt` matter since we logically mask the shift
;; amount to the value's bit width.
(let ((amt_ Reg (lo_reg amt)))
(let ((amt_ Gpr (lo_gpr amt)))
(sar_i128 (put_in_regs src) amt_)))
;; SSE.
@@ -807,9 +844,13 @@
(rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i))
(xmm_mem_imm_new (RegMemImm.Imm (u32_add i 8))))
(rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r))
(mov_rmi_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8)))))
(mov_rmi_to_xmm (RegMemImm.Reg (gpr_to_reg (add ty
(gpr_new r)
(gpr_mem_imm_new (RegMemImm.Imm 8)))))))
(rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m))
(mov_rmi_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi))))
(mov_rmi_to_xmm (RegMemImm.Reg (gpr_to_reg (add ty
(gpr_new (imm ty 8))
(gpr_mem_imm_new rmi))))))
;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure
;; that if the shift amount is in a register, it is in an XMM register.
@@ -834,11 +875,11 @@
(let ((src_ Xmm (put_in_xmm src))
(lo Gpr (pextrd $I64 src_ 0))
(hi Gpr (pextrd $I64 src_ 1))
(amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64))
(shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_))
(shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_)))
(value_xmm (make_i64x2_from_lanes (reg_mem_to_gpr_mem (RegMem.Reg shifted_lo))
(reg_mem_to_gpr_mem (RegMem.Reg shifted_hi))))))
(amt_ Imm8Gpr (put_masked_in_imm8_gpr amt $I64))
(shifted_lo Gpr (sar $I64 lo amt_))
(shifted_hi Gpr (sar $I64 hi amt_)))
(value_xmm (make_i64x2_from_lanes (gpr_to_gpr_mem shifted_lo)
(gpr_to_gpr_mem shifted_hi)))))
;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -846,13 +887,13 @@
;; constant.
(rule (lower (has_type (ty_8_or_16 ty) (rotl src amt)))
(let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero))))
(value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(let ((amt_ Gpr (extend_to_gpr amt $I32 (ExtendKind.Zero))))
(value_gpr (x64_rotl ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
(rule (lower (has_type (ty_8_or_16 ty)
(rotl src (u64_from_iconst amt))))
(value_reg (x64_rotl ty
(put_in_reg src)
(value_gpr (x64_rotl ty
(put_in_gpr src)
(const_to_type_masked_imm8 amt ty))))
;; `i64` and `i32`: we can rely on x86's rotate-amount masking since
@@ -861,13 +902,13 @@
(rule (lower (has_type (ty_32_or_64 ty) (rotl src amt)))
;; NB: Only the low bits of `amt` matter since we logically mask the
;; shift amount to the value's bit width.
(let ((amt_ Reg (lo_reg amt)))
(value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(let ((amt_ Gpr (lo_gpr amt)))
(value_gpr (x64_rotl ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
(rule (lower (has_type (ty_32_or_64 ty)
(rotl src (u64_from_iconst amt))))
(value_reg (x64_rotl ty
(put_in_reg src)
(value_gpr (x64_rotl ty
(put_in_gpr src)
(const_to_type_masked_imm8 amt ty))))
;; `i128`.
@@ -876,9 +917,11 @@
(let ((src_ ValueRegs (put_in_regs src))
;; NB: Only the low bits of `amt` matter since we logically mask the
;; rotation amount to the value's bit width.
(amt_ Reg (lo_reg amt)))
(amt_ Gpr (lo_gpr amt)))
(or_i128 (shl_i128 src_ amt_)
(shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_))))))
(shr_i128 src_ (sub $I64
(gpr_new (imm $I64 128))
(gpr_to_gpr_mem_imm amt_))))))
;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -886,13 +929,13 @@
;; constant.
(rule (lower (has_type (ty_8_or_16 ty) (rotr src amt)))
(let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero))))
(value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(let ((amt_ Gpr (extend_to_gpr amt $I32 (ExtendKind.Zero))))
(value_gpr (x64_rotr ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
(rule (lower (has_type (ty_8_or_16 ty)
(rotr src (u64_from_iconst amt))))
(value_reg (x64_rotr ty
(put_in_reg src)
(value_gpr (x64_rotr ty
(put_in_gpr src)
(const_to_type_masked_imm8 amt ty))))
;; `i64` and `i32`: we can rely on x86's rotate-amount masking since
@@ -901,13 +944,13 @@
(rule (lower (has_type (ty_32_or_64 ty) (rotr src amt)))
;; NB: Only the low bits of `amt` matter since we logically mask the
;; shift amount to the value's bit width.
(let ((amt_ Reg (lo_reg amt)))
(value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(let ((amt_ Gpr (lo_gpr amt)))
(value_gpr (x64_rotr ty (put_in_gpr src) (gpr_to_imm8_gpr amt_)))))
(rule (lower (has_type (ty_32_or_64 ty)
(rotr src (u64_from_iconst amt))))
(value_reg (x64_rotr ty
(put_in_reg src)
(value_gpr (x64_rotr ty
(put_in_gpr src)
(const_to_type_masked_imm8 amt ty))))
;; `i128`.
@@ -916,9 +959,11 @@
(let ((src_ ValueRegs (put_in_regs src))
;; NB: Only the low bits of `amt` matter since we logically mask the
;; rotation amount to the value's bit width.
(amt_ Reg (lo_reg amt)))
(amt_ Gpr (lo_gpr amt)))
(or_i128 (shr_i128 src_ amt_)
(shl_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_))))))
(shl_i128 src_ (sub $I64
(gpr_new (imm $I64 128))
(gpr_to_gpr_mem_imm amt_))))))
;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -961,33 +1006,33 @@
;; Multiply two registers.
(rule (lower (has_type (fits_in_64 ty) (imul x y)))
(value_reg (mul ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
(value_gpr (mul ty
(put_in_gpr x)
(put_in_gpr_mem_imm y))))
;; Multiply a register and an immediate.
(rule (lower (has_type (fits_in_64 ty)
(imul x (simm32_from_value y))))
(value_reg (mul ty (put_in_reg x) y)))
(value_gpr (mul ty (put_in_gpr x) y)))
(rule (lower (has_type (fits_in_64 ty)
(imul (simm32_from_value x) y)))
(value_reg (mul ty (put_in_reg y) x)))
(value_gpr (mul ty (put_in_gpr y) x)))
;; Multiply a register and a memory load.
(rule (lower (has_type (fits_in_64 ty)
(imul x (sinkable_load y))))
(value_reg (mul ty
(put_in_reg x)
(sink_load y))))
(value_gpr (mul ty
(put_in_gpr x)
(sink_load_to_gpr_mem_imm y))))
(rule (lower (has_type (fits_in_64 ty)
(imul (sinkable_load x) y)))
(value_reg (mul ty
(put_in_reg y)
(sink_load x))))
(value_gpr (mul ty
(put_in_gpr y)
(sink_load_to_gpr_mem_imm x))))
;; `i128`.
@@ -1007,25 +1052,25 @@
(rule (lower (has_type $I128 (imul x y)))
;; Put `x` into registers and unpack its hi/lo halves.
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1))
;; Put `y` into registers and unpack its hi/lo halves.
(y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1))
(y_lo Gpr (value_regs_get_gpr y_regs 0))
(y_hi Gpr (value_regs_get_gpr y_regs 1))
;; lo_hi = mul x_lo, y_hi
(lo_hi Reg (mul $I64 x_lo (RegMemImm.Reg y_hi)))
(lo_hi Gpr (mul $I64 x_lo (gpr_to_gpr_mem_imm y_hi)))
;; hi_lo = mul x_hi, y_lo
(hi_lo Reg (mul $I64 x_hi (RegMemImm.Reg y_lo)))
(hi_lo Gpr (mul $I64 x_hi (gpr_to_gpr_mem_imm y_lo)))
;; hilo_hilo = add lo_hi, hi_lo
(hilo_hilo Reg (add $I64 lo_hi (RegMemImm.Reg hi_lo)))
(hilo_hilo Gpr (add $I64 lo_hi (gpr_to_gpr_mem_imm hi_lo)))
;; dst_lo:hi_lolo = mulhi_u x_lo, y_lo
(mul_regs ValueRegs (mulhi_u $I64 x_lo (RegMem.Reg y_lo)))
(dst_lo Reg (value_regs_get mul_regs 0))
(hi_lolo Reg (value_regs_get mul_regs 1))
(mul_regs ValueRegs (mulhi_u $I64 x_lo (gpr_to_gpr_mem y_lo)))
(dst_lo Gpr (value_regs_get_gpr mul_regs 0))
(hi_lolo Gpr (value_regs_get_gpr mul_regs 1))
;; dst_hi = add hilo_hilo, hi_lolo
(dst_hi Reg (add $I64 hilo_hilo (RegMemImm.Reg hi_lolo))))
(value_regs dst_lo dst_hi)))
(dst_hi Gpr (add $I64 hilo_hilo (gpr_to_gpr_mem_imm hi_lolo))))
(value_gprs dst_lo dst_hi)))
;; SSE.
@@ -1310,8 +1355,8 @@
(decl i128_not (Value) ValueRegs)
(rule (i128_not x)
(let ((x_regs ValueRegs (put_in_regs x))
(x_lo Gpr (gpr_new (value_regs_get x_regs 0)))
(x_hi Gpr (gpr_new (value_regs_get x_regs 1))))
(x_lo Gpr (value_regs_get_gpr x_regs 0))
(x_hi Gpr (value_regs_get_gpr x_regs 1)))
(value_gprs (not $I64 x_lo)
(not $I64 x_hi))))
@@ -1420,11 +1465,11 @@
(decl cmp_and_choose (Type CC Value Value) ValueRegs)
(rule (cmp_and_choose (fits_in_64 ty) cc x y)
(let ((x_reg Reg (put_in_reg x))
(y_reg Reg (put_in_reg y))
(let ((x_reg Gpr (put_in_gpr x))
(y_reg Gpr (put_in_gpr y))
(size OperandSize (raw_operand_size_of_type ty)))
(value_reg (with_flags_1 (cmp size (RegMemImm.Reg x_reg) y_reg)
(cmove ty cc (RegMem.Reg y_reg) x_reg)))))
(value_reg (with_flags_1 (cmp size (gpr_to_gpr_mem_imm x_reg) y_reg)
(cmove ty cc (gpr_to_gpr_mem y_reg) x_reg)))))
(rule (lower (has_type (fits_in_64 ty) (umin x y)))
(cmp_and_choose ty (CC.B) x y))

57
cranelift/codegen/src/isa/x64/lower/isle.rs
View File

@@ -79,7 +79,7 @@ where
if let Some(c) = inputs.constant {
if let Some(imm) = to_simm32(c as i64) {
return imm;
return imm.to_reg_mem_imm();
}
// Generate constants fresh at each use to minimize long-range
@@ -120,21 +120,23 @@ where
RegMem::reg(self.put_in_reg(val))
}
fn put_masked_in_imm8_reg(&mut self, val: Value, ty: Type) -> Imm8Reg {
fn put_masked_in_imm8_gpr(&mut self, val: Value, ty: Type) -> Imm8Gpr {
let inputs = self.lower_ctx.get_value_as_source_or_const(val);
if let Some(c) = inputs.constant {
let mask = 1_u64
.checked_shl(ty.bits() as u32)
.map_or(u64::MAX, |x| x - 1);
return Imm8Reg::Imm8 {
return Imm8Gpr::new(Imm8Reg::Imm8 {
imm: (c & mask) as u8,
};
})
.unwrap();
}
Imm8Reg::Reg {
Imm8Gpr::new(Imm8Reg::Reg {
reg: self.put_in_regs(val).regs()[0],
}
})
.unwrap()
}
#[inline]
@@ -178,17 +180,18 @@ where
}
#[inline]
fn const_to_type_masked_imm8(&mut self, c: u64, ty: Type) -> Imm8Reg {
fn const_to_type_masked_imm8(&mut self, c: u64, ty: Type) -> Imm8Gpr {
let mask = 1_u64
.checked_shl(ty.bits() as u32)
.map_or(u64::MAX, |x| x - 1);
Imm8Reg::Imm8 {
Imm8Gpr::new(Imm8Reg::Imm8 {
imm: (c & mask) as u8,
}
})
.unwrap()
}
#[inline]
fn simm32_from_value(&mut self, val: Value) -> Option<RegMemImm> {
fn simm32_from_value(&mut self, val: Value) -> Option<GprMemImm> {
let inst = self.lower_ctx.dfg().value_def(val).inst()?;
let constant: u64 = self.lower_ctx.get_constant(inst)?;
let constant = constant as i64;
@@ -196,7 +199,7 @@ where
}
#[inline]
fn simm32_from_imm64(&mut self, imm: Imm64) -> Option<RegMemImm> {
fn simm32_from_imm64(&mut self, imm: Imm64) -> Option<GprMemImm> {
to_simm32(imm.bits())
}
@@ -412,6 +415,31 @@ where
fn reg_to_gpr_mem(&mut self, r: Reg) -> GprMem {
GprMem::new(RegMem::reg(r)).unwrap()
}
#[inline]
fn imm8_reg_to_imm8_gpr(&mut self, ir: &Imm8Reg) -> Imm8Gpr {
Imm8Gpr::new(ir.clone()).unwrap()
}
#[inline]
fn gpr_to_gpr_mem(&mut self, gpr: Gpr) -> GprMem {
GprMem::from(gpr)
}
#[inline]
fn gpr_to_gpr_mem_imm(&mut self, gpr: Gpr) -> GprMemImm {
GprMemImm::from(gpr)
}
#[inline]
fn gpr_to_imm8_gpr(&mut self, gpr: Gpr) -> Imm8Gpr {
Imm8Gpr::from(gpr)
}
#[inline]
fn imm8_to_imm8_gpr(&mut self, imm: u8) -> Imm8Gpr {
Imm8Gpr::new(Imm8Reg::Imm8 { imm }).unwrap()
}
}
// Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we
@@ -446,11 +474,14 @@ const I8X16_USHR_MASKS: [u8; 128] = [
];
#[inline]
fn to_simm32(constant: i64) -> Option<RegMemImm> {
fn to_simm32(constant: i64) -> Option<GprMemImm> {
if constant == ((constant << 32) >> 32) {
Some(RegMemImm::Imm {
Some(
GprMemImm::new(RegMemImm::Imm {
simm32: constant as u32,
})
.unwrap(),
)
} else {
None
}

4
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle 73285cd431346d53
src/isa/x64/inst.isle 7513533d16948249
src/isa/x64/lower.isle 802b6e750d407100
src/isa/x64/inst.isle 301db31d5f1118ae
src/isa/x64/lower.isle cdc94aec26c0bc5b

1849
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

1
cranelift/codegen/src/machinst/isle.rs
View File

@@ -15,7 +15,6 @@ pub type ValueSlice<'a> = &'a [Value];
pub type ValueArray2 = [Value; 2];
pub type ValueArray3 = [Value; 3];
pub type WritableReg = Writable<Reg>;
pub type OptionWritableReg = Option<WritableReg>;
pub type VecReg = Vec<Reg>;
pub type VecWritableReg = Vec<WritableReg>;
pub type ValueRegs = crate::machinst::ValueRegs<Reg>;