From dc86e7a6dcf95358ccc90e443c7dd2c3d751f76b Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Mon, 14 Feb 2022 12:54:41 -0800 Subject: [PATCH] cranelift: Use GPR newtypes extensively in x64 lowering (#3798) We already defined the `Gpr` newtype and used it in a few places, and we already defined the `Xmm` newtype and used it extensively. This finishes the transition to using the newtypes extensively in lowering by making use of `Gpr` in more places. Fixes #3685 --- cranelift/codegen/src/isa/x64/inst.isle | 293 +-- cranelift/codegen/src/isa/x64/inst/args.rs | 69 +- cranelift/codegen/src/isa/x64/inst/emit.rs | 196 +- cranelift/codegen/src/isa/x64/inst/mod.rs | 304 +-- cranelift/codegen/src/isa/x64/lower.isle | 509 ++--- cranelift/codegen/src/isa/x64/lower/isle.rs | 61 +- .../x64/lower/isle/generated_code.manifest | 4 +- .../src/isa/x64/lower/isle/generated_code.rs | 1849 +++++++++-------- cranelift/codegen/src/machinst/isle.rs | 1 - 9 files changed, 1804 insertions(+), 1482 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 7c23e75a03..9ff8c22ed7 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -16,9 +16,9 @@ ;; Integer arithmetic/bit-twiddling. (AluRmiR (size OperandSize) ;; 4 or 8 (op AluRmiROpcode) - (src1 Reg) - (src2 RegMemImm) - (dst WritableReg)) + (src1 Gpr) + (src2 GprMemImm) + (dst WritableGpr)) ;; Instructions on general-purpose registers that only read src and ;; defines dst (dst is not modified). `bsr`, etc. @@ -40,19 +40,19 @@ ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) (Div (size OperandSize) ;; 1, 2, 4, or 8 (signed bool) - (divisor RegMem) - (dividend Reg) - (dst_quotient WritableReg) - (dst_remainder WritableReg)) + (divisor GprMem) + (dividend Gpr) + (dst_quotient WritableGpr) + (dst_remainder WritableGpr)) ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX * ;; rhs`. (MulHi (size OperandSize) (signed bool) - (src1 Reg) - (src2 RegMem) - (dst_lo WritableReg) - (dst_hi WritableReg)) + (src1 Gpr) + (src2 GprMem) + (dst_lo WritableGpr) + (dst_hi WritableGpr)) ;; A synthetic sequence to implement the right inline checks for ;; remainder and division, assuming the dividend is in %rax. @@ -69,32 +69,32 @@ ;; regalloc failures where %rdx is live before its first def! (CheckedDivOrRemSeq (kind DivOrRemKind) (size OperandSize) - (dividend Reg) + (dividend Gpr) ;; The divisor operand. Note it's marked as modified ;; so that it gets assigned a register different from ;; the temporary. - (divisor WritableReg) - (dst_quotient WritableReg) - (dst_remainder WritableReg) - (tmp OptionWritableReg)) + (divisor WritableGpr) + (dst_quotient WritableGpr) + (dst_remainder WritableGpr) + (tmp OptionWritableGpr)) ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd ;; cdq cqo) or al into ah: (cbw) (SignExtendData (size OperandSize) ;; 1, 2, 4, or 8 - (src Reg) - (dst WritableReg)) + (src Gpr) + (dst WritableGpr)) ;; Constant materialization: (imm32 imm64) reg. ;; ;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. (Imm (dst_size OperandSize) ;; 4 or 8 (simm64 u64) - (dst WritableReg)) + (dst WritableGpr)) ;; GPR to GPR move: mov (64 32) reg reg. (MovRR (size OperandSize) ;; 4 or 8 - (src Reg) - (dst WritableReg)) + (src Gpr) + (dst WritableGpr)) ;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr ;; reg. @@ -103,12 +103,12 @@ ;; zero-extend rule makes it unnecessary. For that case we emit the ;; equivalent "movl AM, reg32". (MovzxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) + (src GprMem) + (dst WritableGpr)) ;; A plain 64-bit integer load, since MovZX_RM_R can't represent that. (Mov64MR (src SyntheticAmode) - (dst WritableReg)) + (dst WritableGpr)) ;; Loads the memory address of addr into dst. (LoadEffectiveAddress (addr SyntheticAmode) @@ -116,22 +116,22 @@ ;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. (MovsxRmR (ext_mode ExtMode) - (src RegMem) - (dst WritableReg)) + (src GprMem) + (dst WritableGpr)) ;; Integer stores: mov (b w l q) reg addr. (MovRM (size OperandSize) ;; 1, 2, 4, or 8 - (src Reg) + (src Gpr) (dst SyntheticAmode)) ;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg. (ShiftR (size OperandSize) ;; 1, 2, 4, or 8 (kind ShiftKind) - (src Reg) - ;; shift count: `Imm8Reg::Imm8(0 .. #bits-in-type - 1)` or - ;; `Imm8Reg::Reg(r)` where `r` get's move mitosis'd into `%cl`. - (num_bits Imm8Reg) - (dst WritableReg)) + (src Gpr) + ;; shift count: `Imm8Gpr::Imm8(0 .. #bits-in-type - 1)` or + ;; `Imm8Reg::Gpr(r)` where `r` get's move mitosis'd into `%cl`. + (num_bits Imm8Gpr) + (dst WritableGpr)) ;; Arithmetic SIMD shifts. (XmmRmiReg (opcode SseOpcode) @@ -142,30 +142,30 @@ ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8 (opcode CmpOpcode) - (src RegMemImm) - (dst Reg)) + (src GprMemImm) + (dst Gpr)) ;; Materializes the requested condition code in the destinaton reg. (Setcc (cc CC) - (dst WritableReg)) + (dst WritableGpr)) ;; Integer conditional move. ;; ;; Overwrites the destination register. (Cmove (size OperandSize) (cc CC) - (consequent RegMem) - (alternative Reg) - (dst WritableReg)) + (consequent GprMem) + (alternative Gpr) + (dst WritableGpr)) ;; ========================================= ;; Stack manipulation. ;; pushq (reg addr imm) - (Push64 (src RegMemImm)) + (Push64 (src GprMemImm)) ;; popq reg - (Pop64 (dst WritableReg)) + (Pop64 (dst WritableGpr)) ;; ========================================= ;; Floating-point operations. @@ -221,7 +221,7 @@ ;; XMM (scalar) unary op (from integer to float reg): movd, movq, ;; cvtsi2s{s,d} (GprToXmm (op SseOpcode) - (src RegMem) + (src GprMem) (dst WritableXmm) (src_size OperandSize)) @@ -272,21 +272,21 @@ ;; registers. (XmmMinMaxSeq (size OperandSize) (is_min bool) - (lhs Reg) - (rhs_dst WritableReg)) + (lhs Xmm) + (rhs_dst WritableXmm)) ;; XMM (scalar) conditional move. ;; ;; Overwrites the destination register if cc is set. (XmmCmove (size OperandSize) (cc CC) - (src RegMem) - (dst WritableReg)) + (src XmmMem) + (dst WritableXmm)) ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. (XmmCmpRmR (op SseOpcode) - (src RegMem) - (dst Reg)) + (src XmmMem) + (dst Xmm)) ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm ;; (reg addr) reg @@ -780,8 +780,8 @@ ;; As a side effect, this marks the value as used. ;; ;; This is used when lowering various shifts and rotates. -(decl put_masked_in_imm8_reg (Value Type) Imm8Reg) -(extern constructor put_masked_in_imm8_reg put_masked_in_imm8_reg) +(decl put_masked_in_imm8_gpr (Value Type) Imm8Gpr) +(extern constructor put_masked_in_imm8_gpr put_masked_in_imm8_gpr) (type CC extern (enum O @@ -825,14 +825,21 @@ (type Gpr (primitive Gpr)) (type WritableGpr (primitive WritableGpr)) +(type OptionWritableGpr (primitive OptionWritableGpr)) (type GprMem extern (enum)) (type GprMemImm extern (enum)) +(type Imm8Gpr extern (enum)) (type Xmm (primitive Xmm)) (type WritableXmm (primitive WritableXmm)) +(type OptionWritableXmm (primitive OptionWritableXmm)) (type XmmMem extern (enum)) (type XmmMemImm extern (enum)) +;; Convert an `Imm8Reg` into an `Imm8Gpr`. +(decl imm8_reg_to_imm8_gpr (Imm8Reg) Imm8Gpr) +(extern constructor imm8_reg_to_imm8_gpr imm8_reg_to_imm8_gpr) + ;; Convert a `WritableGpr` to a `WritableReg`. (decl writable_gpr_to_reg (WritableGpr) WritableReg) (extern constructor writable_gpr_to_reg writable_gpr_to_reg) @@ -857,6 +864,14 @@ (decl gpr_to_reg (Gpr) Reg) (extern constructor gpr_to_reg gpr_to_reg) +;; Convert an `Gpr` to a `GprMem`. +(decl gpr_to_gpr_mem (Gpr) GprMem) +(extern constructor gpr_to_gpr_mem gpr_to_gpr_mem) + +;; Convert an `Gpr` to a `GprMemImm`. +(decl gpr_to_gpr_mem_imm (Gpr) GprMemImm) +(extern constructor gpr_to_gpr_mem_imm gpr_to_gpr_mem_imm) + ;; Convert an `Xmm` to a `Reg`. (decl xmm_to_reg (Xmm) Reg) (extern constructor xmm_to_reg xmm_to_reg) @@ -984,6 +999,25 @@ (rule (value_xmm x) (value_reg (xmm_to_reg x))) +;; Get the `n`th reg in a `ValueRegs` and construct a GPR from it. +;; +;; Asserts that the register is a GPR. +(decl value_regs_get_gpr (ValueRegs usize) Gpr) +(rule (value_regs_get_gpr regs n) + (gpr_new (value_regs_get regs n))) + +;; Convert a `Gpr` to an `Imm8Gpr`. +(decl gpr_to_imm8_gpr (Gpr) Imm8Gpr) +(extern constructor gpr_to_imm8_gpr gpr_to_imm8_gpr) + +;; Convert an 8-bit immediate into an `Imm8Gpr`. +(decl imm8_to_imm8_gpr (u8) Imm8Gpr) +(extern constructor imm8_to_imm8_gpr imm8_to_imm8_gpr) + +;; Get the low half of the given `Value` as a GPR. +(decl lo_gpr (Value) Gpr) +(rule (lo_gpr regs) (gpr_new (lo_reg regs))) + ;;;; Helpers for Getting Particular Physical Registers ;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These should only be used for legalization purposes, when we can't otherwise @@ -1014,15 +1048,15 @@ ;; `Imm8Reg.Imm8`. This is used for shifts and rotates, so that we don't try and ;; shift/rotate more bits than the type has available, per Cranelift's ;; semantics. -(decl const_to_type_masked_imm8 (u64 Type) Imm8Reg) +(decl const_to_type_masked_imm8 (u64 Type) Imm8Gpr) (extern constructor const_to_type_masked_imm8 const_to_type_masked_imm8) -;; Extract a constant `RegMemImm.Imm` from a value operand. -(decl simm32_from_value (RegMemImm) Value) +;; Extract a constant `GprMemImm.Imm` from a value operand. +(decl simm32_from_value (GprMemImm) Value) (extern extractor simm32_from_value simm32_from_value) ;; Extract a constant `RegMemImm.Imm` from an `Imm64` immediate. -(decl simm32_from_imm64 (RegMemImm) Imm64) +(decl simm32_from_imm64 (GprMemImm) Imm64) (extern extractor simm32_from_imm64 simm32_from_imm64) ;; A load that can be sunk into another operation. @@ -1041,6 +1075,10 @@ (decl sink_load (SinkableLoad) RegMemImm) (extern constructor sink_load sink_load) +(decl sink_load_to_gpr_mem_imm (SinkableLoad) GprMemImm) +(rule (sink_load_to_gpr_mem_imm load) + (gpr_mem_imm_new (sink_load load))) + ;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (type ExtKind extern @@ -1057,13 +1095,13 @@ (extern constructor ext_mode ext_mode) ;; Put the given value into a register, but extended as the given type. -(decl extend_to_reg (Value Type ExtendKind) Reg) +(decl extend_to_gpr (Value Type ExtendKind) Gpr) ;; If the value is already of the requested type, no extending is necessary. -(rule (extend_to_reg (and val (value_type ty)) =ty _kind) - (put_in_reg val)) +(rule (extend_to_gpr (and val (value_type ty)) =ty _kind) + (put_in_gpr val)) -(rule (extend_to_reg (and val (value_type from_ty)) +(rule (extend_to_gpr (and val (value_type from_ty)) to_ty kind) (let ((from_bits u16 (ty_bits_u16 from_ty)) @@ -1073,10 +1111,10 @@ (extend kind to_ty (ext_mode from_bits to_bits) - (put_in_reg_mem val)))) + (put_in_gpr_mem val)))) -;; Do a sign or zero extension of the given `RegMem`. -(decl extend (ExtendKind Type ExtMode RegMem) Reg) +;; Do a sign or zero extension of the given `GprMem`. +(decl extend (ExtendKind Type ExtMode GprMem) Gpr) ;; Zero extending uses `movzx`. (rule (extend (ExtendKind.Zero) ty mode src) @@ -1166,14 +1204,14 @@ (decl x64_load (Type SyntheticAmode ExtKind) Reg) (rule (x64_load (fits_in_32 ty) addr (ExtKind.SignExtend)) - (movsx ty - (ext_mode (ty_bytes ty) 8) - (synthetic_amode_to_reg_mem addr))) + (gpr_to_reg (movsx ty + (ext_mode (ty_bytes ty) 8) + (reg_mem_to_gpr_mem (synthetic_amode_to_reg_mem addr))))) (rule (x64_load $I64 addr _ext_kind) - (let ((dst WritableReg (temp_writable_reg $I64)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.Mov64MR addr dst)))) - (writable_reg_to_reg dst))) + (gpr_to_reg (writable_gpr_to_gpr dst)))) (rule (x64_load $F32 addr _ext_kind) (xmm_to_reg (xmm_unary_rm_r (SseOpcode.Movss) @@ -1202,15 +1240,15 @@ ;; only gets defined the once. ;; Helper for emitting `MInst.AluRmiR` instructions. -(decl alu_rmi_r (Type AluRmiROpcode Reg RegMemImm) Reg) +(decl alu_rmi_r (Type AluRmiROpcode Gpr GprMemImm) Gpr) (rule (alu_rmi_r ty opcode src1 src2) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for emitting `add` instructions. -(decl add (Type Reg RegMemImm) Reg) +(decl add (Type Gpr GprMemImm) Gpr) (rule (add ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.Add) @@ -1218,29 +1256,29 @@ src2)) ;; Helper for creating `add` instructions whose flags are also used. -(decl add_with_flags (Type Reg RegMemImm) ProducesFlags) +(decl add_with_flags (Type Gpr GprMemImm) ProducesFlags) (rule (add_with_flags ty src1 src2) - (let ((dst WritableReg (temp_writable_reg ty))) + (let ((dst WritableGpr (temp_writable_gpr))) (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Add) src1 src2 dst) - (writable_reg_to_reg dst)))) + (gpr_to_reg (writable_gpr_to_gpr dst))))) ;; Helper for creating `adc` instructions. -(decl adc (Type Reg RegMemImm) ConsumesFlags) +(decl adc (Type Gpr GprMemImm) ConsumesFlags) (rule (adc ty src1 src2) - (let ((dst WritableReg (temp_writable_reg ty))) + (let ((dst WritableGpr (temp_writable_gpr))) (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Adc) src1 src2 dst) - (writable_reg_to_reg dst)))) + (gpr_to_reg (writable_gpr_to_gpr dst))))) ;; Helper for emitting `sub` instructions. -(decl sub (Type Reg RegMemImm) Reg) +(decl sub (Type Gpr GprMemImm) Gpr) (rule (sub ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.Sub) @@ -1248,29 +1286,29 @@ src2)) ;; Helper for creating `sub` instructions whose flags are also used. -(decl sub_with_flags (Type Reg RegMemImm) ProducesFlags) +(decl sub_with_flags (Type Gpr GprMemImm) ProducesFlags) (rule (sub_with_flags ty src1 src2) - (let ((dst WritableReg (temp_writable_reg ty))) + (let ((dst WritableGpr (temp_writable_gpr))) (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Sub) src1 src2 dst) - (writable_reg_to_reg dst)))) + (gpr_to_reg (writable_gpr_to_gpr dst))))) ;; Helper for creating `sbb` instructions. -(decl sbb (Type Reg RegMemImm) ConsumesFlags) +(decl sbb (Type Gpr GprMemImm) ConsumesFlags) (rule (sbb ty src1 src2) - (let ((dst WritableReg (temp_writable_reg ty))) + (let ((dst WritableGpr (temp_writable_gpr))) (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type_32_64 ty) (AluRmiROpcode.Sbb) src1 src2 dst) - (writable_reg_to_reg dst)))) + (gpr_to_reg (writable_gpr_to_gpr dst))))) ;; Helper for creating `mul` instructions. -(decl mul (Type Reg RegMemImm) Reg) +(decl mul (Type Gpr GprMemImm) Gpr) (rule (mul ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.Mul) @@ -1278,10 +1316,7 @@ src2)) ;; Helper for emitting `and` instructions. -;; -;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin -;; `and` operator. -(decl x64_and (Type Reg RegMemImm) Reg) +(decl x64_and (Type Gpr GprMemImm) Gpr) (rule (x64_and ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.And) @@ -1289,7 +1324,7 @@ src2)) ;; Helper for emitting `or` instructions. -(decl or (Type Reg RegMemImm) Reg) +(decl or (Type Gpr GprMemImm) Gpr) (rule (or ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.Or) @@ -1297,7 +1332,7 @@ src2)) ;; Helper for emitting `xor` instructions. -(decl xor (Type Reg RegMemImm) Reg) +(decl xor (Type Gpr GprMemImm) Gpr) (rule (xor ty src1 src2) (alu_rmi_r ty (AluRmiROpcode.Xor) @@ -1309,10 +1344,10 @@ ;; Integer immediates. (rule (imm (fits_in_64 ty) simm64) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.Imm size simm64 dst)))) - (writable_reg_to_reg dst))) + (gpr_to_reg (writable_gpr_to_gpr dst)))) ;; `f32` immediates. (rule (imm $F32 bits) @@ -1332,21 +1367,21 @@ ;; Special case for when a 64-bit immediate fits into 32-bits. We can use a ;; 32-bit move that zero-extends the value, which has a smaller encoding. (rule (imm $I64 (nonzero_u64_fits_in_u32 x)) - (let ((dst WritableReg (temp_writable_reg $I64)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.Imm (OperandSize.Size32) x dst)))) - (writable_reg_to_reg dst))) + (gpr_to_reg (writable_gpr_to_gpr dst)))) ;; Special case for integer zero immediates: turn them into an `xor r, r`. (rule (imm (fits_in_64 ty) 0) - (let ((wr WritableReg (temp_writable_reg ty)) - (r Reg (writable_reg_to_reg wr)) + (let ((wgpr WritableGpr (temp_writable_gpr)) + (g Gpr (writable_gpr_to_gpr wgpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.AluRmiR size (AluRmiROpcode.Xor) - r - (RegMemImm.Reg r) - wr)))) - r)) + g + (gpr_to_gpr_mem_imm g) + wgpr)))) + (gpr_to_reg g))) ;; Special case for zero immediates with vector types, they turn into an xor ;; specific to the vector type. @@ -1384,44 +1419,42 @@ ;; TODO: use cmpeqpd for all 1s ;; Helper for creating `MInst.ShifR` instructions. -(decl shift_r (Type ShiftKind Reg Imm8Reg) Reg) +(decl shift_r (Type ShiftKind Gpr Imm8Gpr) Gpr) (rule (shift_r ty kind src1 src2) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) ;; Use actual 8/16-bit instructions when appropriate: we ;; rely on their shift-amount-masking semantics. (size OperandSize (raw_operand_size_of_type ty)) (_ Unit (emit (MInst.ShiftR size kind src1 src2 dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) -;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach -;; inst", to disambiguate this from clif's `rotl`). -(decl x64_rotl (Type Reg Imm8Reg) Reg) +;; Helper for creating `rotl` instructions. +(decl x64_rotl (Type Gpr Imm8Gpr) Gpr) (rule (x64_rotl ty src1 src2) (shift_r ty (ShiftKind.RotateLeft) src1 src2)) -;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach -;; inst", to disambiguate this from clif's `rotr`). -(decl x64_rotr (Type Reg Imm8Reg) Reg) +;; Helper for creating `rotr` instructions. +(decl x64_rotr (Type Gpr Imm8Gpr) Gpr) (rule (x64_rotr ty src1 src2) (shift_r ty (ShiftKind.RotateRight) src1 src2)) ;; Helper for creating `shl` instructions. -(decl shl (Type Reg Imm8Reg) Reg) +(decl shl (Type Gpr Imm8Gpr) Gpr) (rule (shl ty src1 src2) (shift_r ty (ShiftKind.ShiftLeft) src1 src2)) ;; Helper for creating logical shift-right instructions. -(decl shr (Type Reg Imm8Reg) Reg) +(decl shr (Type Gpr Imm8Gpr) Gpr) (rule (shr ty src1 src2) (shift_r ty (ShiftKind.ShiftRightLogical) src1 src2)) ;; Helper for creating arithmetic shift-right instructions. -(decl sar (Type Reg Imm8Reg) Reg) +(decl sar (Type Gpr Imm8Gpr) Gpr) (rule (sar ty src1 src2) (shift_r ty (ShiftKind.ShiftRightArithmetic) src1 src2)) ;; Helper for creating `MInst.CmpRmiR` instructions. -(decl cmp_rmi_r (OperandSize CmpOpcode RegMemImm Reg) ProducesFlags) +(decl cmp_rmi_r (OperandSize CmpOpcode GprMemImm Gpr) ProducesFlags) (rule (cmp_rmi_r size opcode src1 src2) (ProducesFlags.ProducesFlags (MInst.CmpRmiR size opcode @@ -1430,36 +1463,36 @@ (invalid_reg))) ;; Helper for creating `cmp` instructions. -(decl cmp (OperandSize RegMemImm Reg) ProducesFlags) +(decl cmp (OperandSize GprMemImm Gpr) ProducesFlags) (rule (cmp size src1 src2) (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2)) ;; Helper for creating `test` instructions. -(decl test (OperandSize RegMemImm Reg) ProducesFlags) +(decl test (OperandSize GprMemImm Gpr) ProducesFlags) (rule (test size src1 src2) (cmp_rmi_r size (CmpOpcode.Test) src1 src2)) ;; Helper for creating `MInst.Cmove` instructions. -(decl cmove (Type CC RegMem Reg) ConsumesFlags) +(decl cmove (Type CC GprMem Gpr) ConsumesFlags) (rule (cmove ty cc consequent alternative) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty))) (ConsumesFlags.ConsumesFlags (MInst.Cmove size cc consequent alternative dst) - (writable_reg_to_reg dst)))) + (gpr_to_reg (writable_gpr_to_gpr dst))))) ;; Helper for creating `MInst.MovzxRmR` instructions. -(decl movzx (Type ExtMode RegMem) Reg) +(decl movzx (Type ExtMode GprMem) Gpr) (rule (movzx ty mode src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.MovzxRmR mode src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `MInst.MovsxRmR` instructions. -(decl movsx (Type ExtMode RegMem) Reg) +(decl movsx (Type ExtMode GprMem) Gpr) (rule (movsx ty mode src) - (let ((dst WritableReg (temp_writable_reg ty)) + (let ((dst WritableGpr (temp_writable_gpr)) (_ Unit (emit (MInst.MovsxRmR mode src dst)))) - (writable_reg_to_reg dst))) + (writable_gpr_to_gpr dst))) ;; Helper for creating `MInst.XmmRmR` instructions. (decl xmm_rm_r (Type SseOpcode Xmm XmmMem) Xmm) @@ -1926,10 +1959,10 @@ ;; Helper for creating `MInst.MulHi` instructions. ;; ;; Returns the (lo, hi) register halves of the multiplication. -(decl mul_hi (Type bool Reg RegMem) ValueRegs) +(decl mul_hi (Type bool Gpr GprMem) ValueRegs) (rule (mul_hi ty signed src1 src2) - (let ((dst_lo WritableReg (temp_writable_reg ty)) - (dst_hi WritableReg (temp_writable_reg ty)) + (let ((dst_lo WritableGpr (temp_writable_gpr)) + (dst_hi WritableGpr (temp_writable_gpr)) (size OperandSize (operand_size_of_type_32_64 ty)) (_ Unit (emit (MInst.MulHi size signed @@ -1937,12 +1970,12 @@ src2 dst_lo dst_hi)))) - (value_regs (writable_reg_to_reg dst_lo) - (writable_reg_to_reg dst_hi)))) + (value_gprs (writable_gpr_to_gpr dst_lo) + (writable_gpr_to_gpr dst_hi)))) ;; Helper for creating `mul` instructions that return both the lower and ;; (unsigned) higher halves of the result. -(decl mulhi_u (Type Reg RegMem) ValueRegs) +(decl mulhi_u (Type Gpr GprMem) ValueRegs) (rule (mulhi_u ty src1 src2) (mul_hi ty $false src1 src2)) @@ -2026,7 +2059,7 @@ (decl gpr_to_xmm (SseOpcode GprMem OperandSize) Xmm) (rule (gpr_to_xmm op src size) (let ((dst WritableXmm (temp_writable_xmm)) - (_ Unit (emit (MInst.GprToXmm op (gpr_mem_to_reg_mem src) dst size)))) + (_ Unit (emit (MInst.GprToXmm op src dst size)))) (writable_xmm_to_xmm dst))) ;; Helper for creating `not` instructions. diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index e27dc3130e..6ad2e0111e 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -48,8 +48,10 @@ macro_rules! newtype_of_reg { ( $newtype_reg:ident, $newtype_writable_reg:ident, + $newtype_option_writable_reg:ident, $newtype_reg_mem:ident, $newtype_reg_mem_imm:ident, + $newtype_imm8_reg:ident, |$check_reg:ident| $check:expr ) => { /// A newtype wrapper around `Reg`. @@ -122,6 +124,9 @@ macro_rules! newtype_of_reg { pub type $newtype_writable_reg = Writable<$newtype_reg>; + #[allow(dead_code)] // Used by some newtypes and not others. + pub type $newtype_option_writable_reg = Option>; + impl ToWritableReg for $newtype_writable_reg { fn to_writable_reg(&self) -> Writable { Writable::from_reg(self.to_reg().to_reg()) @@ -218,6 +223,11 @@ macro_rules! newtype_of_reg { _ => true, }); } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { + self.0.get_regs_as_uses(collector); + } } impl PrettyPrint for $newtype_reg_mem { @@ -290,6 +300,11 @@ macro_rules! newtype_of_reg { _ => true, }); } + + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { + self.0.get_regs_as_uses(collector); + } } impl PrettyPrint for $newtype_reg_mem_imm { @@ -303,18 +318,60 @@ macro_rules! newtype_of_reg { self.0.show_rru_sized(mb_rru, size) } } + + /// A newtype wrapper around `Imm8Reg`. + #[derive(Clone, Debug)] + #[allow(dead_code)] // Used by some newtypes and not others. + pub struct $newtype_imm8_reg(Imm8Reg); + + impl From<$newtype_reg> for $newtype_imm8_reg { + fn from(r: $newtype_reg) -> Self { + Self(Imm8Reg::Reg { reg: r.to_reg() }) + } + } + + impl $newtype_imm8_reg { + /// Construct this newtype from the given `Imm8Reg`, or return + /// `None` if the `Imm8Reg` is not a valid instance of this newtype. + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn new(imm8_reg: Imm8Reg) -> Option { + match imm8_reg { + Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)), + Imm8Reg::Reg { reg: $check_reg } if $check => Some(Self(imm8_reg)), + Imm8Reg::Reg { reg: _ } => None, + } + } + + /// Convert this newtype into its underlying `Imm8Reg`. + #[allow(dead_code)] // Used by some newtypes and not others. + pub fn to_imm8_reg(self) -> Imm8Reg { + self.0 + } + } }; } // Define a newtype of `Reg` for general-purpose registers. -newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| { - reg.get_class() == RegClass::I64 -}); +newtype_of_reg!( + Gpr, + WritableGpr, + OptionWritableGpr, + GprMem, + GprMemImm, + Imm8Gpr, + |reg| reg.get_class() == RegClass::I64 +); // Define a newtype of `Reg` for XMM registers. -newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| { - reg.get_class() == RegClass::V128 -}); +newtype_of_reg!( + Xmm, + WritableXmm, + OptionWritableXmm, + XmmMem, + XmmMemImm, + Imm8Xmm, + |reg| reg.get_class() == RegClass::V128 +); /// A possible addressing mode (amode) that can be used in instructions. /// These denote a 64-bit value only. diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index bda2ddc0a2..c8a8ca71be 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -156,15 +156,15 @@ pub(crate) fn emit( if *op == AluRmiROpcode::Mul { // We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so // we have to special-case it. - match src2 { + match src2.clone().to_reg_mem_imm() { RegMemImm::Reg { reg: reg_e } => { emit_std_reg_reg( sink, LegacyPrefixes::None, 0x0FAF, 2, - reg_g.to_reg(), - *reg_e, + reg_g.to_reg().to_reg(), + reg_e, rex, ); } @@ -178,14 +178,14 @@ pub(crate) fn emit( LegacyPrefixes::None, 0x0FAF, 2, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), &amode, rex, ); } RegMemImm::Imm { simm32 } => { - let use_imm8 = low8_will_sign_extend_to_32(*simm32); + let use_imm8 = low8_will_sign_extend_to_32(simm32); let opcode = if use_imm8 { 0x6B } else { 0x69 }; // Yes, really, reg_g twice. emit_std_reg_reg( @@ -193,11 +193,11 @@ pub(crate) fn emit( LegacyPrefixes::None, opcode, 1, - reg_g.to_reg(), - reg_g.to_reg(), + reg_g.to_reg().to_reg(), + reg_g.to_reg().to_reg(), rex, ); - emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32); + emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32); } } } else { @@ -215,11 +215,11 @@ pub(crate) fn emit( }; assert!(!(is_8bit && *size == OperandSize::Size64)); - match src2 { + match src2.clone().to_reg_mem_imm() { RegMemImm::Reg { reg: reg_e } => { if is_8bit { - rex.always_emit_if_8bit_needed(*reg_e); - rex.always_emit_if_8bit_needed(reg_g.to_reg()); + rex.always_emit_if_8bit_needed(reg_e); + rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg()); } // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R // duality). Do this too, so as to be able to compare generated machine @@ -229,15 +229,15 @@ pub(crate) fn emit( LegacyPrefixes::None, opcode_r, 1, - *reg_e, - reg_g.to_reg(), + reg_e, + reg_g.to_reg().to_reg(), rex, ); } RegMemImm::Mem { addr } => { if is_8bit { - rex.always_emit_if_8bit_needed(reg_g.to_reg()); + rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg()); } // Here we revert to the "normal" G-E ordering. let amode = addr.finalize(state, sink); @@ -248,7 +248,7 @@ pub(crate) fn emit( LegacyPrefixes::None, opcode_m, 1, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), &amode, rex, ); @@ -256,10 +256,10 @@ pub(crate) fn emit( RegMemImm::Imm { simm32 } => { assert!(!is_8bit); - let use_imm8 = low8_will_sign_extend_to_32(*simm32); + let use_imm8 = low8_will_sign_extend_to_32(simm32); let opcode = if use_imm8 { 0x83 } else { 0x81 }; // And also here we use the "normal" G-E ordering. - let enc_g = int_reg_enc(reg_g.to_reg()); + let enc_g = int_reg_enc(reg_g.to_reg().to_reg()); emit_std_enc_enc( sink, LegacyPrefixes::None, @@ -269,7 +269,7 @@ pub(crate) fn emit( enc_g, rex, ); - emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32); + emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32); } } } @@ -377,9 +377,9 @@ pub(crate) fn emit( sink.add_trap(loc, TrapCode::IntegerDivisionByZero); let subopcode = if *signed { 7 } else { 6 }; - match divisor { + match divisor.clone().to_reg_mem() { RegMem::Reg { reg } => { - let src = int_reg_enc(*reg); + let src = int_reg_enc(reg); emit_std_enc_enc( sink, prefix, @@ -387,7 +387,7 @@ pub(crate) fn emit( 1, subopcode, src, - RexFlags::from((*size, *reg)), + RexFlags::from((*size, reg)), ) } RegMem::Mem { addr: src } => { @@ -428,9 +428,9 @@ pub(crate) fn emit( }; let subopcode = if *signed { 5 } else { 4 }; - match src2 { + match src2.clone().to_reg_mem() { RegMem::Reg { reg } => { - let src = int_reg_enc(*reg); + let src = int_reg_enc(reg); emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags) } RegMem::Mem { addr: src } => { @@ -504,7 +504,7 @@ pub(crate) fn emit( // $done: // Check if the divisor is zero, first. - let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg()); + let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg().to_reg()); inst.emit(sink, info, state); let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero); @@ -512,7 +512,8 @@ pub(crate) fn emit( let (do_op, done_label) = if kind.is_signed() { // Now check if the divisor is -1. - let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg()); + let inst = + Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg().to_reg()); inst.emit(sink, info, state); let do_op = sink.get_label(); @@ -537,12 +538,16 @@ pub(crate) fn emit( if *size == OperandSize::Size64 { let tmp = tmp.expect("temporary for i64 sdiv"); - let inst = Inst::imm(OperandSize::Size64, 0x8000000000000000, tmp); + let inst = Inst::imm( + OperandSize::Size64, + 0x8000000000000000, + tmp.to_writable_reg(), + ); inst.emit(sink, info, state); let inst = Inst::cmp_rmi_r( OperandSize::Size64, - RegMemImm::reg(tmp.to_reg()), + RegMemImm::reg(tmp.to_reg().to_reg()), regs::rax(), ); inst.emit(sink, info, state); @@ -576,7 +581,11 @@ pub(crate) fn emit( inst.emit(sink, info, state); } - let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg())); + let inst = Inst::div( + *size, + kind.is_signed(), + RegMem::reg(divisor.to_reg().to_reg()), + ); inst.emit(sink, info, state); // Lowering takes care of moving the result back into the right register, see comment @@ -626,8 +635,8 @@ pub(crate) fn emit( LegacyPrefixes::None, 0x89, 1, - *src, - dst.to_reg(), + src.to_reg(), + dst.to_reg().to_reg(), RexFlags::from(*size), ); } @@ -664,12 +673,12 @@ pub(crate) fn emit( } }; - match src { + match src.clone().to_reg_mem() { RegMem::Reg { reg: src } => { match ext_mode { ExtMode::BL | ExtMode::BQ => { // A redundant REX prefix must be emitted for certain register inputs. - rex_flags.always_emit_if_8bit_needed(*src); + rex_flags.always_emit_if_8bit_needed(src); } _ => {} } @@ -678,8 +687,8 @@ pub(crate) fn emit( LegacyPrefixes::None, opcodes, num_opcodes, - dst.to_reg(), - *src, + dst.to_reg().to_reg(), + src, rex_flags, ) } @@ -694,7 +703,7 @@ pub(crate) fn emit( LegacyPrefixes::None, opcodes, num_opcodes, - dst.to_reg(), + dst.to_reg().to_reg(), src, rex_flags, ) @@ -712,7 +721,7 @@ pub(crate) fn emit( LegacyPrefixes::None, 0x8B, 1, - dst.to_reg(), + dst.to_reg().to_reg(), src, RexFlags::set_w(), ) @@ -758,12 +767,12 @@ pub(crate) fn emit( } }; - match src { + match src.clone().to_reg_mem() { RegMem::Reg { reg: src } => { match ext_mode { ExtMode::BL | ExtMode::BQ => { // A redundant REX prefix must be emitted for certain register inputs. - rex_flags.always_emit_if_8bit_needed(*src); + rex_flags.always_emit_if_8bit_needed(src); } _ => {} } @@ -772,8 +781,8 @@ pub(crate) fn emit( LegacyPrefixes::None, opcodes, num_opcodes, - dst.to_reg(), - *src, + dst.to_reg().to_reg(), + src, rex_flags, ) } @@ -788,7 +797,7 @@ pub(crate) fn emit( LegacyPrefixes::None, opcodes, num_opcodes, - dst.to_reg(), + dst.to_reg().to_reg(), src, rex_flags, ) @@ -812,13 +821,13 @@ pub(crate) fn emit( // This is one of the few places where the presence of a // redundant REX prefix changes the meaning of the // instruction. - let rex = RexFlags::from((*size, *src)); + let rex = RexFlags::from((*size, src.to_reg())); // 8-bit: MOV r8, r/m8 is (REX.W==0) 88 /r // 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r // 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r // 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r - emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *src, dst, rex); + emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src.to_reg(), dst, rex); } Inst::ShiftR { @@ -837,10 +846,10 @@ pub(crate) fn emit( ShiftKind::ShiftRightArithmetic => 7, }; let enc_dst = int_reg_enc(dst.to_reg()); - let rex_flags = RexFlags::from((*size, dst.to_reg())); - match num_bits { + let rex_flags = RexFlags::from((*size, dst.to_reg().to_reg())); + match num_bits.clone().to_imm8_reg() { Imm8Reg::Reg { reg } => { - debug_assert_eq!(*reg, regs::rcx()); + debug_assert_eq!(reg, regs::rcx()); let (opcode, prefix) = match size { OperandSize::Size8 => (0xD2, LegacyPrefixes::None), OperandSize::Size16 => (0xD3, LegacyPrefixes::_66), @@ -870,7 +879,7 @@ pub(crate) fn emit( // When the shift amount is 1, there's an even shorter encoding, but we don't // bother with that nicety here. emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags); - sink.put1(*num_bits); + sink.put1(num_bits); } } } @@ -963,13 +972,13 @@ pub(crate) fn emit( prefix = LegacyPrefixes::_66; } // A redundant REX prefix can change the meaning of this instruction. - let mut rex = RexFlags::from((*size, *reg_g)); + let mut rex = RexFlags::from((*size, reg_g.to_reg())); - match src_e { + match src_e.clone().to_reg_mem_imm() { RegMemImm::Reg { reg: reg_e } => { if *size == OperandSize::Size8 { // Check whether the E register forces the use of a redundant REX. - rex.always_emit_if_8bit_needed(*reg_e); + rex.always_emit_if_8bit_needed(reg_e); } // Use the swapped operands encoding for CMP, to stay consistent with the output of @@ -980,7 +989,7 @@ pub(crate) fn emit( (OperandSize::Size8, false) => 0x84, (_, false) => 0x85, }; - emit_std_reg_reg(sink, prefix, opcode, 1, *reg_e, *reg_g, rex); + emit_std_reg_reg(sink, prefix, opcode, 1, reg_e, reg_g.to_reg(), rex); } RegMemImm::Mem { addr } => { @@ -992,13 +1001,23 @@ pub(crate) fn emit( (OperandSize::Size8, false) => 0x84, (_, false) => 0x85, }; - emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *reg_g, addr, rex); + emit_std_reg_mem( + sink, + state, + info, + prefix, + opcode, + 1, + reg_g.to_reg(), + addr, + rex, + ); } RegMemImm::Imm { simm32 } => { // FIXME JRS 2020Feb11: there are shorter encodings for // cmp $imm, rax/eax/ax/al. - let use_imm8 = is_cmp && low8_will_sign_extend_to_32(*simm32); + let use_imm8 = is_cmp && low8_will_sign_extend_to_32(simm32); // And also here we use the "normal" G-E ordering. let opcode = if is_cmp { @@ -1018,9 +1037,9 @@ pub(crate) fn emit( }; let subopcode = if is_cmp { 7 } else { 0 }; - let enc_g = int_reg_enc(*reg_g); + let enc_g = int_reg_enc(reg_g.to_reg()); emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_g, rex); - emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, *simm32); + emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, simm32); } } } @@ -1056,9 +1075,17 @@ pub(crate) fn emit( _ => unreachable!("invalid size spec for cmove"), }; let opcode = 0x0F40 + cc.get_enc() as u32; - match consequent { + match consequent.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex_flags); + emit_std_reg_reg( + sink, + prefix, + opcode, + 2, + reg_g.to_reg().to_reg(), + reg_e, + rex_flags, + ); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1069,7 +1096,7 @@ pub(crate) fn emit( prefix, opcode, 2, - reg_g.to_reg(), + reg_g.to_reg().to_reg(), addr, rex_flags, ); @@ -1090,7 +1117,7 @@ pub(crate) fn emit( } else { SseOpcode::Movss }; - let inst = Inst::xmm_unary_rm_r(op, src.clone(), *dst); + let inst = Inst::xmm_unary_rm_r(op, src.clone().to_reg_mem(), dst.to_writable_reg()); inst.emit(sink, info, state); sink.bind_label(next); @@ -1101,9 +1128,9 @@ pub(crate) fn emit( sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow); } - match src { + match src.clone().to_reg_mem_imm() { RegMemImm::Reg { reg } => { - let enc_reg = int_reg_enc(*reg); + let enc_reg = int_reg_enc(reg); let rex = 0x40 | ((enc_reg >> 3) & 1); if rex != 0x40 { sink.put1(rex); @@ -1127,12 +1154,12 @@ pub(crate) fn emit( } RegMemImm::Imm { simm32 } => { - if low8_will_sign_extend_to_64(*simm32) { + if low8_will_sign_extend_to_64(simm32) { sink.put1(0x6A); - sink.put1(*simm32 as u8); + sink.put1(simm32 as u8); } else { sink.put1(0x68); - sink.put4(*simm32); + sink.put4(simm32); } } } @@ -1711,7 +1738,8 @@ pub(crate) fn emit( _ => unreachable!(), }; - let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(*lhs), rhs_dst.to_reg()); + let inst = + Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_reg().to_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::NZ, do_min_max); @@ -1721,7 +1749,7 @@ pub(crate) fn emit( // and negative zero. These instructions merge the sign bits in that // case, and are no-ops otherwise. let op = if *is_min { or_op } else { and_op }; - let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst); + let inst = Inst::xmm_rm_r(op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg()); inst.emit(sink, info, state); let inst = Inst::jmp_known(done); @@ -1731,13 +1759,17 @@ pub(crate) fn emit( // read-only operand: perform an addition between the two operands, which has the // desired NaN propagation effects. sink.bind_label(propagate_nan); - let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst); + let inst = Inst::xmm_rm_r(add_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg()); inst.emit(sink, info, state); one_way_jmp(sink, CC::P, done); sink.bind_label(do_min_max); - let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst); + let inst = Inst::xmm_rm_r( + min_max_op, + RegMem::reg(lhs.to_reg()), + rhs_dst.to_writable_reg(), + ); inst.emit(sink, info, state); sink.bind_label(done); @@ -1890,17 +1922,9 @@ pub(crate) fn emit( _ => panic!("unexpected opcode {:?}", op), }; let rex = RexFlags::from(*src_size); - match src_e { + match src_e.clone().to_reg_mem() { RegMem::Reg { reg: reg_e } => { - emit_std_reg_reg( - sink, - prefix, - opcode, - 2, - reg_g.to_reg().to_reg(), - *reg_e, - rex, - ); + emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg().to_reg(), reg_e, rex); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); @@ -1928,13 +1952,23 @@ pub(crate) fn emit( _ => unimplemented!("Emit xmm cmp rm r"), }; - match src { + match src.clone().to_reg_mem() { RegMem::Reg { reg } => { - emit_std_reg_reg(sink, prefix, opcode, len, *dst, *reg, rex); + emit_std_reg_reg(sink, prefix, opcode, len, dst.to_reg(), reg, rex); } RegMem::Mem { addr } => { let addr = &addr.finalize(state, sink); - emit_std_reg_mem(sink, state, info, prefix, opcode, len, *dst, addr, rex); + emit_std_reg_mem( + sink, + state, + info, + prefix, + opcode, + len, + dst.to_reg(), + addr, + rex, + ); } } } diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index bd7c039b94..20b02f2280 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -133,9 +133,9 @@ impl Inst { Self::AluRmiR { size, op, - src1: dst.to_reg(), - src2: src, - dst, + src1: Gpr::new(dst.to_reg()).unwrap(), + src2: GprMemImm::new(src).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -174,10 +174,10 @@ impl Inst { Inst::Div { size, signed, - divisor, - dividend: regs::rax(), - dst_quotient: Writable::from_reg(regs::rax()), - dst_remainder: Writable::from_reg(regs::rdx()), + divisor: GprMem::new(divisor).unwrap(), + dividend: Gpr::new(regs::rax()).unwrap(), + dst_quotient: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()), + dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()), } } @@ -191,10 +191,10 @@ impl Inst { Inst::MulHi { size, signed, - src1: regs::rax(), - src2: rhs, - dst_lo: Writable::from_reg(regs::rax()), - dst_hi: Writable::from_reg(regs::rdx()), + src1: Gpr::new(regs::rax()).unwrap(), + src2: GprMem::new(rhs).unwrap(), + dst_lo: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()), + dst_hi: WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()), } } @@ -211,19 +211,19 @@ impl Inst { Inst::CheckedDivOrRemSeq { kind, size, - divisor, - dividend: regs::rax(), - dst_quotient: Writable::from_reg(regs::rax()), - dst_remainder: Writable::from_reg(regs::rdx()), - tmp, + divisor: WritableGpr::from_writable_reg(divisor).unwrap(), + dividend: Gpr::new(regs::rax()).unwrap(), + dst_quotient: Writable::from_reg(Gpr::new(regs::rax()).unwrap()), + dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()), + tmp: tmp.map(|tmp| WritableGpr::from_writable_reg(tmp).unwrap()), } } pub(crate) fn sign_extend_data(size: OperandSize) -> Inst { Inst::SignExtendData { size, - src: regs::rax(), - dst: Writable::from_reg(regs::rdx()), + src: Gpr::new(regs::rax()).unwrap(), + dst: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()), } } @@ -239,7 +239,7 @@ impl Inst { Inst::Imm { dst_size, simm64, - dst, + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -247,6 +247,8 @@ impl Inst { debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); debug_assert!(src.get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); + let src = Gpr::new(src).unwrap(); + let dst = WritableGpr::from_writable_reg(dst).unwrap(); Inst::MovRR { size, src, dst } } @@ -360,7 +362,7 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); Inst::GprToXmm { op, - src, + src: GprMem::new(src).unwrap(), dst: WritableXmm::from_writable_reg(dst).unwrap(), src_size, } @@ -369,6 +371,8 @@ impl Inst { pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.get_class() == RegClass::V128); + let src = XmmMem::new(src).unwrap(); + let dst = Xmm::new(dst).unwrap(); Inst::XmmCmpRmR { op, src, dst } } @@ -457,8 +461,8 @@ impl Inst { Inst::XmmMinMaxSeq { size, is_min, - lhs, - rhs_dst, + lhs: Xmm::new(lhs).unwrap(), + rhs_dst: WritableXmm::from_writable_reg(rhs_dst).unwrap(), } } @@ -483,6 +487,8 @@ impl Inst { pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); + let src = GprMem::new(src).unwrap(); + let dst = WritableGpr::from_writable_reg(dst).unwrap(); Inst::MovzxRmR { ext_mode, src, dst } } @@ -500,6 +506,8 @@ impl Inst { pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); + let src = GprMem::new(src).unwrap(); + let dst = WritableGpr::from_writable_reg(dst).unwrap(); Inst::MovsxRmR { ext_mode, src, dst } } @@ -507,7 +515,7 @@ impl Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::Mov64MR { src: src.into(), - dst, + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -524,7 +532,7 @@ impl Inst { debug_assert!(src.get_class() == RegClass::I64); Inst::MovRM { size, - src, + src: Gpr::new(src).unwrap(), dst: dst.into(), } } @@ -552,12 +560,13 @@ impl Inst { Inst::ShiftR { size, kind, - src: dst.to_reg(), - num_bits: match num_bits { + src: Gpr::new(dst.to_reg()).unwrap(), + num_bits: Imm8Gpr::new(match num_bits { Some(imm) => Imm8Reg::Imm8 { imm }, None => Imm8Reg::Reg { reg: regs::rcx() }, - }, - dst, + }) + .unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -568,8 +577,8 @@ impl Inst { debug_assert_eq!(dst.get_class(), RegClass::I64); Inst::CmpRmiR { size, - src, - dst, + src: GprMemImm::new(src).unwrap(), + dst: Gpr::new(dst).unwrap(), opcode: CmpOpcode::Cmp, } } @@ -580,8 +589,8 @@ impl Inst { debug_assert_eq!(dst.get_class(), RegClass::I64); Inst::CmpRmiR { size, - src, - dst, + src: GprMemImm::new(src).unwrap(), + dst: Gpr::new(dst).unwrap(), opcode: CmpOpcode::Test, } } @@ -594,6 +603,7 @@ impl Inst { pub(crate) fn setcc(cc: CC, dst: Writable) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); + let dst = WritableGpr::from_writable_reg(dst).unwrap(); Inst::Setcc { cc, dst } } @@ -607,9 +617,9 @@ impl Inst { Inst::Cmove { size, cc, - consequent: src, - alternative: dst.to_reg(), - dst, + consequent: GprMem::new(src).unwrap(), + alternative: Gpr::new(dst.to_reg()).unwrap(), + dst: WritableGpr::from_writable_reg(dst).unwrap(), } } @@ -617,16 +627,20 @@ impl Inst { debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); + let src = XmmMem::new(src).unwrap(); + let dst = WritableXmm::from_writable_reg(dst).unwrap(); Inst::XmmCmove { size, cc, src, dst } } pub(crate) fn push64(src: RegMemImm) -> Inst { src.assert_regclass_is(RegClass::I64); + let src = GprMemImm::new(src).unwrap(); Inst::Push64 { src } } pub(crate) fn pop64(dst: Writable) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); + let dst = WritableGpr::from_writable_reg(dst).unwrap(); Inst::Pop64 { dst } } @@ -780,7 +794,7 @@ impl Inst { fn produces_const(&self) -> bool { match self { Self::AluRmiR { op, src2, dst, .. } => { - src2.to_reg() == Some(dst.to_reg()) + src2.clone().to_reg_mem_imm().to_reg() == Some(dst.to_reg().to_reg()) && (*op == AluRmiROpcode::Xor || *op == AluRmiROpcode::Sub) } @@ -838,7 +852,11 @@ impl Inst { Inst::AluRmiR { src1, dst, .. } => { if *src1 != dst.to_reg() { debug_assert!(src1.is_virtual()); - insts.push(Self::gen_move(*dst, *src1, types::I64)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src1.to_reg(), + types::I64, + )); *src1 = dst.to_reg(); } insts.push(self); @@ -883,7 +901,11 @@ impl Inst { } => { if *alternative != dst.to_reg() { debug_assert!(alternative.is_virtual()); - insts.push(Self::mov_r_r(*size, *alternative, *dst)); + insts.push(Self::mov_r_r( + *size, + alternative.to_reg(), + dst.to_writable_reg(), + )); *alternative = dst.to_reg(); } insts.push(self); @@ -916,22 +938,30 @@ impl Inst { debug_assert!(dividend.is_virtual()); insts.push(Self::gen_move( Writable::from_reg(regs::rax()), - *dividend, + dividend.to_reg(), types::I64, )); - *dividend = regs::rax(); + *dividend = Gpr::new(regs::rax()).unwrap(); } let mut quotient_mov = None; if dst_quotient.to_reg() != regs::rax() { debug_assert!(dst_quotient.to_reg().is_virtual()); - quotient_mov = Some(Self::gen_move(*dst_quotient, regs::rax(), types::I64)); - *dst_quotient = Writable::from_reg(regs::rax()); + quotient_mov = Some(Self::gen_move( + dst_quotient.to_writable_reg(), + regs::rax(), + types::I64, + )); + *dst_quotient = Writable::from_reg(Gpr::new(regs::rax()).unwrap()); } let mut remainder_mov = None; if dst_remainder.to_reg() != regs::rdx() { debug_assert!(dst_remainder.to_reg().is_virtual()); - remainder_mov = Some(Self::gen_move(*dst_remainder, regs::rdx(), types::I64)); - *dst_remainder = Writable::from_reg(regs::rdx()); + remainder_mov = Some(Self::gen_move( + dst_remainder.to_writable_reg(), + regs::rdx(), + types::I64, + )); + *dst_remainder = Writable::from_reg(Gpr::new(regs::rdx()).unwrap()); } insts.push(self); insts.extend(quotient_mov); @@ -947,22 +977,30 @@ impl Inst { debug_assert!(src1.is_virtual()); insts.push(Self::gen_move( Writable::from_reg(regs::rax()), - *src1, + src1.to_reg(), types::I64, )); - *src1 = regs::rax(); + *src1 = Gpr::new(regs::rax()).unwrap(); } let mut dst_lo_mov = None; if dst_lo.to_reg() != regs::rax() { debug_assert!(dst_lo.to_reg().is_virtual()); - dst_lo_mov = Some(Self::gen_move(*dst_lo, regs::rax(), types::I64)); - *dst_lo = Writable::from_reg(regs::rax()); + dst_lo_mov = Some(Self::gen_move( + dst_lo.to_writable_reg(), + regs::rax(), + types::I64, + )); + *dst_lo = Writable::from_reg(Gpr::new(regs::rax()).unwrap()); } let mut dst_hi_mov = None; if dst_hi.to_reg() != regs::rdx() { debug_assert!(dst_hi.to_reg().is_virtual()); - dst_hi_mov = Some(Self::gen_move(*dst_hi, regs::rdx(), types::I64)); - *dst_hi = Writable::from_reg(regs::rdx()); + dst_hi_mov = Some(Self::gen_move( + dst_hi.to_writable_reg(), + regs::rdx(), + types::I64, + )); + *dst_hi = Writable::from_reg(Gpr::new(regs::rdx()).unwrap()); } insts.push(self); insts.extend(dst_lo_mov); @@ -973,16 +1011,20 @@ impl Inst { debug_assert!(src.is_virtual()); insts.push(Self::gen_move( Writable::from_reg(regs::rax()), - *src, + src.to_reg(), types::I64, )); - *src = regs::rax(); + *src = Gpr::new(regs::rax()).unwrap(); } let mut dst_mov = None; if dst.to_reg() != regs::rax() { debug_assert!(dst.to_reg().is_virtual()); - dst_mov = Some(Self::gen_move(*dst, dst.to_reg(), types::I64)); - *dst = Writable::from_reg(regs::rax()); + dst_mov = Some(Self::gen_move( + dst.to_writable_reg(), + dst.to_reg().to_reg(), + types::I64, + )); + *dst = Writable::from_reg(Gpr::new(regs::rax()).unwrap()); } insts.push(self); insts.extend(dst_mov); @@ -992,18 +1034,22 @@ impl Inst { } => { if *src != dst.to_reg() { debug_assert!(src.is_virtual()); - insts.push(Self::gen_move(*dst, *src, types::I64)); + insts.push(Self::gen_move( + dst.to_writable_reg(), + src.to_reg(), + types::I64, + )); *src = dst.to_reg(); } - if let Imm8Reg::Reg { reg } = num_bits { - if *reg != regs::rcx() { + if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() { + if reg != regs::rcx() { debug_assert!(reg.is_virtual()); insts.push(Self::gen_move( Writable::from_reg(regs::rcx()), - *reg, + reg, types::I64, )); - *reg = regs::rcx(); + *num_bits = Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(); } } insts.push(self); @@ -1146,7 +1192,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())), src2.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())), - show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*size, op.is_8bit())), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size_lqb(*size, op.is_8bit())), ), Inst::UnaryRmR { src, dst, op, size } => format!( @@ -1208,7 +1254,7 @@ impl PrettyPrint for Inst { DivOrRemKind::SignedRem => "srem", DivOrRemKind::UnsignedRem => "urem", }, - show_ireg_sized(divisor.to_reg(), mb_rru, size.to_bytes()), + show_ireg_sized(divisor.to_reg().to_reg(), mb_rru, size.to_bytes()), ), Inst::SignExtendData { size, .. } => match size { @@ -1276,8 +1322,8 @@ impl PrettyPrint for Inst { }, format!("f{}", size.to_bits()) ), - show_ireg_sized(*lhs, mb_rru, 8), - show_ireg_sized(rhs_dst.to_reg(), mb_rru, 8), + show_ireg_sized(lhs.to_reg(), mb_rru, 8), + show_ireg_sized(rhs_dst.to_reg().to_reg(), mb_rru, 8), ), Inst::XmmRmRImm { @@ -1342,7 +1388,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, 8), - show_ireg_sized(*dst, mb_rru, 8), + show_ireg_sized(dst.to_reg(), mb_rru, 8), ), Inst::CvtUint64ToFloatSeq { @@ -1405,14 +1451,14 @@ impl PrettyPrint for Inst { "{} ${}, {}", ljustify("movabsq".to_string()), *simm64 as i64, - show_ireg_sized(dst.to_reg(), mb_rru, 8) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8) ) } else { format!( "{} ${}, {}", ljustify("movl".to_string()), (*simm64 as u32) as i32, - show_ireg_sized(dst.to_reg(), mb_rru, 4) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4) ) } } @@ -1420,8 +1466,8 @@ impl PrettyPrint for Inst { Inst::MovRR { size, src, dst } => format!( "{} {}, {}", ljustify2("mov".to_string(), suffix_lq(*size)), - show_ireg_sized(*src, mb_rru, size.to_bytes()), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::MovzxRmR { @@ -1432,14 +1478,14 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify("movl".to_string()), src.show_rru_sized(mb_rru, ext_mode.src_size()), - show_ireg_sized(dst.to_reg(), mb_rru, 4) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4) ) } else { format!( "{} {}, {}", ljustify2("movz".to_string(), ext_mode.to_string()), src.show_rru_sized(mb_rru, ext_mode.src_size()), - show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size()) ) } } @@ -1464,13 +1510,13 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify2("movs".to_string(), ext_mode.to_string()), src.show_rru_sized(mb_rru, ext_mode.src_size()), - show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size()) ), Inst::MovRM { size, src, dst, .. } => format!( "{} {}, {}", ljustify2("mov".to_string(), suffix_bwlq(*size)), - show_ireg_sized(*src, mb_rru, size.to_bytes()), + show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()), dst.show_rru(mb_rru) ), @@ -1480,19 +1526,19 @@ impl PrettyPrint for Inst { num_bits, dst, .. - } => match num_bits { + } => match num_bits.clone().to_imm8_reg() { Imm8Reg::Reg { reg } => format!( "{} {}, {}", ljustify2(kind.to_string(), suffix_bwlq(*size)), - show_ireg_sized(*reg, mb_rru, 1), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(reg, mb_rru, 1), + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Imm8Reg::Imm8 { imm: num_bits } => format!( "{} ${}, {}", ljustify2(kind.to_string(), suffix_bwlq(*size)), num_bits, - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), }, @@ -1519,14 +1565,14 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify2(op.to_string(), suffix_bwlq(*size)), src.show_rru_sized(mb_rru, size.to_bytes()), - show_ireg_sized(*dst, mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) ) } Inst::Setcc { cc, dst } => format!( "{} {}", ljustify2("set".to_string(), cc.to_string()), - show_ireg_sized(dst.to_reg(), mb_rru, 1) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 1) ), Inst::Cmove { @@ -1539,7 +1585,7 @@ impl PrettyPrint for Inst { "{} {}, {}", ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))), src.show_rru_sized(mb_rru, size.to_bytes()), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ), Inst::XmmCmove { size, cc, src, dst } => { @@ -1552,7 +1598,7 @@ impl PrettyPrint for Inst { "ss" }, src.show_rru_sized(mb_rru, size.to_bytes()), - show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes()) ) } @@ -1693,10 +1739,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { // No need to account for src2, since src2 == dst. - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } else { src2.get_regs_as_uses(collector); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } } Inst::Not { src, dst, .. } => { @@ -1760,9 +1806,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { // the rdx register *before* the instruction, which is not too bad. collector.add_mod(Writable::from_reg(regs::rax())); collector.add_mod(Writable::from_reg(regs::rdx())); - collector.add_mod(*divisor); + collector.add_mod(divisor.to_writable_reg()); if let Some(tmp) = tmp { - collector.add_def(*tmp); + collector.add_def(tmp.to_writable_reg()); } } Inst::SignExtendData { size, src, dst } => { @@ -1852,8 +1898,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()), Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst), Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => { - collector.add_use(*lhs); - collector.add_mod(*rhs_dst); + collector.add_use(lhs.to_reg()); + collector.add_mod(rhs_dst.to_writable_reg()); } Inst::XmmRmiReg { src1, src2, dst, .. @@ -1868,14 +1914,14 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } Inst::XmmCmpRmR { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_use(*dst); + collector.add_use(dst.to_reg()); } Inst::Imm { dst, .. } => { - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::MovRR { src, dst, .. } => { - collector.add_use(*src); - collector.add_def(*dst); + collector.add_use(src.to_reg()); + collector.add_def(dst.to_writable_reg()); } Inst::XmmToGpr { src, dst, .. } => { collector.add_use(src.to_reg()); @@ -1918,11 +1964,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } Inst::MovzxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::Mov64MR { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_def(*dst) + collector.add_def(dst.to_writable_reg()) } Inst::LoadEffectiveAddress { addr: src, dst } => { src.get_regs_as_uses(collector); @@ -1930,41 +1976,44 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { } Inst::MovsxRmR { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::MovRM { src, dst, .. } => { - collector.add_use(*src); + collector.add_use(src.to_reg()); dst.get_regs_as_uses(collector); } Inst::ShiftR { num_bits, dst, .. } => { - if let Imm8Reg::Reg { reg } = num_bits { - debug_assert_eq!(*reg, regs::rcx()); + if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() { + debug_assert_eq!(reg, regs::rcx()); collector.add_use(regs::rcx()); } - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); } Inst::CmpRmiR { src, dst, .. } => { src.get_regs_as_uses(collector); - collector.add_use(*dst); // yes, really `add_use` + collector.add_use(dst.to_reg()); // yes, really `add_use` } Inst::Setcc { dst, .. } => { - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::Cmove { consequent: src, dst, .. - } - | Inst::XmmCmove { src, dst, .. } => { + } => { src.get_regs_as_uses(collector); - collector.add_mod(*dst); + collector.add_mod(dst.to_writable_reg()); + } + Inst::XmmCmove { src, dst, .. } => { + src.get_regs_as_uses(collector); + collector.add_mod(dst.to_writable_reg()); } Inst::Push64 { src } => { src.get_regs_as_uses(collector); collector.add_mod(Writable::from_reg(regs::rsp())); } Inst::Pop64 { dst } => { - collector.add_def(*dst); + collector.add_def(dst.to_writable_reg()); } Inst::CallKnown { @@ -2149,11 +2198,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - mapper.map_def(dst); + dst.map_def(mapper); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *src1 = dst.to_reg(); } } @@ -2165,9 +2214,9 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { Inst::Div { divisor, .. } => divisor.map_uses(mapper), Inst::MulHi { src2, .. } => src2.map_uses(mapper), Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => { - mapper.map_mod(divisor); + divisor.map_mod(mapper); if let Some(tmp) = tmp { - mapper.map_def(tmp) + tmp.map_def(mapper) } } Inst::SignExtendData { .. } => {} @@ -2275,8 +2324,8 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut rhs_dst, .. } => { - mapper.map_use(lhs); - mapper.map_mod(rhs_dst); + lhs.map_use(mapper); + rhs_dst.map_mod(mapper); } Inst::XmmMovRM { ref mut src, @@ -2292,16 +2341,16 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_use(dst); + dst.map_use(mapper); } - Inst::Imm { ref mut dst, .. } => mapper.map_def(dst), + Inst::Imm { ref mut dst, .. } => dst.map_def(mapper), Inst::MovRR { ref mut src, ref mut dst, .. } => { - mapper.map_use(src); - mapper.map_def(dst); + src.map_use(mapper); + dst.map_def(mapper); } Inst::XmmToGpr { ref mut src, @@ -2356,11 +2405,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::Mov64MR { src, dst, .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::LoadEffectiveAddress { addr: src, dst } => { src.map_uses(mapper); @@ -2372,14 +2421,14 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_def(dst); + dst.map_def(mapper); } Inst::MovRM { ref mut src, ref mut dst, .. } => { - mapper.map_use(src); + src.map_use(mapper); dst.map_uses(mapper); } Inst::ShiftR { @@ -2388,7 +2437,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { debug_assert_eq!(*src, dst.to_reg()); - mapper.map_mod(dst); + dst.map_mod(mapper); *src = dst.to_reg(); } Inst::CmpRmiR { @@ -2397,9 +2446,9 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_use(dst); + dst.map_use(mapper); } - Inst::Setcc { ref mut dst, .. } => mapper.map_def(dst), + Inst::Setcc { ref mut dst, .. } => dst.map_def(mapper), Inst::Cmove { consequent: ref mut src, ref mut dst, @@ -2407,7 +2456,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); *alternative = dst.to_reg(); } Inst::XmmCmove { @@ -2416,11 +2465,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - mapper.map_mod(dst); + dst.map_mod(mapper); } Inst::Push64 { ref mut src } => src.map_uses(mapper), Inst::Pop64 { ref mut dst } => { - mapper.map_def(dst); + dst.map_def(mapper); } Inst::CallKnown { @@ -2520,7 +2569,7 @@ impl MachInst for Inst { // conceivably use `movl %reg, %reg` to zero out the top 32 bits of // %reg. Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => { - Some((*dst, *src)) + Some((dst.to_writable_reg(), src.to_reg())) } // Note as well that MOVS[S|D] when used in the `XmmUnaryRmR` context are pure moves of // scalar floating-point values (and annotate `dst` as `def`s to the register allocator) @@ -2578,12 +2627,15 @@ impl MachInst for Inst { size: OperandSize::Size8, src, dst: SyntheticAmode::NominalSPOffset { simm32 }, - } => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)), + } => Some(MachInstStackOpInfo::StoreNomSPOff( + src.to_reg(), + *simm32 as i64, + )), Self::Mov64MR { src: SyntheticAmode::NominalSPOffset { simm32 }, dst, } => Some(MachInstStackOpInfo::LoadNomSPOff( - dst.to_reg(), + dst.to_reg().to_reg(), *simm32 as i64, )), _ => None, diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index c3d84b7f49..78d0bada8c 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -63,33 +63,33 @@ ;; Add two registers. (rule (lower (has_type (fits_in_64 ty) (iadd x y))) - (value_reg (add ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_gpr (add ty + (put_in_gpr x) + (gpr_to_gpr_mem_imm (put_in_gpr y))))) ;; Add a register and an immediate. (rule (lower (has_type (fits_in_64 ty) (iadd x (simm32_from_value y)))) - (value_reg (add ty (put_in_reg x) y))) + (value_gpr (add ty (put_in_gpr x) y))) (rule (lower (has_type (fits_in_64 ty) (iadd (simm32_from_value x) y))) - (value_reg (add ty (put_in_reg y) x))) + (value_gpr (add ty (put_in_gpr y) x))) ;; Add a register and memory. (rule (lower (has_type (fits_in_64 ty) (iadd x (sinkable_load y)))) - (value_reg (add ty - (put_in_reg x) - (sink_load y)))) + (value_gpr (add ty + (put_in_gpr x) + (sink_load_to_gpr_mem_imm y)))) (rule (lower (has_type (fits_in_64 ty) (iadd (sinkable_load x) y))) - (value_reg (add ty - (put_in_reg y) - (sink_load x)))) + (value_gpr (add ty + (put_in_gpr y) + (sink_load_to_gpr_mem_imm x)))) ;; SSE. @@ -117,15 +117,15 @@ (rule (lower (has_type $I128 (iadd x y))) ;; Get the high/low registers for `x`. (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1))) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1))) ;; Get the high/low registers for `y`. (let ((y_regs ValueRegs (put_in_regs y)) - (y_lo Reg (value_regs_get y_regs 0)) - (y_hi Reg (value_regs_get y_regs 1))) + (y_lo Gpr (value_regs_get_gpr y_regs 0)) + (y_hi Gpr (value_regs_get_gpr y_regs 1))) ;; Do an add followed by an add-with-carry. - (with_flags (add_with_flags $I64 x_lo (RegMemImm.Reg y_lo)) - (adc $I64 x_hi (RegMemImm.Reg y_hi)))))) + (with_flags (add_with_flags $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) + (adc $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))) ;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -166,42 +166,42 @@ ;; Add two registers. (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x y))) - (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64)))) - (value_regs (add ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))) - unused_iflags))) + (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr)))) + (value_gprs (add ty + (put_in_gpr x) + (put_in_gpr_mem_imm y)) + unused_iflags))) ;; Add a register and an immediate. (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (simm32_from_value y)))) - (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64)))) - (value_regs (add ty (put_in_reg x) y) + (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr)))) + (value_gprs (add ty (put_in_gpr x) y) unused_iflags))) (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (simm32_from_value x) y))) - (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64)))) - (value_regs (add ty (put_in_reg y) x) + (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr)))) + (value_gprs (add ty (put_in_gpr y) x) unused_iflags))) ;; Add a register and memory. (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (sinkable_load y)))) - (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64)))) - (value_regs (add ty - (put_in_reg x) - (sink_load y)) + (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr)))) + (value_gprs (add ty + (put_in_gpr x) + (sink_load_to_gpr_mem_imm y)) unused_iflags))) (rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (sinkable_load x) y))) - (let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64)))) - (value_regs (add ty - (put_in_reg y) - (sink_load x)) + (let ((unused_iflags Gpr (writable_gpr_to_gpr (temp_writable_gpr)))) + (value_gprs (add ty + (put_in_gpr y) + (sink_load_to_gpr_mem_imm x)) unused_iflags))) ;; (No `iadd_ifcout` for `i128`.) @@ -212,30 +212,30 @@ ;; When the immediate fits in a `RegMemImm.Imm`, use that. (rule (lower (has_type (fits_in_64 ty) (iadd_imm y (simm32_from_imm64 x)))) - (value_reg (add ty (put_in_reg y) x))) + (value_gpr (add ty (put_in_gpr y) x))) ;; Otherwise, put the immediate into a register. (rule (lower (has_type (fits_in_64 ty) (iadd_imm y (u64_from_imm64 x)))) - (value_reg (add ty (put_in_reg y) (RegMemImm.Reg (imm ty x))))) + (value_gpr (add ty (put_in_gpr y) (gpr_to_gpr_mem_imm (gpr_new (imm ty x)))))) ;; `i128` ;; When the immediate fits in a `RegMemImm.Imm`, use that. (rule (lower (has_type $I128 (iadd_imm y (simm32_from_imm64 x)))) (let ((y_regs ValueRegs (put_in_regs y)) - (y_lo Reg (value_regs_get y_regs 0)) - (y_hi Reg (value_regs_get y_regs 1))) + (y_lo Gpr (value_regs_get_gpr y_regs 0)) + (y_hi Gpr (value_regs_get_gpr y_regs 1))) (with_flags (add_with_flags $I64 y_lo x) - (adc $I64 y_hi (RegMemImm.Imm 0))))) + (adc $I64 y_hi (gpr_mem_imm_new (RegMemImm.Imm 0)))))) ;; Otherwise, put the immediate into a register. (rule (lower (has_type $I128 (iadd_imm y (u64_from_imm64 x)))) (let ((y_regs ValueRegs (put_in_regs y)) - (y_lo Reg (value_regs_get y_regs 0)) - (y_hi Reg (value_regs_get y_regs 1)) - (x_lo Reg (imm $I64 x))) - (with_flags (add_with_flags $I64 y_lo (RegMemImm.Reg x_lo)) - (adc $I64 y_hi (RegMemImm.Imm 0))))) + (y_lo Gpr (value_regs_get_gpr y_regs 0)) + (y_hi Gpr (value_regs_get_gpr y_regs 1)) + (x_lo Gpr (gpr_new (imm $I64 x)))) + (with_flags (add_with_flags $I64 y_lo (gpr_to_gpr_mem_imm x_lo)) + (adc $I64 y_hi (gpr_mem_imm_new (RegMemImm.Imm 0)))))) ;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -244,21 +244,21 @@ ;; Sub two registers. (rule (lower (has_type (fits_in_64 ty) (isub x y))) - (value_reg (sub ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_gpr (sub ty + (put_in_gpr x) + (put_in_gpr_mem_imm y)))) ;; Sub a register and an immediate. (rule (lower (has_type (fits_in_64 ty) (isub x (simm32_from_value y)))) - (value_reg (sub ty (put_in_reg x) y))) + (value_gpr (sub ty (put_in_gpr x) y))) ;; Sub a register and memory. (rule (lower (has_type (fits_in_64 ty) (isub x (sinkable_load y)))) - (value_reg (sub ty - (put_in_reg x) - (sink_load y)))) + (value_gpr (sub ty + (put_in_gpr x) + (sink_load_to_gpr_mem_imm y)))) ;; SSE. @@ -286,15 +286,15 @@ (rule (lower (has_type $I128 (isub x y))) ;; Get the high/low registers for `x`. (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1))) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1))) ;; Get the high/low registers for `y`. (let ((y_regs ValueRegs (put_in_regs y)) - (y_lo Reg (value_regs_get y_regs 0)) - (y_hi Reg (value_regs_get y_regs 1))) + (y_lo Gpr (value_regs_get_gpr y_regs 0)) + (y_hi Gpr (value_regs_get_gpr y_regs 1))) ;; Do a sub followed by an sub-with-borrow. - (with_flags (sub_with_flags $I64 x_lo (RegMemImm.Reg y_lo)) - (sbb $I64 x_hi (RegMemImm.Reg y_hi)))))) + (with_flags (sub_with_flags $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) + (sbb $I64 x_hi (gpr_to_gpr_mem_imm y_hi)))))) ;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -326,36 +326,36 @@ ;; And two registers. (rule (lower (has_type (fits_in_64 ty) (band x y))) - (value_reg (x64_and ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_gpr (x64_and ty + (put_in_gpr x) + (put_in_gpr_mem_imm y)))) ;; And with a memory operand. (rule (lower (has_type (fits_in_64 ty) (band x (sinkable_load y)))) - (value_reg (x64_and ty - (put_in_reg x) - (sink_load y)))) + (value_gpr (x64_and ty + (put_in_gpr x) + (sink_load_to_gpr_mem_imm y)))) (rule (lower (has_type (fits_in_64 ty) (band (sinkable_load x) y))) - (value_reg (x64_and ty - (put_in_reg y) - (sink_load x)))) + (value_gpr (x64_and ty + (put_in_gpr y) + (sink_load_to_gpr_mem_imm x)))) ;; And with an immediate. (rule (lower (has_type (fits_in_64 ty) (band x (simm32_from_value y)))) - (value_reg (x64_and ty - (put_in_reg x) + (value_gpr (x64_and ty + (put_in_gpr x) y))) (rule (lower (has_type (fits_in_64 ty) (band (simm32_from_value x) y))) - (value_reg (x64_and ty - (put_in_reg y) + (value_gpr (x64_and ty + (put_in_gpr y) x))) ;; SSE. @@ -375,23 +375,23 @@ (rule (lower (has_type $I128 (band x y))) (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1)) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1)) (y_regs ValueRegs (put_in_regs y)) - (y_lo Reg (value_regs_get y_regs 0)) - (y_hi Reg (value_regs_get y_regs 1))) - (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo)) - (x64_and $I64 x_hi (RegMemImm.Reg y_hi))))) + (y_lo Gpr (value_regs_get_gpr y_regs 0)) + (y_hi Gpr (value_regs_get_gpr y_regs 1))) + (value_gprs (x64_and $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) + (x64_and $I64 x_hi (gpr_to_gpr_mem_imm y_hi))))) (rule (lower (has_type $B128 (band x y))) ;; Booleans are always `0` or `1`, so we only need to do the `and` on the ;; low half. The high half is always zero but, rather than generate a new ;; zero, we just reuse `x`'s high half which is already zero. (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1)) - (y_lo Reg (lo_reg y))) - (value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo)) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1)) + (y_lo Gpr (lo_gpr y))) + (value_gprs (x64_and $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) x_hi))) ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -400,36 +400,36 @@ ;; Or two registers. (rule (lower (has_type (fits_in_64 ty) (bor x y))) - (value_reg (or ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_gpr (or ty + (put_in_gpr x) + (put_in_gpr_mem_imm y)))) ;; Or with a memory operand. (rule (lower (has_type (fits_in_64 ty) (bor x (sinkable_load y)))) - (value_reg (or ty - (put_in_reg x) - (sink_load y)))) + (value_gpr (or ty + (put_in_gpr x) + (sink_load_to_gpr_mem_imm y)))) (rule (lower (has_type (fits_in_64 ty) (bor (sinkable_load x) y))) - (value_reg (or ty - (put_in_reg y) - (sink_load x)))) + (value_gpr (or ty + (put_in_gpr y) + (sink_load_to_gpr_mem_imm x)))) ;; Or with an immediate. (rule (lower (has_type (fits_in_64 ty) (bor x (simm32_from_value y)))) - (value_reg (or ty - (put_in_reg x) + (value_gpr (or ty + (put_in_gpr x) y))) (rule (lower (has_type (fits_in_64 ty) (bor (simm32_from_value x) y))) - (value_reg (or ty - (put_in_reg y) + (value_gpr (or ty + (put_in_gpr y) x))) ;; SSE. @@ -449,12 +449,12 @@ (decl or_i128 (ValueRegs ValueRegs) ValueRegs) (rule (or_i128 x y) - (let ((x_lo Reg (value_regs_get x 0)) - (x_hi Reg (value_regs_get x 1)) - (y_lo Reg (value_regs_get y 0)) - (y_hi Reg (value_regs_get y 1))) - (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo)) - (or $I64 x_hi (RegMemImm.Reg y_hi))))) + (let ((x_lo Gpr (value_regs_get_gpr x 0)) + (x_hi Gpr (value_regs_get_gpr x 1)) + (y_lo Gpr (value_regs_get_gpr y 0)) + (y_hi Gpr (value_regs_get_gpr y 1))) + (value_gprs (or $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) + (or $I64 x_hi (gpr_to_gpr_mem_imm y_hi))))) (rule (lower (has_type $I128 (bor x y))) (or_i128 (put_in_regs x) (put_in_regs y))) @@ -464,10 +464,10 @@ ;; low half. The high half is always zero but, rather than generate a new ;; zero, we just reuse `x`'s high half which is already zero. (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1)) - (y_lo Reg (lo_reg y))) - (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo)) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1)) + (y_lo Gpr (lo_gpr y))) + (value_gprs (or $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) x_hi))) ;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -476,36 +476,36 @@ ;; Xor two registers. (rule (lower (has_type (fits_in_64 ty) (bxor x y))) - (value_reg (xor ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_gpr (xor ty + (put_in_gpr x) + (put_in_gpr_mem_imm y)))) ;; Xor with a memory operand. (rule (lower (has_type (fits_in_64 ty) (bxor x (sinkable_load y)))) - (value_reg (xor ty - (put_in_reg x) - (sink_load y)))) + (value_gpr (xor ty + (put_in_gpr x) + (sink_load_to_gpr_mem_imm y)))) (rule (lower (has_type (fits_in_64 ty) (bxor (sinkable_load x) y))) - (value_reg (xor ty - (put_in_reg y) - (sink_load x)))) + (value_gpr (xor ty + (put_in_gpr y) + (sink_load_to_gpr_mem_imm x)))) ;; Xor with an immediate. (rule (lower (has_type (fits_in_64 ty) (bxor x (simm32_from_value y)))) - (value_reg (xor ty - (put_in_reg x) + (value_gpr (xor ty + (put_in_gpr x) y))) (rule (lower (has_type (fits_in_64 ty) (bxor (simm32_from_value x) y))) - (value_reg (xor ty - (put_in_reg y) + (value_gpr (xor ty + (put_in_gpr y) x))) ;; SSE. @@ -517,23 +517,23 @@ (rule (lower (has_type $I128 (bxor x y))) (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1)) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1)) (y_regs ValueRegs (put_in_regs y)) - (y_lo Reg (value_regs_get y_regs 0)) - (y_hi Reg (value_regs_get y_regs 1))) - (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo)) - (xor $I64 x_hi (RegMemImm.Reg y_hi))))) + (y_lo Gpr (value_regs_get_gpr y_regs 0)) + (y_hi Gpr (value_regs_get_gpr y_regs 1))) + (value_gprs (xor $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) + (xor $I64 x_hi (gpr_to_gpr_mem_imm y_hi))))) (rule (lower (has_type $B128 (bxor x y))) ;; Booleans are always `0` or `1`, so we only need to do the `xor` on the ;; low half. The high half is always zero but, rather than generate a new ;; zero, we just reuse `x`'s high half which is already zero. (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1)) - (y_lo Reg (lo_reg y))) - (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo)) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1)) + (y_lo Gpr (lo_gpr y))) + (value_gprs (xor $I64 x_lo (gpr_to_gpr_mem_imm y_lo)) x_hi))) ;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -541,38 +541,49 @@ ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (ishl src amt))) - (value_reg (shl ty (put_in_reg src) (put_masked_in_imm8_reg amt ty)))) + (value_gpr (shl ty (put_in_gpr src) (put_masked_in_imm8_gpr amt ty)))) ;; `i128`. -(decl shl_i128 (ValueRegs Reg) ValueRegs) +(decl shl_i128 (ValueRegs Gpr) ValueRegs) (rule (shl_i128 src amt) ;; Unpack the registers that make up the 128-bit value being shifted. - (let ((src_lo Reg (value_regs_get src 0)) - (src_hi Reg (value_regs_get src 1)) + (let ((src_lo Gpr (value_regs_get_gpr src 0)) + (src_hi Gpr (value_regs_get_gpr src 1)) ;; Do two 64-bit shifts. - (lo_shifted Reg (shl $I64 src_lo (Imm8Reg.Reg amt))) - (hi_shifted Reg (shl $I64 src_hi (Imm8Reg.Reg amt))) + (lo_shifted Gpr (shl $I64 src_lo (gpr_to_imm8_gpr amt))) + (hi_shifted Gpr (shl $I64 src_hi (gpr_to_imm8_gpr amt))) ;; `src_lo >> (64 - amt)` are the bits to carry over from the lo ;; into the hi. - (carry Reg (shr $I64 src_lo (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt))))) - (zero Reg (imm $I64 0)) + (carry Gpr (shr $I64 + src_lo + (gpr_to_imm8_gpr (sub $I64 + (gpr_new (imm $I64 64)) + (gpr_to_gpr_mem_imm amt))))) + (zero Gpr (gpr_new (imm $I64 0))) ;; Nullify the carry if we are shifting in by a multiple of 128. - (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt) - (cmove $I64 (CC.Z) (RegMem.Reg zero) carry))) + (carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64) + (gpr_mem_imm_new (RegMemImm.Imm 127)) + amt) + (cmove $I64 + (CC.Z) + (gpr_to_gpr_mem zero) + carry)))) ;; Add the carry into the high half. - (hi_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg hi_shifted)))) + (hi_shifted_ Gpr (or $I64 carry_ (gpr_to_gpr_mem_imm hi_shifted)))) ;; Combine the two shifted halves. However, if we are shifting by >= 64 ;; (modulo 128), then the low bits are zero and the high bits are our ;; low bits. - (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt) - (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted) zero) - (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted_) lo_shifted)))) + (with_flags_2 (test (OperandSize.Size64) + (gpr_mem_imm_new (RegMemImm.Imm 64)) + amt) + (cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted) zero) + (cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted_) lo_shifted)))) (rule (lower (has_type $I128 (ishl src amt))) ;; NB: Only the low bits of `amt` matter since we logically mask the shift ;; amount to the value's bit width. - (let ((amt_ Reg (lo_reg amt))) + (let ((amt_ Gpr (lo_gpr amt))) (shl_i128 (put_in_regs src) amt_))) ;; SSE. @@ -613,10 +624,12 @@ (rule (ishl_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ishl_i8x16_mask_table)) (base_mask_addr Gpr (lea mask_table)) - (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) + (mask_offset Gpr (shl $I64 + (gpr_new amt) + (imm8_to_imm8_gpr 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - (gpr_new mask_offset) + mask_offset 0)))) (rule (ishl_i8x16_mask (RegMemImm.Mem amt)) (ishl_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) @@ -640,38 +653,49 @@ ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (ushr src amt))) - (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Zero)))) - (value_reg (shr ty src_ (put_masked_in_imm8_reg amt ty))))) + (let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Zero)))) + (value_gpr (shr ty src_ (put_masked_in_imm8_gpr amt ty))))) ;; `i128`. -(decl shr_i128 (ValueRegs Reg) ValueRegs) +(decl shr_i128 (ValueRegs Gpr) ValueRegs) (rule (shr_i128 src amt) ;; Unpack the lo/hi halves of `src`. - (let ((src_lo Reg (value_regs_get src 0)) - (src_hi Reg (value_regs_get src 1)) + (let ((src_lo Gpr (value_regs_get_gpr src 0)) + (src_hi Gpr (value_regs_get_gpr src 1)) ;; Do a shift on each half. - (lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt))) - (hi_shifted Reg (shr $I64 src_hi (Imm8Reg.Reg amt))) + (lo_shifted Gpr (shr $I64 src_lo (gpr_to_imm8_gpr amt))) + (hi_shifted Gpr (shr $I64 src_hi (gpr_to_imm8_gpr amt))) ;; `src_hi << (64 - amt)` are the bits to carry over from the hi ;; into the lo. - (carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt))))) + (carry Gpr (shl $I64 + src_hi + (gpr_to_imm8_gpr (sub $I64 + (gpr_new (imm $I64 64)) + (gpr_to_gpr_mem_imm amt))))) ;; Nullify the carry if we are shifting by a multiple of 128. - (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt) - (cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry))) + (carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64) + (gpr_mem_imm_new (RegMemImm.Imm 127)) + amt) + (cmove $I64 + (CC.Z) + (gpr_to_gpr_mem (gpr_new (imm $I64 0))) + carry)))) ;; Add the carry bits into the lo. - (lo_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg lo_shifted)))) + (lo_shifted_ Gpr (or $I64 carry_ (gpr_to_gpr_mem_imm lo_shifted)))) ;; Combine the two shifted halves. However, if we are shifting by >= 64 ;; (modulo 128), then the hi bits are zero and the lo bits are what ;; would otherwise be our hi bits. - (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt) - (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted) - (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) (imm $I64 0))))) + (with_flags_2 (test (OperandSize.Size64) + (gpr_mem_imm_new (RegMemImm.Imm 64)) + amt) + (cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted_) hi_shifted) + (cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted) (gpr_new (imm $I64 0)))))) (rule (lower (has_type $I128 (ushr src amt))) ;; NB: Only the low bits of `amt` matter since we logically mask the shift ;; amount to the value's bit width. - (let ((amt_ Reg (lo_reg amt))) + (let ((amt_ Gpr (lo_gpr amt))) (shr_i128 (put_in_regs src) amt_))) ;; SSE. @@ -712,10 +736,12 @@ (rule (ushr_i8x16_mask (RegMemImm.Reg amt)) (let ((mask_table SyntheticAmode (ushr_i8x16_mask_table)) (base_mask_addr Gpr (lea mask_table)) - (mask_offset Reg (shl $I64 amt (Imm8Reg.Imm8 4)))) + (mask_offset Gpr (shl $I64 + (gpr_new amt) + (imm8_to_imm8_gpr 4)))) (amode_to_synthetic_amode (amode_imm_reg_reg_shift 0 base_mask_addr - (gpr_new mask_offset) + mask_offset 0)))) (rule (ushr_i8x16_mask (RegMemImm.Mem amt)) (ushr_i8x16_mask (RegMemImm.Reg (x64_load $I64 amt (ExtKind.None))))) @@ -739,41 +765,52 @@ ;; `i64` and smaller. (rule (lower (has_type (fits_in_64 ty) (sshr src amt))) - (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Sign)))) - (value_reg (sar ty src_ (put_masked_in_imm8_reg amt ty))))) + (let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Sign)))) + (value_gpr (sar ty src_ (put_masked_in_imm8_gpr amt ty))))) ;; `i128`. -(decl sar_i128 (ValueRegs Reg) ValueRegs) +(decl sar_i128 (ValueRegs Gpr) ValueRegs) (rule (sar_i128 src amt) ;; Unpack the low/high halves of `src`. - (let ((src_lo Reg (value_regs_get src 0)) - (src_hi Reg (value_regs_get src 1)) + (let ((src_lo Gpr (value_regs_get_gpr src 0)) + (src_hi Gpr (value_regs_get_gpr src 1)) ;; Do a shift of each half. NB: the low half uses an unsigned shift ;; because its MSB is not a sign bit. - (lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt))) - (hi_shifted Reg (sar $I64 src_hi (Imm8Reg.Reg amt))) + (lo_shifted Gpr (shr $I64 src_lo (gpr_to_imm8_gpr amt))) + (hi_shifted Gpr (sar $I64 src_hi (gpr_to_imm8_gpr amt))) ;; `src_hi << (64 - amt)` are the bits to carry over from the low ;; half to the high half. - (carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt))))) + (carry Gpr (shl $I64 + src_hi + (gpr_to_imm8_gpr (sub $I64 + (gpr_new (imm $I64 64)) + (gpr_to_gpr_mem_imm amt))))) ;; Nullify the carry if we are shifting by a multiple of 128. - (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt) - (cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry))) + (carry_ Gpr (gpr_new (with_flags_1 (test (OperandSize.Size64) + (gpr_mem_imm_new (RegMemImm.Imm 127)) + amt) + (cmove $I64 + (CC.Z) + (gpr_to_gpr_mem (gpr_new (imm $I64 0))) + carry)))) ;; Add the carry into the low half. - (lo_shifted_ Reg (or $I64 lo_shifted (RegMemImm.Reg carry_))) + (lo_shifted_ Gpr (or $I64 lo_shifted (gpr_to_gpr_mem_imm carry_))) ;; Get all sign bits. - (sign_bits Reg (sar $I64 src_hi (Imm8Reg.Imm8 63)))) + (sign_bits Gpr (sar $I64 src_hi (imm8_to_imm8_gpr 63)))) ;; Combine the two shifted halves. However, if we are shifting by >= 64 ;; (modulo 128), then the hi bits are all sign bits and the lo bits are ;; what would otherwise be our hi bits. - (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt) - (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted) - (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) sign_bits)))) + (with_flags_2 (test (OperandSize.Size64) + (gpr_mem_imm_new (RegMemImm.Imm 64)) + amt) + (cmove $I64 (CC.Z) (gpr_to_gpr_mem lo_shifted_) hi_shifted) + (cmove $I64 (CC.Z) (gpr_to_gpr_mem hi_shifted) sign_bits)))) (rule (lower (has_type $I128 (sshr src amt))) ;; NB: Only the low bits of `amt` matter since we logically mask the shift ;; amount to the value's bit width. - (let ((amt_ Reg (lo_reg amt))) + (let ((amt_ Gpr (lo_gpr amt))) (sar_i128 (put_in_regs src) amt_))) ;; SSE. @@ -807,9 +844,13 @@ (rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i)) (xmm_mem_imm_new (RegMemImm.Imm (u32_add i 8)))) (rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r)) - (mov_rmi_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) + (mov_rmi_to_xmm (RegMemImm.Reg (gpr_to_reg (add ty + (gpr_new r) + (gpr_mem_imm_new (RegMemImm.Imm 8))))))) (rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m)) - (mov_rmi_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) + (mov_rmi_to_xmm (RegMemImm.Reg (gpr_to_reg (add ty + (gpr_new (imm ty 8)) + (gpr_mem_imm_new rmi)))))) ;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure ;; that if the shift amount is in a register, it is in an XMM register. @@ -834,11 +875,11 @@ (let ((src_ Xmm (put_in_xmm src)) (lo Gpr (pextrd $I64 src_ 0)) (hi Gpr (pextrd $I64 src_ 1)) - (amt_ Imm8Reg (put_masked_in_imm8_reg amt $I64)) - (shifted_lo Reg (sar $I64 (gpr_to_reg lo) amt_)) - (shifted_hi Reg (sar $I64 (gpr_to_reg hi) amt_))) - (value_xmm (make_i64x2_from_lanes (reg_mem_to_gpr_mem (RegMem.Reg shifted_lo)) - (reg_mem_to_gpr_mem (RegMem.Reg shifted_hi)))))) + (amt_ Imm8Gpr (put_masked_in_imm8_gpr amt $I64)) + (shifted_lo Gpr (sar $I64 lo amt_)) + (shifted_hi Gpr (sar $I64 hi amt_))) + (value_xmm (make_i64x2_from_lanes (gpr_to_gpr_mem shifted_lo) + (gpr_to_gpr_mem shifted_hi))))) ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -846,13 +887,13 @@ ;; constant. (rule (lower (has_type (ty_8_or_16 ty) (rotl src amt))) - (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) - (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (let ((amt_ Gpr (extend_to_gpr amt $I32 (ExtendKind.Zero)))) + (value_gpr (x64_rotl ty (put_in_gpr src) (gpr_to_imm8_gpr amt_))))) (rule (lower (has_type (ty_8_or_16 ty) (rotl src (u64_from_iconst amt)))) - (value_reg (x64_rotl ty - (put_in_reg src) + (value_gpr (x64_rotl ty + (put_in_gpr src) (const_to_type_masked_imm8 amt ty)))) ;; `i64` and `i32`: we can rely on x86's rotate-amount masking since @@ -861,13 +902,13 @@ (rule (lower (has_type (ty_32_or_64 ty) (rotl src amt))) ;; NB: Only the low bits of `amt` matter since we logically mask the ;; shift amount to the value's bit width. - (let ((amt_ Reg (lo_reg amt))) - (value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (let ((amt_ Gpr (lo_gpr amt))) + (value_gpr (x64_rotl ty (put_in_gpr src) (gpr_to_imm8_gpr amt_))))) (rule (lower (has_type (ty_32_or_64 ty) (rotl src (u64_from_iconst amt)))) - (value_reg (x64_rotl ty - (put_in_reg src) + (value_gpr (x64_rotl ty + (put_in_gpr src) (const_to_type_masked_imm8 amt ty)))) ;; `i128`. @@ -876,9 +917,11 @@ (let ((src_ ValueRegs (put_in_regs src)) ;; NB: Only the low bits of `amt` matter since we logically mask the ;; rotation amount to the value's bit width. - (amt_ Reg (lo_reg amt))) + (amt_ Gpr (lo_gpr amt))) (or_i128 (shl_i128 src_ amt_) - (shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) + (shr_i128 src_ (sub $I64 + (gpr_new (imm $I64 128)) + (gpr_to_gpr_mem_imm amt_)))))) ;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -886,13 +929,13 @@ ;; constant. (rule (lower (has_type (ty_8_or_16 ty) (rotr src amt))) - (let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero)))) - (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (let ((amt_ Gpr (extend_to_gpr amt $I32 (ExtendKind.Zero)))) + (value_gpr (x64_rotr ty (put_in_gpr src) (gpr_to_imm8_gpr amt_))))) (rule (lower (has_type (ty_8_or_16 ty) (rotr src (u64_from_iconst amt)))) - (value_reg (x64_rotr ty - (put_in_reg src) + (value_gpr (x64_rotr ty + (put_in_gpr src) (const_to_type_masked_imm8 amt ty)))) ;; `i64` and `i32`: we can rely on x86's rotate-amount masking since @@ -901,13 +944,13 @@ (rule (lower (has_type (ty_32_or_64 ty) (rotr src amt))) ;; NB: Only the low bits of `amt` matter since we logically mask the ;; shift amount to the value's bit width. - (let ((amt_ Reg (lo_reg amt))) - (value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + (let ((amt_ Gpr (lo_gpr amt))) + (value_gpr (x64_rotr ty (put_in_gpr src) (gpr_to_imm8_gpr amt_))))) (rule (lower (has_type (ty_32_or_64 ty) (rotr src (u64_from_iconst amt)))) - (value_reg (x64_rotr ty - (put_in_reg src) + (value_gpr (x64_rotr ty + (put_in_gpr src) (const_to_type_masked_imm8 amt ty)))) ;; `i128`. @@ -916,9 +959,11 @@ (let ((src_ ValueRegs (put_in_regs src)) ;; NB: Only the low bits of `amt` matter since we logically mask the ;; rotation amount to the value's bit width. - (amt_ Reg (lo_reg amt))) + (amt_ Gpr (lo_gpr amt))) (or_i128 (shr_i128 src_ amt_) - (shl_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) + (shl_i128 src_ (sub $I64 + (gpr_new (imm $I64 128)) + (gpr_to_gpr_mem_imm amt_)))))) ;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -961,33 +1006,33 @@ ;; Multiply two registers. (rule (lower (has_type (fits_in_64 ty) (imul x y))) - (value_reg (mul ty - (put_in_reg x) - (RegMemImm.Reg (put_in_reg y))))) + (value_gpr (mul ty + (put_in_gpr x) + (put_in_gpr_mem_imm y)))) ;; Multiply a register and an immediate. (rule (lower (has_type (fits_in_64 ty) (imul x (simm32_from_value y)))) - (value_reg (mul ty (put_in_reg x) y))) + (value_gpr (mul ty (put_in_gpr x) y))) (rule (lower (has_type (fits_in_64 ty) (imul (simm32_from_value x) y))) - (value_reg (mul ty (put_in_reg y) x))) + (value_gpr (mul ty (put_in_gpr y) x))) ;; Multiply a register and a memory load. (rule (lower (has_type (fits_in_64 ty) (imul x (sinkable_load y)))) - (value_reg (mul ty - (put_in_reg x) - (sink_load y)))) + (value_gpr (mul ty + (put_in_gpr x) + (sink_load_to_gpr_mem_imm y)))) (rule (lower (has_type (fits_in_64 ty) (imul (sinkable_load x) y))) - (value_reg (mul ty - (put_in_reg y) - (sink_load x)))) + (value_gpr (mul ty + (put_in_gpr y) + (sink_load_to_gpr_mem_imm x)))) ;; `i128`. @@ -1007,25 +1052,25 @@ (rule (lower (has_type $I128 (imul x y))) ;; Put `x` into registers and unpack its hi/lo halves. (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Reg (value_regs_get x_regs 0)) - (x_hi Reg (value_regs_get x_regs 1)) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1)) ;; Put `y` into registers and unpack its hi/lo halves. (y_regs ValueRegs (put_in_regs y)) - (y_lo Reg (value_regs_get y_regs 0)) - (y_hi Reg (value_regs_get y_regs 1)) + (y_lo Gpr (value_regs_get_gpr y_regs 0)) + (y_hi Gpr (value_regs_get_gpr y_regs 1)) ;; lo_hi = mul x_lo, y_hi - (lo_hi Reg (mul $I64 x_lo (RegMemImm.Reg y_hi))) + (lo_hi Gpr (mul $I64 x_lo (gpr_to_gpr_mem_imm y_hi))) ;; hi_lo = mul x_hi, y_lo - (hi_lo Reg (mul $I64 x_hi (RegMemImm.Reg y_lo))) + (hi_lo Gpr (mul $I64 x_hi (gpr_to_gpr_mem_imm y_lo))) ;; hilo_hilo = add lo_hi, hi_lo - (hilo_hilo Reg (add $I64 lo_hi (RegMemImm.Reg hi_lo))) + (hilo_hilo Gpr (add $I64 lo_hi (gpr_to_gpr_mem_imm hi_lo))) ;; dst_lo:hi_lolo = mulhi_u x_lo, y_lo - (mul_regs ValueRegs (mulhi_u $I64 x_lo (RegMem.Reg y_lo))) - (dst_lo Reg (value_regs_get mul_regs 0)) - (hi_lolo Reg (value_regs_get mul_regs 1)) + (mul_regs ValueRegs (mulhi_u $I64 x_lo (gpr_to_gpr_mem y_lo))) + (dst_lo Gpr (value_regs_get_gpr mul_regs 0)) + (hi_lolo Gpr (value_regs_get_gpr mul_regs 1)) ;; dst_hi = add hilo_hilo, hi_lolo - (dst_hi Reg (add $I64 hilo_hilo (RegMemImm.Reg hi_lolo)))) - (value_regs dst_lo dst_hi))) + (dst_hi Gpr (add $I64 hilo_hilo (gpr_to_gpr_mem_imm hi_lolo)))) + (value_gprs dst_lo dst_hi))) ;; SSE. @@ -1310,8 +1355,8 @@ (decl i128_not (Value) ValueRegs) (rule (i128_not x) (let ((x_regs ValueRegs (put_in_regs x)) - (x_lo Gpr (gpr_new (value_regs_get x_regs 0))) - (x_hi Gpr (gpr_new (value_regs_get x_regs 1)))) + (x_lo Gpr (value_regs_get_gpr x_regs 0)) + (x_hi Gpr (value_regs_get_gpr x_regs 1))) (value_gprs (not $I64 x_lo) (not $I64 x_hi)))) @@ -1420,11 +1465,11 @@ (decl cmp_and_choose (Type CC Value Value) ValueRegs) (rule (cmp_and_choose (fits_in_64 ty) cc x y) - (let ((x_reg Reg (put_in_reg x)) - (y_reg Reg (put_in_reg y)) + (let ((x_reg Gpr (put_in_gpr x)) + (y_reg Gpr (put_in_gpr y)) (size OperandSize (raw_operand_size_of_type ty))) - (value_reg (with_flags_1 (cmp size (RegMemImm.Reg x_reg) y_reg) - (cmove ty cc (RegMem.Reg y_reg) x_reg))))) + (value_reg (with_flags_1 (cmp size (gpr_to_gpr_mem_imm x_reg) y_reg) + (cmove ty cc (gpr_to_gpr_mem y_reg) x_reg))))) (rule (lower (has_type (fits_in_64 ty) (umin x y))) (cmp_and_choose ty (CC.B) x y)) diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 8de77cead3..1d31c79d29 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -79,7 +79,7 @@ where if let Some(c) = inputs.constant { if let Some(imm) = to_simm32(c as i64) { - return imm; + return imm.to_reg_mem_imm(); } // Generate constants fresh at each use to minimize long-range @@ -120,21 +120,23 @@ where RegMem::reg(self.put_in_reg(val)) } - fn put_masked_in_imm8_reg(&mut self, val: Value, ty: Type) -> Imm8Reg { + fn put_masked_in_imm8_gpr(&mut self, val: Value, ty: Type) -> Imm8Gpr { let inputs = self.lower_ctx.get_value_as_source_or_const(val); if let Some(c) = inputs.constant { let mask = 1_u64 .checked_shl(ty.bits() as u32) .map_or(u64::MAX, |x| x - 1); - return Imm8Reg::Imm8 { + return Imm8Gpr::new(Imm8Reg::Imm8 { imm: (c & mask) as u8, - }; + }) + .unwrap(); } - Imm8Reg::Reg { + Imm8Gpr::new(Imm8Reg::Reg { reg: self.put_in_regs(val).regs()[0], - } + }) + .unwrap() } #[inline] @@ -178,17 +180,18 @@ where } #[inline] - fn const_to_type_masked_imm8(&mut self, c: u64, ty: Type) -> Imm8Reg { + fn const_to_type_masked_imm8(&mut self, c: u64, ty: Type) -> Imm8Gpr { let mask = 1_u64 .checked_shl(ty.bits() as u32) .map_or(u64::MAX, |x| x - 1); - Imm8Reg::Imm8 { + Imm8Gpr::new(Imm8Reg::Imm8 { imm: (c & mask) as u8, - } + }) + .unwrap() } #[inline] - fn simm32_from_value(&mut self, val: Value) -> Option { + fn simm32_from_value(&mut self, val: Value) -> Option { let inst = self.lower_ctx.dfg().value_def(val).inst()?; let constant: u64 = self.lower_ctx.get_constant(inst)?; let constant = constant as i64; @@ -196,7 +199,7 @@ where } #[inline] - fn simm32_from_imm64(&mut self, imm: Imm64) -> Option { + fn simm32_from_imm64(&mut self, imm: Imm64) -> Option { to_simm32(imm.bits()) } @@ -412,6 +415,31 @@ where fn reg_to_gpr_mem(&mut self, r: Reg) -> GprMem { GprMem::new(RegMem::reg(r)).unwrap() } + + #[inline] + fn imm8_reg_to_imm8_gpr(&mut self, ir: &Imm8Reg) -> Imm8Gpr { + Imm8Gpr::new(ir.clone()).unwrap() + } + + #[inline] + fn gpr_to_gpr_mem(&mut self, gpr: Gpr) -> GprMem { + GprMem::from(gpr) + } + + #[inline] + fn gpr_to_gpr_mem_imm(&mut self, gpr: Gpr) -> GprMemImm { + GprMemImm::from(gpr) + } + + #[inline] + fn gpr_to_imm8_gpr(&mut self, gpr: Gpr) -> Imm8Gpr { + Imm8Gpr::from(gpr) + } + + #[inline] + fn imm8_to_imm8_gpr(&mut self, imm: u8) -> Imm8Gpr { + Imm8Gpr::new(Imm8Reg::Imm8 { imm }).unwrap() + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we @@ -446,11 +474,14 @@ const I8X16_USHR_MASKS: [u8; 128] = [ ]; #[inline] -fn to_simm32(constant: i64) -> Option { +fn to_simm32(constant: i64) -> Option { if constant == ((constant << 32) >> 32) { - Some(RegMemImm::Imm { - simm32: constant as u32, - }) + Some( + GprMemImm::new(RegMemImm::Imm { + simm32: constant as u32, + }) + .unwrap(), + ) } else { None } diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index f3599d8258..2ad2966635 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 src/prelude.isle 73285cd431346d53 -src/isa/x64/inst.isle 7513533d16948249 -src/isa/x64/lower.isle 802b6e750d407100 +src/isa/x64/inst.isle 301db31d5f1118ae +src/isa/x64/lower.isle cdc94aec26c0bc5b diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 0a5bb3bacd..0856e4405e 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -78,14 +78,17 @@ pub trait Context { fn synthetic_amode_to_reg_mem(&mut self, arg0: &SyntheticAmode) -> RegMem; fn amode_imm_reg_reg_shift(&mut self, arg0: u32, arg1: Gpr, arg2: Gpr, arg3: u8) -> Amode; fn amode_to_synthetic_amode(&mut self, arg0: &Amode) -> SyntheticAmode; - fn put_masked_in_imm8_reg(&mut self, arg0: Value, arg1: Type) -> Imm8Reg; + fn put_masked_in_imm8_gpr(&mut self, arg0: Value, arg1: Type) -> Imm8Gpr; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; + fn imm8_reg_to_imm8_gpr(&mut self, arg0: &Imm8Reg) -> Imm8Gpr; fn writable_gpr_to_reg(&mut self, arg0: WritableGpr) -> WritableReg; fn writable_xmm_to_reg(&mut self, arg0: WritableXmm) -> WritableReg; fn writable_reg_to_xmm(&mut self, arg0: WritableReg) -> WritableXmm; fn writable_xmm_to_xmm(&mut self, arg0: WritableXmm) -> Xmm; fn writable_gpr_to_gpr(&mut self, arg0: WritableGpr) -> Gpr; fn gpr_to_reg(&mut self, arg0: Gpr) -> Reg; + fn gpr_to_gpr_mem(&mut self, arg0: Gpr) -> GprMem; + fn gpr_to_gpr_mem_imm(&mut self, arg0: Gpr) -> GprMemImm; fn xmm_to_reg(&mut self, arg0: Xmm) -> Reg; fn xmm_to_xmm_mem_imm(&mut self, arg0: Xmm) -> XmmMemImm; fn temp_writable_gpr(&mut self) -> WritableGpr; @@ -100,14 +103,16 @@ pub trait Context { fn gpr_new(&mut self, arg0: Reg) -> Gpr; fn reg_mem_to_gpr_mem(&mut self, arg0: &RegMem) -> GprMem; fn reg_to_gpr_mem(&mut self, arg0: Reg) -> GprMem; + fn gpr_to_imm8_gpr(&mut self, arg0: Gpr) -> Imm8Gpr; + fn imm8_to_imm8_gpr(&mut self, arg0: u8) -> Imm8Gpr; fn xmm0(&mut self) -> WritableXmm; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; fn imm8_from_value(&mut self, arg0: Value) -> Option; - fn const_to_type_masked_imm8(&mut self, arg0: u64, arg1: Type) -> Imm8Reg; - fn simm32_from_value(&mut self, arg0: Value) -> Option; - fn simm32_from_imm64(&mut self, arg0: Imm64) -> Option; + fn const_to_type_masked_imm8(&mut self, arg0: u64, arg1: Type) -> Imm8Gpr; + fn simm32_from_value(&mut self, arg0: Value) -> Option; + fn simm32_from_imm64(&mut self, arg0: Imm64) -> Option; fn sinkable_load(&mut self, arg0: Value) -> Option; fn sink_load(&mut self, arg0: &SinkableLoad) -> RegMemImm; fn ext_mode(&mut self, arg0: u16, arg1: u16) -> ExtMode; @@ -146,9 +151,9 @@ pub enum MInst { AluRmiR { size: OperandSize, op: AluRmiROpcode, - src1: Reg, - src2: RegMemImm, - dst: WritableReg, + src1: Gpr, + src2: GprMemImm, + dst: WritableGpr, }, UnaryRmR { size: OperandSize, @@ -169,51 +174,51 @@ pub enum MInst { Div { size: OperandSize, signed: bool, - divisor: RegMem, - dividend: Reg, - dst_quotient: WritableReg, - dst_remainder: WritableReg, + divisor: GprMem, + dividend: Gpr, + dst_quotient: WritableGpr, + dst_remainder: WritableGpr, }, MulHi { size: OperandSize, signed: bool, - src1: Reg, - src2: RegMem, - dst_lo: WritableReg, - dst_hi: WritableReg, + src1: Gpr, + src2: GprMem, + dst_lo: WritableGpr, + dst_hi: WritableGpr, }, CheckedDivOrRemSeq { kind: DivOrRemKind, size: OperandSize, - dividend: Reg, - divisor: WritableReg, - dst_quotient: WritableReg, - dst_remainder: WritableReg, - tmp: OptionWritableReg, + dividend: Gpr, + divisor: WritableGpr, + dst_quotient: WritableGpr, + dst_remainder: WritableGpr, + tmp: OptionWritableGpr, }, SignExtendData { size: OperandSize, - src: Reg, - dst: WritableReg, + src: Gpr, + dst: WritableGpr, }, Imm { dst_size: OperandSize, simm64: u64, - dst: WritableReg, + dst: WritableGpr, }, MovRR { size: OperandSize, - src: Reg, - dst: WritableReg, + src: Gpr, + dst: WritableGpr, }, MovzxRmR { ext_mode: ExtMode, - src: RegMem, - dst: WritableReg, + src: GprMem, + dst: WritableGpr, }, Mov64MR { src: SyntheticAmode, - dst: WritableReg, + dst: WritableGpr, }, LoadEffectiveAddress { addr: SyntheticAmode, @@ -221,20 +226,20 @@ pub enum MInst { }, MovsxRmR { ext_mode: ExtMode, - src: RegMem, - dst: WritableReg, + src: GprMem, + dst: WritableGpr, }, MovRM { size: OperandSize, - src: Reg, + src: Gpr, dst: SyntheticAmode, }, ShiftR { size: OperandSize, kind: ShiftKind, - src: Reg, - num_bits: Imm8Reg, - dst: WritableReg, + src: Gpr, + num_bits: Imm8Gpr, + dst: WritableGpr, }, XmmRmiReg { opcode: SseOpcode, @@ -245,25 +250,25 @@ pub enum MInst { CmpRmiR { size: OperandSize, opcode: CmpOpcode, - src: RegMemImm, - dst: Reg, + src: GprMemImm, + dst: Gpr, }, Setcc { cc: CC, - dst: WritableReg, + dst: WritableGpr, }, Cmove { size: OperandSize, cc: CC, - consequent: RegMem, - alternative: Reg, - dst: WritableReg, + consequent: GprMem, + alternative: Gpr, + dst: WritableGpr, }, Push64 { - src: RegMemImm, + src: GprMemImm, }, Pop64 { - dst: WritableReg, + dst: WritableGpr, }, XmmRmR { op: SseOpcode, @@ -305,7 +310,7 @@ pub enum MInst { }, GprToXmm { op: SseOpcode, - src: RegMem, + src: GprMem, dst: WritableXmm, src_size: OperandSize, }, @@ -337,19 +342,19 @@ pub enum MInst { XmmMinMaxSeq { size: OperandSize, is_min: bool, - lhs: Reg, - rhs_dst: WritableReg, + lhs: Xmm, + rhs_dst: WritableXmm, }, XmmCmove { size: OperandSize, cc: CC, - src: RegMem, - dst: WritableReg, + src: XmmMem, + dst: WritableXmm, }, XmmCmpRmR { op: SseOpcode, - src: RegMem, - dst: Reg, + src: XmmMem, + dst: Xmm, }, XmmRmRImm { op: SseOpcode, @@ -448,7 +453,7 @@ pub enum MInst { }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1051. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1089. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ExtendKind { Sign, @@ -634,7 +639,7 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize // Generated as internal constructor for term put_in_gpr. pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 949. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = C::gpr_new(ctx, expr0_0); return Some(expr1_0); @@ -643,7 +648,7 @@ pub fn constructor_put_in_gpr(ctx: &mut C, arg0: Value) -> Option(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 941. + // Rule at src/isa/x64/inst.isle line 956. let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_gpr_mem(ctx, &expr0_0); return Some(expr1_0); @@ -652,7 +657,7 @@ pub fn constructor_put_in_gpr_mem(ctx: &mut C, arg0: Value) -> Optio // Generated as internal constructor for term put_in_gpr_mem_imm. pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 948. + // Rule at src/isa/x64/inst.isle line 963. let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); return Some(expr1_0); @@ -661,7 +666,7 @@ pub fn constructor_put_in_gpr_mem_imm(ctx: &mut C, arg0: Value) -> O // Generated as internal constructor for term put_in_xmm. pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 955. + // Rule at src/isa/x64/inst.isle line 970. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = C::xmm_new(ctx, expr0_0); return Some(expr1_0); @@ -670,7 +675,7 @@ pub fn constructor_put_in_xmm(ctx: &mut C, arg0: Value) -> Option(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 962. + // Rule at src/isa/x64/inst.isle line 977. let expr0_0 = C::put_in_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); return Some(expr1_0); @@ -679,7 +684,7 @@ pub fn constructor_put_in_xmm_mem(ctx: &mut C, arg0: Value) -> Optio // Generated as internal constructor for term put_in_xmm_mem_imm. pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 969. + // Rule at src/isa/x64/inst.isle line 984. let expr0_0 = C::put_in_reg_mem_imm(ctx, pattern0_0); let expr1_0 = C::xmm_mem_imm_new(ctx, &expr0_0); return Some(expr1_0); @@ -688,7 +693,7 @@ pub fn constructor_put_in_xmm_mem_imm(ctx: &mut C, arg0: Value) -> O // Generated as internal constructor for term value_gpr. pub fn constructor_value_gpr(ctx: &mut C, arg0: Gpr) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 974. + // Rule at src/isa/x64/inst.isle line 989. let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); @@ -698,7 +703,7 @@ pub fn constructor_value_gpr(ctx: &mut C, arg0: Gpr) -> Option(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 979. + // Rule at src/isa/x64/inst.isle line 994. let expr0_0 = C::gpr_to_reg(ctx, pattern0_0); let expr1_0 = C::gpr_to_reg(ctx, pattern1_0); let expr2_0 = C::value_regs(ctx, expr0_0, expr1_0); @@ -708,35 +713,70 @@ pub fn constructor_value_gprs(ctx: &mut C, arg0: Gpr, arg1: Gpr) -> // Generated as internal constructor for term value_xmm. pub fn constructor_value_xmm(ctx: &mut C, arg0: Xmm) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 984. + // Rule at src/isa/x64/inst.isle line 999. let expr0_0 = C::xmm_to_reg(ctx, pattern0_0); let expr1_0 = C::value_reg(ctx, expr0_0); return Some(expr1_0); } -// Generated as internal constructor for term extend_to_reg. -pub fn constructor_extend_to_reg( +// Generated as internal constructor for term value_regs_get_gpr. +pub fn constructor_value_regs_get_gpr( + ctx: &mut C, + arg0: ValueRegs, + arg1: usize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1006. + let expr0_0 = C::value_regs_get(ctx, pattern0_0, pattern1_0); + let expr1_0 = C::gpr_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term lo_gpr. +pub fn constructor_lo_gpr(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 1019. + let expr0_0 = constructor_lo_reg(ctx, pattern0_0)?; + let expr1_0 = C::gpr_new(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term sink_load_to_gpr_mem_imm. +pub fn constructor_sink_load_to_gpr_mem_imm( + ctx: &mut C, + arg0: &SinkableLoad, +) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 1079. + let expr0_0 = C::sink_load(ctx, pattern0_0); + let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term extend_to_gpr. +pub fn constructor_extend_to_gpr( ctx: &mut C, arg0: Value, arg1: Type, arg2: &ExtendKind, -) -> Option { +) -> Option { let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1063. - let expr0_0 = C::put_in_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1101. + let expr0_0 = constructor_put_in_gpr(ctx, pattern0_0)?; return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1066. + // Rule at src/isa/x64/inst.isle line 1104. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; let expr3_0 = C::ext_mode(ctx, expr0_0, expr2_0); - let expr4_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr4_0 = constructor_put_in_gpr_mem(ctx, pattern0_0)?; let expr5_0 = constructor_extend(ctx, pattern3_0, pattern2_0, &expr3_0, &expr4_0)?; return Some(expr5_0); } @@ -747,15 +787,15 @@ pub fn constructor_extend( arg0: &ExtendKind, arg1: Type, arg2: &ExtMode, - arg3: &RegMem, -) -> Option { + arg3: &GprMem, +) -> Option { let pattern0_0 = arg0; match pattern0_0 { &ExtendKind::Sign => { let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1086. + // Rule at src/isa/x64/inst.isle line 1124. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -763,7 +803,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1082. + // Rule at src/isa/x64/inst.isle line 1120. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -776,17 +816,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1093. + // Rule at src/isa/x64/inst.isle line 1131. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1094. + // Rule at src/isa/x64/inst.isle line 1132. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1095. + // Rule at src/isa/x64/inst.isle line 1133. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -803,7 +843,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1099. + // Rule at src/isa/x64/inst.isle line 1137. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -813,40 +853,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1108. + // Rule at src/isa/x64/inst.isle line 1146. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1109. + // Rule at src/isa/x64/inst.isle line 1147. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 1104. + // Rule at src/isa/x64/inst.isle line 1142. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 1105. + // Rule at src/isa/x64/inst.isle line 1143. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 1106. + // Rule at src/isa/x64/inst.isle line 1144. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 1107. + // Rule at src/isa/x64/inst.isle line 1145. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -858,7 +898,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1123. + // Rule at src/isa/x64/inst.isle line 1161. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -882,7 +922,7 @@ pub fn constructor_make_i64x2_from_lanes( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1134. + // Rule at src/isa/x64/inst.isle line 1172. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = C::writable_xmm_to_xmm(ctx, expr0_0); @@ -923,12 +963,12 @@ pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 1157. + // Rule at src/isa/x64/inst.isle line 1195. let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 1158. + // Rule at src/isa/x64/inst.isle line 1196. let expr0_0 = SseOpcode::Movd; let expr1_0 = C::reg_to_gpr_mem(ctx, pattern1_0); let expr2_0 = OperandSize::Size32; @@ -939,7 +979,7 @@ pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 1156. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); return Some(expr0_0); } @@ -959,21 +999,21 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1173. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = MInst::Mov64MR { + // Rule at src/isa/x64/inst.isle line 1211. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = MInst::Mov64MR { src: pattern2_0.clone(), - dst: expr1_0, + dst: expr0_0, }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr4_0 = C::gpr_to_reg(ctx, expr3_0); return Some(expr4_0); } if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1178. + // Rule at src/isa/x64/inst.isle line 1216. let expr0_0 = SseOpcode::Movss; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); @@ -984,7 +1024,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1182. + // Rule at src/isa/x64/inst.isle line 1220. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); @@ -995,7 +1035,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1186. + // Rule at src/isa/x64/inst.isle line 1224. let expr0_0 = SseOpcode::Movups; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); @@ -1006,7 +1046,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1190. + // Rule at src/isa/x64/inst.isle line 1228. let expr0_0 = SseOpcode::Movupd; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); @@ -1017,7 +1057,7 @@ pub fn constructor_x64_load( if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1194. + // Rule at src/isa/x64/inst.isle line 1232. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); let expr2_0 = C::reg_mem_to_xmm_mem(ctx, &expr1_0); @@ -1029,13 +1069,15 @@ pub fn constructor_x64_load( let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 1168. + // Rule at src/isa/x64/inst.isle line 1206. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); let expr3_0 = C::synthetic_amode_to_reg_mem(ctx, pattern2_0); - let expr4_0 = constructor_movsx(ctx, pattern1_0, &expr2_0, &expr3_0)?; - return Some(expr4_0); + let expr4_0 = C::reg_mem_to_gpr_mem(ctx, &expr3_0); + let expr5_0 = constructor_movsx(ctx, pattern1_0, &expr2_0, &expr4_0)?; + let expr6_0 = C::gpr_to_reg(ctx, expr5_0); + return Some(expr6_0); } } return None; @@ -1046,15 +1088,15 @@ pub fn constructor_alu_rmi_r( ctx: &mut C, arg0: Type, arg1: &AluRmiROpcode, - arg2: Reg, - arg3: &RegMemImm, -) -> Option { + arg2: Gpr, + arg3: &GprMemImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1206. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1244. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { size: expr1_0, @@ -1064,7 +1106,7 @@ pub fn constructor_alu_rmi_r( dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } @@ -1072,13 +1114,13 @@ pub fn constructor_alu_rmi_r( pub fn constructor_add( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { + arg1: Gpr, + arg2: &GprMemImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1214. + // Rule at src/isa/x64/inst.isle line 1252. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1088,14 +1130,14 @@ pub fn constructor_add( pub fn constructor_add_with_flags( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, + arg1: Gpr, + arg2: &GprMemImm, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1222. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1260. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; let expr3_0 = MInst::AluRmiR { @@ -1105,26 +1147,27 @@ pub fn constructor_add_with_flags( src2: pattern2_0.clone(), dst: expr0_0, }; - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr5_0 = ProducesFlags::ProducesFlags { + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + let expr6_0 = ProducesFlags::ProducesFlags { inst: expr3_0, - result: expr4_0, + result: expr5_0, }; - return Some(expr5_0); + return Some(expr6_0); } // Generated as internal constructor for term adc. pub fn constructor_adc( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, + arg1: Gpr, + arg2: &GprMemImm, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1233. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1271. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; let expr3_0 = MInst::AluRmiR { @@ -1134,25 +1177,26 @@ pub fn constructor_adc( src2: pattern2_0.clone(), dst: expr0_0, }; - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr5_0 = ConsumesFlags::ConsumesFlags { + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + let expr6_0 = ConsumesFlags::ConsumesFlags { inst: expr3_0, - result: expr4_0, + result: expr5_0, }; - return Some(expr5_0); + return Some(expr6_0); } // Generated as internal constructor for term sub. pub fn constructor_sub( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { + arg1: Gpr, + arg2: &GprMemImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1244. + // Rule at src/isa/x64/inst.isle line 1282. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1162,14 +1206,14 @@ pub fn constructor_sub( pub fn constructor_sub_with_flags( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, + arg1: Gpr, + arg2: &GprMemImm, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1252. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1290. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; let expr3_0 = MInst::AluRmiR { @@ -1179,26 +1223,27 @@ pub fn constructor_sub_with_flags( src2: pattern2_0.clone(), dst: expr0_0, }; - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr5_0 = ProducesFlags::ProducesFlags { + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + let expr6_0 = ProducesFlags::ProducesFlags { inst: expr3_0, - result: expr4_0, + result: expr5_0, }; - return Some(expr5_0); + return Some(expr6_0); } // Generated as internal constructor for term sbb. pub fn constructor_sbb( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, + arg1: Gpr, + arg2: &GprMemImm, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1263. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1301. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; let expr3_0 = MInst::AluRmiR { @@ -1208,25 +1253,26 @@ pub fn constructor_sbb( src2: pattern2_0.clone(), dst: expr0_0, }; - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr5_0 = ConsumesFlags::ConsumesFlags { + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + let expr6_0 = ConsumesFlags::ConsumesFlags { inst: expr3_0, - result: expr4_0, + result: expr5_0, }; - return Some(expr5_0); + return Some(expr6_0); } // Generated as internal constructor for term mul. pub fn constructor_mul( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { + arg1: Gpr, + arg2: &GprMemImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1274. + // Rule at src/isa/x64/inst.isle line 1312. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1236,13 +1282,13 @@ pub fn constructor_mul( pub fn constructor_x64_and( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { + arg1: Gpr, + arg2: &GprMemImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1285. + // Rule at src/isa/x64/inst.isle line 1320. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1252,13 +1298,13 @@ pub fn constructor_x64_and( pub fn constructor_or( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { + arg1: Gpr, + arg2: &GprMemImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1293. + // Rule at src/isa/x64/inst.isle line 1328. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1268,13 +1314,13 @@ pub fn constructor_or( pub fn constructor_xor( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMemImm, -) -> Option { + arg1: Gpr, + arg2: &GprMemImm, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1301. + // Rule at src/isa/x64/inst.isle line 1336. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1286,24 +1332,24 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 1334. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = OperandSize::Size32; - let expr3_0 = MInst::Imm { - dst_size: expr2_0, + // Rule at src/isa/x64/inst.isle line 1369. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = OperandSize::Size32; + let expr2_0 = MInst::Imm { + dst_size: expr1_0, simm64: pattern3_0, - dst: expr1_0, + dst: expr0_0, }; - let expr4_0 = C::emit(ctx, &expr3_0); - let expr5_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); return Some(expr5_0); } } if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1363. + // Rule at src/isa/x64/inst.isle line 1398. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = SseOpcode::Xorps; @@ -1318,7 +1364,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::xmm_to_reg(ctx, expr1_0); return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 1318. + // Rule at src/isa/x64/inst.isle line 1353. let expr0_0 = SseOpcode::Movd; let expr1_0: Type = I32; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; @@ -1332,7 +1378,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1375. + // Rule at src/isa/x64/inst.isle line 1410. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = SseOpcode::Xorpd; @@ -1347,7 +1393,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::xmm_to_reg(ctx, expr1_0); return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 1324. + // Rule at src/isa/x64/inst.isle line 1359. let expr0_0 = SseOpcode::Movq; let expr1_0: Type = I64; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; @@ -1361,7 +1407,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1353. + // Rule at src/isa/x64/inst.isle line 1388. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -1380,12 +1426,12 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1340. - let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); - let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + // Rule at src/isa/x64/inst.isle line 1375. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr3_0 = AluRmiROpcode::Xor; - let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr4_0 = C::gpr_to_gpr_mem_imm(ctx, expr1_0); let expr5_0 = MInst::AluRmiR { size: expr2_0, op: expr3_0, @@ -1394,10 +1440,11 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option dst: expr0_0, }; let expr6_0 = C::emit(ctx, &expr5_0); - return Some(expr1_0); + let expr7_0 = C::gpr_to_reg(ctx, expr1_0); + return Some(expr7_0); } - // Rule at src/isa/x64/inst.isle line 1311. - let expr0_0 = C::temp_writable_reg(ctx, pattern1_0); + // Rule at src/isa/x64/inst.isle line 1346. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { dst_size: expr1_0, @@ -1405,8 +1452,9 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); - return Some(expr4_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + return Some(expr5_0); } return None; } @@ -1416,15 +1464,15 @@ pub fn constructor_shift_r( ctx: &mut C, arg0: Type, arg1: &ShiftKind, - arg2: Reg, - arg3: &Imm8Reg, -) -> Option { + arg2: Gpr, + arg3: &Imm8Gpr, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1388. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1423. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { size: expr1_0, @@ -1434,7 +1482,7 @@ pub fn constructor_shift_r( dst: expr0_0, }; let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr4_0); } @@ -1442,13 +1490,13 @@ pub fn constructor_shift_r( pub fn constructor_x64_rotl( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &Imm8Reg, -) -> Option { + arg1: Gpr, + arg2: &Imm8Gpr, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1399. + // Rule at src/isa/x64/inst.isle line 1433. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1458,13 +1506,13 @@ pub fn constructor_x64_rotl( pub fn constructor_x64_rotr( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &Imm8Reg, -) -> Option { + arg1: Gpr, + arg2: &Imm8Gpr, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1405. + // Rule at src/isa/x64/inst.isle line 1438. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1474,13 +1522,13 @@ pub fn constructor_x64_rotr( pub fn constructor_shl( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &Imm8Reg, -) -> Option { + arg1: Gpr, + arg2: &Imm8Gpr, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1410. + // Rule at src/isa/x64/inst.isle line 1443. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1490,13 +1538,13 @@ pub fn constructor_shl( pub fn constructor_shr( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &Imm8Reg, -) -> Option { + arg1: Gpr, + arg2: &Imm8Gpr, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1415. + // Rule at src/isa/x64/inst.isle line 1448. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1506,13 +1554,13 @@ pub fn constructor_shr( pub fn constructor_sar( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &Imm8Reg, -) -> Option { + arg1: Gpr, + arg2: &Imm8Gpr, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1420. + // Rule at src/isa/x64/inst.isle line 1453. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1523,14 +1571,14 @@ pub fn constructor_cmp_rmi_r( ctx: &mut C, arg0: &OperandSize, arg1: &CmpOpcode, - arg2: &RegMemImm, - arg3: Reg, + arg2: &GprMemImm, + arg3: Gpr, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1425. + // Rule at src/isa/x64/inst.isle line 1458. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1549,13 +1597,13 @@ pub fn constructor_cmp_rmi_r( pub fn constructor_cmp( ctx: &mut C, arg0: &OperandSize, - arg1: &RegMemImm, - arg2: Reg, + arg1: &GprMemImm, + arg2: Gpr, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1434. + // Rule at src/isa/x64/inst.isle line 1467. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1565,13 +1613,13 @@ pub fn constructor_cmp( pub fn constructor_test( ctx: &mut C, arg0: &OperandSize, - arg1: &RegMemImm, - arg2: Reg, + arg1: &GprMemImm, + arg2: Gpr, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1439. + // Rule at src/isa/x64/inst.isle line 1472. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1582,15 +1630,15 @@ pub fn constructor_cmove( ctx: &mut C, arg0: Type, arg1: &CC, - arg2: &RegMem, - arg3: Reg, + arg2: &GprMem, + arg3: Gpr, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1444. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1477. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { size: expr1_0, @@ -1599,12 +1647,13 @@ pub fn constructor_cmove( alternative: pattern3_0, dst: expr0_0, }; - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr4_0 = ConsumesFlags::ConsumesFlags { + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr4_0 = C::gpr_to_reg(ctx, expr3_0); + let expr5_0 = ConsumesFlags::ConsumesFlags { inst: expr2_0, - result: expr3_0, + result: expr4_0, }; - return Some(expr4_0); + return Some(expr5_0); } // Generated as internal constructor for term movzx. @@ -1612,20 +1661,20 @@ pub fn constructor_movzx( ctx: &mut C, arg0: Type, arg1: &ExtMode, - arg2: &RegMem, -) -> Option { + arg2: &GprMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1452. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1485. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), src: pattern2_0.clone(), dst: expr0_0, }; let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr3_0); } @@ -1634,20 +1683,20 @@ pub fn constructor_movsx( ctx: &mut C, arg0: Type, arg1: &ExtMode, - arg2: &RegMem, -) -> Option { + arg2: &GprMem, +) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1459. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1492. + let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), src: pattern2_0.clone(), dst: expr0_0, }; let expr2_0 = C::emit(ctx, &expr1_0); - let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr3_0 = C::writable_gpr_to_gpr(ctx, expr0_0); return Some(expr3_0); } @@ -1663,7 +1712,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1466. + // Rule at src/isa/x64/inst.isle line 1499. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1680,7 +1729,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1473. + // Rule at src/isa/x64/inst.isle line 1506. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1691,7 +1740,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1478. + // Rule at src/isa/x64/inst.isle line 1511. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1702,7 +1751,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1483. + // Rule at src/isa/x64/inst.isle line 1516. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1713,7 +1762,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1488. + // Rule at src/isa/x64/inst.isle line 1521. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1724,7 +1773,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1493. + // Rule at src/isa/x64/inst.isle line 1526. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1735,7 +1784,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1498. + // Rule at src/isa/x64/inst.isle line 1531. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1746,7 +1795,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1503. + // Rule at src/isa/x64/inst.isle line 1536. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1757,7 +1806,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1508. + // Rule at src/isa/x64/inst.isle line 1541. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1768,7 +1817,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1513. + // Rule at src/isa/x64/inst.isle line 1546. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1779,7 +1828,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1518. + // Rule at src/isa/x64/inst.isle line 1551. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1790,7 +1839,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1523. + // Rule at src/isa/x64/inst.isle line 1556. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1801,7 +1850,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1528. + // Rule at src/isa/x64/inst.isle line 1561. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1812,7 +1861,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1533. + // Rule at src/isa/x64/inst.isle line 1566. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1823,7 +1872,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1538. + // Rule at src/isa/x64/inst.isle line 1571. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1834,7 +1883,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1543. + // Rule at src/isa/x64/inst.isle line 1576. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1845,7 +1894,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1548. + // Rule at src/isa/x64/inst.isle line 1581. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1856,7 +1905,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1553. + // Rule at src/isa/x64/inst.isle line 1586. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1867,7 +1916,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1558. + // Rule at src/isa/x64/inst.isle line 1591. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1878,7 +1927,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1563. + // Rule at src/isa/x64/inst.isle line 1596. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1889,7 +1938,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1568. + // Rule at src/isa/x64/inst.isle line 1601. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1900,7 +1949,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1573. + // Rule at src/isa/x64/inst.isle line 1606. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1911,7 +1960,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1578. + // Rule at src/isa/x64/inst.isle line 1611. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1922,7 +1971,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1583. + // Rule at src/isa/x64/inst.isle line 1616. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1933,7 +1982,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1588. + // Rule at src/isa/x64/inst.isle line 1621. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1944,7 +1993,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1593. + // Rule at src/isa/x64/inst.isle line 1626. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1955,7 +2004,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1598. + // Rule at src/isa/x64/inst.isle line 1631. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1966,7 +2015,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1603. + // Rule at src/isa/x64/inst.isle line 1636. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1977,7 +2026,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1608. + // Rule at src/isa/x64/inst.isle line 1641. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1988,7 +2037,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1613. + // Rule at src/isa/x64/inst.isle line 1646. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1999,7 +2048,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1618. + // Rule at src/isa/x64/inst.isle line 1651. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2010,7 +2059,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1623. + // Rule at src/isa/x64/inst.isle line 1656. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2021,7 +2070,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1628. + // Rule at src/isa/x64/inst.isle line 1661. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2032,7 +2081,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1633. + // Rule at src/isa/x64/inst.isle line 1666. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2043,7 +2092,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1638. + // Rule at src/isa/x64/inst.isle line 1671. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2054,7 +2103,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1643. + // Rule at src/isa/x64/inst.isle line 1676. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2065,7 +2114,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1648. + // Rule at src/isa/x64/inst.isle line 1681. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2076,7 +2125,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1653. + // Rule at src/isa/x64/inst.isle line 1686. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2087,7 +2136,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1658. + // Rule at src/isa/x64/inst.isle line 1691. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2098,17 +2147,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1662. + // Rule at src/isa/x64/inst.isle line 1695. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1663. + // Rule at src/isa/x64/inst.isle line 1696. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1664. + // Rule at src/isa/x64/inst.isle line 1697. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -2119,17 +2168,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1667. + // Rule at src/isa/x64/inst.isle line 1700. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1668. + // Rule at src/isa/x64/inst.isle line 1701. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1669. + // Rule at src/isa/x64/inst.isle line 1702. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -2148,7 +2197,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1673. + // Rule at src/isa/x64/inst.isle line 1706. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -2172,7 +2221,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1687. + // Rule at src/isa/x64/inst.isle line 1720. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); @@ -2192,7 +2241,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1701. + // Rule at src/isa/x64/inst.isle line 1734. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2203,7 +2252,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1706. + // Rule at src/isa/x64/inst.isle line 1739. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2214,7 +2263,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1711. + // Rule at src/isa/x64/inst.isle line 1744. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2225,7 +2274,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1716. + // Rule at src/isa/x64/inst.isle line 1749. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2236,7 +2285,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1721. + // Rule at src/isa/x64/inst.isle line 1754. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2247,7 +2296,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1726. + // Rule at src/isa/x64/inst.isle line 1759. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2258,7 +2307,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1731. + // Rule at src/isa/x64/inst.isle line 1764. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2269,7 +2318,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1736. + // Rule at src/isa/x64/inst.isle line 1769. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2280,7 +2329,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1741. + // Rule at src/isa/x64/inst.isle line 1774. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2291,7 +2340,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1746. + // Rule at src/isa/x64/inst.isle line 1779. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2302,7 +2351,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1751. + // Rule at src/isa/x64/inst.isle line 1784. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2313,7 +2362,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1756. + // Rule at src/isa/x64/inst.isle line 1789. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2324,7 +2373,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1761. + // Rule at src/isa/x64/inst.isle line 1794. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2335,7 +2384,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1766. + // Rule at src/isa/x64/inst.isle line 1799. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2346,7 +2395,7 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1771. + // Rule at src/isa/x64/inst.isle line 1804. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2357,7 +2406,7 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1776. + // Rule at src/isa/x64/inst.isle line 1809. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2368,7 +2417,7 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1781. + // Rule at src/isa/x64/inst.isle line 1814. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2389,7 +2438,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1786. + // Rule at src/isa/x64/inst.isle line 1819. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -2417,7 +2466,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1798. + // Rule at src/isa/x64/inst.isle line 1831. let expr0_0 = SseOpcode::Palignr; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -2436,7 +2485,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1807. + // Rule at src/isa/x64/inst.isle line 1840. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -2456,7 +2505,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1816. + // Rule at src/isa/x64/inst.isle line 1849. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -2475,7 +2524,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1825. + // Rule at src/isa/x64/inst.isle line 1858. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -2496,7 +2545,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1834. + // Rule at src/isa/x64/inst.isle line 1867. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -2515,7 +2564,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1843. + // Rule at src/isa/x64/inst.isle line 1876. let expr0_0 = SseOpcode::Insertps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -2534,7 +2583,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1852. + // Rule at src/isa/x64/inst.isle line 1885. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = SseOpcode::Pshufd; @@ -2561,7 +2610,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1865. + // Rule at src/isa/x64/inst.isle line 1898. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), @@ -2576,7 +2625,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1872. + // Rule at src/isa/x64/inst.isle line 1905. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2585,7 +2634,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1877. + // Rule at src/isa/x64/inst.isle line 1910. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2594,7 +2643,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1882. + // Rule at src/isa/x64/inst.isle line 1915. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2603,7 +2652,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1887. + // Rule at src/isa/x64/inst.isle line 1920. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2612,7 +2661,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1892. + // Rule at src/isa/x64/inst.isle line 1925. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2626,7 +2675,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1897. + // Rule at src/isa/x64/inst.isle line 1930. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), @@ -2641,7 +2690,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1904. + // Rule at src/isa/x64/inst.isle line 1937. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -2657,7 +2706,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1909. + // Rule at src/isa/x64/inst.isle line 1942. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), @@ -2674,7 +2723,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1921. + // Rule at src/isa/x64/inst.isle line 1954. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2685,16 +2734,16 @@ pub fn constructor_mul_hi( ctx: &mut C, arg0: Type, arg1: bool, - arg2: Reg, - arg3: &RegMem, + arg2: Gpr, + arg3: &GprMem, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1930. - let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); - let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); + // Rule at src/isa/x64/inst.isle line 1963. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::temp_writable_gpr(ctx); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr3_0 = MInst::MulHi { size: expr2_0, @@ -2705,9 +2754,9 @@ pub fn constructor_mul_hi( dst_hi: expr1_0, }; let expr4_0 = C::emit(ctx, &expr3_0); - let expr5_0 = C::writable_reg_to_reg(ctx, expr0_0); - let expr6_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr7_0 = C::value_regs(ctx, expr5_0, expr6_0); + let expr5_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr6_0 = C::writable_gpr_to_gpr(ctx, expr1_0); + let expr7_0 = constructor_value_gprs(ctx, expr5_0, expr6_0)?; return Some(expr7_0); } @@ -2715,13 +2764,13 @@ pub fn constructor_mul_hi( pub fn constructor_mulhi_u( ctx: &mut C, arg0: Type, - arg1: Reg, - arg2: &RegMem, + arg1: Gpr, + arg2: &GprMem, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1946. + // Rule at src/isa/x64/inst.isle line 1979. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -2737,7 +2786,7 @@ pub fn constructor_xmm_rmi_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1951. + // Rule at src/isa/x64/inst.isle line 1984. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmiReg { opcode: pattern0_0.clone(), @@ -2754,7 +2803,7 @@ pub fn constructor_xmm_rmi_xmm( pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1961. + // Rule at src/isa/x64/inst.isle line 1994. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2764,7 +2813,7 @@ pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1966. + // Rule at src/isa/x64/inst.isle line 1999. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2774,7 +2823,7 @@ pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1971. + // Rule at src/isa/x64/inst.isle line 2004. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2784,7 +2833,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1976. + // Rule at src/isa/x64/inst.isle line 2009. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2794,7 +2843,7 @@ pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1981. + // Rule at src/isa/x64/inst.isle line 2014. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2804,7 +2853,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1986. + // Rule at src/isa/x64/inst.isle line 2019. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2814,7 +2863,7 @@ pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1991. + // Rule at src/isa/x64/inst.isle line 2024. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2824,7 +2873,7 @@ pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1996. + // Rule at src/isa/x64/inst.isle line 2029. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -2835,7 +2884,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Xmm, arg2: let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2001. + // Rule at src/isa/x64/inst.isle line 2034. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); let expr2_0 = SseOpcode::Pextrd; @@ -2867,7 +2916,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2018. + // Rule at src/isa/x64/inst.isle line 2051. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -2887,25 +2936,24 @@ pub fn constructor_gpr_to_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2027. + // Rule at src/isa/x64/inst.isle line 2060. let expr0_0 = C::temp_writable_xmm(ctx); - let expr1_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); - let expr2_0 = MInst::GprToXmm { + let expr1_0 = MInst::GprToXmm { op: pattern0_0.clone(), - src: expr1_0, + src: pattern1_0.clone(), dst: expr0_0, src_size: pattern2_0.clone(), }; - let expr3_0 = C::emit(ctx, &expr2_0); - let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); - return Some(expr4_0); + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr3_0); } // Generated as internal constructor for term not. pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2034. + // Rule at src/isa/x64/inst.isle line 2067. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -2922,7 +2970,7 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2042. + // Rule at src/isa/x64/inst.isle line 2075. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { @@ -2938,7 +2986,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option // Generated as internal constructor for term lea. pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2049. + // Rule at src/isa/x64/inst.isle line 2082. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), @@ -2952,7 +3000,7 @@ pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option // Generated as internal constructor for term ud2. pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2056. + // Rule at src/isa/x64/inst.isle line 2089. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -2997,13 +3045,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern2_0 { &Opcode::Trap => { - // Rule at src/isa/x64/lower.isle line 1487. + // Rule at src/isa/x64/lower.isle line 1532. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_safepoint(ctx, &expr0_0)?; return Some(expr1_0); } &Opcode::ResumableTrap => { - // Rule at src/isa/x64/lower.isle line 1492. + // Rule at src/isa/x64/lower.isle line 1537. let expr0_0 = constructor_ud2(ctx, &pattern2_1)?; let expr1_0 = constructor_safepoint(ctx, &expr0_0)?; return Some(expr1_0); @@ -3020,7 +3068,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { @@ -3092,14 +3140,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { @@ -3108,14 +3156,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} @@ -3126,7 +3174,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1321. + // Rule at src/isa/x64/lower.isle line 1366. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -3164,20 +3212,20 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1007. + // Rule at src/isa/x64/lower.isle line 1052. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; - let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; let expr3_0: usize = 1; - let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr4_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr3_0)?; let expr5_0 = C::put_in_regs(ctx, pattern7_1); let expr6_0: usize = 0; - let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr7_0 = constructor_value_regs_get_gpr(ctx, expr5_0, expr6_0)?; let expr8_0: usize = 1; - let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr9_0 = constructor_value_regs_get_gpr(ctx, expr5_0, expr8_0)?; let expr10_0: Type = I64; - let expr11_0 = RegMemImm::Reg { reg: expr9_0 }; + let expr11_0 = C::gpr_to_gpr_mem_imm(ctx, expr9_0); let expr12_0 = constructor_mul(ctx, expr10_0, expr2_0, &expr11_0)?; let expr13_0: Type = I64; - let expr14_0 = RegMemImm::Reg { reg: expr7_0 }; + let expr14_0 = C::gpr_to_gpr_mem_imm(ctx, expr7_0); let expr15_0 = constructor_mul(ctx, expr13_0, expr4_0, &expr14_0)?; let expr16_0: Type = I64; - let expr17_0 = RegMemImm::Reg { reg: expr15_0 }; + let expr17_0 = C::gpr_to_gpr_mem_imm(ctx, expr15_0); let expr18_0 = constructor_add(ctx, expr16_0, expr12_0, &expr17_0)?; let expr19_0: Type = I64; - let expr20_0 = RegMem::Reg { reg: expr7_0 }; + let expr20_0 = C::gpr_to_gpr_mem(ctx, expr7_0); let expr21_0 = constructor_mulhi_u(ctx, expr19_0, expr2_0, &expr20_0)?; let expr22_0: usize = 0; - let expr23_0 = C::value_regs_get(ctx, expr21_0, expr22_0); + let expr23_0 = constructor_value_regs_get_gpr(ctx, expr21_0, expr22_0)?; let expr24_0: usize = 1; - let expr25_0 = C::value_regs_get(ctx, expr21_0, expr24_0); + let expr25_0 = constructor_value_regs_get_gpr(ctx, expr21_0, expr24_0)?; let expr26_0: Type = I64; - let expr27_0 = RegMemImm::Reg { reg: expr25_0 }; + let expr27_0 = C::gpr_to_gpr_mem_imm(ctx, expr25_0); let expr28_0 = constructor_add(ctx, expr26_0, expr18_0, &expr27_0)?; - let expr29_0 = C::value_regs(ctx, expr23_0, expr28_0); + let expr29_0 = constructor_value_gprs(ctx, expr23_0, expr28_0)?; return Some(expr29_0); } &Opcode::Band => { @@ -3248,21 +3296,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { @@ -3280,62 +3328,64 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 875. + // Rule at src/isa/x64/lower.isle line 916. let expr0_0 = C::put_in_regs(ctx, pattern7_0); - let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = constructor_lo_gpr(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; let expr3_0: Type = I64; let expr4_0: Type = I64; let expr5_0: u64 = 128; let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?; - let expr7_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr8_0 = constructor_sub(ctx, expr3_0, expr6_0, &expr7_0)?; - let expr9_0 = constructor_shr_i128(ctx, expr0_0, expr8_0)?; - let expr10_0 = constructor_or_i128(ctx, expr2_0, expr9_0)?; - return Some(expr10_0); + let expr7_0 = C::gpr_new(ctx, expr6_0); + let expr8_0 = C::gpr_to_gpr_mem_imm(ctx, expr1_0); + let expr9_0 = constructor_sub(ctx, expr3_0, expr7_0, &expr8_0)?; + let expr10_0 = constructor_shr_i128(ctx, expr0_0, expr9_0)?; + let expr11_0 = constructor_or_i128(ctx, expr2_0, expr10_0)?; + return Some(expr11_0); } &Opcode::Rotr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 915. + // Rule at src/isa/x64/lower.isle line 958. let expr0_0 = C::put_in_regs(ctx, pattern7_0); - let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = constructor_lo_gpr(ctx, pattern7_1)?; let expr2_0 = constructor_shr_i128(ctx, expr0_0, expr1_0)?; let expr3_0: Type = I64; let expr4_0: Type = I64; let expr5_0: u64 = 128; let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?; - let expr7_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr8_0 = constructor_sub(ctx, expr3_0, expr6_0, &expr7_0)?; - let expr9_0 = constructor_shl_i128(ctx, expr0_0, expr8_0)?; - let expr10_0 = constructor_or_i128(ctx, expr2_0, expr9_0)?; - return Some(expr10_0); + let expr7_0 = C::gpr_new(ctx, expr6_0); + let expr8_0 = C::gpr_to_gpr_mem_imm(ctx, expr1_0); + let expr9_0 = constructor_sub(ctx, expr3_0, expr7_0, &expr8_0)?; + let expr10_0 = constructor_shl_i128(ctx, expr0_0, expr9_0)?; + let expr11_0 = constructor_or_i128(ctx, expr2_0, expr10_0)?; + return Some(expr11_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 572. - let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + // Rule at src/isa/x64/lower.isle line 583. + let expr0_0 = constructor_lo_gpr(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?; return Some(expr2_0); @@ -3343,8 +3393,8 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 671. - let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + // Rule at src/isa/x64/lower.isle line 695. + let expr0_0 = constructor_lo_gpr(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?; return Some(expr2_0); @@ -3352,8 +3402,8 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 773. - let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + // Rule at src/isa/x64/lower.isle line 810. + let expr0_0 = constructor_lo_gpr(ctx, pattern7_1)?; let expr1_0 = C::put_in_regs(ctx, pattern7_0); let expr2_0 = constructor_sar_i128(ctx, expr1_0, expr0_0)?; return Some(expr2_0); @@ -3366,7 +3416,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1318. + // Rule at src/isa/x64/lower.isle line 1363. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -3381,20 +3431,22 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} @@ -3411,7 +3463,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1454. + // Rule at src/isa/x64/lower.isle line 1499. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -3421,7 +3473,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1476. + // Rule at src/isa/x64/lower.isle line 1521. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -3431,7 +3483,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1443. + // Rule at src/isa/x64/lower.isle line 1488. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -3441,7 +3493,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1465. + // Rule at src/isa/x64/lower.isle line 1510. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -3451,7 +3503,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 584. + // Rule at src/isa/x64/lower.isle line 595. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3470,7 +3522,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 681. + // Rule at src/isa/x64/lower.isle line 705. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3490,7 +3542,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 932. + // Rule at src/isa/x64/lower.isle line 977. let expr0_0: Type = I8X16; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -3526,7 +3578,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1262. + // Rule at src/isa/x64/lower.isle line 1307. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; @@ -3549,7 +3601,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1457. + // Rule at src/isa/x64/lower.isle line 1502. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -3559,7 +3611,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1479. + // Rule at src/isa/x64/lower.isle line 1524. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -3569,7 +3621,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1446. + // Rule at src/isa/x64/lower.isle line 1491. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -3579,7 +3631,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1468. + // Rule at src/isa/x64/lower.isle line 1513. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -3589,7 +3641,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 626. + // Rule at src/isa/x64/lower.isle line 639. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3600,7 +3652,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 725. + // Rule at src/isa/x64/lower.isle line 751. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3611,7 +3663,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 817. + // Rule at src/isa/x64/lower.isle line 858. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3628,7 +3680,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 936. + // Rule at src/isa/x64/lower.isle line 981. let expr0_0: Type = I16X8; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -3639,7 +3691,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1265. + // Rule at src/isa/x64/lower.isle line 1310. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; @@ -3662,7 +3714,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1460. + // Rule at src/isa/x64/lower.isle line 1505. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -3672,7 +3724,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1482. + // Rule at src/isa/x64/lower.isle line 1527. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -3682,7 +3734,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1449. + // Rule at src/isa/x64/lower.isle line 1494. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -3692,7 +3744,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1471. + // Rule at src/isa/x64/lower.isle line 1516. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -3702,7 +3754,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 630. + // Rule at src/isa/x64/lower.isle line 643. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3713,7 +3765,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 729. + // Rule at src/isa/x64/lower.isle line 755. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3724,7 +3776,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 821. + // Rule at src/isa/x64/lower.isle line 862. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3741,7 +3793,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 940. + // Rule at src/isa/x64/lower.isle line 985. let expr0_0: Type = I32X4; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -3752,7 +3804,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1268. + // Rule at src/isa/x64/lower.isle line 1313. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_1)?; let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; let expr2_0 = constructor_value_xmm(ctx, expr1_0)?; @@ -3775,7 +3827,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 634. + // Rule at src/isa/x64/lower.isle line 647. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3786,7 +3838,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 733. + // Rule at src/isa/x64/lower.isle line 759. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); let expr2_0 = constructor_mov_rmi_to_xmm(ctx, &expr1_0)?; @@ -3797,7 +3849,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 833. + // Rule at src/isa/x64/lower.isle line 874. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0: Type = I64; let expr2_0: u8 = 0; @@ -3806,21 +3858,17 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } @@ -3831,7 +3879,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 944. + // Rule at src/isa/x64/lower.isle line 989. let expr0_0: Type = I64X2; let expr1_0: u64 = 0; let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -3842,7 +3890,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1282. + // Rule at src/isa/x64/lower.isle line 1327. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0: Type = I64X2; let expr2_0: u64 = 0; @@ -3869,7 +3917,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 1255. + // Rule at src/isa/x64/lower.isle line 1300. let expr0_0 = constructor_put_in_xmm(ctx, pattern6_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern6_0)?; let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; @@ -3950,7 +3998,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 950. + // Rule at src/isa/x64/lower.isle line 995. let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -4078,7 +4126,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 954. + // Rule at src/isa/x64/lower.isle line 999. let expr0_0 = constructor_put_in_xmm(ctx, pattern9_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern9_1)?; let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -4191,7 +4239,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1331. + // Rule at src/isa/x64/lower.isle line 1376. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr0_0); @@ -5085,7 +5133,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1345. + // Rule at src/isa/x64/lower.isle line 1390. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_2)?; @@ -5103,7 +5151,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1326. + // Rule at src/isa/x64/lower.isle line 1371. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); @@ -5159,7 +5207,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1435. + // Rule at src/isa/x64/lower.isle line 1480. let expr0_0 = CC::L; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -5169,7 +5217,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1429. + // Rule at src/isa/x64/lower.isle line 1474. let expr0_0 = CC::B; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -5179,7 +5227,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1438. + // Rule at src/isa/x64/lower.isle line 1483. let expr0_0 = CC::NL; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -5189,7 +5237,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1432. + // Rule at src/isa/x64/lower.isle line 1477. let expr0_0 = CC::NB; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -5201,42 +5249,44 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { @@ -5244,292 +5294,292 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { - // Rule at src/isa/x64/lower.isle line 974. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 1019. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; let expr1_0 = constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { - // Rule at src/isa/x64/lower.isle line 986. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + // Rule at src/isa/x64/lower.isle line 1031. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { - // Rule at src/isa/x64/lower.isle line 970. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + // Rule at src/isa/x64/lower.isle line 1015. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; let expr1_0 = constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { - // Rule at src/isa/x64/lower.isle line 980. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + // Rule at src/isa/x64/lower.isle line 1025. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } - // Rule at src/isa/x64/lower.isle line 963. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr3_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); + // Rule at src/isa/x64/lower.isle line 1008. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_gpr_mem_imm(ctx, pattern7_1)?; + let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; + return Some(expr3_0); } &Opcode::IaddIfcout => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 183. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = C::put_in_reg(ctx, pattern7_1); - let expr4_0 = - constructor_add(ctx, pattern3_0, expr3_0, &pattern8_0)?; - let expr5_0 = C::value_regs(ctx, expr4_0, expr2_0); - return Some(expr5_0); + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = constructor_put_in_gpr(ctx, pattern7_1)?; + let expr3_0 = + constructor_add(ctx, pattern3_0, expr2_0, &pattern8_0)?; + let expr4_0 = constructor_value_gprs(ctx, expr3_0, expr1_0)?; + return Some(expr4_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 199. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = C::put_in_reg(ctx, pattern7_1); - let expr4_0 = C::sink_load(ctx, &pattern8_0); - let expr5_0 = constructor_add(ctx, pattern3_0, expr3_0, &expr4_0)?; - let expr6_0 = C::value_regs(ctx, expr5_0, expr2_0); - return Some(expr6_0); + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = constructor_put_in_gpr(ctx, pattern7_1)?; + let expr3_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; + let expr4_0 = constructor_add(ctx, pattern3_0, expr2_0, &expr3_0)?; + let expr5_0 = constructor_value_gprs(ctx, expr4_0, expr1_0)?; + return Some(expr5_0); } if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 177. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = C::put_in_reg(ctx, pattern7_0); - let expr4_0 = - constructor_add(ctx, pattern3_0, expr3_0, &pattern8_0)?; - let expr5_0 = C::value_regs(ctx, expr4_0, expr2_0); - return Some(expr5_0); + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr3_0 = + constructor_add(ctx, pattern3_0, expr2_0, &pattern8_0)?; + let expr4_0 = constructor_value_gprs(ctx, expr3_0, expr1_0)?; + return Some(expr4_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 191. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = C::put_in_reg(ctx, pattern7_0); - let expr4_0 = C::sink_load(ctx, &pattern8_0); - let expr5_0 = constructor_add(ctx, pattern3_0, expr3_0, &expr4_0)?; - let expr6_0 = C::value_regs(ctx, expr5_0, expr2_0); - return Some(expr6_0); + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr3_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; + let expr4_0 = constructor_add(ctx, pattern3_0, expr2_0, &expr3_0)?; + let expr5_0 = constructor_value_gprs(ctx, expr4_0, expr1_0)?; + return Some(expr5_0); } // Rule at src/isa/x64/lower.isle line 167. - let expr0_0: Type = I64; - let expr1_0 = C::temp_writable_reg(ctx, expr0_0); - let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); - let expr3_0 = C::put_in_reg(ctx, pattern7_0); - let expr4_0 = C::put_in_reg(ctx, pattern7_1); - let expr5_0 = RegMemImm::Reg { reg: expr4_0 }; - let expr6_0 = constructor_add(ctx, pattern3_0, expr3_0, &expr5_0)?; - let expr7_0 = C::value_regs(ctx, expr6_0, expr2_0); - return Some(expr7_0); + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + let expr2_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr3_0 = constructor_put_in_gpr_mem_imm(ctx, pattern7_1)?; + let expr4_0 = constructor_add(ctx, pattern3_0, expr2_0, &expr3_0)?; + let expr5_0 = constructor_value_gprs(ctx, expr4_0, expr1_0)?; + return Some(expr5_0); } &Opcode::Band => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 355. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; let expr1_0 = constructor_x64_and(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 341. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_x64_and(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 349. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; let expr1_0 = constructor_x64_and(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 335. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_x64_and(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } // Rule at src/isa/x64/lower.isle line 328. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr3_0 = constructor_x64_and(ctx, pattern3_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_gpr_mem_imm(ctx, pattern7_1)?; + let expr2_0 = constructor_x64_and(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; + return Some(expr3_0); } &Opcode::Bor => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 429. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; let expr1_0 = constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 415. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 423. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; let expr1_0 = constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 409. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } // Rule at src/isa/x64/lower.isle line 402. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr3_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_gpr_mem_imm(ctx, pattern7_1)?; + let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; + return Some(expr3_0); } &Opcode::Bxor => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 505. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; let expr1_0 = constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { // Rule at src/isa/x64/lower.isle line 491. - let expr0_0 = C::put_in_reg(ctx, pattern7_1); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_1)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 499. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; let expr1_0 = constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?; - let expr2_0 = C::value_reg(ctx, expr1_0); + let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { // Rule at src/isa/x64/lower.isle line 485. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = + constructor_sink_load_to_gpr_mem_imm(ctx, &pattern8_0)?; let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } // Rule at src/isa/x64/lower.isle line 478. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_in_reg(ctx, pattern7_1); - let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; - let expr3_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); - return Some(expr4_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_gpr_mem_imm(ctx, pattern7_1)?; + let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; + return Some(expr3_0); } &Opcode::Ishl => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); // Rule at src/isa/x64/lower.isle line 543. - let expr0_0 = C::put_in_reg(ctx, pattern7_0); - let expr1_0 = C::put_masked_in_imm8_reg(ctx, pattern7_1, pattern3_0); + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = C::put_masked_in_imm8_gpr(ctx, pattern7_1, pattern3_0); let expr2_0 = constructor_shl(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = C::value_reg(ctx, expr2_0); + let expr3_0 = constructor_value_gpr(ctx, expr2_0)?; return Some(expr3_0); } &Opcode::Ushr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 642. + // Rule at src/isa/x64/lower.isle line 655. let expr0_0 = ExtendKind::Zero; let expr1_0 = - constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; - let expr2_0 = C::put_masked_in_imm8_reg(ctx, pattern7_1, pattern3_0); + constructor_extend_to_gpr(ctx, pattern7_0, pattern3_0, &expr0_0)?; + let expr2_0 = C::put_masked_in_imm8_gpr(ctx, pattern7_1, pattern3_0); let expr3_0 = constructor_shr(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_gpr(ctx, expr3_0)?; return Some(expr4_0); } &Opcode::Sshr => { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 741. + // Rule at src/isa/x64/lower.isle line 767. let expr0_0 = ExtendKind::Sign; let expr1_0 = - constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; - let expr2_0 = C::put_masked_in_imm8_reg(ctx, pattern7_1, pattern3_0); + constructor_extend_to_gpr(ctx, pattern7_0, pattern3_0, &expr0_0)?; + let expr2_0 = C::put_masked_in_imm8_gpr(ctx, pattern7_1, pattern3_0); let expr3_0 = constructor_sar(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = C::value_reg(ctx, expr3_0); + let expr4_0 = constructor_value_gpr(ctx, expr3_0)?; return Some(expr4_0); } _ => {} @@ -5541,14 +5591,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match &pattern5_0 { &Opcode::Ineg => { - // Rule at src/isa/x64/lower.isle line 927. + // Rule at src/isa/x64/lower.isle line 972. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_neg(ctx, pattern3_0, expr0_0)?; let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Bnot => { - // Rule at src/isa/x64/lower.isle line 1305. + // Rule at src/isa/x64/lower.isle line 1350. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; let expr2_0 = constructor_value_gpr(ctx, expr1_0)?; @@ -5565,12 +5615,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} @@ -5595,23 +5646,23 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { @@ -5625,23 +5676,23 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} @@ -5667,26 +5718,26 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { @@ -5700,26 +5751,26 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} @@ -5804,20 +5855,20 @@ pub fn constructor_or_i128( let pattern1_0 = arg1; // Rule at src/isa/x64/lower.isle line 451. let expr0_0: usize = 0; - let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr1_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr0_0)?; let expr2_0: usize = 1; - let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr3_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr2_0)?; let expr4_0: usize = 0; - let expr5_0 = C::value_regs_get(ctx, pattern1_0, expr4_0); + let expr5_0 = constructor_value_regs_get_gpr(ctx, pattern1_0, expr4_0)?; let expr6_0: usize = 1; - let expr7_0 = C::value_regs_get(ctx, pattern1_0, expr6_0); + let expr7_0 = constructor_value_regs_get_gpr(ctx, pattern1_0, expr6_0)?; let expr8_0: Type = I64; - let expr9_0 = RegMemImm::Reg { reg: expr5_0 }; + let expr9_0 = C::gpr_to_gpr_mem_imm(ctx, expr5_0); let expr10_0 = constructor_or(ctx, expr8_0, expr1_0, &expr9_0)?; let expr11_0: Type = I64; - let expr12_0 = RegMemImm::Reg { reg: expr7_0 }; + let expr12_0 = C::gpr_to_gpr_mem_imm(ctx, expr7_0); let expr13_0 = constructor_or(ctx, expr11_0, expr3_0, &expr12_0)?; - let expr14_0 = C::value_regs(ctx, expr10_0, expr13_0); + let expr14_0 = constructor_value_gprs(ctx, expr10_0, expr13_0)?; return Some(expr14_0); } @@ -5825,59 +5876,64 @@ pub fn constructor_or_i128( pub fn constructor_shl_i128( ctx: &mut C, arg0: ValueRegs, - arg1: Reg, + arg1: Gpr, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; // Rule at src/isa/x64/lower.isle line 549. let expr0_0: usize = 0; - let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr1_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr0_0)?; let expr2_0: usize = 1; - let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr3_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr2_0)?; let expr4_0: Type = I64; - let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr5_0 = C::gpr_to_imm8_gpr(ctx, pattern1_0); let expr6_0 = constructor_shl(ctx, expr4_0, expr1_0, &expr5_0)?; let expr7_0: Type = I64; - let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr8_0 = C::gpr_to_imm8_gpr(ctx, pattern1_0); let expr9_0 = constructor_shl(ctx, expr7_0, expr3_0, &expr8_0)?; let expr10_0: Type = I64; let expr11_0: Type = I64; let expr12_0: Type = I64; let expr13_0: u64 = 64; let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?; - let expr15_0 = RegMemImm::Reg { reg: pattern1_0 }; - let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?; - let expr17_0 = Imm8Reg::Reg { reg: expr16_0 }; - let expr18_0 = constructor_shr(ctx, expr10_0, expr1_0, &expr17_0)?; - let expr19_0: Type = I64; - let expr20_0: u64 = 0; - let expr21_0 = constructor_imm(ctx, expr19_0, expr20_0)?; - let expr22_0 = OperandSize::Size64; - let expr23_0: u32 = 127; - let expr24_0 = RegMemImm::Imm { simm32: expr23_0 }; - let expr25_0 = constructor_test(ctx, &expr22_0, &expr24_0, pattern1_0)?; - let expr26_0: Type = I64; - let expr27_0 = CC::Z; - let expr28_0 = RegMem::Reg { reg: expr21_0 }; - let expr29_0 = constructor_cmove(ctx, expr26_0, &expr27_0, &expr28_0, expr18_0)?; - let expr30_0 = constructor_with_flags_1(ctx, &expr25_0, &expr29_0)?; - let expr31_0: Type = I64; - let expr32_0 = RegMemImm::Reg { reg: expr9_0 }; - let expr33_0 = constructor_or(ctx, expr31_0, expr30_0, &expr32_0)?; - let expr34_0 = OperandSize::Size64; - let expr35_0: u32 = 64; - let expr36_0 = RegMemImm::Imm { simm32: expr35_0 }; - let expr37_0 = constructor_test(ctx, &expr34_0, &expr36_0, pattern1_0)?; - let expr38_0: Type = I64; - let expr39_0 = CC::Z; - let expr40_0 = RegMem::Reg { reg: expr6_0 }; - let expr41_0 = constructor_cmove(ctx, expr38_0, &expr39_0, &expr40_0, expr21_0)?; - let expr42_0: Type = I64; - let expr43_0 = CC::Z; - let expr44_0 = RegMem::Reg { reg: expr33_0 }; - let expr45_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr6_0)?; - let expr46_0 = constructor_with_flags_2(ctx, &expr37_0, &expr41_0, &expr45_0)?; - return Some(expr46_0); + let expr15_0 = C::gpr_new(ctx, expr14_0); + let expr16_0 = C::gpr_to_gpr_mem_imm(ctx, pattern1_0); + let expr17_0 = constructor_sub(ctx, expr11_0, expr15_0, &expr16_0)?; + let expr18_0 = C::gpr_to_imm8_gpr(ctx, expr17_0); + let expr19_0 = constructor_shr(ctx, expr10_0, expr1_0, &expr18_0)?; + let expr20_0: Type = I64; + let expr21_0: u64 = 0; + let expr22_0 = constructor_imm(ctx, expr20_0, expr21_0)?; + let expr23_0 = C::gpr_new(ctx, expr22_0); + let expr24_0 = OperandSize::Size64; + let expr25_0: u32 = 127; + let expr26_0 = RegMemImm::Imm { simm32: expr25_0 }; + let expr27_0 = C::gpr_mem_imm_new(ctx, &expr26_0); + let expr28_0 = constructor_test(ctx, &expr24_0, &expr27_0, pattern1_0)?; + let expr29_0: Type = I64; + let expr30_0 = CC::Z; + let expr31_0 = C::gpr_to_gpr_mem(ctx, expr23_0); + let expr32_0 = constructor_cmove(ctx, expr29_0, &expr30_0, &expr31_0, expr19_0)?; + let expr33_0 = constructor_with_flags_1(ctx, &expr28_0, &expr32_0)?; + let expr34_0 = C::gpr_new(ctx, expr33_0); + let expr35_0: Type = I64; + let expr36_0 = C::gpr_to_gpr_mem_imm(ctx, expr9_0); + let expr37_0 = constructor_or(ctx, expr35_0, expr34_0, &expr36_0)?; + let expr38_0 = OperandSize::Size64; + let expr39_0: u32 = 64; + let expr40_0 = RegMemImm::Imm { simm32: expr39_0 }; + let expr41_0 = C::gpr_mem_imm_new(ctx, &expr40_0); + let expr42_0 = constructor_test(ctx, &expr38_0, &expr41_0, pattern1_0)?; + let expr43_0: Type = I64; + let expr44_0 = CC::Z; + let expr45_0 = C::gpr_to_gpr_mem(ctx, expr6_0); + let expr46_0 = constructor_cmove(ctx, expr43_0, &expr44_0, &expr45_0, expr23_0)?; + let expr47_0: Type = I64; + let expr48_0 = CC::Z; + let expr49_0 = C::gpr_to_gpr_mem(ctx, expr37_0); + let expr50_0 = constructor_cmove(ctx, expr47_0, &expr48_0, &expr49_0, expr6_0)?; + let expr51_0 = constructor_with_flags_2(ctx, &expr42_0, &expr46_0, &expr50_0)?; + return Some(expr51_0); } // Generated as internal constructor for term ishl_i8x16_mask. @@ -5888,29 +5944,29 @@ pub fn constructor_ishl_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 604. + // Rule at src/isa/x64/lower.isle line 615. let expr0_0 = C::ishl_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 613. + // Rule at src/isa/x64/lower.isle line 624. let expr0_0 = C::ishl_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; - let expr3_0: u8 = 4; - let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; - let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; - let expr6_0: u32 = 0; - let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr3_0 = C::gpr_new(ctx, pattern1_0); + let expr4_0: u8 = 4; + let expr5_0 = C::imm8_to_imm8_gpr(ctx, expr4_0); + let expr6_0 = constructor_shl(ctx, expr2_0, expr3_0, &expr5_0)?; + let expr7_0: u32 = 0; let expr8_0: u8 = 0; - let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr7_0, expr1_0, expr6_0, expr8_0); let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 621. + // Rule at src/isa/x64/lower.isle line 634. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -5927,62 +5983,68 @@ pub fn constructor_ishl_i8x16_mask( pub fn constructor_shr_i128( ctx: &mut C, arg0: ValueRegs, - arg1: Reg, + arg1: Gpr, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 649. + // Rule at src/isa/x64/lower.isle line 662. let expr0_0: usize = 0; - let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr1_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr0_0)?; let expr2_0: usize = 1; - let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr3_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr2_0)?; let expr4_0: Type = I64; - let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr5_0 = C::gpr_to_imm8_gpr(ctx, pattern1_0); let expr6_0 = constructor_shr(ctx, expr4_0, expr1_0, &expr5_0)?; let expr7_0: Type = I64; - let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr8_0 = C::gpr_to_imm8_gpr(ctx, pattern1_0); let expr9_0 = constructor_shr(ctx, expr7_0, expr3_0, &expr8_0)?; let expr10_0: Type = I64; let expr11_0: Type = I64; let expr12_0: Type = I64; let expr13_0: u64 = 64; let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?; - let expr15_0 = RegMemImm::Reg { reg: pattern1_0 }; - let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?; - let expr17_0 = Imm8Reg::Reg { reg: expr16_0 }; - let expr18_0 = constructor_shl(ctx, expr10_0, expr3_0, &expr17_0)?; - let expr19_0 = OperandSize::Size64; - let expr20_0: u32 = 127; - let expr21_0 = RegMemImm::Imm { simm32: expr20_0 }; - let expr22_0 = constructor_test(ctx, &expr19_0, &expr21_0, pattern1_0)?; - let expr23_0: Type = I64; - let expr24_0 = CC::Z; + let expr15_0 = C::gpr_new(ctx, expr14_0); + let expr16_0 = C::gpr_to_gpr_mem_imm(ctx, pattern1_0); + let expr17_0 = constructor_sub(ctx, expr11_0, expr15_0, &expr16_0)?; + let expr18_0 = C::gpr_to_imm8_gpr(ctx, expr17_0); + let expr19_0 = constructor_shl(ctx, expr10_0, expr3_0, &expr18_0)?; + let expr20_0 = OperandSize::Size64; + let expr21_0: u32 = 127; + let expr22_0 = RegMemImm::Imm { simm32: expr21_0 }; + let expr23_0 = C::gpr_mem_imm_new(ctx, &expr22_0); + let expr24_0 = constructor_test(ctx, &expr20_0, &expr23_0, pattern1_0)?; let expr25_0: Type = I64; - let expr26_0: u64 = 0; - let expr27_0 = constructor_imm(ctx, expr25_0, expr26_0)?; - let expr28_0 = RegMem::Reg { reg: expr27_0 }; - let expr29_0 = constructor_cmove(ctx, expr23_0, &expr24_0, &expr28_0, expr18_0)?; - let expr30_0 = constructor_with_flags_1(ctx, &expr22_0, &expr29_0)?; - let expr31_0: Type = I64; - let expr32_0 = RegMemImm::Reg { reg: expr6_0 }; - let expr33_0 = constructor_or(ctx, expr31_0, expr30_0, &expr32_0)?; - let expr34_0 = OperandSize::Size64; - let expr35_0: u32 = 64; - let expr36_0 = RegMemImm::Imm { simm32: expr35_0 }; - let expr37_0 = constructor_test(ctx, &expr34_0, &expr36_0, pattern1_0)?; - let expr38_0: Type = I64; - let expr39_0 = CC::Z; - let expr40_0 = RegMem::Reg { reg: expr33_0 }; - let expr41_0 = constructor_cmove(ctx, expr38_0, &expr39_0, &expr40_0, expr9_0)?; - let expr42_0: Type = I64; - let expr43_0 = CC::Z; - let expr44_0 = RegMem::Reg { reg: expr9_0 }; - let expr45_0: Type = I64; - let expr46_0: u64 = 0; - let expr47_0 = constructor_imm(ctx, expr45_0, expr46_0)?; - let expr48_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr47_0)?; - let expr49_0 = constructor_with_flags_2(ctx, &expr37_0, &expr41_0, &expr48_0)?; - return Some(expr49_0); + let expr26_0 = CC::Z; + let expr27_0: Type = I64; + let expr28_0: u64 = 0; + let expr29_0 = constructor_imm(ctx, expr27_0, expr28_0)?; + let expr30_0 = C::gpr_new(ctx, expr29_0); + let expr31_0 = C::gpr_to_gpr_mem(ctx, expr30_0); + let expr32_0 = constructor_cmove(ctx, expr25_0, &expr26_0, &expr31_0, expr19_0)?; + let expr33_0 = constructor_with_flags_1(ctx, &expr24_0, &expr32_0)?; + let expr34_0 = C::gpr_new(ctx, expr33_0); + let expr35_0: Type = I64; + let expr36_0 = C::gpr_to_gpr_mem_imm(ctx, expr6_0); + let expr37_0 = constructor_or(ctx, expr35_0, expr34_0, &expr36_0)?; + let expr38_0 = OperandSize::Size64; + let expr39_0: u32 = 64; + let expr40_0 = RegMemImm::Imm { simm32: expr39_0 }; + let expr41_0 = C::gpr_mem_imm_new(ctx, &expr40_0); + let expr42_0 = constructor_test(ctx, &expr38_0, &expr41_0, pattern1_0)?; + let expr43_0: Type = I64; + let expr44_0 = CC::Z; + let expr45_0 = C::gpr_to_gpr_mem(ctx, expr37_0); + let expr46_0 = constructor_cmove(ctx, expr43_0, &expr44_0, &expr45_0, expr9_0)?; + let expr47_0: Type = I64; + let expr48_0 = CC::Z; + let expr49_0 = C::gpr_to_gpr_mem(ctx, expr9_0); + let expr50_0: Type = I64; + let expr51_0: u64 = 0; + let expr52_0 = constructor_imm(ctx, expr50_0, expr51_0)?; + let expr53_0 = C::gpr_new(ctx, expr52_0); + let expr54_0 = constructor_cmove(ctx, expr47_0, &expr48_0, &expr49_0, expr53_0)?; + let expr55_0 = constructor_with_flags_2(ctx, &expr42_0, &expr46_0, &expr54_0)?; + return Some(expr55_0); } // Generated as internal constructor for term ushr_i8x16_mask. @@ -5993,29 +6055,29 @@ pub fn constructor_ushr_i8x16_mask( let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 703. + // Rule at src/isa/x64/lower.isle line 727. let expr0_0 = C::ushr_i8x16_mask_for_const(ctx, pattern1_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/lower.isle line 712. + // Rule at src/isa/x64/lower.isle line 736. let expr0_0 = C::ushr_i8x16_mask_table(ctx); let expr1_0 = constructor_lea(ctx, &expr0_0)?; let expr2_0: Type = I64; - let expr3_0: u8 = 4; - let expr4_0 = Imm8Reg::Imm8 { imm: expr3_0 }; - let expr5_0 = constructor_shl(ctx, expr2_0, pattern1_0, &expr4_0)?; - let expr6_0: u32 = 0; - let expr7_0 = C::gpr_new(ctx, expr5_0); + let expr3_0 = C::gpr_new(ctx, pattern1_0); + let expr4_0: u8 = 4; + let expr5_0 = C::imm8_to_imm8_gpr(ctx, expr4_0); + let expr6_0 = constructor_shl(ctx, expr2_0, expr3_0, &expr5_0)?; + let expr7_0: u32 = 0; let expr8_0: u8 = 0; - let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr6_0, expr1_0, expr7_0, expr8_0); + let expr9_0 = C::amode_imm_reg_reg_shift(ctx, expr7_0, expr1_0, expr6_0, expr8_0); let expr10_0 = C::amode_to_synthetic_amode(ctx, &expr9_0); return Some(expr10_0); } &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/lower.isle line 720. + // Rule at src/isa/x64/lower.isle line 746. let expr0_0: Type = I64; let expr1_0 = ExtKind::None; let expr2_0 = constructor_x64_load(ctx, expr0_0, &pattern1_0, &expr1_0)?; @@ -6032,63 +6094,68 @@ pub fn constructor_ushr_i8x16_mask( pub fn constructor_sar_i128( ctx: &mut C, arg0: ValueRegs, - arg1: Reg, + arg1: Gpr, ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 748. + // Rule at src/isa/x64/lower.isle line 774. let expr0_0: usize = 0; - let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr1_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr0_0)?; let expr2_0: usize = 1; - let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr3_0 = constructor_value_regs_get_gpr(ctx, pattern0_0, expr2_0)?; let expr4_0: Type = I64; - let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr5_0 = C::gpr_to_imm8_gpr(ctx, pattern1_0); let expr6_0 = constructor_shr(ctx, expr4_0, expr1_0, &expr5_0)?; let expr7_0: Type = I64; - let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr8_0 = C::gpr_to_imm8_gpr(ctx, pattern1_0); let expr9_0 = constructor_sar(ctx, expr7_0, expr3_0, &expr8_0)?; let expr10_0: Type = I64; let expr11_0: Type = I64; let expr12_0: Type = I64; let expr13_0: u64 = 64; let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?; - let expr15_0 = RegMemImm::Reg { reg: pattern1_0 }; - let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?; - let expr17_0 = Imm8Reg::Reg { reg: expr16_0 }; - let expr18_0 = constructor_shl(ctx, expr10_0, expr3_0, &expr17_0)?; - let expr19_0 = OperandSize::Size64; - let expr20_0: u32 = 127; - let expr21_0 = RegMemImm::Imm { simm32: expr20_0 }; - let expr22_0 = constructor_test(ctx, &expr19_0, &expr21_0, pattern1_0)?; - let expr23_0: Type = I64; - let expr24_0 = CC::Z; + let expr15_0 = C::gpr_new(ctx, expr14_0); + let expr16_0 = C::gpr_to_gpr_mem_imm(ctx, pattern1_0); + let expr17_0 = constructor_sub(ctx, expr11_0, expr15_0, &expr16_0)?; + let expr18_0 = C::gpr_to_imm8_gpr(ctx, expr17_0); + let expr19_0 = constructor_shl(ctx, expr10_0, expr3_0, &expr18_0)?; + let expr20_0 = OperandSize::Size64; + let expr21_0: u32 = 127; + let expr22_0 = RegMemImm::Imm { simm32: expr21_0 }; + let expr23_0 = C::gpr_mem_imm_new(ctx, &expr22_0); + let expr24_0 = constructor_test(ctx, &expr20_0, &expr23_0, pattern1_0)?; let expr25_0: Type = I64; - let expr26_0: u64 = 0; - let expr27_0 = constructor_imm(ctx, expr25_0, expr26_0)?; - let expr28_0 = RegMem::Reg { reg: expr27_0 }; - let expr29_0 = constructor_cmove(ctx, expr23_0, &expr24_0, &expr28_0, expr18_0)?; - let expr30_0 = constructor_with_flags_1(ctx, &expr22_0, &expr29_0)?; - let expr31_0: Type = I64; - let expr32_0 = RegMemImm::Reg { reg: expr30_0 }; - let expr33_0 = constructor_or(ctx, expr31_0, expr6_0, &expr32_0)?; - let expr34_0: Type = I64; - let expr35_0: u8 = 63; - let expr36_0 = Imm8Reg::Imm8 { imm: expr35_0 }; - let expr37_0 = constructor_sar(ctx, expr34_0, expr3_0, &expr36_0)?; - let expr38_0 = OperandSize::Size64; - let expr39_0: u32 = 64; - let expr40_0 = RegMemImm::Imm { simm32: expr39_0 }; - let expr41_0 = constructor_test(ctx, &expr38_0, &expr40_0, pattern1_0)?; - let expr42_0: Type = I64; - let expr43_0 = CC::Z; - let expr44_0 = RegMem::Reg { reg: expr33_0 }; - let expr45_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr9_0)?; - let expr46_0: Type = I64; - let expr47_0 = CC::Z; - let expr48_0 = RegMem::Reg { reg: expr9_0 }; - let expr49_0 = constructor_cmove(ctx, expr46_0, &expr47_0, &expr48_0, expr37_0)?; - let expr50_0 = constructor_with_flags_2(ctx, &expr41_0, &expr45_0, &expr49_0)?; - return Some(expr50_0); + let expr26_0 = CC::Z; + let expr27_0: Type = I64; + let expr28_0: u64 = 0; + let expr29_0 = constructor_imm(ctx, expr27_0, expr28_0)?; + let expr30_0 = C::gpr_new(ctx, expr29_0); + let expr31_0 = C::gpr_to_gpr_mem(ctx, expr30_0); + let expr32_0 = constructor_cmove(ctx, expr25_0, &expr26_0, &expr31_0, expr19_0)?; + let expr33_0 = constructor_with_flags_1(ctx, &expr24_0, &expr32_0)?; + let expr34_0 = C::gpr_new(ctx, expr33_0); + let expr35_0: Type = I64; + let expr36_0 = C::gpr_to_gpr_mem_imm(ctx, expr34_0); + let expr37_0 = constructor_or(ctx, expr35_0, expr6_0, &expr36_0)?; + let expr38_0: Type = I64; + let expr39_0: u8 = 63; + let expr40_0 = C::imm8_to_imm8_gpr(ctx, expr39_0); + let expr41_0 = constructor_sar(ctx, expr38_0, expr3_0, &expr40_0)?; + let expr42_0 = OperandSize::Size64; + let expr43_0: u32 = 64; + let expr44_0 = RegMemImm::Imm { simm32: expr43_0 }; + let expr45_0 = C::gpr_mem_imm_new(ctx, &expr44_0); + let expr46_0 = constructor_test(ctx, &expr42_0, &expr45_0, pattern1_0)?; + let expr47_0: Type = I64; + let expr48_0 = CC::Z; + let expr49_0 = C::gpr_to_gpr_mem(ctx, expr37_0); + let expr50_0 = constructor_cmove(ctx, expr47_0, &expr48_0, &expr49_0, expr9_0)?; + let expr51_0: Type = I64; + let expr52_0 = CC::Z; + let expr53_0 = C::gpr_to_gpr_mem(ctx, expr9_0); + let expr54_0 = constructor_cmove(ctx, expr51_0, &expr52_0, &expr53_0, expr41_0)?; + let expr55_0 = constructor_with_flags_2(ctx, &expr46_0, &expr50_0, &expr54_0)?; + return Some(expr55_0); } // Generated as internal constructor for term sshr_i8x16_bigger_shift. @@ -6101,7 +6168,7 @@ pub fn constructor_sshr_i8x16_bigger_shift( let pattern1_0 = arg1; match pattern1_0 { &RegMemImm::Imm { simm32: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 807. + // Rule at src/isa/x64/lower.isle line 844. let expr0_0: u32 = 8; let expr1_0 = C::u32_add(ctx, pattern2_0, expr0_0); let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; @@ -6109,24 +6176,30 @@ pub fn constructor_sshr_i8x16_bigger_shift( return Some(expr3_0); } &RegMemImm::Reg { reg: pattern2_0 } => { - // Rule at src/isa/x64/lower.isle line 809. - let expr0_0: u32 = 8; - let expr1_0 = RegMemImm::Imm { simm32: expr0_0 }; - let expr2_0 = constructor_add(ctx, pattern0_0, pattern2_0, &expr1_0)?; - let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; - return Some(expr4_0); + // Rule at src/isa/x64/lower.isle line 846. + let expr0_0 = C::gpr_new(ctx, pattern2_0); + let expr1_0: u32 = 8; + let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; + let expr3_0 = C::gpr_mem_imm_new(ctx, &expr2_0); + let expr4_0 = constructor_add(ctx, pattern0_0, expr0_0, &expr3_0)?; + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + let expr6_0 = RegMemImm::Reg { reg: expr5_0 }; + let expr7_0 = constructor_mov_rmi_to_xmm(ctx, &expr6_0)?; + return Some(expr7_0); } &RegMemImm::Mem { addr: ref pattern2_0, } => { - // Rule at src/isa/x64/lower.isle line 811. + // Rule at src/isa/x64/lower.isle line 850. let expr0_0: u64 = 8; let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; - let expr2_0 = constructor_add(ctx, pattern0_0, expr1_0, pattern1_0)?; - let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; - let expr4_0 = constructor_mov_rmi_to_xmm(ctx, &expr3_0)?; - return Some(expr4_0); + let expr2_0 = C::gpr_new(ctx, expr1_0); + let expr3_0 = C::gpr_mem_imm_new(ctx, pattern1_0); + let expr4_0 = constructor_add(ctx, pattern0_0, expr2_0, &expr3_0)?; + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + let expr6_0 = RegMemImm::Reg { reg: expr5_0 }; + let expr7_0 = constructor_mov_rmi_to_xmm(ctx, &expr6_0)?; + return Some(expr7_0); } _ => {} } @@ -6144,21 +6217,21 @@ pub fn constructor_sse_and_not( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1244. + // Rule at src/isa/x64/lower.isle line 1289. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1245. + // Rule at src/isa/x64/lower.isle line 1290. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1246. + // Rule at src/isa/x64/lower.isle line 1291. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -6168,20 +6241,18 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1311. + // Rule at src/isa/x64/lower.isle line 1356. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; - let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); - let expr3_0 = C::gpr_new(ctx, expr2_0); - let expr4_0: usize = 1; - let expr5_0 = C::value_regs_get(ctx, expr0_0, expr4_0); - let expr6_0 = C::gpr_new(ctx, expr5_0); + let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; + let expr3_0: usize = 1; + let expr4_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr3_0)?; + let expr5_0: Type = I64; + let expr6_0 = constructor_not(ctx, expr5_0, expr2_0)?; let expr7_0: Type = I64; - let expr8_0 = constructor_not(ctx, expr7_0, expr3_0)?; - let expr9_0: Type = I64; - let expr10_0 = constructor_not(ctx, expr9_0, expr6_0)?; - let expr11_0 = constructor_value_gprs(ctx, expr8_0, expr10_0)?; - return Some(expr11_0); + let expr8_0 = constructor_not(ctx, expr7_0, expr4_0)?; + let expr9_0 = constructor_value_gprs(ctx, expr6_0, expr8_0)?; + return Some(expr9_0); } // Generated as internal constructor for term vec_insert_lane. @@ -6197,7 +6268,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1365. + // Rule at src/isa/x64/lower.isle line 1410. let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = constructor_pinsrb(ctx, pattern2_0, &expr0_0, pattern4_0)?; return Some(expr1_0); @@ -6206,7 +6277,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1369. + // Rule at src/isa/x64/lower.isle line 1414. let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = constructor_pinsrw(ctx, pattern2_0, &expr0_0, pattern4_0)?; return Some(expr1_0); @@ -6215,7 +6286,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1373. + // Rule at src/isa/x64/lower.isle line 1418. let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = OperandSize::Size32; let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; @@ -6225,7 +6296,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1377. + // Rule at src/isa/x64/lower.isle line 1422. let expr0_0 = C::reg_mem_to_gpr_mem(ctx, pattern3_0); let expr1_0 = OperandSize::Size64; let expr2_0 = constructor_pinsrd(ctx, pattern2_0, &expr0_0, pattern4_0, &expr1_0)?; @@ -6235,7 +6306,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1381. + // Rule at src/isa/x64/lower.isle line 1426. let expr0_0 = C::reg_mem_to_xmm_mem(ctx, pattern3_0); let expr1_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr2_0 = constructor_insertps(ctx, pattern2_0, &expr0_0, expr1_0)?; @@ -6247,7 +6318,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1403. + // Rule at src/isa/x64/lower.isle line 1448. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); let expr2_0 = constructor_movsd(ctx, pattern2_0, &expr1_0)?; @@ -6256,7 +6327,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1405. + // Rule at src/isa/x64/lower.isle line 1450. let expr0_0 = SseOpcode::Movsd; let expr1_0 = C::reg_mem_to_xmm_mem(ctx, pattern3_0); let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -6265,7 +6336,7 @@ pub fn constructor_vec_insert_lane( return Some(expr4_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1414. + // Rule at src/isa/x64/lower.isle line 1459. let expr0_0 = C::reg_mem_to_xmm_mem(ctx, pattern3_0); let expr1_0 = constructor_movlhps(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -6287,13 +6358,13 @@ pub fn constructor_cmp_and_choose( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1422. - let expr0_0 = C::put_in_reg(ctx, pattern3_0); - let expr1_0 = C::put_in_reg(ctx, pattern4_0); + // Rule at src/isa/x64/lower.isle line 1467. + let expr0_0 = constructor_put_in_gpr(ctx, pattern3_0)?; + let expr1_0 = constructor_put_in_gpr(ctx, pattern4_0)?; let expr2_0 = C::raw_operand_size_of_type(ctx, pattern1_0); - let expr3_0 = RegMemImm::Reg { reg: expr0_0 }; + let expr3_0 = C::gpr_to_gpr_mem_imm(ctx, expr0_0); let expr4_0 = constructor_cmp(ctx, &expr2_0, &expr3_0, expr1_0)?; - let expr5_0 = RegMem::Reg { reg: expr1_0 }; + let expr5_0 = C::gpr_to_gpr_mem(ctx, expr1_0); let expr6_0 = constructor_cmove(ctx, pattern1_0, pattern2_0, &expr5_0, expr0_0)?; let expr7_0 = constructor_with_flags_1(ctx, &expr4_0, &expr6_0)?; let expr8_0 = C::value_reg(ctx, expr7_0); diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index aca0cf135f..b8b8825635 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -15,7 +15,6 @@ pub type ValueSlice<'a> = &'a [Value]; pub type ValueArray2 = [Value; 2]; pub type ValueArray3 = [Value; 3]; pub type WritableReg = Writable; -pub type OptionWritableReg = Option; pub type VecReg = Vec; pub type VecWritableReg = Vec; pub type ValueRegs = crate::machinst::ValueRegs;