diff --git a/.gitattributes b/.gitattributes index 928feac126..abbcc83672 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,4 +11,4 @@ # Tell GitHub this is generated code, and doesn't need to be shown in diffs by # default. -cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs linguist-generated +cranelift/codegen/src/isa/*/lower/isle/generated_code.rs linguist-generated diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index a5cc2eaf91..34e0a4a64b 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -198,6 +198,8 @@ fn get_isle_compilations(crate_dir: &std::path::Path) -> Result Result integer. + (FpuToInt + (op FpuToIntOp) + (rd WritableReg) + (rn Reg)) + + ;; Conversion: integer -> FP. + (IntToFpu + (op IntToFpuOp) + (rd WritableReg) + (rn Reg)) + + ;; FP conditional select, 32 bit. + (FpuCSel32 + (rd WritableReg) + (rn Reg) + (rm Reg) + (cond Cond)) + + ;; FP conditional select, 64 bit. + (FpuCSel64 + (rd WritableReg) + (rn Reg) + (rm Reg) + (cond Cond)) + + ;; Round to integer. + (FpuRound + (op FpuRoundMode) + (rd WritableReg) + (rn Reg)) + + ;; Move from a GPR to a vector register. The scalar value is parked in the lowest lane + ;; of the destination, and all other lanes are zeroed out. Currently only 32- and 64-bit + ;; transactions are supported. + (MovToFpu + (rd WritableReg) + (rn Reg) + (size ScalarSize)) + + ;; Move to a vector element from a GPR. + (MovToVec + (rd WritableReg) + (rn Reg) + (idx u8) + (size VectorSize)) + + ;; Unsigned move from a vector element to a GPR. + (MovFromVec + (rd WritableReg) + (rn Reg) + (idx u8) + (size VectorSize)) + + ;; Signed move from a vector element to a GPR. + (MovFromVecSigned + (rd WritableReg) + (rn Reg) + (idx u8) + (size VectorSize) + (scalar_size OperandSize)) + + ;; Duplicate general-purpose register to vector. + (VecDup + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Duplicate scalar to vector. + (VecDupFromFpu + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Duplicate FP immediate to vector. + (VecDupFPImm + (rd WritableReg) + (imm ASIMDFPModImm) + (size VectorSize)) + + ;; Duplicate immediate to vector. + (VecDupImm + (rd WritableReg) + (imm ASIMDMovModImm) + (invert bool) + (size VectorSize)) + + ;; Vector extend. + (VecExtend + (t VecExtendOp) + (rd WritableReg) + (rn Reg) + (high_half bool)) + + ;; Move vector element to another vector element. + (VecMovElement + (rd WritableReg) + (rn Reg) + (dest_idx u8) + (src_idx u8) + (size VectorSize)) + + ;; Vector widening operation. + (VecRRLong + (op VecRRLongOp) + (rd WritableReg) + (rn Reg) + (high_half bool)) + + ;; Vector narrowing operation. + (VecRRNarrow + (op VecRRNarrowOp) + (rd WritableReg) + (rn Reg) + (high_half bool)) + + ;; 1-operand vector instruction that operates on a pair of elements. + (VecRRPair + (op VecPairOp) + (rd WritableReg) + (rn Reg)) + + ;; 2-operand vector instruction that produces a result with twice the + ;; lane width and half the number of lanes. + (VecRRRLong + (alu_op VecRRRLongOp) + (rd WritableReg) + (rn Reg) + (rm Reg) + (high_half bool)) + + ;; 1-operand vector instruction that extends elements of the input + ;; register and operates on a pair of elements. The output lane width + ;; is double that of the input. + (VecRRPairLong + (op VecRRPairLongOp) + (rd WritableReg) + (rn Reg)) + + ;; A vector ALU op. + (VecRRR + (alu_op VecALUOp) + (rd WritableReg) + (rn Reg) + (rm Reg) + (size VectorSize)) + + ;; Vector two register miscellaneous instruction. + (VecMisc + (op VecMisc2) + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Vector instruction across lanes. + (VecLanes + (op VecLanesOp) + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Vector shift by immediate Shift Left (immediate), Unsigned Shift Right (immediate) + ;; Signed Shift Right (immediate). These are somewhat unusual in that, for right shifts, + ;; the allowed range of `imm` values is 1 to lane-size-in-bits, inclusive. A zero + ;; right-shift cannot be encoded. Left shifts are "normal", though, having valid `imm` + ;; values from 0 to lane-size-in-bits - 1 inclusive. + (VecShiftImm + (op VecShiftImmOp) + (rd WritableReg) + (rn Reg) + (size VectorSize) + (imm u8)) + + ;; Vector extract - create a new vector, being the concatenation of the lowest `imm4` bytes + ;; of `rm` followed by the uppermost `16 - imm4` bytes of `rn`. + (VecExtract + (rd WritableReg) + (rn Reg) + (rm Reg) + (imm4 u8)) + + ;; Table vector lookup - single register table. The table consists of 8-bit elements and is + ;; stored in `rn`, while `rm` contains 8-bit element indices. `is_extension` specifies whether + ;; to emit a TBX or a TBL instruction, i.e. whether to leave the elements in the destination + ;; vector that correspond to out-of-range indices (greater than 15) unmodified or to set them + ;; to 0. + (VecTbl + (rd WritableReg) + (rn Reg) + (rm Reg) + (is_extension bool)) + + ;; Table vector lookup - two register table. The table consists of 8-bit elements and is + ;; stored in `rn` and `rn2`, while `rm` contains 8-bit element indices. `is_extension` + ;; specifies whether to emit a TBX or a TBL instruction, i.e. whether to leave the elements in + ;; the destination vector that correspond to out-of-range indices (greater than 31) unmodified + ;; or to set them to 0. The table registers `rn` and `rn2` must have consecutive numbers + ;; modulo 32, that is v31 and v0 (in that order) are consecutive registers. + (VecTbl2 + (rd WritableReg) + (rn Reg) + (rn2 Reg) + (rm Reg) + (is_extension bool)) + + ;; Load an element and replicate to all lanes of a vector. + (VecLoadReplicate + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Vector conditional select, 128 bit. A synthetic instruction, which generates a 4-insn + ;; control-flow diamond. + (VecCSel + (rd WritableReg) + (rn Reg) + (rm Reg) + (cond Cond)) + + ;; Move to the NZCV flags (actually a `MSR NZCV, Xn` insn). + (MovToNZCV + (rn Reg)) + + ;; Move from the NZCV flags (actually a `MRS Xn, NZCV` insn). + (MovFromNZCV + (rd WritableReg)) + + ;; A machine call instruction. N.B.: this allows only a +/- 128MB offset (it uses a relocation + ;; of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the + ;; code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit + ;; target. + (Call + (info BoxCallInfo)) + + ;; A machine indirect-call instruction. + (CallInd + (info BoxCallIndInfo)) + + ;; ---- branches (exactly one must appear at end of BB) ---- + + ;; A machine return instruction. + (Ret) + + ;; A placeholder instruction, generating no code, meaning that a function epilogue must be + ;; inserted there. + (EpiloguePlaceholder) + + ;; An unconditional branch. + (Jump + (dest BranchTarget)) + + ;; A conditional branch. Contains two targets; at emission time, both are emitted, but + ;; the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the + ;; choice of taken/not_taken (inverting the branch polarity as needed) based on the + ;; fallthrough at the time of lowering. + (CondBr + (taken BranchTarget) + (not_taken BranchTarget) + (kind CondBrKind)) + + ;; A conditional trap: execute a `udf` if the condition is true. This is + ;; one VCode instruction because it uses embedded control flow; it is + ;; logically a single-in, single-out region, but needs to appear as one + ;; unit to the register allocator. + ;; + ;; The `CondBrKind` gives the conditional-branch condition that will + ;; *execute* the embedded `Inst`. (In the emitted code, we use the inverse + ;; of this condition in a branch that skips the trap instruction.) + (TrapIf + (kind CondBrKind) + (trap_code TrapCode)) + + ;; An indirect branch through a register, augmented with set of all + ;; possible successors. + (IndirectBr + (rn Reg) + (targets VecMachLabel)) + + ;; A "break" instruction, used for e.g. traps and debug breakpoints. + (Brk) + + ;; An instruction guaranteed to always be undefined and to trigger an illegal instruction at + ;; runtime. + (Udf + (trap_code TrapCode)) + + ;; Compute the address (using a PC-relative offset) of a memory location, using the `ADR` + ;; instruction. Note that we take a simple offset, not a `MemLabel`, here, because `Adr` is + ;; only used for now in fixed lowering sequences with hardcoded offsets. In the future we may + ;; need full `MemLabel` support. + (Adr + (rd WritableReg) + ;; Offset in range -2^20 .. 2^20. + (off i32)) + + ;; Raw 32-bit word, used for inline constants and jump-table entries. + (Word4 + (data u32)) + + ;; Raw 64-bit word, used for inline constants. + (Word8 + (data u64)) + + ;; Jump-table sequence, as one compound instruction (see note in lower_inst.rs for rationale). + (JTSequence + (info BoxJTSequenceInfo) + (ridx Reg) + (rtmp1 WritableReg) + (rtmp2 WritableReg)) + + ;; Load an inline symbol reference. + (LoadExtName + (rd WritableReg) + (name BoxExternalName) + (offset i64)) + + ;; Load address referenced by `mem` into `rd`. + (LoadAddr + (rd WritableReg) + (mem AMode)) + + ;; Marker, no-op in generated code: SP "virtual offset" is adjusted. This + ;; controls how AMode::NominalSPOffset args are lowered. + (VirtualSPOffsetAdj + (offset i64)) + + ;; Meta-insn, no-op in generated code: emit constant/branch veneer island + ;; at this point (with a guard jump around it) if less than the needed + ;; space is available before the next branch deadline. See the `MachBuffer` + ;; implementation in `machinst/buffer.rs` for the overall algorithm. In + ;; brief, we retain a set of "pending/unresolved label references" from + ;; branches as we scan forward through instructions to emit machine code; + ;; if we notice we're about to go out of range on an unresolved reference, + ;; we stop, emit a bunch of "veneers" (branches in a form that has a longer + ;; range, e.g. a 26-bit-offset unconditional jump), and point the original + ;; label references to those. This is an "island" because it comes in the + ;; middle of the code. + ;; + ;; This meta-instruction is a necessary part of the logic that determines + ;; where to place islands. Ordinarily, we want to place them between basic + ;; blocks, so we compute the worst-case size of each block, and emit the + ;; island before starting a block if we would exceed a deadline before the + ;; end of the block. However, some sequences (such as an inline jumptable) + ;; are variable-length and not accounted for by this logic; so these + ;; lowered sequences include an `EmitIsland` to trigger island generation + ;; where necessary. + (EmitIsland + ;; The needed space before the next deadline. + (needed_space CodeOffset)) + + ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol in x0. + (ElfTlsGetAddr + (symbol ExternalName)) + + ;; A definition of a value label. + (ValueLabelMarker + (reg Reg) + (label ValueLabel)) + + ;; An unwind pseudo-instruction. + (Unwind + (inst UnwindInst)) +)) + +;; An ALU operation. This can be paired with several instruction formats +;; below (see `Inst`) in any combination. +(type ALUOp + (enum + (Add32) + (Add64) + (Sub32) + (Sub64) + (Orr32) + (Orr64) + (OrrNot32) + (OrrNot64) + (And32) + (And64) + (AndS32) + (AndS64) + (AndNot32) + (AndNot64) + ;; XOR (AArch64 calls this "EOR") + (Eor32) + ;; XOR (AArch64 calls this "EOR") + (Eor64) + ;; XNOR (AArch64 calls this "EOR-NOT") + (EorNot32) + ;; XNOR (AArch64 calls this "EOR-NOT") + (EorNot64) + ;; Add, setting flags + (AddS32) + ;; Add, setting flags + (AddS64) + ;; Sub, setting flags + (SubS32) + ;; Sub, setting flags + (SubS64) + ;; Signed multiply, high-word result + (SMulH) + ;; Unsigned multiply, high-word result + (UMulH) + (SDiv64) + (UDiv64) + (RotR32) + (RotR64) + (Lsr32) + (Lsr64) + (Asr32) + (Asr64) + (Lsl32) + (Lsl64) + ;; Add with carry + (Adc32) + (Adc64) + ;; Add with carry, settings flags + (AdcS32) + (AdcS64) + ;; Subtract with carry + (Sbc32) + (Sbc64) + ;; Subtract with carry, settings flags + (SbcS32) + (SbcS64) +)) + +;; An ALU operation with three arguments. +(type ALUOp3 + (enum + ;; Multiply-add + (MAdd32) + ;; Multiply-add + (MAdd64) + ;; Multiply-sub + (MSub32) + ;; Multiply-sub + (MSub64) +)) + +(type UImm5 (primitive UImm5)) +(type Imm12 (primitive Imm12)) +(type ImmLogic (primitive ImmLogic)) +(type ImmShift (primitive ImmShift)) +(type ShiftOpAndAmt (primitive ShiftOpAndAmt)) +(type MoveWideConst (primitive MoveWideConst)) +(type NZCV (primitive NZCV)) +(type ASIMDFPModImm (primitive ASIMDFPModImm)) +(type ASIMDMovModImm (primitive ASIMDMovModImm)) + +(type BoxCallInfo (primitive BoxCallInfo)) +(type BoxCallIndInfo (primitive BoxCallIndInfo)) +(type VecMachLabel (primitive VecMachLabel)) +(type CondBrKind (primitive CondBrKind)) +(type BranchTarget (primitive BranchTarget)) +(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) +(type BoxExternalName (primitive BoxExternalName)) +(type CodeOffset (primitive CodeOffset)) +(type ExternalName (primitive ExternalName)) +(type ValueLabel (primitive ValueLabel)) +(type UnwindInst (primitive UnwindInst)) + +(type ExtendOp extern + (enum + (UXTB) + (UXTH) + (UXTW) + (UXTX) + (SXTB) + (SXTH) + (SXTW) + (SXTX) +)) + +;; An operation on the bits of a register. This can be paired with several instruction formats +;; below (see `Inst`) in any combination. +(type BitOp + (enum + ;; Bit reverse + (RBit32) + ;; Bit reverse + (RBit64) + (Clz32) + (Clz64) + (Cls32) + (Cls64) +)) + +(type AMode extern (enum)) +(type PairAMode extern (enum)) +(type FPUOpRI extern (enum)) + +(type OperandSize extern + (enum Size32 + Size64)) + +(type ScalarSize extern + (enum Size8 + Size16 + Size32 + Size64 + Size128)) + +(type Cond extern + (enum + (Eq) + (Ne) + (Hs) + (Lo) + (Mi) + (Pl) + (Vs) + (Vc) + (Hi) + (Ls) + (Ge) + (Lt) + (Gt) + (Le) + (Al) + (Nv) +)) + +(type VectorSize extern + (enum + (Size8x8) + (Size8x16) + (Size16x4) + (Size16x8) + (Size32x2) + (Size32x4) + (Size64x2) +)) + +;; A floating-point unit (FPU) operation with one arg. +(type FPUOp1 + (enum + (Abs32) + (Abs64) + (Neg32) + (Neg64) + (Sqrt32) + (Sqrt64) + (Cvt32To64) + (Cvt64To32) +)) + +;; A floating-point unit (FPU) operation with two args. +(type FPUOp2 + (enum + (Add32) + (Add64) + (Sub32) + (Sub64) + (Mul32) + (Mul64) + (Div32) + (Div64) + (Max32) + (Max64) + (Min32) + (Min64) + ;; Signed saturating add + (Sqadd64) + ;; Unsigned saturating add + (Uqadd64) + ;; Signed saturating subtract + (Sqsub64) + ;; Unsigned saturating subtract + (Uqsub64) +)) + +;; A floating-point unit (FPU) operation with three args. +(type FPUOp3 + (enum + (MAdd32) + (MAdd64) +)) + +;; A conversion from an FP to an integer value. +(type FpuToIntOp + (enum + (F32ToU32) + (F32ToI32) + (F32ToU64) + (F32ToI64) + (F64ToU32) + (F64ToI32) + (F64ToU64) + (F64ToI64) +)) + +;; A conversion from an integer to an FP value. +(type IntToFpuOp + (enum + (U32ToF32) + (I32ToF32) + (U32ToF64) + (I32ToF64) + (U64ToF32) + (I64ToF32) + (U64ToF64) + (I64ToF64) +)) + +;; Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to +;; nearest, and for 32- or 64-bit FP values. +(type FpuRoundMode + (enum + (Minus32) + (Minus64) + (Plus32) + (Plus64) + (Zero32) + (Zero64) + (Nearest32) + (Nearest64) +)) + +;; Type of vector element extensions. +(type VecExtendOp + (enum + ;; Signed extension of 8-bit elements + (Sxtl8) + ;; Signed extension of 16-bit elements + (Sxtl16) + ;; Signed extension of 32-bit elements + (Sxtl32) + ;; Unsigned extension of 8-bit elements + (Uxtl8) + ;; Unsigned extension of 16-bit elements + (Uxtl16) + ;; Unsigned extension of 32-bit elements + (Uxtl32) +)) + +;; A vector ALU operation. +(type VecALUOp + (enum + ;; Signed saturating add + (Sqadd) + ;; Unsigned saturating add + (Uqadd) + ;; Signed saturating subtract + (Sqsub) + ;; Unsigned saturating subtract + (Uqsub) + ;; Compare bitwise equal + (Cmeq) + ;; Compare signed greater than or equal + (Cmge) + ;; Compare signed greater than + (Cmgt) + ;; Compare unsigned higher + (Cmhs) + ;; Compare unsigned higher or same + (Cmhi) + ;; Floating-point compare equal + (Fcmeq) + ;; Floating-point compare greater than + (Fcmgt) + ;; Floating-point compare greater than or equal + (Fcmge) + ;; Bitwise and + (And) + ;; Bitwise bit clear + (Bic) + ;; Bitwise inclusive or + (Orr) + ;; Bitwise exclusive or + (Eor) + ;; Bitwise select + (Bsl) + ;; Unsigned maximum pairwise + (Umaxp) + ;; Add + (Add) + ;; Subtract + (Sub) + ;; Multiply + (Mul) + ;; Signed shift left + (Sshl) + ;; Unsigned shift left + (Ushl) + ;; Unsigned minimum + (Umin) + ;; Signed minimum + (Smin) + ;; Unsigned maximum + (Umax) + ;; Signed maximum + (Smax) + ;; Unsigned rounding halving add + (Urhadd) + ;; Floating-point add + (Fadd) + ;; Floating-point subtract + (Fsub) + ;; Floating-point divide + (Fdiv) + ;; Floating-point maximum + (Fmax) + ;; Floating-point minimum + (Fmin) + ;; Floating-point multiply + (Fmul) + ;; Add pairwise + (Addp) + ;; Zip vectors (primary) [meaning, high halves] + (Zip1) + ;; Signed saturating rounding doubling multiply returning high half + (Sqrdmulh) +)) + +;; A Vector miscellaneous operation with two registers. +(type VecMisc2 + (enum + ;; Bitwise NOT + (Not) + ;; Negate + (Neg) + ;; Absolute value + (Abs) + ;; Floating-point absolute value + (Fabs) + ;; Floating-point negate + (Fneg) + ;; Floating-point square root + (Fsqrt) + ;; Reverse elements in 64-bit doublewords + (Rev64) + ;; Floating-point convert to signed integer, rounding toward zero + (Fcvtzs) + ;; Floating-point convert to unsigned integer, rounding toward zero + (Fcvtzu) + ;; Signed integer convert to floating-point + (Scvtf) + ;; Unsigned integer convert to floating-point + (Ucvtf) + ;; Floating point round to integral, rounding towards nearest + (Frintn) + ;; Floating point round to integral, rounding towards zero + (Frintz) + ;; Floating point round to integral, rounding towards minus infinity + (Frintm) + ;; Floating point round to integral, rounding towards plus infinity + (Frintp) + ;; Population count per byte + (Cnt) + ;; Compare bitwise equal to 0 + (Cmeq0) +)) + +;; A vector widening operation with one argument. +(type VecRRLongOp + (enum + ;; Floating-point convert to higher precision long, 16-bit elements + (Fcvtl16) + ;; Floating-point convert to higher precision long, 32-bit elements + (Fcvtl32) + ;; Shift left long (by element size), 8-bit elements + (Shll8) + ;; Shift left long (by element size), 16-bit elements + (Shll16) + ;; Shift left long (by element size), 32-bit elements + (Shll32) +)) + +;; A vector narrowing operation with one argument. +(type VecRRNarrowOp + (enum + ;; Extract narrow, 16-bit elements + (Xtn16) + ;; Extract narrow, 32-bit elements + (Xtn32) + ;; Extract narrow, 64-bit elements + (Xtn64) + ;; Signed saturating extract narrow, 16-bit elements + (Sqxtn16) + ;; Signed saturating extract narrow, 32-bit elements + (Sqxtn32) + ;; Signed saturating extract narrow, 64-bit elements + (Sqxtn64) + ;; Signed saturating extract unsigned narrow, 16-bit elements + (Sqxtun16) + ;; Signed saturating extract unsigned narrow, 32-bit elements + (Sqxtun32) + ;; Signed saturating extract unsigned narrow, 64-bit elements + (Sqxtun64) + ;; Unsigned saturating extract narrow, 16-bit elements + (Uqxtn16) + ;; Unsigned saturating extract narrow, 32-bit elements + (Uqxtn32) + ;; Unsigned saturating extract narrow, 64-bit elements + (Uqxtn64) + ;; Floating-point convert to lower precision narrow, 32-bit elements + (Fcvtn32) + ;; Floating-point convert to lower precision narrow, 64-bit elements + (Fcvtn64) +)) + +(type VecRRRLongOp + (enum + ;; Signed multiply long. + (Smull8) + (Smull16) + (Smull32) + ;; Unsigned multiply long. + (Umull8) + (Umull16) + (Umull32) + ;; Unsigned multiply add long + (Umlal8) + (Umlal16) + (Umlal32) +)) + +;; A vector operation on a pair of elements with one register. +(type VecPairOp + (enum + ;; Add pair of elements + (Addp) +)) + +;; 1-operand vector instruction that extends elements of the input register +;; and operates on a pair of elements. +(type VecRRPairLongOp + (enum + ;; Sign extend and add pair of elements + (Saddlp8) + (Saddlp16) + ;; Unsigned extend and add pair of elements + (Uaddlp8) + (Uaddlp16) +)) + +;; An operation across the lanes of vectors. +(type VecLanesOp + (enum + ;; Integer addition across a vector + (Addv) + ;; Unsigned minimum across a vector + (Uminv) +)) + +;; A shift-by-immediate operation on each lane of a vector. +(type VecShiftImmOp + (enum + ;; Unsigned shift left + (Shl) + ;; Unsigned shift right + (Ushr) + ;; Signed shift right + (Sshr) +)) + +;; Atomic read-modify-write operations with acquire-release semantics +(type AtomicRMWOp + (enum + (Add) + (Clr) + (Eor) + (Set) + (Smax) + (Smin) + (Umax) + (Umin) +)) + +;; Extractor helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl move_wide_const_from_u64 (MoveWideConst) u64) +(extern extractor move_wide_const_from_u64 move_wide_const_from_u64) + +(decl move_wide_const_from_negated_u64 (MoveWideConst) u64) +(extern extractor move_wide_const_from_negated_u64 move_wide_const_from_negated_u64) + +(decl imm_logic_from_u64 (ImmLogic) u64) +(extern extractor imm_logic_from_u64 imm_logic_from_u64) + +(decl integral_ty (Type) Type) +(extern extractor integral_ty integral_ty) + +;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Emit an instruction. +;; +;; This is low-level and side-effectful; it should only be used as an +;; implementation detail by helpers that preserve the SSA facade themselves. +(decl emit (MInst) Unit) +(extern constructor emit emit) + +;; Helper for creating the zero register. +(decl zero_reg () Reg) +(extern constructor zero_reg zero_reg) + +;; Helper for emitting `MInst.MovZ` instructions. +(decl movz (MoveWideConst OperandSize) Reg) +(rule (movz imm size) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.MovZ dst imm size)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.MovN` instructions. +(decl movn (MoveWideConst OperandSize) Reg) +(rule (movn imm size) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.MovN dst imm size)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRImmLogic` instructions. +(decl alu_rr_imm_logic (ALUOp Reg ImmLogic) Reg) +(rule (alu_rr_imm_logic op src imm) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRImmLogic op dst src imm)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `orr64` instructions. +(decl orr64 (Reg ImmLogic) Reg) +(rule (orr64 src imm) (alu_rr_imm_logic (ALUOp.Orr64) src imm)) + +;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl imm (Type u64) Reg) + +;; 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ +(rule (imm (integral_ty _ty) (move_wide_const_from_u64 n)) + (movz n (OperandSize.Size64))) + +;; 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVN +(rule (imm (integral_ty _ty) (move_wide_const_from_negated_u64 n)) + (movn n (OperandSize.Size64))) + +;; Weird logical-instruction immediate in ORI using zero register +(rule (imm (integral_ty _ty) (imm_logic_from_u64 n)) + (orr64 (zero_reg) n)) + +(decl load_constant64_full (u64) Reg) +(extern constructor load_constant64_full load_constant64_full) + +;; Fallback for integral 64-bit constants that uses lots of `movk` +(rule (imm (integral_ty _ty) n) + (load_constant64_full n)) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 5b8fe374d8..915657c2f5 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -1,4 +1,5 @@ use crate::ir::types::*; +use crate::ir::TrapCode; use crate::isa::aarch64::inst::*; use crate::isa::test_utils; use crate::isa::CallConv; diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rs index 08fc75431d..79473375dd 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs @@ -327,7 +327,7 @@ impl Imm12 { } /// An immediate for logical instructions. -#[derive(Clone, Debug, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq)] pub struct ImmLogic { /// The actual value. value: u64, diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 98e3a85cff..83d878b746 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -7,16 +7,14 @@ use crate::binemit::{Addend, CodeOffset, Reloc}; use crate::ir::types::{ B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64, }; -use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; -use crate::isa::unwind::UnwindInst; +use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, Type, ValueLabel}; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; +use regalloc::RegUsageCollector; use regalloc::{PrettyPrint, RealRegUniverse, Reg, RegClass, SpillSlot, VirtualReg, Writable}; -use regalloc::{RegUsageCollector, RegUsageMapper}; -use alloc::boxed::Box; use alloc::vec::Vec; use core::convert::TryFrom; use smallvec::{smallvec, SmallVec}; @@ -40,118 +38,11 @@ mod emit_tests; //============================================================================= // Instructions (top level): definition -/// An ALU operation. This can be paired with several instruction formats -/// below (see `Inst`) in any combination. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum ALUOp { - Add32, - Add64, - Sub32, - Sub64, - Orr32, - Orr64, - OrrNot32, - OrrNot64, - And32, - And64, - AndS32, - AndS64, - AndNot32, - AndNot64, - /// XOR (AArch64 calls this "EOR") - Eor32, - /// XOR (AArch64 calls this "EOR") - Eor64, - /// XNOR (AArch64 calls this "EOR-NOT") - EorNot32, - /// XNOR (AArch64 calls this "EOR-NOT") - EorNot64, - /// Add, setting flags - AddS32, - /// Add, setting flags - AddS64, - /// Sub, setting flags - SubS32, - /// Sub, setting flags - SubS64, - /// Signed multiply, high-word result - SMulH, - /// Unsigned multiply, high-word result - UMulH, - SDiv64, - UDiv64, - RotR32, - RotR64, - Lsr32, - Lsr64, - Asr32, - Asr64, - Lsl32, - Lsl64, - /// Add with carry - Adc32, - Adc64, - /// Add with carry, settings flags - AdcS32, - AdcS64, - /// Subtract with carry - Sbc32, - Sbc64, - /// Subtract with carry, settings flags - SbcS32, - SbcS64, -} - -/// An ALU operation with three arguments. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum ALUOp3 { - /// Multiply-add - MAdd32, - /// Multiply-add - MAdd64, - /// Multiply-sub - MSub32, - /// Multiply-sub - MSub64, -} - -/// A floating-point unit (FPU) operation with one arg. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FPUOp1 { - Abs32, - Abs64, - Neg32, - Neg64, - Sqrt32, - Sqrt64, - Cvt32To64, - Cvt64To32, -} - -/// A floating-point unit (FPU) operation with two args. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FPUOp2 { - Add32, - Add64, - Sub32, - Sub64, - Mul32, - Mul64, - Div32, - Div64, - Max32, - Max64, - Min32, - Min64, - /// Signed saturating add - Sqadd64, - /// Unsigned saturating add - Uqadd64, - /// Signed saturating subtract - Sqsub64, - /// Unsigned saturating subtract - Uqsub64, -} +pub use crate::isa::aarch64::lower::isle::generated_code::{ + ALUOp, ALUOp3, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, + IntToFpuOp, MInst as Inst, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, + VecRRNarrowOp, VecRRPairLongOp, VecRRRLongOp, VecShiftImmOp, +}; /// A floating-point unit (FPU) operation with two args, a register and an immediate. #[derive(Copy, Clone, Debug)] @@ -166,318 +57,6 @@ pub enum FPUOpRI { Sli64(FPULeftShiftImm), } -/// A floating-point unit (FPU) operation with three args. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FPUOp3 { - MAdd32, - MAdd64, -} - -/// A conversion from an FP to an integer value. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FpuToIntOp { - F32ToU32, - F32ToI32, - F32ToU64, - F32ToI64, - F64ToU32, - F64ToI32, - F64ToU64, - F64ToI64, -} - -/// A conversion from an integer to an FP value. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum IntToFpuOp { - U32ToF32, - I32ToF32, - U32ToF64, - I32ToF64, - U64ToF32, - I64ToF32, - U64ToF64, - I64ToF64, -} - -/// Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to -/// nearest, and for 32- or 64-bit FP values. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FpuRoundMode { - Minus32, - Minus64, - Plus32, - Plus64, - Zero32, - Zero64, - Nearest32, - Nearest64, -} - -/// Type of vector element extensions. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecExtendOp { - /// Signed extension of 8-bit elements - Sxtl8, - /// Signed extension of 16-bit elements - Sxtl16, - /// Signed extension of 32-bit elements - Sxtl32, - /// Unsigned extension of 8-bit elements - Uxtl8, - /// Unsigned extension of 16-bit elements - Uxtl16, - /// Unsigned extension of 32-bit elements - Uxtl32, -} - -/// A vector ALU operation. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecALUOp { - /// Signed saturating add - Sqadd, - /// Unsigned saturating add - Uqadd, - /// Signed saturating subtract - Sqsub, - /// Unsigned saturating subtract - Uqsub, - /// Compare bitwise equal - Cmeq, - /// Compare signed greater than or equal - Cmge, - /// Compare signed greater than - Cmgt, - /// Compare unsigned higher - Cmhs, - /// Compare unsigned higher or same - Cmhi, - /// Floating-point compare equal - Fcmeq, - /// Floating-point compare greater than - Fcmgt, - /// Floating-point compare greater than or equal - Fcmge, - /// Bitwise and - And, - /// Bitwise bit clear - Bic, - /// Bitwise inclusive or - Orr, - /// Bitwise exclusive or - Eor, - /// Bitwise select - Bsl, - /// Unsigned maximum pairwise - Umaxp, - /// Add - Add, - /// Subtract - Sub, - /// Multiply - Mul, - /// Signed shift left - Sshl, - /// Unsigned shift left - Ushl, - /// Unsigned minimum - Umin, - /// Signed minimum - Smin, - /// Unsigned maximum - Umax, - /// Signed maximum - Smax, - /// Unsigned rounding halving add - Urhadd, - /// Floating-point add - Fadd, - /// Floating-point subtract - Fsub, - /// Floating-point divide - Fdiv, - /// Floating-point maximum - Fmax, - /// Floating-point minimum - Fmin, - /// Floating-point multiply - Fmul, - /// Add pairwise - Addp, - /// Zip vectors (primary) [meaning, high halves] - Zip1, - /// Signed saturating rounding doubling multiply returning high half - Sqrdmulh, -} - -/// A Vector miscellaneous operation with two registers. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecMisc2 { - /// Bitwise NOT - Not, - /// Negate - Neg, - /// Absolute value - Abs, - /// Floating-point absolute value - Fabs, - /// Floating-point negate - Fneg, - /// Floating-point square root - Fsqrt, - /// Reverse elements in 64-bit doublewords - Rev64, - /// Floating-point convert to signed integer, rounding toward zero - Fcvtzs, - /// Floating-point convert to unsigned integer, rounding toward zero - Fcvtzu, - /// Signed integer convert to floating-point - Scvtf, - /// Unsigned integer convert to floating-point - Ucvtf, - /// Floating point round to integral, rounding towards nearest - Frintn, - /// Floating point round to integral, rounding towards zero - Frintz, - /// Floating point round to integral, rounding towards minus infinity - Frintm, - /// Floating point round to integral, rounding towards plus infinity - Frintp, - /// Population count per byte - Cnt, - /// Compare bitwise equal to 0 - Cmeq0, -} - -/// A vector widening operation with one argument. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRLongOp { - /// Floating-point convert to higher precision long, 16-bit elements - Fcvtl16, - /// Floating-point convert to higher precision long, 32-bit elements - Fcvtl32, - /// Shift left long (by element size), 8-bit elements - Shll8, - /// Shift left long (by element size), 16-bit elements - Shll16, - /// Shift left long (by element size), 32-bit elements - Shll32, -} - -/// A vector narrowing operation with one argument. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRNarrowOp { - /// Extract narrow, 16-bit elements - Xtn16, - /// Extract narrow, 32-bit elements - Xtn32, - /// Extract narrow, 64-bit elements - Xtn64, - /// Signed saturating extract narrow, 16-bit elements - Sqxtn16, - /// Signed saturating extract narrow, 32-bit elements - Sqxtn32, - /// Signed saturating extract narrow, 64-bit elements - Sqxtn64, - /// Signed saturating extract unsigned narrow, 16-bit elements - Sqxtun16, - /// Signed saturating extract unsigned narrow, 32-bit elements - Sqxtun32, - /// Signed saturating extract unsigned narrow, 64-bit elements - Sqxtun64, - /// Unsigned saturating extract narrow, 16-bit elements - Uqxtn16, - /// Unsigned saturating extract narrow, 32-bit elements - Uqxtn32, - /// Unsigned saturating extract narrow, 64-bit elements - Uqxtn64, - /// Floating-point convert to lower precision narrow, 32-bit elements - Fcvtn32, - /// Floating-point convert to lower precision narrow, 64-bit elements - Fcvtn64, -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRRLongOp { - /// Signed multiply long. - Smull8, - Smull16, - Smull32, - /// Unsigned multiply long. - Umull8, - Umull16, - Umull32, - /// Unsigned multiply add long - Umlal8, - Umlal16, - Umlal32, -} - -/// A vector operation on a pair of elements with one register. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecPairOp { - /// Add pair of elements - Addp, -} - -/// 1-operand vector instruction that extends elements of the input register -/// and operates on a pair of elements. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRPairLongOp { - /// Sign extend and add pair of elements - Saddlp8, - Saddlp16, - /// Unsigned extend and add pair of elements - Uaddlp8, - Uaddlp16, -} - -/// An operation across the lanes of vectors. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecLanesOp { - /// Integer addition across a vector - Addv, - /// Unsigned minimum across a vector - Uminv, -} - -/// A shift-by-immediate operation on each lane of a vector. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecShiftImmOp { - // Unsigned shift left - Shl, - // Unsigned shift right - Ushr, - // Signed shift right - Sshr, -} - -/// Atomic read-modify-write operations with acquire-release semantics -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum AtomicRMWOp { - Add, - Clr, - Eor, - Set, - Smax, - Smin, - Umax, - Umin, -} - -/// An operation on the bits of a register. This can be paired with several instruction formats -/// below (see `Inst`) in any combination. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum BitOp { - /// Bit reverse - RBit32, - /// Bit reverse - RBit64, - Clz32, - Clz64, - Cls32, - Cls64, -} - impl BitOp { /// What is the opcode's native width? pub fn operand_size(&self) -> OperandSize { @@ -545,864 +124,6 @@ pub struct JTSequenceInfo { pub targets_for_term: Vec, // needed for MachTerminator. } -/// Instruction formats. -#[derive(Clone, Debug)] -pub enum Inst { - /// A no-op of zero size. - Nop0, - - /// A no-op that is one instruction large. - Nop4, - - /// An ALU operation with two register sources and a register destination. - AluRRR { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - }, - /// An ALU operation with three register sources and a register destination. - AluRRRR { - alu_op: ALUOp3, - rd: Writable, - rn: Reg, - rm: Reg, - ra: Reg, - }, - /// An ALU operation with a register source and an immediate-12 source, and a register - /// destination. - AluRRImm12 { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - imm12: Imm12, - }, - /// An ALU operation with a register source and an immediate-logic source, and a register destination. - AluRRImmLogic { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - imml: ImmLogic, - }, - /// An ALU operation with a register source and an immediate-shiftamt source, and a register destination. - AluRRImmShift { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - immshift: ImmShift, - }, - /// An ALU operation with two register sources, one of which can be shifted, and a register - /// destination. - AluRRRShift { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - shiftop: ShiftOpAndAmt, - }, - /// An ALU operation with two register sources, one of which can be {zero,sign}-extended and - /// shifted, and a register destination. - AluRRRExtend { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - extendop: ExtendOp, - }, - - /// A bit op instruction with a single register source. - BitRR { - op: BitOp, - rd: Writable, - rn: Reg, - }, - - /// An unsigned (zero-extending) 8-bit load. - ULoad8 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A signed (sign-extending) 8-bit load. - SLoad8 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// An unsigned (zero-extending) 16-bit load. - ULoad16 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A signed (sign-extending) 16-bit load. - SLoad16 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// An unsigned (zero-extending) 32-bit load. - ULoad32 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A signed (sign-extending) 32-bit load. - SLoad32 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A 64-bit load. - ULoad64 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - - /// An 8-bit store. - Store8 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A 16-bit store. - Store16 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A 32-bit store. - Store32 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A 64-bit store. - Store64 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - - /// A store of a pair of registers. - StoreP64 { - rt: Reg, - rt2: Reg, - mem: PairAMode, - flags: MemFlags, - }, - /// A load of a pair of registers. - LoadP64 { - rt: Writable, - rt2: Writable, - mem: PairAMode, - flags: MemFlags, - }, - - /// A MOV instruction. These are encoded as ORR's (AluRRR form) but we - /// keep them separate at the `Inst` level for better pretty-printing - /// and faster `is_move()` logic. - Mov64 { - rd: Writable, - rm: Reg, - }, - - /// A 32-bit MOV. Zeroes the top 32 bits of the destination. This is - /// effectively an alias for an unsigned 32-to-64-bit extension. - Mov32 { - rd: Writable, - rm: Reg, - }, - - /// A MOVZ with a 16-bit immediate. - MovZ { - rd: Writable, - imm: MoveWideConst, - size: OperandSize, - }, - - /// A MOVN with a 16-bit immediate. - MovN { - rd: Writable, - imm: MoveWideConst, - size: OperandSize, - }, - - /// A MOVK with a 16-bit immediate. - MovK { - rd: Writable, - imm: MoveWideConst, - size: OperandSize, - }, - - /// A sign- or zero-extend operation. - Extend { - rd: Writable, - rn: Reg, - signed: bool, - from_bits: u8, - to_bits: u8, - }, - - /// A conditional-select operation. - CSel { - rd: Writable, - cond: Cond, - rn: Reg, - rm: Reg, - }, - - /// A conditional-set operation. - CSet { - rd: Writable, - cond: Cond, - }, - - /// A conditional-set-mask operation. - CSetm { - rd: Writable, - cond: Cond, - }, - - /// A conditional comparison with an immediate. - CCmpImm { - size: OperandSize, - rn: Reg, - imm: UImm5, - nzcv: NZCV, - cond: Cond, - }, - - /// A synthetic insn, which is a load-linked store-conditional loop, that has the overall - /// effect of atomically modifying a memory location in a particular way. Because we have - /// no way to explain to the regalloc about earlyclobber registers, this instruction has - /// completely fixed operand registers, and we rely on the RA's coalescing to remove copies - /// in the surrounding code to the extent it can. The sequence is both preceded and - /// followed by a fence which is at least as comprehensive as that of the `Fence` - /// instruction below. This instruction is sequentially consistent. The operand - /// conventions are: - /// - /// x25 (rd) address - /// x26 (rd) second operand for `op` - /// x27 (wr) old value - /// x24 (wr) scratch reg; value afterwards has no meaning - /// x28 (wr) scratch reg; value afterwards has no meaning - AtomicRMWLoop { - ty: Type, // I8, I16, I32 or I64 - op: inst_common::AtomicRmwOp, - }, - - /// An atomic read-modify-write operation. These instructions require the - /// Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have - /// acquire-release semantics. - AtomicRMW { - op: AtomicRMWOp, - rs: Reg, - rt: Writable, - rn: Reg, - ty: Type, - }, - - /// An atomic compare-and-swap operation. This instruction is sequentially consistent. - AtomicCAS { - rs: Writable, - rt: Reg, - rn: Reg, - ty: Type, - }, - - /// Similar to AtomicRMWLoop, a compare-and-swap operation implemented using a load-linked - /// store-conditional loop. - /// This instruction is sequentially consistent. - /// Note that the operand conventions, although very similar to AtomicRMWLoop, are different: - /// - /// x25 (rd) address - /// x26 (rd) expected value - /// x28 (rd) replacement value - /// x27 (wr) old value - /// x24 (wr) scratch reg; value afterwards has no meaning - AtomicCASLoop { - ty: Type, // I8, I16, I32 or I64 - }, - - /// Read `access_ty` bits from address `rt`, either 8, 16, 32 or 64-bits, and put - /// it in `rn`, optionally zero-extending to fill a word or double word result. - /// This instruction is sequentially consistent. - LoadAcquire { - access_ty: Type, // I8, I16, I32 or I64 - rt: Writable, - rn: Reg, - }, - - /// Write the lowest `ty` bits of `rt` to address `rn`. - /// This instruction is sequentially consistent. - StoreRelease { - access_ty: Type, // I8, I16, I32 or I64 - rt: Reg, - rn: Reg, - }, - - /// A memory fence. This must provide ordering to ensure that, at a minimum, neither loads - /// nor stores may move forwards or backwards across the fence. Currently emitted as "dmb - /// ish". This instruction is sequentially consistent. - Fence, - - /// FPU move. Note that this is distinct from a vector-register - /// move; moving just 64 bits seems to be significantly faster. - FpuMove64 { - rd: Writable, - rn: Reg, - }, - - /// Vector register move. - FpuMove128 { - rd: Writable, - rn: Reg, - }, - - /// Move to scalar from a vector element. - FpuMoveFromVec { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - }, - - /// Zero-extend a SIMD & FP scalar to the full width of a vector register. - /// 16-bit scalars require half-precision floating-point support (FEAT_FP16). - FpuExtend { - rd: Writable, - rn: Reg, - size: ScalarSize, - }, - - /// 1-op FPU instruction. - FpuRR { - fpu_op: FPUOp1, - rd: Writable, - rn: Reg, - }, - - /// 2-op FPU instruction. - FpuRRR { - fpu_op: FPUOp2, - rd: Writable, - rn: Reg, - rm: Reg, - }, - - FpuRRI { - fpu_op: FPUOpRI, - rd: Writable, - rn: Reg, - }, - - /// 3-op FPU instruction. - FpuRRRR { - fpu_op: FPUOp3, - rd: Writable, - rn: Reg, - rm: Reg, - ra: Reg, - }, - - /// FPU comparison, single-precision (32 bit). - FpuCmp32 { - rn: Reg, - rm: Reg, - }, - - /// FPU comparison, double-precision (64 bit). - FpuCmp64 { - rn: Reg, - rm: Reg, - }, - - /// Floating-point load, single-precision (32 bit). - FpuLoad32 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point store, single-precision (32 bit). - FpuStore32 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point load, double-precision (64 bit). - FpuLoad64 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point store, double-precision (64 bit). - FpuStore64 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point/vector load, 128 bit. - FpuLoad128 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point/vector store, 128 bit. - FpuStore128 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A load of a pair of floating-point registers, double precision (64-bit). - FpuLoadP64 { - rt: Writable, - rt2: Writable, - mem: PairAMode, - flags: MemFlags, - }, - /// A store of a pair of floating-point registers, double precision (64-bit). - FpuStoreP64 { - rt: Reg, - rt2: Reg, - mem: PairAMode, - flags: MemFlags, - }, - /// A load of a pair of floating-point registers, 128-bit. - FpuLoadP128 { - rt: Writable, - rt2: Writable, - mem: PairAMode, - flags: MemFlags, - }, - /// A store of a pair of floating-point registers, 128-bit. - FpuStoreP128 { - rt: Reg, - rt2: Reg, - mem: PairAMode, - flags: MemFlags, - }, - LoadFpuConst64 { - rd: Writable, - const_data: u64, - }, - - LoadFpuConst128 { - rd: Writable, - const_data: u128, - }, - - /// Conversion: FP -> integer. - FpuToInt { - op: FpuToIntOp, - rd: Writable, - rn: Reg, - }, - - /// Conversion: integer -> FP. - IntToFpu { - op: IntToFpuOp, - rd: Writable, - rn: Reg, - }, - - /// FP conditional select, 32 bit. - FpuCSel32 { - rd: Writable, - rn: Reg, - rm: Reg, - cond: Cond, - }, - /// FP conditional select, 64 bit. - FpuCSel64 { - rd: Writable, - rn: Reg, - rm: Reg, - cond: Cond, - }, - - /// Round to integer. - FpuRound { - op: FpuRoundMode, - rd: Writable, - rn: Reg, - }, - - /// Move from a GPR to a vector register. The scalar value is parked in the lowest lane - /// of the destination, and all other lanes are zeroed out. Currently only 32- and 64-bit - /// transactions are supported. - MovToFpu { - rd: Writable, - rn: Reg, - size: ScalarSize, - }, - - /// Move to a vector element from a GPR. - MovToVec { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - }, - - /// Unsigned move from a vector element to a GPR. - MovFromVec { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - }, - - /// Signed move from a vector element to a GPR. - MovFromVecSigned { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - scalar_size: OperandSize, - }, - - /// Duplicate general-purpose register to vector. - VecDup { - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Duplicate scalar to vector. - VecDupFromFpu { - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Duplicate FP immediate to vector. - VecDupFPImm { - rd: Writable, - imm: ASIMDFPModImm, - size: VectorSize, - }, - - /// Duplicate immediate to vector. - VecDupImm { - rd: Writable, - imm: ASIMDMovModImm, - invert: bool, - size: VectorSize, - }, - - /// Vector extend. - VecExtend { - t: VecExtendOp, - rd: Writable, - rn: Reg, - high_half: bool, - }, - - /// Move vector element to another vector element. - VecMovElement { - rd: Writable, - rn: Reg, - dest_idx: u8, - src_idx: u8, - size: VectorSize, - }, - - /// Vector widening operation. - VecRRLong { - op: VecRRLongOp, - rd: Writable, - rn: Reg, - high_half: bool, - }, - - /// Vector narrowing operation. - VecRRNarrow { - op: VecRRNarrowOp, - rd: Writable, - rn: Reg, - high_half: bool, - }, - - /// 1-operand vector instruction that operates on a pair of elements. - VecRRPair { - op: VecPairOp, - rd: Writable, - rn: Reg, - }, - - /// 2-operand vector instruction that produces a result with twice the - /// lane width and half the number of lanes. - VecRRRLong { - alu_op: VecRRRLongOp, - rd: Writable, - rn: Reg, - rm: Reg, - high_half: bool, - }, - - /// 1-operand vector instruction that extends elements of the input - /// register and operates on a pair of elements. The output lane width - /// is double that of the input. - VecRRPairLong { - op: VecRRPairLongOp, - rd: Writable, - rn: Reg, - }, - - /// A vector ALU op. - VecRRR { - alu_op: VecALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - size: VectorSize, - }, - - /// Vector two register miscellaneous instruction. - VecMisc { - op: VecMisc2, - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Vector instruction across lanes. - VecLanes { - op: VecLanesOp, - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Vector shift by immediate: Shift Left (immediate), Unsigned Shift Right (immediate), - /// Signed Shift Right (immediate). These are somewhat unusual in that, for right shifts, - /// the allowed range of `imm` values is 1 to lane-size-in-bits, inclusive. A zero - /// right-shift cannot be encoded. Left shifts are "normal", though, having valid `imm` - /// values from 0 to lane-size-in-bits - 1 inclusive. - VecShiftImm { - op: VecShiftImmOp, - rd: Writable, - rn: Reg, - size: VectorSize, - imm: u8, - }, - - /// Vector extract - create a new vector, being the concatenation of the lowest `imm4` bytes - /// of `rm` followed by the uppermost `16 - imm4` bytes of `rn`. - VecExtract { - rd: Writable, - rn: Reg, - rm: Reg, - imm4: u8, - }, - - /// Table vector lookup - single register table. The table consists of 8-bit elements and is - /// stored in `rn`, while `rm` contains 8-bit element indices. `is_extension` specifies whether - /// to emit a TBX or a TBL instruction, i.e. whether to leave the elements in the destination - /// vector that correspond to out-of-range indices (greater than 15) unmodified or to set them - /// to 0. - VecTbl { - rd: Writable, - rn: Reg, - rm: Reg, - is_extension: bool, - }, - - /// Table vector lookup - two register table. The table consists of 8-bit elements and is - /// stored in `rn` and `rn2`, while `rm` contains 8-bit element indices. `is_extension` - /// specifies whether to emit a TBX or a TBL instruction, i.e. whether to leave the elements in - /// the destination vector that correspond to out-of-range indices (greater than 31) unmodified - /// or to set them to 0. The table registers `rn` and `rn2` must have consecutive numbers - /// modulo 32, that is v31 and v0 (in that order) are consecutive registers. - VecTbl2 { - rd: Writable, - rn: Reg, - rn2: Reg, - rm: Reg, - is_extension: bool, - }, - - /// Load an element and replicate to all lanes of a vector. - VecLoadReplicate { - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Vector conditional select, 128 bit. A synthetic instruction, which generates a 4-insn - /// control-flow diamond. - VecCSel { - rd: Writable, - rn: Reg, - rm: Reg, - cond: Cond, - }, - - /// Move to the NZCV flags (actually a `MSR NZCV, Xn` insn). - MovToNZCV { - rn: Reg, - }, - - /// Move from the NZCV flags (actually a `MRS Xn, NZCV` insn). - MovFromNZCV { - rd: Writable, - }, - - /// A machine call instruction. N.B.: this allows only a +/- 128MB offset (it uses a relocation - /// of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the - /// code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit - /// target. - Call { - info: Box, - }, - /// A machine indirect-call instruction. - CallInd { - info: Box, - }, - - // ---- branches (exactly one must appear at end of BB) ---- - /// A machine return instruction. - Ret, - - /// A placeholder instruction, generating no code, meaning that a function epilogue must be - /// inserted there. - EpiloguePlaceholder, - - /// An unconditional branch. - Jump { - dest: BranchTarget, - }, - - /// A conditional branch. Contains two targets; at emission time, both are emitted, but - /// the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the - /// choice of taken/not_taken (inverting the branch polarity as needed) based on the - /// fallthrough at the time of lowering. - CondBr { - taken: BranchTarget, - not_taken: BranchTarget, - kind: CondBrKind, - }, - - /// A conditional trap: execute a `udf` if the condition is true. This is - /// one VCode instruction because it uses embedded control flow; it is - /// logically a single-in, single-out region, but needs to appear as one - /// unit to the register allocator. - /// - /// The `CondBrKind` gives the conditional-branch condition that will - /// *execute* the embedded `Inst`. (In the emitted code, we use the inverse - /// of this condition in a branch that skips the trap instruction.) - TrapIf { - kind: CondBrKind, - trap_code: TrapCode, - }, - - /// An indirect branch through a register, augmented with set of all - /// possible successors. - IndirectBr { - rn: Reg, - targets: Vec, - }, - - /// A "break" instruction, used for e.g. traps and debug breakpoints. - Brk, - - /// An instruction guaranteed to always be undefined and to trigger an illegal instruction at - /// runtime. - Udf { - trap_code: TrapCode, - }, - - /// Compute the address (using a PC-relative offset) of a memory location, using the `ADR` - /// instruction. Note that we take a simple offset, not a `MemLabel`, here, because `Adr` is - /// only used for now in fixed lowering sequences with hardcoded offsets. In the future we may - /// need full `MemLabel` support. - Adr { - rd: Writable, - /// Offset in range -2^20 .. 2^20. - off: i32, - }, - - /// Raw 32-bit word, used for inline constants and jump-table entries. - Word4 { - data: u32, - }, - - /// Raw 64-bit word, used for inline constants. - Word8 { - data: u64, - }, - - /// Jump-table sequence, as one compound instruction (see note in lower_inst.rs for rationale). - JTSequence { - info: Box, - ridx: Reg, - rtmp1: Writable, - rtmp2: Writable, - }, - - /// Load an inline symbol reference. - LoadExtName { - rd: Writable, - name: Box, - offset: i64, - }, - - /// Load address referenced by `mem` into `rd`. - LoadAddr { - rd: Writable, - mem: AMode, - }, - - /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This - /// controls how AMode::NominalSPOffset args are lowered. - VirtualSPOffsetAdj { - offset: i64, - }, - - /// Meta-insn, no-op in generated code: emit constant/branch veneer island - /// at this point (with a guard jump around it) if less than the needed - /// space is available before the next branch deadline. See the `MachBuffer` - /// implementation in `machinst/buffer.rs` for the overall algorithm. In - /// brief, we retain a set of "pending/unresolved label references" from - /// branches as we scan forward through instructions to emit machine code; - /// if we notice we're about to go out of range on an unresolved reference, - /// we stop, emit a bunch of "veneers" (branches in a form that has a longer - /// range, e.g. a 26-bit-offset unconditional jump), and point the original - /// label references to those. This is an "island" because it comes in the - /// middle of the code. - /// - /// This meta-instruction is a necessary part of the logic that determines - /// where to place islands. Ordinarily, we want to place them between basic - /// blocks, so we compute the worst-case size of each block, and emit the - /// island before starting a block if we would exceed a deadline before the - /// end of the block. However, some sequences (such as an inline jumptable) - /// are variable-length and not accounted for by this logic; so these - /// lowered sequences include an `EmitIsland` to trigger island generation - /// where necessary. - EmitIsland { - /// The needed space before the next deadline. - needed_space: CodeOffset, - }, - - /// A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol in x0. - ElfTlsGetAddr { - symbol: ExternalName, - }, - - /// A definition of a value label. - ValueLabelMarker { - reg: Reg, - label: ValueLabel, - }, - - /// An unwind pseudo-instruction. - Unwind { - inst: UnwindInst, - }, -} - fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize { let mut count = 0; for _ in 0..num_half_words { @@ -1426,6 +147,10 @@ impl Inst { /// Create an instruction that loads a constant, using one of serveral options (MOVZ, MOVN, /// logical immediate, or constant pool). pub fn load_constant(rd: Writable, value: u64) -> SmallVec<[Inst; 4]> { + // NB: this is duplicated in `lower/isle.rs` and `inst.isle` right now, + // if modifications are made here before this is deleted after moving to + // ISLE then those locations should be updated as well. + if let Some(imm) = MoveWideConst::maybe_from_u64(value) { // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ smallvec![Inst::MovZ { @@ -2304,65 +1029,44 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { //============================================================================= // Instructions: map_regs -fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { - fn map_use(m: &RUM, r: &mut Reg) { - if r.is_virtual() { - let new = m.get_use(r.to_virtual_reg()).unwrap().to_reg(); - *r = new; - } - } - - fn map_def(m: &RUM, r: &mut Writable) { - if r.to_reg().is_virtual() { - let new = m.get_def(r.to_reg().to_virtual_reg()).unwrap().to_reg(); - *r = Writable::from_reg(new); - } - } - - fn map_mod(m: &RUM, r: &mut Writable) { - if r.to_reg().is_virtual() { - let new = m.get_mod(r.to_reg().to_virtual_reg()).unwrap().to_reg(); - *r = Writable::from_reg(new); - } - } - - fn map_mem(m: &RUM, mem: &mut AMode) { +pub fn aarch64_map_regs(inst: &mut Inst, mapper: &RM) { + fn map_mem(m: &RM, mem: &mut AMode) { // N.B.: we take only the pre-map here, but this is OK because the // only addressing modes that update registers (pre/post-increment on // AArch64) both read and write registers, so they are "mods" rather // than "defs", so must be the same in both the pre- and post-map. match mem { - &mut AMode::Unscaled(ref mut reg, ..) => map_use(m, reg), - &mut AMode::UnsignedOffset(ref mut reg, ..) => map_use(m, reg), + &mut AMode::Unscaled(ref mut reg, ..) => m.map_use(reg), + &mut AMode::UnsignedOffset(ref mut reg, ..) => m.map_use(reg), &mut AMode::RegReg(ref mut r1, ref mut r2) | &mut AMode::RegScaled(ref mut r1, ref mut r2, ..) | &mut AMode::RegScaledExtended(ref mut r1, ref mut r2, ..) | &mut AMode::RegExtended(ref mut r1, ref mut r2, ..) => { - map_use(m, r1); - map_use(m, r2); + m.map_use(r1); + m.map_use(r2); } &mut AMode::Label(..) => {} - &mut AMode::PreIndexed(ref mut r, ..) => map_mod(m, r), - &mut AMode::PostIndexed(ref mut r, ..) => map_mod(m, r), + &mut AMode::PreIndexed(ref mut r, ..) => m.map_mod(r), + &mut AMode::PostIndexed(ref mut r, ..) => m.map_mod(r), &mut AMode::FPOffset(..) | &mut AMode::SPOffset(..) | &mut AMode::NominalSPOffset(..) => {} - &mut AMode::RegOffset(ref mut r, ..) => map_use(m, r), + &mut AMode::RegOffset(ref mut r, ..) => m.map_use(r), }; } - fn map_pairmem(m: &RUM, mem: &mut PairAMode) { + fn map_pairmem(m: &RM, mem: &mut PairAMode) { match mem { - &mut PairAMode::SignedOffset(ref mut reg, ..) => map_use(m, reg), - &mut PairAMode::PreIndexed(ref mut reg, ..) => map_def(m, reg), - &mut PairAMode::PostIndexed(ref mut reg, ..) => map_def(m, reg), + &mut PairAMode::SignedOffset(ref mut reg, ..) => m.map_use(reg), + &mut PairAMode::PreIndexed(ref mut reg, ..) => m.map_def(reg), + &mut PairAMode::PostIndexed(ref mut reg, ..) => m.map_def(reg), } } - fn map_br(m: &RUM, br: &mut CondBrKind) { + fn map_br(m: &RM, br: &mut CondBrKind) { match br { - &mut CondBrKind::Zero(ref mut reg) => map_use(m, reg), - &mut CondBrKind::NotZero(ref mut reg) => map_use(m, reg), + &mut CondBrKind::Zero(ref mut reg) => m.map_use(reg), + &mut CondBrKind::NotZero(ref mut reg) => m.map_use(reg), &mut CondBrKind::Cond(..) => {} }; } @@ -2374,9 +1078,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::AluRRRR { ref mut rd, @@ -2385,34 +1089,34 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut ra, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); - map_use(mapper, ra); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); + mapper.map_use(ra); } &mut Inst::AluRRImm12 { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::AluRRImmLogic { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::AluRRImmShift { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::AluRRRShift { ref mut rd, @@ -2420,9 +1124,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::AluRRRExtend { ref mut rd, @@ -2430,24 +1134,24 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::BitRR { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::ULoad8 { ref mut rd, ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::SLoad8 { @@ -2455,7 +1159,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::ULoad16 { @@ -2463,7 +1167,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::SLoad16 { @@ -2471,7 +1175,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::ULoad32 { @@ -2479,7 +1183,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::SLoad32 { @@ -2487,7 +1191,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } @@ -2496,7 +1200,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::Store8 { @@ -2504,7 +1208,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::Store16 { @@ -2512,7 +1216,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::Store32 { @@ -2520,7 +1224,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::Store64 { @@ -2528,7 +1232,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } @@ -2538,8 +1242,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rt); - map_use(mapper, rt2); + mapper.map_use(rt); + mapper.map_use(rt2); map_pairmem(mapper, mem); } &mut Inst::LoadP64 { @@ -2548,32 +1252,32 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rt); - map_def(mapper, rt2); + mapper.map_def(rt); + mapper.map_def(rt2); map_pairmem(mapper, mem); } &mut Inst::Mov64 { ref mut rd, ref mut rm, } => { - map_def(mapper, rd); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rm); } &mut Inst::Mov32 { ref mut rd, ref mut rm, } => { - map_def(mapper, rd); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rm); } &mut Inst::MovZ { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::MovN { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::MovK { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::CSel { ref mut rd, @@ -2581,15 +1285,15 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::CSet { ref mut rd, .. } | &mut Inst::CSetm { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::CCmpImm { ref mut rn, .. } => { - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::AtomicRMWLoop { .. } => { // There are no vregs to map in this insn. @@ -2600,9 +1304,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_use(mapper, rs); - map_def(mapper, rt); - map_use(mapper, rn); + mapper.map_use(rs); + mapper.map_def(rt); + mapper.map_use(rn); } &mut Inst::AtomicCAS { ref mut rs, @@ -2610,9 +1314,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_mod(mapper, rs); - map_use(mapper, rt); - map_use(mapper, rn); + mapper.map_mod(rs); + mapper.map_use(rt); + mapper.map_use(rn); } &mut Inst::AtomicCASLoop { .. } => { // There are no vregs to map in this insn. @@ -2622,55 +1326,55 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rt); - map_use(mapper, rn); + mapper.map_def(rt); + mapper.map_use(rn); } &mut Inst::StoreRelease { ref mut rt, ref mut rn, .. } => { - map_use(mapper, rt); - map_use(mapper, rn); + mapper.map_use(rt); + mapper.map_use(rn); } &mut Inst::Fence {} => {} &mut Inst::FpuMove64 { ref mut rd, ref mut rn, } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuMove128 { ref mut rd, ref mut rn, } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuMoveFromVec { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuExtend { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuRR { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuRRR { ref mut rd, @@ -2678,9 +1382,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuRRI { fpu_op, @@ -2689,10 +1393,10 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { .. } => { match fpu_op { - FPUOpRI::UShr32(..) | FPUOpRI::UShr64(..) => map_def(mapper, rd), - FPUOpRI::Sli32(..) | FPUOpRI::Sli64(..) => map_mod(mapper, rd), + FPUOpRI::UShr32(..) | FPUOpRI::UShr64(..) => mapper.map_def(rd), + FPUOpRI::Sli32(..) | FPUOpRI::Sli64(..) => mapper.map_mod(rd), } - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::FpuRRRR { ref mut rd, @@ -2701,34 +1405,34 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut ra, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); - map_use(mapper, ra); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); + mapper.map_use(ra); } &mut Inst::VecMisc { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecLanes { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecShiftImm { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecExtract { ref mut rd, @@ -2736,9 +1440,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::VecTbl { ref mut rd, @@ -2746,13 +1450,13 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, is_extension, } => { - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); if is_extension { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } } &mut Inst::VecTbl2 { @@ -2762,14 +1466,14 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, is_extension, } => { - map_use(mapper, rn); - map_use(mapper, rn2); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rn2); + mapper.map_use(rm); if is_extension { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } } &mut Inst::VecLoadReplicate { @@ -2777,8 +1481,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecCSel { ref mut rd, @@ -2786,30 +1490,30 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuCmp32 { ref mut rn, ref mut rm, } => { - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuCmp64 { ref mut rn, ref mut rm, } => { - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuLoad32 { ref mut rd, ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::FpuLoad64 { @@ -2817,7 +1521,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::FpuLoad128 { @@ -2825,7 +1529,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::FpuStore32 { @@ -2833,7 +1537,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::FpuStore64 { @@ -2841,7 +1545,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::FpuStore128 { @@ -2849,7 +1553,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::FpuLoadP64 { @@ -2858,8 +1562,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rt); - map_def(mapper, rt2); + mapper.map_def(rt); + mapper.map_def(rt2); map_pairmem(mapper, mem); } &mut Inst::FpuStoreP64 { @@ -2868,8 +1572,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rt); - map_use(mapper, rt2); + mapper.map_use(rt); + mapper.map_use(rt2); map_pairmem(mapper, mem); } &mut Inst::FpuLoadP128 { @@ -2878,8 +1582,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rt); - map_def(mapper, rt2); + mapper.map_def(rt); + mapper.map_def(rt2); map_pairmem(mapper, mem); } &mut Inst::FpuStoreP128 { @@ -2888,31 +1592,31 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rt); - map_use(mapper, rt2); + mapper.map_use(rt); + mapper.map_use(rt2); map_pairmem(mapper, mem); } &mut Inst::LoadFpuConst64 { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::LoadFpuConst128 { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::FpuToInt { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::IntToFpu { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuCSel32 { ref mut rd, @@ -2920,9 +1624,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuCSel64 { ref mut rd, @@ -2930,33 +1634,33 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuRound { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::MovToFpu { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::MovToVec { ref mut rd, ref mut rn, .. } => { - map_mod(mapper, rd); - map_use(mapper, rn); + mapper.map_mod(rd); + mapper.map_use(rn); } &mut Inst::MovFromVec { ref mut rd, @@ -2968,54 +1672,54 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecDup { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecDupFromFpu { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecDupFPImm { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::VecDupImm { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::VecExtend { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecMovElement { ref mut rd, ref mut rn, .. } => { - map_mod(mapper, rd); - map_use(mapper, rn); + mapper.map_mod(rd); + mapper.map_use(rn); } &mut Inst::VecRRLong { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecRRNarrow { ref mut rd, @@ -3023,12 +1727,12 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { high_half, .. } => { - map_use(mapper, rn); + mapper.map_use(rn); if high_half { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } } &mut Inst::VecRRPair { @@ -3036,8 +1740,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecRRRLong { alu_op, @@ -3048,20 +1752,20 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { } => { match alu_op { VecRRRLongOp::Umlal8 | VecRRRLongOp::Umlal16 | VecRRRLongOp::Umlal32 => { - map_mod(mapper, rd) + mapper.map_mod(rd) } - _ => map_def(mapper, rd), + _ => mapper.map_def(rd), }; - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::VecRRPairLong { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecRRR { alu_op, @@ -3071,58 +1775,58 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { .. } => { if alu_op == VecALUOp::Bsl { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::MovToNZCV { ref mut rn } => { - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::MovFromNZCV { ref mut rd } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::Extend { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::Jump { .. } => {} &mut Inst::Call { ref mut info } => { for r in info.uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in info.defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } } &mut Inst::Ret | &mut Inst::EpiloguePlaceholder => {} &mut Inst::CallInd { ref mut info, .. } => { for r in info.uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in info.defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } - map_use(mapper, &mut info.rn); + mapper.map_use(&mut info.rn); } &mut Inst::CondBr { ref mut kind, .. } => { map_br(mapper, kind); } &mut Inst::IndirectBr { ref mut rn, .. } => { - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::Nop0 | &mut Inst::Nop4 | &mut Inst::Brk | &mut Inst::Udf { .. } => {} &mut Inst::TrapIf { ref mut kind, .. } => { map_br(mapper, kind); } &mut Inst::Adr { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::Word4 { .. } | &mut Inst::Word8 { .. } => {} &mut Inst::JTSequence { @@ -3131,25 +1835,25 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rtmp2, .. } => { - map_use(mapper, ridx); - map_def(mapper, rtmp1); - map_def(mapper, rtmp2); + mapper.map_use(ridx); + mapper.map_def(rtmp1); + mapper.map_def(rtmp2); } &mut Inst::LoadExtName { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::LoadAddr { ref mut rd, ref mut mem, } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::VirtualSPOffsetAdj { .. } => {} &mut Inst::EmitIsland { .. } => {} &mut Inst::ElfTlsGetAddr { .. } => {} &mut Inst::ValueLabelMarker { ref mut reg, .. } => { - map_use(mapper, reg); + mapper.map_use(reg); } &mut Inst::Unwind { .. } => {} } @@ -3165,7 +1869,7 @@ impl MachInst for Inst { aarch64_get_regs(self, collector) } - fn map_regs(&mut self, mapper: &RUM) { + fn map_regs(&mut self, mapper: &RM) { aarch64_map_regs(self, mapper); } diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle new file mode 100644 index 0000000000..7b793dc271 --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -0,0 +1,23 @@ +;; aarch64 instruction selection and CLIF-to-MachInst lowering. + +;; The main lowering constructor term: takes a clif `Inst` and returns the +;; register(s) within which the lowered instruction's result values live. +(decl lower (Inst) ValueRegs) + +;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty (iconst (u64_from_imm64 n)))) + (value_reg (imm ty n))) + +;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty (bconst $false))) + (value_reg (imm ty 0))) + +(rule (lower (has_type ty (bconst $true))) + (value_reg (imm ty 1))) + +;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty (null))) + (value_reg (imm ty 0))) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 34a627dfba..4697411afa 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -25,6 +25,8 @@ use regalloc::{Reg, Writable}; use smallvec::SmallVec; use std::cmp; +pub mod isle; + //============================================================================ // Result enum types. // diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs new file mode 100644 index 0000000000..6708431eb9 --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -0,0 +1,187 @@ +//! ISLE integration glue code for aarch64 lowering. + +// Pull in the ISLE generated code. +pub mod generated_code; + +// Types that the generated ISLE code uses via `use super::*`. +use super::{ + zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, AtomicRmwOp, BranchTarget, CallIndInfo, + CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, Imm12, ImmLogic, ImmShift, Inst as MInst, + JTSequenceInfo, MachLabel, MoveWideConst, Opcode, OperandSize, PairAMode, Reg, ScalarSize, + ShiftOpAndAmt, UImm5, VectorSize, NZCV, +}; +use crate::isa::aarch64::settings as aarch64_settings; +use crate::machinst::isle::*; +use crate::{ + binemit::CodeOffset, + ir::{ + immediates::*, types::*, ExternalName, Inst, InstructionData, MemFlags, TrapCode, Value, + ValueLabel, ValueList, + }, + isa::aarch64::inst::aarch64_map_regs, + isa::unwind::UnwindInst, + machinst::{get_output_reg, InsnOutput, LowerCtx, RegRenamer}, +}; +use smallvec::SmallVec; +use std::boxed::Box; +use std::vec::Vec; + +type BoxCallInfo = Box; +type BoxCallIndInfo = Box; +type VecMachLabel = Vec; +type BoxJTSequenceInfo = Box; +type BoxExternalName = Box; + +/// The main entry point for lowering with ISLE. +pub(crate) fn lower( + lower_ctx: &mut C, + isa_flags: &aarch64_settings::Flags, + outputs: &[InsnOutput], + inst: Inst, +) -> Result<(), ()> +where + C: LowerCtx, +{ + // TODO: reuse the ISLE context across lowerings so we can reuse its + // internal heap allocations. + let mut isle_ctx = IsleContext::new(lower_ctx, isa_flags); + + let temp_regs = generated_code::constructor_lower(&mut isle_ctx, inst).ok_or(())?; + let mut temp_regs = temp_regs.regs().iter(); + + // The ISLE generated code emits its own registers to define the + // instruction's lowered values in. We rename those registers to the + // registers they were assigned when their value was used as an operand in + // earlier lowerings. + let mut renamer = RegRenamer::default(); + for output in outputs { + let dsts = get_output_reg(isle_ctx.lower_ctx, *output); + for (temp, dst) in temp_regs.by_ref().zip(dsts.regs()) { + renamer.add_rename(*temp, dst.to_reg()); + } + } + + for mut inst in isle_ctx.into_emitted_insts() { + aarch64_map_regs(&mut inst, &renamer); + lower_ctx.emit(inst); + } + + Ok(()) +} + +pub struct IsleContext<'a, C> { + lower_ctx: &'a mut C, + #[allow(dead_code)] // dead for now, but probably not for long + isa_flags: &'a aarch64_settings::Flags, + emitted_insts: SmallVec<[MInst; 6]>, +} + +impl<'a, C> IsleContext<'a, C> { + pub fn new(lower_ctx: &'a mut C, isa_flags: &'a aarch64_settings::Flags) -> Self { + IsleContext { + lower_ctx, + isa_flags, + emitted_insts: SmallVec::new(), + } + } + + pub fn into_emitted_insts(self) -> SmallVec<[MInst; 6]> { + self.emitted_insts + } +} + +impl<'a, C> generated_code::Context for IsleContext<'a, C> +where + C: LowerCtx, +{ + isle_prelude_methods!(); + + fn move_wide_const_from_u64(&mut self, n: u64) -> Option { + MoveWideConst::maybe_from_u64(n) + } + + fn move_wide_const_from_negated_u64(&mut self, n: u64) -> Option { + MoveWideConst::maybe_from_u64(!n) + } + + fn imm_logic_from_u64(&mut self, n: u64) -> Option { + ImmLogic::maybe_from_u64(n, I64) + } + + fn integral_ty(&mut self, ty: Type) -> Option { + match ty { + I8 | I16 | I32 | I64 | R64 => Some(ty), + ty if ty.is_bool() => Some(ty), + _ => None, + } + } + + /// This is the fallback case for loading a 64-bit integral constant into a + /// register. + /// + /// The logic here is nontrivial enough that it's not really worth porting + /// this over to ISLE. + fn load_constant64_full(&mut self, value: u64) -> Reg { + // If the top 32 bits are zero, use 32-bit `mov` operations. + let (num_half_words, size, negated) = if value >> 32 == 0 { + (2, OperandSize::Size32, (!value << 32) >> 32) + } else { + (4, OperandSize::Size64, !value) + }; + // If the number of 0xffff half words is greater than the number of 0x0000 half words + // it is more efficient to use `movn` for the first instruction. + let first_is_inverted = count_zero_half_words(negated, num_half_words) + > count_zero_half_words(value, num_half_words); + // Either 0xffff or 0x0000 half words can be skipped, depending on the first + // instruction used. + let ignored_halfword = if first_is_inverted { 0xffff } else { 0 }; + let mut first_mov_emitted = false; + + let rd = self.temp_writable_reg(I64); + + for i in 0..num_half_words { + let imm16 = (value >> (16 * i)) & 0xffff; + if imm16 != ignored_halfword { + if !first_mov_emitted { + first_mov_emitted = true; + if first_is_inverted { + let imm = + MoveWideConst::maybe_with_shift(((!imm16) & 0xffff) as u16, i * 16) + .unwrap(); + self.emitted_insts.push(MInst::MovN { rd, imm, size }); + } else { + let imm = MoveWideConst::maybe_with_shift(imm16 as u16, i * 16).unwrap(); + self.emitted_insts.push(MInst::MovZ { rd, imm, size }); + } + } else { + let imm = MoveWideConst::maybe_with_shift(imm16 as u16, i * 16).unwrap(); + self.emitted_insts.push(MInst::MovK { rd, imm, size }); + } + } + } + + assert!(first_mov_emitted); + + return self.writable_reg_to_reg(rd); + + fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize { + let mut count = 0; + for _ in 0..num_half_words { + if value & 0xffff == 0 { + count += 1; + } + value >>= 16; + } + + count + } + } + + fn zero_reg(&mut self) -> Reg { + zero_reg() + } + + fn emit(&mut self, inst: &MInst) -> Unit { + self.emitted_insts.push(inst.clone()); + } +} diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest new file mode 100644 index 0000000000..f02a0fefe3 --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -0,0 +1,4 @@ +src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb +src/prelude.isle a069d14321601afc63959af23086709d67d189dafcdc7d1fc8534b32d89d49008acb8368b7b5a7bc51a353736a378197ac352ccce2bb3be89d93afb6979e480a +src/isa/aarch64/inst.isle 841748c9c5900821b7086a09a41c6dcdb2172eb47a45293b6ef10f2e1f1389620bf6a2c75152af807d8bc8929029a357af5191f5d87bac2c9ec54bf63a9a2a8f +src/isa/aarch64/lower.isle b3cd0834484e543f39d477d47ee66042276e99955c21fb8c9340a5f27ac317936acb2907a30f758bf596066e36db801a179fda6dbcecaee758a0187a5a5f1412 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs new file mode 100644 index 0000000000..137cff579a --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -0,0 +1,1108 @@ +// GENERATED BY ISLE. DO NOT EDIT! +// +// Generated automatically from the instruction-selection DSL code in: +// - src/clif.isle +// - src/prelude.isle +// - src/isa/aarch64/inst.isle +// - src/isa/aarch64/lower.isle + +#![allow(dead_code, unreachable_code, unreachable_patterns)] +#![allow(unused_imports, unused_variables, non_snake_case)] +#![allow(irrefutable_let_patterns)] + +use super::*; // Pulls in all external types. + +/// Context during lowering: an implementation of this trait +/// must be provided with all external constructors and extractors. +/// A mutable borrow is passed along through all lowering logic. +pub trait Context { + fn unpack_value_array_2(&mut self, arg0: &ValueArray2) -> (Value, Value); + fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> ValueArray2; + fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value); + fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3; + fn value_reg(&mut self, arg0: Reg) -> ValueRegs; + fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; + fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; + fn invalid_reg(&mut self) -> Reg; + fn put_in_reg(&mut self, arg0: Value) -> Reg; + fn put_in_regs(&mut self, arg0: Value) -> ValueRegs; + fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg; + fn u8_as_u64(&mut self, arg0: u8) -> u64; + fn u16_as_u64(&mut self, arg0: u16) -> u64; + fn u32_as_u64(&mut self, arg0: u32) -> u64; + fn ty_bits(&mut self, arg0: Type) -> u16; + fn fits_in_64(&mut self, arg0: Type) -> Option; + fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; + fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); + fn unwrap_head_value_list_2(&mut self, arg0: ValueList) -> (Value, Value, ValueSlice); + fn writable_reg_to_reg(&mut self, arg0: WritableReg) -> Reg; + fn u64_from_imm64(&mut self, arg0: Imm64) -> u64; + fn u64_from_ieee32(&mut self, arg0: Ieee32) -> u64; + fn u64_from_ieee64(&mut self, arg0: Ieee64) -> u64; + fn inst_results(&mut self, arg0: Inst) -> ValueSlice; + fn first_result(&mut self, arg0: Inst) -> Option; + fn inst_data(&mut self, arg0: Inst) -> InstructionData; + fn value_type(&mut self, arg0: Value) -> Type; + fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>; + fn def_inst(&mut self, arg0: Value) -> Option; + fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option; + fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option; + fn imm_logic_from_u64(&mut self, arg0: u64) -> Option; + fn integral_ty(&mut self, arg0: Type) -> Option; + fn emit(&mut self, arg0: &MInst) -> Unit; + fn zero_reg(&mut self) -> Reg; + fn load_constant64_full(&mut self, arg0: u64) -> Reg; +} + +/// Internal type MInst: defined at src/isa/aarch64/inst.isle line 2. +#[derive(Clone, Debug)] +pub enum MInst { + Nop0, + Nop4, + AluRRR { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + }, + AluRRRR { + alu_op: ALUOp3, + rd: WritableReg, + rn: Reg, + rm: Reg, + ra: Reg, + }, + AluRRImm12 { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + imm12: Imm12, + }, + AluRRImmLogic { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + imml: ImmLogic, + }, + AluRRImmShift { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + immshift: ImmShift, + }, + AluRRRShift { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + shiftop: ShiftOpAndAmt, + }, + AluRRRExtend { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + extendop: ExtendOp, + }, + BitRR { + op: BitOp, + rd: WritableReg, + rn: Reg, + }, + ULoad8 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + SLoad8 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + ULoad16 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + SLoad16 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + ULoad32 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + SLoad32 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + ULoad64 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + Store8 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + Store16 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + Store32 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + Store64 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + StoreP64 { + rt: Reg, + rt2: Reg, + mem: PairAMode, + flags: MemFlags, + }, + LoadP64 { + rt: WritableReg, + rt2: WritableReg, + mem: PairAMode, + flags: MemFlags, + }, + Mov64 { + rd: WritableReg, + rm: Reg, + }, + Mov32 { + rd: WritableReg, + rm: Reg, + }, + MovZ { + rd: WritableReg, + imm: MoveWideConst, + size: OperandSize, + }, + MovN { + rd: WritableReg, + imm: MoveWideConst, + size: OperandSize, + }, + MovK { + rd: WritableReg, + imm: MoveWideConst, + size: OperandSize, + }, + Extend { + rd: WritableReg, + rn: Reg, + signed: bool, + from_bits: u8, + to_bits: u8, + }, + CSel { + rd: WritableReg, + cond: Cond, + rn: Reg, + rm: Reg, + }, + CSet { + rd: WritableReg, + cond: Cond, + }, + CSetm { + rd: WritableReg, + cond: Cond, + }, + CCmpImm { + size: OperandSize, + rn: Reg, + imm: UImm5, + nzcv: NZCV, + cond: Cond, + }, + AtomicRMWLoop { + ty: Type, + op: AtomicRmwOp, + }, + AtomicRMW { + op: AtomicRMWOp, + rs: Reg, + rt: WritableReg, + rn: Reg, + ty: Type, + }, + AtomicCAS { + rs: WritableReg, + rt: Reg, + rn: Reg, + ty: Type, + }, + AtomicCASLoop { + ty: Type, + }, + LoadAcquire { + access_ty: Type, + rt: WritableReg, + rn: Reg, + }, + StoreRelease { + access_ty: Type, + rt: Reg, + rn: Reg, + }, + Fence, + FpuMove64 { + rd: WritableReg, + rn: Reg, + }, + FpuMove128 { + rd: WritableReg, + rn: Reg, + }, + FpuMoveFromVec { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + }, + FpuExtend { + rd: WritableReg, + rn: Reg, + size: ScalarSize, + }, + FpuRR { + fpu_op: FPUOp1, + rd: WritableReg, + rn: Reg, + }, + FpuRRR { + fpu_op: FPUOp2, + rd: WritableReg, + rn: Reg, + rm: Reg, + }, + FpuRRI { + fpu_op: FPUOpRI, + rd: WritableReg, + rn: Reg, + }, + FpuRRRR { + fpu_op: FPUOp3, + rd: WritableReg, + rn: Reg, + rm: Reg, + ra: Reg, + }, + FpuCmp32 { + rn: Reg, + rm: Reg, + }, + FpuCmp64 { + rn: Reg, + rm: Reg, + }, + FpuLoad32 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + FpuStore32 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + FpuLoad64 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + FpuStore64 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + FpuLoad128 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + FpuStore128 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + FpuLoadP64 { + rt: WritableReg, + rt2: WritableReg, + mem: PairAMode, + flags: MemFlags, + }, + FpuStoreP64 { + rt: Reg, + rt2: Reg, + mem: PairAMode, + flags: MemFlags, + }, + FpuLoadP128 { + rt: WritableReg, + rt2: WritableReg, + mem: PairAMode, + flags: MemFlags, + }, + FpuStoreP128 { + rt: Reg, + rt2: Reg, + mem: PairAMode, + flags: MemFlags, + }, + LoadFpuConst64 { + rd: WritableReg, + const_data: u64, + }, + LoadFpuConst128 { + rd: WritableReg, + const_data: u128, + }, + FpuToInt { + op: FpuToIntOp, + rd: WritableReg, + rn: Reg, + }, + IntToFpu { + op: IntToFpuOp, + rd: WritableReg, + rn: Reg, + }, + FpuCSel32 { + rd: WritableReg, + rn: Reg, + rm: Reg, + cond: Cond, + }, + FpuCSel64 { + rd: WritableReg, + rn: Reg, + rm: Reg, + cond: Cond, + }, + FpuRound { + op: FpuRoundMode, + rd: WritableReg, + rn: Reg, + }, + MovToFpu { + rd: WritableReg, + rn: Reg, + size: ScalarSize, + }, + MovToVec { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + }, + MovFromVec { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + }, + MovFromVecSigned { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + scalar_size: OperandSize, + }, + VecDup { + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecDupFromFpu { + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecDupFPImm { + rd: WritableReg, + imm: ASIMDFPModImm, + size: VectorSize, + }, + VecDupImm { + rd: WritableReg, + imm: ASIMDMovModImm, + invert: bool, + size: VectorSize, + }, + VecExtend { + t: VecExtendOp, + rd: WritableReg, + rn: Reg, + high_half: bool, + }, + VecMovElement { + rd: WritableReg, + rn: Reg, + dest_idx: u8, + src_idx: u8, + size: VectorSize, + }, + VecRRLong { + op: VecRRLongOp, + rd: WritableReg, + rn: Reg, + high_half: bool, + }, + VecRRNarrow { + op: VecRRNarrowOp, + rd: WritableReg, + rn: Reg, + high_half: bool, + }, + VecRRPair { + op: VecPairOp, + rd: WritableReg, + rn: Reg, + }, + VecRRRLong { + alu_op: VecRRRLongOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + high_half: bool, + }, + VecRRPairLong { + op: VecRRPairLongOp, + rd: WritableReg, + rn: Reg, + }, + VecRRR { + alu_op: VecALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + size: VectorSize, + }, + VecMisc { + op: VecMisc2, + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecLanes { + op: VecLanesOp, + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecShiftImm { + op: VecShiftImmOp, + rd: WritableReg, + rn: Reg, + size: VectorSize, + imm: u8, + }, + VecExtract { + rd: WritableReg, + rn: Reg, + rm: Reg, + imm4: u8, + }, + VecTbl { + rd: WritableReg, + rn: Reg, + rm: Reg, + is_extension: bool, + }, + VecTbl2 { + rd: WritableReg, + rn: Reg, + rn2: Reg, + rm: Reg, + is_extension: bool, + }, + VecLoadReplicate { + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecCSel { + rd: WritableReg, + rn: Reg, + rm: Reg, + cond: Cond, + }, + MovToNZCV { + rn: Reg, + }, + MovFromNZCV { + rd: WritableReg, + }, + Call { + info: BoxCallInfo, + }, + CallInd { + info: BoxCallIndInfo, + }, + Ret, + EpiloguePlaceholder, + Jump { + dest: BranchTarget, + }, + CondBr { + taken: BranchTarget, + not_taken: BranchTarget, + kind: CondBrKind, + }, + TrapIf { + kind: CondBrKind, + trap_code: TrapCode, + }, + IndirectBr { + rn: Reg, + targets: VecMachLabel, + }, + Brk, + Udf { + trap_code: TrapCode, + }, + Adr { + rd: WritableReg, + off: i32, + }, + Word4 { + data: u32, + }, + Word8 { + data: u64, + }, + JTSequence { + info: BoxJTSequenceInfo, + ridx: Reg, + rtmp1: WritableReg, + rtmp2: WritableReg, + }, + LoadExtName { + rd: WritableReg, + name: BoxExternalName, + offset: i64, + }, + LoadAddr { + rd: WritableReg, + mem: AMode, + }, + VirtualSPOffsetAdj { + offset: i64, + }, + EmitIsland { + needed_space: CodeOffset, + }, + ElfTlsGetAddr { + symbol: ExternalName, + }, + ValueLabelMarker { + reg: Reg, + label: ValueLabel, + }, + Unwind { + inst: UnwindInst, + }, +} + +/// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 783. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ALUOp { + Add32, + Add64, + Sub32, + Sub64, + Orr32, + Orr64, + OrrNot32, + OrrNot64, + And32, + And64, + AndS32, + AndS64, + AndNot32, + AndNot64, + Eor32, + Eor64, + EorNot32, + EorNot64, + AddS32, + AddS64, + SubS32, + SubS64, + SMulH, + UMulH, + SDiv64, + UDiv64, + RotR32, + RotR64, + Lsr32, + Lsr64, + Asr32, + Asr64, + Lsl32, + Lsl64, + Adc32, + Adc64, + AdcS32, + AdcS64, + Sbc32, + Sbc64, + SbcS32, + SbcS64, +} + +/// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 844. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ALUOp3 { + MAdd32, + MAdd64, + MSub32, + MSub64, +} + +/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 892. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum BitOp { + RBit32, + RBit64, + Clz32, + Clz64, + Cls32, + Cls64, +} + +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 951. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FPUOp1 { + Abs32, + Abs64, + Neg32, + Neg64, + Sqrt32, + Sqrt64, + Cvt32To64, + Cvt64To32, +} + +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 964. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FPUOp2 { + Add32, + Add64, + Sub32, + Sub64, + Mul32, + Mul64, + Div32, + Div64, + Max32, + Max64, + Min32, + Min64, + Sqadd64, + Uqadd64, + Sqsub64, + Uqsub64, +} + +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 989. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FPUOp3 { + MAdd32, + MAdd64, +} + +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 996. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FpuToIntOp { + F32ToU32, + F32ToI32, + F32ToU64, + F32ToI64, + F64ToU32, + F64ToI32, + F64ToU64, + F64ToI64, +} + +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1009. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum IntToFpuOp { + U32ToF32, + I32ToF32, + U32ToF64, + I32ToF64, + U64ToF32, + I64ToF32, + U64ToF64, + I64ToF64, +} + +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1023. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FpuRoundMode { + Minus32, + Minus64, + Plus32, + Plus64, + Zero32, + Zero64, + Nearest32, + Nearest64, +} + +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1036. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecExtendOp { + Sxtl8, + Sxtl16, + Sxtl32, + Uxtl8, + Uxtl16, + Uxtl32, +} + +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1053. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecALUOp { + Sqadd, + Uqadd, + Sqsub, + Uqsub, + Cmeq, + Cmge, + Cmgt, + Cmhs, + Cmhi, + Fcmeq, + Fcmgt, + Fcmge, + And, + Bic, + Orr, + Eor, + Bsl, + Umaxp, + Add, + Sub, + Mul, + Sshl, + Ushl, + Umin, + Smin, + Umax, + Smax, + Urhadd, + Fadd, + Fsub, + Fdiv, + Fmax, + Fmin, + Fmul, + Addp, + Zip1, + Sqrdmulh, +} + +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1132. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecMisc2 { + Not, + Neg, + Abs, + Fabs, + Fneg, + Fsqrt, + Rev64, + Fcvtzs, + Fcvtzu, + Scvtf, + Ucvtf, + Frintn, + Frintz, + Frintm, + Frintp, + Cnt, + Cmeq0, +} + +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1171. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRLongOp { + Fcvtl16, + Fcvtl32, + Shll8, + Shll16, + Shll32, +} + +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1186. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRNarrowOp { + Xtn16, + Xtn32, + Xtn64, + Sqxtn16, + Sqxtn32, + Sqxtn64, + Sqxtun16, + Sqxtun32, + Sqxtun64, + Uqxtn16, + Uqxtn32, + Uqxtn64, + Fcvtn32, + Fcvtn64, +} + +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1218. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRRLongOp { + Smull8, + Smull16, + Smull32, + Umull8, + Umull16, + Umull32, + Umlal8, + Umlal16, + Umlal32, +} + +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1235. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecPairOp { + Addp, +} + +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1243. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRPairLongOp { + Saddlp8, + Saddlp16, + Uaddlp8, + Uaddlp16, +} + +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1254. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecLanesOp { + Addv, + Uminv, +} + +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1263. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecShiftImmOp { + Shl, + Ushr, + Sshr, +} + +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1274. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum AtomicRMWOp { + Add, + Clr, + Eor, + Set, + Smax, + Smin, + Umax, + Umin, +} + +// Generated as internal constructor for term temp_reg. +pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + // Rule at src/prelude.isle line 60. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term lo_reg. +pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/prelude.isle line 95. + let expr0_0 = C::put_in_regs(ctx, pattern0_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term movz. +pub fn constructor_movz( + ctx: &mut C, + arg0: MoveWideConst, + arg1: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1315. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::MovZ { + rd: expr1_0, + imm: pattern0_0, + size: pattern1_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term movn. +pub fn constructor_movn( + ctx: &mut C, + arg0: MoveWideConst, + arg1: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1322. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::MovN { + rd: expr1_0, + imm: pattern0_0, + size: pattern1_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rr_imm_logic. +pub fn constructor_alu_rr_imm_logic( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: ImmLogic, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1329. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRImmLogic { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + imml: pattern2_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term orr64. +pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1336. + let expr0_0 = ALUOp::Orr64; + let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term imm. +pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) { + let pattern2_0 = arg1; + if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) { + // Rule at src/isa/aarch64/inst.isle line 1351. + let expr0_0 = C::zero_reg(ctx); + let expr1_0 = constructor_orr64(ctx, expr0_0, pattern3_0)?; + return Some(expr1_0); + } + if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { + // Rule at src/isa/aarch64/inst.isle line 1343. + let expr0_0 = OperandSize::Size64; + let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; + return Some(expr1_0); + } + if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { + // Rule at src/isa/aarch64/inst.isle line 1347. + let expr0_0 = OperandSize::Size64; + let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; + return Some(expr1_0); + } + // Rule at src/isa/aarch64/inst.isle line 1358. + let expr0_0 = C::load_constant64_full(ctx, pattern2_0); + return Some(expr0_0); + } + return None; +} + +// Generated as internal constructor for term lower. +pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) { + let pattern2_0 = C::value_type(ctx, pattern1_0); + let pattern3_0 = C::inst_data(ctx, pattern0_0); + match &pattern3_0 { + &InstructionData::NullAry { + opcode: ref pattern4_0, + } => { + if let &Opcode::Null = &pattern4_0 { + // Rule at src/isa/aarch64/lower.isle line 22. + let expr0_0: u64 = 0; + let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + } + &InstructionData::UnaryImm { + opcode: ref pattern4_0, + imm: pattern4_1, + } => { + if let &Opcode::Iconst = &pattern4_0 { + let pattern6_0 = C::u64_from_imm64(ctx, pattern4_1); + // Rule at src/isa/aarch64/lower.isle line 9. + let expr0_0 = constructor_imm(ctx, pattern2_0, pattern6_0)?; + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); + } + } + &InstructionData::UnaryBool { + opcode: ref pattern4_0, + imm: pattern4_1, + } => { + if let &Opcode::Bconst = &pattern4_0 { + if pattern4_1 == true { + // Rule at src/isa/aarch64/lower.isle line 17. + let expr0_0: u64 = 1; + let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if pattern4_1 == false { + // Rule at src/isa/aarch64/lower.isle line 14. + let expr0_0: u64 = 0; + let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + } + } + _ => {} + } + } + return None; +} diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index c2ae72055e..74d1506c74 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -38,22 +38,21 @@ pub(crate) fn lower_insn_to_regs>( None }; + if let Ok(()) = super::lower::isle::lower(ctx, isa_flags, &outputs, insn) { + return Ok(()); + } + + let implemented_in_isle = || { + unreachable!( + "implemented in ISLE: inst = `{}`, type = `{:?}`", + ctx.dfg().display_inst(insn), + ty + ); + }; + match op { - Opcode::Iconst | Opcode::Bconst | Opcode::Null => { - let value = ctx.get_constant(insn).unwrap(); - match ty.unwrap() { - I8 | I16 | I32 | I64 | R64 => {} - ty if ty.is_bool() => {} - ty => { - return Err(CodegenError::Unsupported(format!( - "{}: Unsupported type: {:?}", - op, ty - ))); - } - } - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - lower_constant_u64(ctx, rd, value); - } + Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(), + Opcode::F32const => { let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32); let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 74e3373c6c..9b9cf2fd0c 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -1,7 +1,7 @@ //! Instruction operand sub-components (aka "parts"): definitions and printing. use super::regs::{self, show_ireg_sized}; -use super::{EmitState, RegMapper}; +use super::EmitState; use crate::ir::condcodes::{FloatCC, IntCC}; use crate::ir::{MemFlags, Type}; use crate::isa::x64::inst::Inst; diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index db1b48226f..c65c3c7523 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2565,64 +2565,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { //============================================================================= // Instructions and subcomponents: map_regs -// Define our own register-mapping trait so we can do arbitrary register -// renaming that are more free form than what `regalloc` constrains us to with -// its `RegUsageMapper` trait definition. -pub trait RegMapper { - fn get_use(&self, reg: Reg) -> Option; - fn get_def(&self, reg: Reg) -> Option; - fn get_mod(&self, reg: Reg) -> Option; -} - -impl RegMapper for T -where - T: regalloc::RegUsageMapper, -{ - fn get_use(&self, reg: Reg) -> Option { - let v = reg.as_virtual_reg()?; - self.get_use(v).map(|r| r.to_reg()) - } - - fn get_def(&self, reg: Reg) -> Option { - let v = reg.as_virtual_reg()?; - self.get_def(v).map(|r| r.to_reg()) - } - - fn get_mod(&self, reg: Reg) -> Option { - let v = reg.as_virtual_reg()?; - self.get_mod(v).map(|r| r.to_reg()) - } -} - -fn map_use(m: &RM, r: &mut Reg) { - if let Some(new) = m.get_use(*r) { - *r = new; - } -} - -fn map_def(m: &RM, r: &mut Writable) { - if let Some(new) = m.get_def(r.to_reg()) { - *r = Writable::from_reg(new); - } -} - -fn map_mod(m: &RM, r: &mut Writable) { - if let Some(new) = m.get_mod(r.to_reg()) { - *r = Writable::from_reg(new); - } -} - impl Amode { fn map_uses(&mut self, map: &RM) { match self { - Amode::ImmReg { ref mut base, .. } => map_use(map, base), + Amode::ImmReg { ref mut base, .. } => map.map_use(base), Amode::ImmRegRegShift { ref mut base, ref mut index, .. } => { - map_use(map, base); - map_use(map, index); + map.map_use(base); + map.map_use(index); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -2645,7 +2598,7 @@ impl Amode { impl RegMemImm { fn map_uses(&mut self, map: &RM) { match self { - RegMemImm::Reg { ref mut reg } => map_use(map, reg), + RegMemImm::Reg { ref mut reg } => map.map_use(reg), RegMemImm::Mem { ref mut addr } => addr.map_uses(map), RegMemImm::Imm { .. } => {} } @@ -2655,7 +2608,7 @@ impl RegMemImm { match self { Self::Reg { reg } => { let mut writable_src = Writable::from_reg(*reg); - map_def(mapper, &mut writable_src); + mapper.map_def(&mut writable_src); *self = Self::reg(writable_src.to_reg()); } _ => panic!("unexpected RegMemImm kind in map_src_reg_as_def"), @@ -2666,7 +2619,7 @@ impl RegMemImm { impl RegMem { fn map_uses(&mut self, map: &RM) { match self { - RegMem::Reg { ref mut reg } => map_use(map, reg), + RegMem::Reg { ref mut reg } => map.map_use(reg), RegMem::Mem { ref mut addr, .. } => addr.map_uses(map), } } @@ -2675,7 +2628,7 @@ impl RegMem { match self { Self::Reg { reg } => { let mut writable_src = Writable::from_reg(*reg); - map_def(mapper, &mut writable_src); + mapper.map_def(&mut writable_src); *self = Self::reg(writable_src.to_reg()); } _ => panic!("unexpected RegMem kind in map_src_reg_as_def"), @@ -2698,25 +2651,25 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } } Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - map_mod(mapper, dst); + mapper.map_mod(dst); *src = dst.to_reg(); } Inst::Div { divisor, .. } => divisor.map_uses(mapper), Inst::MulHi { src2, .. } => src2.map_uses(mapper), Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => { - map_mod(mapper, divisor); + mapper.map_mod(divisor); if let Some(tmp) = tmp { - map_def(mapper, tmp) + mapper.map_def(tmp) } } Inst::SignExtendData { .. } => {} @@ -2736,7 +2689,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::XmmRmRImm { ref op, @@ -2748,7 +2701,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else if *op == SseOpcode::Pextrb || *op == SseOpcode::Pextrw @@ -2760,11 +2713,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { || *op == SseOpcode::Roundpd { src2.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } } @@ -2777,11 +2730,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } } @@ -2793,10 +2746,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src1.map_uses(mapper); - map_use(mapper, src2); + mapper.map_use(src2); match *op { - Avx512Opcode::Vpermi2b => map_mod(mapper, dst), - _ => map_def(mapper, dst), + Avx512Opcode::Vpermi2b => mapper.map_mod(dst), + _ => mapper.map_def(dst), } } Inst::XmmRmiReg { @@ -2807,29 +2760,29 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } => { debug_assert_eq!(*src1, dst.to_reg()); src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } Inst::XmmUninitializedValue { ref mut dst, .. } => { - map_def(mapper, dst); + mapper.map_def(dst); } Inst::XmmLoadConst { ref mut dst, .. } => { - map_def(mapper, dst); + mapper.map_def(dst); } Inst::XmmMinMaxSeq { ref mut lhs, ref mut rhs_dst, .. } => { - map_use(mapper, lhs); - map_mod(mapper, rhs_dst); + mapper.map_use(lhs); + mapper.map_mod(rhs_dst); } Inst::XmmMovRM { ref mut src, ref mut dst, .. } => { - map_use(mapper, src); + mapper.map_use(src); dst.map_uses(mapper); } Inst::XmmCmpRmR { @@ -2838,9 +2791,9 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_use(mapper, dst); + mapper.map_use(dst); } - Inst::Imm { ref mut dst, .. } => map_def(mapper, dst), + Inst::Imm { ref mut dst, .. } => mapper.map_def(dst), Inst::MovRR { ref mut src, ref mut dst, @@ -2851,8 +2804,8 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut dst, .. } => { - map_use(mapper, src); - map_def(mapper, dst); + mapper.map_use(src); + mapper.map_def(dst); } Inst::GprToXmm { ref mut src, @@ -2860,7 +2813,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::CvtUint64ToFloatSeq { ref mut src, @@ -2869,10 +2822,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_gpr2, .. } => { - map_mod(mapper, src); - map_def(mapper, dst); - map_def(mapper, tmp_gpr1); - map_def(mapper, tmp_gpr2); + mapper.map_mod(src); + mapper.map_def(dst); + mapper.map_def(tmp_gpr1); + mapper.map_def(tmp_gpr2); } Inst::CvtFloatToSintSeq { ref mut src, @@ -2888,10 +2841,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_xmm, .. } => { - map_mod(mapper, src); - map_def(mapper, dst); - map_def(mapper, tmp_gpr); - map_def(mapper, tmp_xmm); + mapper.map_mod(src); + mapper.map_def(dst); + mapper.map_def(tmp_gpr); + mapper.map_def(tmp_xmm); } Inst::MovzxRmR { ref mut src, @@ -2899,11 +2852,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::MovsxRmR { ref mut src, @@ -2911,14 +2864,14 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::MovRM { ref mut src, ref mut dst, .. } => { - map_use(mapper, src); + mapper.map_use(src); dst.map_uses(mapper); } Inst::ShiftR { @@ -2927,7 +2880,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { debug_assert_eq!(*src, dst.to_reg()); - map_mod(mapper, dst); + mapper.map_mod(dst); *src = dst.to_reg(); } Inst::CmpRmiR { @@ -2936,9 +2889,9 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_use(mapper, dst); + mapper.map_use(dst); } - Inst::Setcc { ref mut dst, .. } => map_def(mapper, dst), + Inst::Setcc { ref mut dst, .. } => mapper.map_def(dst), Inst::Cmove { consequent: ref mut src, ref mut dst, @@ -2946,7 +2899,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *alternative = dst.to_reg(); } Inst::XmmCmove { @@ -2955,11 +2908,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); } Inst::Push64 { ref mut src } => src.map_uses(mapper), Inst::Pop64 { ref mut dst } => { - map_def(mapper, dst); + mapper.map_def(dst); } Inst::CallKnown { @@ -2968,10 +2921,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { for r in uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } } @@ -2982,10 +2935,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { for r in uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } dest.map_uses(mapper); } @@ -2996,25 +2949,25 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp2, .. } => { - map_use(mapper, idx); - map_def(mapper, tmp1); - map_def(mapper, tmp2); + mapper.map_use(idx); + mapper.map_def(tmp1); + mapper.map_def(tmp2); } Inst::JmpUnknown { ref mut target } => target.map_uses(mapper), - Inst::LoadExtName { ref mut dst, .. } => map_def(mapper, dst), + Inst::LoadExtName { ref mut dst, .. } => mapper.map_def(dst), Inst::LockCmpxchg { ref mut replacement, ref mut mem, .. } => { - map_use(mapper, replacement); + mapper.map_use(replacement); mem.map_uses(mapper); } - Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg), + Inst::ValueLabelMarker { ref mut reg, .. } => mapper.map_use(reg), Inst::Ret | Inst::EpiloguePlaceholder diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 118a0cdb25..179b95fa72 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -6,70 +6,29 @@ mod generated_code; // Types that the generated ISLE code uses via `use super::*`. use super::{ is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm, - Writable, }; use crate::isa::x64::inst::args::SyntheticAmode; use crate::isa::x64::settings as x64_settings; +use crate::machinst::isle::*; use crate::{ ir::{immediates::*, types::*, Inst, InstructionData, Opcode, Value, ValueList}, isa::x64::inst::{ args::{ Avx512Opcode, CmpOpcode, ExtMode, FcmpImm, Imm8Reg, RegMem, ShiftKind, SseOpcode, CC, }, - x64_map_regs, RegMapper, + x64_map_regs, }, - machinst::{get_output_reg, InsnInput, InsnOutput, LowerCtx}, + machinst::{get_output_reg, InsnInput, InsnOutput, LowerCtx, RegRenamer}, }; use smallvec::SmallVec; use std::convert::TryFrom; -type Unit = (); -type ValueSlice<'a> = &'a [Value]; -type ValueArray2 = [Value; 2]; -type ValueArray3 = [Value; 3]; -type WritableReg = Writable; -type ValueRegs = crate::machinst::ValueRegs; - pub struct SinkableLoad { inst: Inst, addr_input: InsnInput, offset: i32, } -#[derive(Default)] -struct RegRenamer { - // Map of `(old, new)` register names. Use a `SmallVec` because we typically - // only have one or two renamings. - renames: SmallVec<[(Reg, Reg); 2]>, -} - -impl RegRenamer { - fn add_rename(&mut self, old: Reg, new: Reg) { - self.renames.push((old, new)); - } - - fn get_rename(&self, reg: Reg) -> Option { - self.renames - .iter() - .find(|(old, _)| reg == *old) - .map(|(_, new)| *new) - } -} - -impl RegMapper for RegRenamer { - fn get_use(&self, reg: Reg) -> Option { - self.get_rename(reg) - } - - fn get_def(&self, reg: Reg) -> Option { - self.get_rename(reg) - } - - fn get_mod(&self, reg: Reg) -> Option { - self.get_rename(reg) - } -} - /// The main entry point for lowering with ISLE. pub(crate) fn lower( lower_ctx: &mut C, @@ -131,159 +90,7 @@ impl<'a, C> generated_code::Context for IsleContext<'a, C> where C: LowerCtx, { - #[inline] - fn unpack_value_array_2(&mut self, arr: &ValueArray2) -> (Value, Value) { - let [a, b] = *arr; - (a, b) - } - - #[inline] - fn pack_value_array_2(&mut self, a: Value, b: Value) -> ValueArray2 { - [a, b] - } - - #[inline] - fn unpack_value_array_3(&mut self, arr: &ValueArray3) -> (Value, Value, Value) { - let [a, b, c] = *arr; - (a, b, c) - } - - #[inline] - fn pack_value_array_3(&mut self, a: Value, b: Value, c: Value) -> ValueArray3 { - [a, b, c] - } - - #[inline] - fn value_reg(&mut self, reg: Reg) -> ValueRegs { - ValueRegs::one(reg) - } - - #[inline] - fn value_regs(&mut self, r1: Reg, r2: Reg) -> ValueRegs { - ValueRegs::two(r1, r2) - } - - #[inline] - fn temp_writable_reg(&mut self, ty: Type) -> WritableReg { - let value_regs = self.lower_ctx.alloc_tmp(ty); - value_regs.only_reg().unwrap() - } - - #[inline] - fn invalid_reg(&mut self) -> Reg { - Reg::invalid() - } - - #[inline] - fn put_in_reg(&mut self, val: Value) -> Reg { - self.lower_ctx.put_value_in_regs(val).only_reg().unwrap() - } - - #[inline] - fn put_in_regs(&mut self, val: Value) -> ValueRegs { - self.lower_ctx.put_value_in_regs(val) - } - - #[inline] - fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg { - regs.regs()[i] - } - - #[inline] - fn u8_as_u64(&mut self, x: u8) -> u64 { - x.into() - } - - #[inline] - fn u16_as_u64(&mut self, x: u16) -> u64 { - x.into() - } - - #[inline] - fn u32_as_u64(&mut self, x: u32) -> u64 { - x.into() - } - - #[inline] - fn ty_bits(&mut self, ty: Type) -> u16 { - ty.bits() - } - - #[inline] - fn fits_in_64(&mut self, ty: Type) -> Option { - if ty.bits() <= 64 { - Some(ty) - } else { - None - } - } - - #[inline] - fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { - list.as_slice(&self.lower_ctx.dfg().value_lists) - } - - #[inline] - fn unwrap_head_value_list_1(&mut self, list: ValueList) -> (Value, ValueSlice) { - match self.value_list_slice(list) { - [head, tail @ ..] => (*head, tail), - _ => out_of_line_panic("`unwrap_head_value_list_1` on empty `ValueList`"), - } - } - - #[inline] - fn unwrap_head_value_list_2(&mut self, list: ValueList) -> (Value, Value, ValueSlice) { - match self.value_list_slice(list) { - [head1, head2, tail @ ..] => (*head1, *head2, tail), - _ => out_of_line_panic( - "`unwrap_head_value_list_2` on list without at least two elements", - ), - } - } - - #[inline] - fn writable_reg_to_reg(&mut self, r: WritableReg) -> Reg { - r.to_reg() - } - - #[inline] - fn u64_from_imm64(&mut self, imm: Imm64) -> u64 { - imm.bits() as u64 - } - - #[inline] - fn inst_results(&mut self, inst: Inst) -> ValueSlice { - self.lower_ctx.dfg().inst_results(inst) - } - - #[inline] - fn first_result(&mut self, inst: Inst) -> Option { - self.lower_ctx.dfg().inst_results(inst).first().copied() - } - - #[inline] - fn inst_data(&mut self, inst: Inst) -> InstructionData { - self.lower_ctx.dfg()[inst].clone() - } - - #[inline] - fn value_type(&mut self, val: Value) -> Type { - self.lower_ctx.dfg().value_type(val) - } - - #[inline] - fn multi_lane(&mut self, ty: Type) -> Option<(u8, u16)> { - if ty.lane_count() > 1 { - Some((ty.lane_bits(), ty.lane_count())) - } else { - None - } - } - - #[inline] - fn def_inst(&mut self, val: Value) -> Option { - self.lower_ctx.dfg().value_def(val).inst() - } + isle_prelude_methods!(); #[inline] fn operand_size_of_type(&mut self, ty: Type) -> OperandSize { @@ -400,14 +207,6 @@ where None } } - - fn u64_from_ieee32(&mut self, val: Ieee32) -> u64 { - val.bits().into() - } - - fn u64_from_ieee64(&mut self, val: Ieee64) -> u64 { - val.bits() - } } #[inline] @@ -420,10 +219,3 @@ fn to_simm32(constant: i64) -> Option { None } } - -#[inline(never)] -#[cold] -#[track_caller] -fn out_of_line_panic(msg: &str) -> ! { - panic!("{}", msg); -} diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 2c10784cd3..89fd2808d2 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle 21143c73c97885afd9c34acedb8dbc653dca3b5a3e2e73754a5761762f7fe6ce2eefa520227e64c2b18f5626ca0ffa0a6aff54971099e619a464a23adf365bd2 +src/prelude.isle a069d14321601afc63959af23086709d67d189dafcdc7d1fc8534b32d89d49008acb8368b7b5a7bc51a353736a378197ac352ccce2bb3be89d93afb6979e480a src/isa/x64/inst.isle fdfbfc6dfad1fc5ed252e0a14ccc69baba51d0538e05cfb9916f6213e5a6fcfc9d22605a29bd684d6a66f6d5e1c8ec36a963660d52c2e8b3fb6e0758f7adb7b5 src/isa/x64/lower.isle 8555abdae385431c96aaabc392b7b3a8b1bbe733be08b007ef776850860cb77e85a140db02f427586c155c0b0129f9ffd531abd2e4a772c72667535cc015e609 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index ba6a6f36c8..61bac2a6b8 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -73,7 +73,7 @@ pub enum ConsumesFlags { } /// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 433. -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, Zero, @@ -82,7 +82,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 57. + // Rule at src/prelude.isle line 60. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -91,7 +91,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 92. + // Rule at src/prelude.isle line 95. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index a19d94407c..426a9c0c4d 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -68,6 +68,9 @@ pub use cranelift_entity as entity; #[cfg(feature = "unwind")] pub use gimli; +#[macro_use] +mod machinst; + pub mod binemit; pub mod cfg_printer; pub mod cursor; @@ -99,7 +102,6 @@ mod iterators; mod legalizer; mod licm; mod log; -mod machinst; mod nan_canonicalization; mod remove_constant_phis; mod result; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs new file mode 100644 index 0000000000..99e4c8f49f --- /dev/null +++ b/cranelift/codegen/src/machinst/isle.rs @@ -0,0 +1,185 @@ +use crate::ir::Value; +use regalloc::{Reg, Writable}; + +pub type Unit = (); +pub type ValueSlice<'a> = &'a [Value]; +pub type ValueArray2 = [Value; 2]; +pub type ValueArray3 = [Value; 3]; +pub type WritableReg = Writable; +pub type ValueRegs = crate::machinst::ValueRegs; + +#[macro_export] +#[doc(hidden)] +macro_rules! isle_prelude_methods { + () => { + #[inline] + fn unpack_value_array_2(&mut self, arr: &ValueArray2) -> (Value, Value) { + let [a, b] = *arr; + (a, b) + } + + #[inline] + fn pack_value_array_2(&mut self, a: Value, b: Value) -> ValueArray2 { + [a, b] + } + + #[inline] + fn unpack_value_array_3(&mut self, arr: &ValueArray3) -> (Value, Value, Value) { + let [a, b, c] = *arr; + (a, b, c) + } + + #[inline] + fn pack_value_array_3(&mut self, a: Value, b: Value, c: Value) -> ValueArray3 { + [a, b, c] + } + + #[inline] + fn value_reg(&mut self, reg: Reg) -> ValueRegs { + ValueRegs::one(reg) + } + + #[inline] + fn value_regs(&mut self, r1: Reg, r2: Reg) -> ValueRegs { + ValueRegs::two(r1, r2) + } + + #[inline] + fn temp_writable_reg(&mut self, ty: Type) -> WritableReg { + let value_regs = self.lower_ctx.alloc_tmp(ty); + value_regs.only_reg().unwrap() + } + + #[inline] + fn invalid_reg(&mut self) -> Reg { + Reg::invalid() + } + + #[inline] + fn put_in_reg(&mut self, val: Value) -> Reg { + self.lower_ctx.put_value_in_regs(val).only_reg().unwrap() + } + + #[inline] + fn put_in_regs(&mut self, val: Value) -> ValueRegs { + self.lower_ctx.put_value_in_regs(val) + } + + #[inline] + fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg { + regs.regs()[i] + } + + #[inline] + fn u8_as_u64(&mut self, x: u8) -> u64 { + x.into() + } + + #[inline] + fn u16_as_u64(&mut self, x: u16) -> u64 { + x.into() + } + + #[inline] + fn u32_as_u64(&mut self, x: u32) -> u64 { + x.into() + } + + #[inline] + fn ty_bits(&mut self, ty: Type) -> u16 { + ty.bits() + } + + #[inline] + fn fits_in_64(&mut self, ty: Type) -> Option { + if ty.bits() <= 64 { + Some(ty) + } else { + None + } + } + + #[inline] + fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { + list.as_slice(&self.lower_ctx.dfg().value_lists) + } + + #[inline] + fn unwrap_head_value_list_1(&mut self, list: ValueList) -> (Value, ValueSlice) { + match self.value_list_slice(list) { + [head, tail @ ..] => (*head, tail), + _ => crate::machinst::isle::out_of_line_panic( + "`unwrap_head_value_list_1` on empty `ValueList`", + ), + } + } + + #[inline] + fn unwrap_head_value_list_2(&mut self, list: ValueList) -> (Value, Value, ValueSlice) { + match self.value_list_slice(list) { + [head1, head2, tail @ ..] => (*head1, *head2, tail), + _ => crate::machinst::isle::out_of_line_panic( + "`unwrap_head_value_list_2` on list without at least two elements", + ), + } + } + + #[inline] + fn writable_reg_to_reg(&mut self, r: WritableReg) -> Reg { + r.to_reg() + } + + #[inline] + fn u64_from_imm64(&mut self, imm: Imm64) -> u64 { + imm.bits() as u64 + } + + #[inline] + fn inst_results(&mut self, inst: Inst) -> ValueSlice { + self.lower_ctx.dfg().inst_results(inst) + } + + #[inline] + fn first_result(&mut self, inst: Inst) -> Option { + self.lower_ctx.dfg().inst_results(inst).first().copied() + } + + #[inline] + fn inst_data(&mut self, inst: Inst) -> InstructionData { + self.lower_ctx.dfg()[inst].clone() + } + + #[inline] + fn value_type(&mut self, val: Value) -> Type { + self.lower_ctx.dfg().value_type(val) + } + + #[inline] + fn multi_lane(&mut self, ty: Type) -> Option<(u8, u16)> { + if ty.lane_count() > 1 { + Some((ty.lane_bits(), ty.lane_count())) + } else { + None + } + } + + #[inline] + fn def_inst(&mut self, val: Value) -> Option { + self.lower_ctx.dfg().value_def(val).inst() + } + + fn u64_from_ieee32(&mut self, val: Ieee32) -> u64 { + val.bits().into() + } + + fn u64_from_ieee64(&mut self, val: Ieee64) -> u64 { + val.bits() + } + }; +} + +#[inline(never)] +#[cold] +pub fn out_of_line_panic(msg: &str) -> ! { + panic!("{}", msg); +} diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index b0122dfe9d..20a78b6c87 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -81,6 +81,9 @@ use target_lexicon::Triple; #[cfg(feature = "unwind")] use crate::isa::unwind::systemv::RegisterMappingError; +#[macro_use] +pub mod isle; + pub mod lower; pub use lower::*; pub mod vcode; @@ -104,6 +107,8 @@ pub use inst_common::*; pub mod valueregs; pub use valueregs::*; pub mod debug; +pub use regmapping::*; +pub mod regmapping; /// A machine instruction. pub trait MachInst: Clone + Debug { diff --git a/cranelift/codegen/src/machinst/regmapping.rs b/cranelift/codegen/src/machinst/regmapping.rs new file mode 100644 index 0000000000..0f132e47b4 --- /dev/null +++ b/cranelift/codegen/src/machinst/regmapping.rs @@ -0,0 +1,83 @@ +use regalloc::{Reg, RegUsageMapper, Writable}; +use smallvec::SmallVec; + +// Define our own register-mapping trait so we can do arbitrary register +// renaming that are more free form than what `regalloc` constrains us to with +// its `RegUsageMapper` trait definition. +pub trait RegMapper { + fn get_use(&self, reg: Reg) -> Option; + fn get_def(&self, reg: Reg) -> Option; + fn get_mod(&self, reg: Reg) -> Option; + + fn map_use(&self, r: &mut Reg) { + if let Some(new) = self.get_use(*r) { + *r = new; + } + } + + fn map_def(&self, r: &mut Writable) { + if let Some(new) = self.get_def(r.to_reg()) { + *r = Writable::from_reg(new); + } + } + + fn map_mod(&self, r: &mut Writable) { + if let Some(new) = self.get_mod(r.to_reg()) { + *r = Writable::from_reg(new); + } + } +} + +impl RegMapper for T +where + T: RegUsageMapper, +{ + fn get_use(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_use(v).map(|r| r.to_reg()) + } + + fn get_def(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_def(v).map(|r| r.to_reg()) + } + + fn get_mod(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_mod(v).map(|r| r.to_reg()) + } +} + +#[derive(Default)] +pub struct RegRenamer { + // Map of `(old, new)` register names. Use a `SmallVec` because we typically + // only have one or two renamings. + renames: SmallVec<[(Reg, Reg); 2]>, +} + +impl RegRenamer { + pub fn add_rename(&mut self, old: Reg, new: Reg) { + self.renames.push((old, new)); + } + + fn get_rename(&self, reg: Reg) -> Option { + self.renames + .iter() + .find(|(old, _)| reg == *old) + .map(|(_, new)| *new) + } +} + +impl RegMapper for RegRenamer { + fn get_use(&self, reg: Reg) -> Option { + self.get_rename(reg) + } + + fn get_def(&self, reg: Reg) -> Option { + self.get_rename(reg) + } + + fn get_mod(&self, reg: Reg) -> Option { + self.get_rename(reg) + } +} diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 3f6f8ef7f7..c1367cc575 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -1,5 +1,8 @@ ;; This is a prelude of standard definitions for ISLE, the instruction-selector ;; DSL, as we use it bound to our interfaces. +;; +;; Note that all `extern` functions here are typically defined in the +;; `isle_prelude_methods` macro defined in `src/isa/isle.rs` ;;;; Primitive and External Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/isle/isle/src/codegen.rs b/cranelift/isle/isle/src/codegen.rs index 74874b6a1d..23a76d39e7 100644 --- a/cranelift/isle/isle/src/codegen.rs +++ b/cranelift/isle/isle/src/codegen.rs @@ -150,7 +150,11 @@ impl<'a> Codegen<'a> { pos.pretty_print_line(&self.typeenv.filenames[..]) ) .unwrap(); - writeln!(code, "#[derive(Clone, Debug)]").unwrap(); + if variants.iter().all(|v| v.fields.is_empty()) { + writeln!(code, "#[derive(Copy, Clone, Debug, PartialEq, Eq)]").unwrap(); + } else { + writeln!(code, "#[derive(Clone, Debug)]").unwrap(); + } writeln!(code, "pub enum {} {{", name).unwrap(); for variant in variants { let name = &self.typeenv.syms[variant.name.index()];