From 1141169ff8c72e9b375acdbb1aaa2c27ee135a4c Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 18 Nov 2021 10:38:16 -0600 Subject: [PATCH] aarch64: Initial work to transition backend to ISLE (#3541) * aarch64: Initial work to transition backend to ISLE This commit is what is hoped to be the initial commit towards migrating the aarch64 backend to ISLE. There's seemingly a lot of changes here but it's intended to largely be code motion. The current thinking is to closely follow the x64 backend for how all this is handled and organized. Major changes in this PR are: * The `Inst` enum is now defined in ISLE. This avoids having to define it in two places (once in Rust and once in ISLE). I've preserved all the comments in the ISLE and otherwise this isn't actually a functional change from the Rust perspective, it's still the same enum according to Rust. * Lots of little enums and things were moved to ISLE as well. As with `Inst` their definitions didn't change, only where they're defined. This will give future ISLE PRs access to all these operations. * Initial code for lowering `iconst`, `null`, and `bconst` are implemented. Ironically none of this is actually used right now because constant lowering is handled in `put_input_in_regs` which specially handles constants. Nonetheless I wanted to get at least something simple working which shows off how to special case various things that are specific to AArch64. In a future PR I plan to hook up const-lowering in ISLE to this path so even though `iconst`-the-clif-instruction is never lowered this should use the const lowering defined in ISLE rather than elsewhere in the backend (eventually leading to the deletion of the non-ISLE lowering). * The `IsleContext` skeleton is created and set up for future additions. * Some code for ISLE that's shared across all backends now lives in `isle_prelude_methods!()` and is deduplicated between the AArch64 backend and the x64 backend. * Register mapping is tweaked to do the same thing for AArch64 that it does for x64. Namely mapping virtual registers is supported instead of just virtual to machine registers. My main goal with this PR was to get AArch64 into a place where new instructions can be added with relative ease. Additionally I'm hoping to figure out as part of this change how much to share for ISLE between AArch64 and x64 (and other backends). * Don't use priorities with rules * Update .gitattributes with concise syntax * Deduplicate some type definitions * Rebuild ISLE * Move isa::isle to machinst::isle --- .gitattributes | 2 +- cranelift/codegen/build.rs | 22 +- cranelift/codegen/src/isa/aarch64/inst.isle | 1359 +++++++++++++ .../src/isa/aarch64/inst/emit_tests.rs | 1 + .../codegen/src/isa/aarch64/inst/imms.rs | 2 +- cranelift/codegen/src/isa/aarch64/inst/mod.rs | 1718 ++--------------- cranelift/codegen/src/isa/aarch64/lower.isle | 23 + cranelift/codegen/src/isa/aarch64/lower.rs | 2 + .../codegen/src/isa/aarch64/lower/isle.rs | 187 ++ .../lower/isle/generated_code.manifest | 4 + .../isa/aarch64/lower/isle/generated_code.rs | 1108 +++++++++++ .../codegen/src/isa/aarch64/lower_inst.rs | 29 +- cranelift/codegen/src/isa/x64/inst/args.rs | 2 +- cranelift/codegen/src/isa/x64/inst/mod.rs | 167 +- cranelift/codegen/src/isa/x64/lower/isle.rs | 216 +-- .../x64/lower/isle/generated_code.manifest | 2 +- .../src/isa/x64/lower/isle/generated_code.rs | 6 +- cranelift/codegen/src/lib.rs | 4 +- cranelift/codegen/src/machinst/isle.rs | 185 ++ cranelift/codegen/src/machinst/mod.rs | 5 + cranelift/codegen/src/machinst/regmapping.rs | 83 + cranelift/codegen/src/prelude.isle | 3 + cranelift/isle/isle/src/codegen.rs | 6 +- 23 files changed, 3283 insertions(+), 1853 deletions(-) create mode 100644 cranelift/codegen/src/isa/aarch64/inst.isle create mode 100644 cranelift/codegen/src/isa/aarch64/lower.isle create mode 100644 cranelift/codegen/src/isa/aarch64/lower/isle.rs create mode 100644 cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest create mode 100644 cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs create mode 100644 cranelift/codegen/src/machinst/isle.rs create mode 100644 cranelift/codegen/src/machinst/regmapping.rs diff --git a/.gitattributes b/.gitattributes index 928feac126..abbcc83672 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,4 +11,4 @@ # Tell GitHub this is generated code, and doesn't need to be shown in diffs by # default. -cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs linguist-generated +cranelift/codegen/src/isa/*/lower/isle/generated_code.rs linguist-generated diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index a5cc2eaf91..34e0a4a64b 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -198,6 +198,8 @@ fn get_isle_compilations(crate_dir: &std::path::Path) -> Result Result integer. + (FpuToInt + (op FpuToIntOp) + (rd WritableReg) + (rn Reg)) + + ;; Conversion: integer -> FP. + (IntToFpu + (op IntToFpuOp) + (rd WritableReg) + (rn Reg)) + + ;; FP conditional select, 32 bit. + (FpuCSel32 + (rd WritableReg) + (rn Reg) + (rm Reg) + (cond Cond)) + + ;; FP conditional select, 64 bit. + (FpuCSel64 + (rd WritableReg) + (rn Reg) + (rm Reg) + (cond Cond)) + + ;; Round to integer. + (FpuRound + (op FpuRoundMode) + (rd WritableReg) + (rn Reg)) + + ;; Move from a GPR to a vector register. The scalar value is parked in the lowest lane + ;; of the destination, and all other lanes are zeroed out. Currently only 32- and 64-bit + ;; transactions are supported. + (MovToFpu + (rd WritableReg) + (rn Reg) + (size ScalarSize)) + + ;; Move to a vector element from a GPR. + (MovToVec + (rd WritableReg) + (rn Reg) + (idx u8) + (size VectorSize)) + + ;; Unsigned move from a vector element to a GPR. + (MovFromVec + (rd WritableReg) + (rn Reg) + (idx u8) + (size VectorSize)) + + ;; Signed move from a vector element to a GPR. + (MovFromVecSigned + (rd WritableReg) + (rn Reg) + (idx u8) + (size VectorSize) + (scalar_size OperandSize)) + + ;; Duplicate general-purpose register to vector. + (VecDup + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Duplicate scalar to vector. + (VecDupFromFpu + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Duplicate FP immediate to vector. + (VecDupFPImm + (rd WritableReg) + (imm ASIMDFPModImm) + (size VectorSize)) + + ;; Duplicate immediate to vector. + (VecDupImm + (rd WritableReg) + (imm ASIMDMovModImm) + (invert bool) + (size VectorSize)) + + ;; Vector extend. + (VecExtend + (t VecExtendOp) + (rd WritableReg) + (rn Reg) + (high_half bool)) + + ;; Move vector element to another vector element. + (VecMovElement + (rd WritableReg) + (rn Reg) + (dest_idx u8) + (src_idx u8) + (size VectorSize)) + + ;; Vector widening operation. + (VecRRLong + (op VecRRLongOp) + (rd WritableReg) + (rn Reg) + (high_half bool)) + + ;; Vector narrowing operation. + (VecRRNarrow + (op VecRRNarrowOp) + (rd WritableReg) + (rn Reg) + (high_half bool)) + + ;; 1-operand vector instruction that operates on a pair of elements. + (VecRRPair + (op VecPairOp) + (rd WritableReg) + (rn Reg)) + + ;; 2-operand vector instruction that produces a result with twice the + ;; lane width and half the number of lanes. + (VecRRRLong + (alu_op VecRRRLongOp) + (rd WritableReg) + (rn Reg) + (rm Reg) + (high_half bool)) + + ;; 1-operand vector instruction that extends elements of the input + ;; register and operates on a pair of elements. The output lane width + ;; is double that of the input. + (VecRRPairLong + (op VecRRPairLongOp) + (rd WritableReg) + (rn Reg)) + + ;; A vector ALU op. + (VecRRR + (alu_op VecALUOp) + (rd WritableReg) + (rn Reg) + (rm Reg) + (size VectorSize)) + + ;; Vector two register miscellaneous instruction. + (VecMisc + (op VecMisc2) + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Vector instruction across lanes. + (VecLanes + (op VecLanesOp) + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Vector shift by immediate Shift Left (immediate), Unsigned Shift Right (immediate) + ;; Signed Shift Right (immediate). These are somewhat unusual in that, for right shifts, + ;; the allowed range of `imm` values is 1 to lane-size-in-bits, inclusive. A zero + ;; right-shift cannot be encoded. Left shifts are "normal", though, having valid `imm` + ;; values from 0 to lane-size-in-bits - 1 inclusive. + (VecShiftImm + (op VecShiftImmOp) + (rd WritableReg) + (rn Reg) + (size VectorSize) + (imm u8)) + + ;; Vector extract - create a new vector, being the concatenation of the lowest `imm4` bytes + ;; of `rm` followed by the uppermost `16 - imm4` bytes of `rn`. + (VecExtract + (rd WritableReg) + (rn Reg) + (rm Reg) + (imm4 u8)) + + ;; Table vector lookup - single register table. The table consists of 8-bit elements and is + ;; stored in `rn`, while `rm` contains 8-bit element indices. `is_extension` specifies whether + ;; to emit a TBX or a TBL instruction, i.e. whether to leave the elements in the destination + ;; vector that correspond to out-of-range indices (greater than 15) unmodified or to set them + ;; to 0. + (VecTbl + (rd WritableReg) + (rn Reg) + (rm Reg) + (is_extension bool)) + + ;; Table vector lookup - two register table. The table consists of 8-bit elements and is + ;; stored in `rn` and `rn2`, while `rm` contains 8-bit element indices. `is_extension` + ;; specifies whether to emit a TBX or a TBL instruction, i.e. whether to leave the elements in + ;; the destination vector that correspond to out-of-range indices (greater than 31) unmodified + ;; or to set them to 0. The table registers `rn` and `rn2` must have consecutive numbers + ;; modulo 32, that is v31 and v0 (in that order) are consecutive registers. + (VecTbl2 + (rd WritableReg) + (rn Reg) + (rn2 Reg) + (rm Reg) + (is_extension bool)) + + ;; Load an element and replicate to all lanes of a vector. + (VecLoadReplicate + (rd WritableReg) + (rn Reg) + (size VectorSize)) + + ;; Vector conditional select, 128 bit. A synthetic instruction, which generates a 4-insn + ;; control-flow diamond. + (VecCSel + (rd WritableReg) + (rn Reg) + (rm Reg) + (cond Cond)) + + ;; Move to the NZCV flags (actually a `MSR NZCV, Xn` insn). + (MovToNZCV + (rn Reg)) + + ;; Move from the NZCV flags (actually a `MRS Xn, NZCV` insn). + (MovFromNZCV + (rd WritableReg)) + + ;; A machine call instruction. N.B.: this allows only a +/- 128MB offset (it uses a relocation + ;; of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the + ;; code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit + ;; target. + (Call + (info BoxCallInfo)) + + ;; A machine indirect-call instruction. + (CallInd + (info BoxCallIndInfo)) + + ;; ---- branches (exactly one must appear at end of BB) ---- + + ;; A machine return instruction. + (Ret) + + ;; A placeholder instruction, generating no code, meaning that a function epilogue must be + ;; inserted there. + (EpiloguePlaceholder) + + ;; An unconditional branch. + (Jump + (dest BranchTarget)) + + ;; A conditional branch. Contains two targets; at emission time, both are emitted, but + ;; the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the + ;; choice of taken/not_taken (inverting the branch polarity as needed) based on the + ;; fallthrough at the time of lowering. + (CondBr + (taken BranchTarget) + (not_taken BranchTarget) + (kind CondBrKind)) + + ;; A conditional trap: execute a `udf` if the condition is true. This is + ;; one VCode instruction because it uses embedded control flow; it is + ;; logically a single-in, single-out region, but needs to appear as one + ;; unit to the register allocator. + ;; + ;; The `CondBrKind` gives the conditional-branch condition that will + ;; *execute* the embedded `Inst`. (In the emitted code, we use the inverse + ;; of this condition in a branch that skips the trap instruction.) + (TrapIf + (kind CondBrKind) + (trap_code TrapCode)) + + ;; An indirect branch through a register, augmented with set of all + ;; possible successors. + (IndirectBr + (rn Reg) + (targets VecMachLabel)) + + ;; A "break" instruction, used for e.g. traps and debug breakpoints. + (Brk) + + ;; An instruction guaranteed to always be undefined and to trigger an illegal instruction at + ;; runtime. + (Udf + (trap_code TrapCode)) + + ;; Compute the address (using a PC-relative offset) of a memory location, using the `ADR` + ;; instruction. Note that we take a simple offset, not a `MemLabel`, here, because `Adr` is + ;; only used for now in fixed lowering sequences with hardcoded offsets. In the future we may + ;; need full `MemLabel` support. + (Adr + (rd WritableReg) + ;; Offset in range -2^20 .. 2^20. + (off i32)) + + ;; Raw 32-bit word, used for inline constants and jump-table entries. + (Word4 + (data u32)) + + ;; Raw 64-bit word, used for inline constants. + (Word8 + (data u64)) + + ;; Jump-table sequence, as one compound instruction (see note in lower_inst.rs for rationale). + (JTSequence + (info BoxJTSequenceInfo) + (ridx Reg) + (rtmp1 WritableReg) + (rtmp2 WritableReg)) + + ;; Load an inline symbol reference. + (LoadExtName + (rd WritableReg) + (name BoxExternalName) + (offset i64)) + + ;; Load address referenced by `mem` into `rd`. + (LoadAddr + (rd WritableReg) + (mem AMode)) + + ;; Marker, no-op in generated code: SP "virtual offset" is adjusted. This + ;; controls how AMode::NominalSPOffset args are lowered. + (VirtualSPOffsetAdj + (offset i64)) + + ;; Meta-insn, no-op in generated code: emit constant/branch veneer island + ;; at this point (with a guard jump around it) if less than the needed + ;; space is available before the next branch deadline. See the `MachBuffer` + ;; implementation in `machinst/buffer.rs` for the overall algorithm. In + ;; brief, we retain a set of "pending/unresolved label references" from + ;; branches as we scan forward through instructions to emit machine code; + ;; if we notice we're about to go out of range on an unresolved reference, + ;; we stop, emit a bunch of "veneers" (branches in a form that has a longer + ;; range, e.g. a 26-bit-offset unconditional jump), and point the original + ;; label references to those. This is an "island" because it comes in the + ;; middle of the code. + ;; + ;; This meta-instruction is a necessary part of the logic that determines + ;; where to place islands. Ordinarily, we want to place them between basic + ;; blocks, so we compute the worst-case size of each block, and emit the + ;; island before starting a block if we would exceed a deadline before the + ;; end of the block. However, some sequences (such as an inline jumptable) + ;; are variable-length and not accounted for by this logic; so these + ;; lowered sequences include an `EmitIsland` to trigger island generation + ;; where necessary. + (EmitIsland + ;; The needed space before the next deadline. + (needed_space CodeOffset)) + + ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol in x0. + (ElfTlsGetAddr + (symbol ExternalName)) + + ;; A definition of a value label. + (ValueLabelMarker + (reg Reg) + (label ValueLabel)) + + ;; An unwind pseudo-instruction. + (Unwind + (inst UnwindInst)) +)) + +;; An ALU operation. This can be paired with several instruction formats +;; below (see `Inst`) in any combination. +(type ALUOp + (enum + (Add32) + (Add64) + (Sub32) + (Sub64) + (Orr32) + (Orr64) + (OrrNot32) + (OrrNot64) + (And32) + (And64) + (AndS32) + (AndS64) + (AndNot32) + (AndNot64) + ;; XOR (AArch64 calls this "EOR") + (Eor32) + ;; XOR (AArch64 calls this "EOR") + (Eor64) + ;; XNOR (AArch64 calls this "EOR-NOT") + (EorNot32) + ;; XNOR (AArch64 calls this "EOR-NOT") + (EorNot64) + ;; Add, setting flags + (AddS32) + ;; Add, setting flags + (AddS64) + ;; Sub, setting flags + (SubS32) + ;; Sub, setting flags + (SubS64) + ;; Signed multiply, high-word result + (SMulH) + ;; Unsigned multiply, high-word result + (UMulH) + (SDiv64) + (UDiv64) + (RotR32) + (RotR64) + (Lsr32) + (Lsr64) + (Asr32) + (Asr64) + (Lsl32) + (Lsl64) + ;; Add with carry + (Adc32) + (Adc64) + ;; Add with carry, settings flags + (AdcS32) + (AdcS64) + ;; Subtract with carry + (Sbc32) + (Sbc64) + ;; Subtract with carry, settings flags + (SbcS32) + (SbcS64) +)) + +;; An ALU operation with three arguments. +(type ALUOp3 + (enum + ;; Multiply-add + (MAdd32) + ;; Multiply-add + (MAdd64) + ;; Multiply-sub + (MSub32) + ;; Multiply-sub + (MSub64) +)) + +(type UImm5 (primitive UImm5)) +(type Imm12 (primitive Imm12)) +(type ImmLogic (primitive ImmLogic)) +(type ImmShift (primitive ImmShift)) +(type ShiftOpAndAmt (primitive ShiftOpAndAmt)) +(type MoveWideConst (primitive MoveWideConst)) +(type NZCV (primitive NZCV)) +(type ASIMDFPModImm (primitive ASIMDFPModImm)) +(type ASIMDMovModImm (primitive ASIMDMovModImm)) + +(type BoxCallInfo (primitive BoxCallInfo)) +(type BoxCallIndInfo (primitive BoxCallIndInfo)) +(type VecMachLabel (primitive VecMachLabel)) +(type CondBrKind (primitive CondBrKind)) +(type BranchTarget (primitive BranchTarget)) +(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) +(type BoxExternalName (primitive BoxExternalName)) +(type CodeOffset (primitive CodeOffset)) +(type ExternalName (primitive ExternalName)) +(type ValueLabel (primitive ValueLabel)) +(type UnwindInst (primitive UnwindInst)) + +(type ExtendOp extern + (enum + (UXTB) + (UXTH) + (UXTW) + (UXTX) + (SXTB) + (SXTH) + (SXTW) + (SXTX) +)) + +;; An operation on the bits of a register. This can be paired with several instruction formats +;; below (see `Inst`) in any combination. +(type BitOp + (enum + ;; Bit reverse + (RBit32) + ;; Bit reverse + (RBit64) + (Clz32) + (Clz64) + (Cls32) + (Cls64) +)) + +(type AMode extern (enum)) +(type PairAMode extern (enum)) +(type FPUOpRI extern (enum)) + +(type OperandSize extern + (enum Size32 + Size64)) + +(type ScalarSize extern + (enum Size8 + Size16 + Size32 + Size64 + Size128)) + +(type Cond extern + (enum + (Eq) + (Ne) + (Hs) + (Lo) + (Mi) + (Pl) + (Vs) + (Vc) + (Hi) + (Ls) + (Ge) + (Lt) + (Gt) + (Le) + (Al) + (Nv) +)) + +(type VectorSize extern + (enum + (Size8x8) + (Size8x16) + (Size16x4) + (Size16x8) + (Size32x2) + (Size32x4) + (Size64x2) +)) + +;; A floating-point unit (FPU) operation with one arg. +(type FPUOp1 + (enum + (Abs32) + (Abs64) + (Neg32) + (Neg64) + (Sqrt32) + (Sqrt64) + (Cvt32To64) + (Cvt64To32) +)) + +;; A floating-point unit (FPU) operation with two args. +(type FPUOp2 + (enum + (Add32) + (Add64) + (Sub32) + (Sub64) + (Mul32) + (Mul64) + (Div32) + (Div64) + (Max32) + (Max64) + (Min32) + (Min64) + ;; Signed saturating add + (Sqadd64) + ;; Unsigned saturating add + (Uqadd64) + ;; Signed saturating subtract + (Sqsub64) + ;; Unsigned saturating subtract + (Uqsub64) +)) + +;; A floating-point unit (FPU) operation with three args. +(type FPUOp3 + (enum + (MAdd32) + (MAdd64) +)) + +;; A conversion from an FP to an integer value. +(type FpuToIntOp + (enum + (F32ToU32) + (F32ToI32) + (F32ToU64) + (F32ToI64) + (F64ToU32) + (F64ToI32) + (F64ToU64) + (F64ToI64) +)) + +;; A conversion from an integer to an FP value. +(type IntToFpuOp + (enum + (U32ToF32) + (I32ToF32) + (U32ToF64) + (I32ToF64) + (U64ToF32) + (I64ToF32) + (U64ToF64) + (I64ToF64) +)) + +;; Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to +;; nearest, and for 32- or 64-bit FP values. +(type FpuRoundMode + (enum + (Minus32) + (Minus64) + (Plus32) + (Plus64) + (Zero32) + (Zero64) + (Nearest32) + (Nearest64) +)) + +;; Type of vector element extensions. +(type VecExtendOp + (enum + ;; Signed extension of 8-bit elements + (Sxtl8) + ;; Signed extension of 16-bit elements + (Sxtl16) + ;; Signed extension of 32-bit elements + (Sxtl32) + ;; Unsigned extension of 8-bit elements + (Uxtl8) + ;; Unsigned extension of 16-bit elements + (Uxtl16) + ;; Unsigned extension of 32-bit elements + (Uxtl32) +)) + +;; A vector ALU operation. +(type VecALUOp + (enum + ;; Signed saturating add + (Sqadd) + ;; Unsigned saturating add + (Uqadd) + ;; Signed saturating subtract + (Sqsub) + ;; Unsigned saturating subtract + (Uqsub) + ;; Compare bitwise equal + (Cmeq) + ;; Compare signed greater than or equal + (Cmge) + ;; Compare signed greater than + (Cmgt) + ;; Compare unsigned higher + (Cmhs) + ;; Compare unsigned higher or same + (Cmhi) + ;; Floating-point compare equal + (Fcmeq) + ;; Floating-point compare greater than + (Fcmgt) + ;; Floating-point compare greater than or equal + (Fcmge) + ;; Bitwise and + (And) + ;; Bitwise bit clear + (Bic) + ;; Bitwise inclusive or + (Orr) + ;; Bitwise exclusive or + (Eor) + ;; Bitwise select + (Bsl) + ;; Unsigned maximum pairwise + (Umaxp) + ;; Add + (Add) + ;; Subtract + (Sub) + ;; Multiply + (Mul) + ;; Signed shift left + (Sshl) + ;; Unsigned shift left + (Ushl) + ;; Unsigned minimum + (Umin) + ;; Signed minimum + (Smin) + ;; Unsigned maximum + (Umax) + ;; Signed maximum + (Smax) + ;; Unsigned rounding halving add + (Urhadd) + ;; Floating-point add + (Fadd) + ;; Floating-point subtract + (Fsub) + ;; Floating-point divide + (Fdiv) + ;; Floating-point maximum + (Fmax) + ;; Floating-point minimum + (Fmin) + ;; Floating-point multiply + (Fmul) + ;; Add pairwise + (Addp) + ;; Zip vectors (primary) [meaning, high halves] + (Zip1) + ;; Signed saturating rounding doubling multiply returning high half + (Sqrdmulh) +)) + +;; A Vector miscellaneous operation with two registers. +(type VecMisc2 + (enum + ;; Bitwise NOT + (Not) + ;; Negate + (Neg) + ;; Absolute value + (Abs) + ;; Floating-point absolute value + (Fabs) + ;; Floating-point negate + (Fneg) + ;; Floating-point square root + (Fsqrt) + ;; Reverse elements in 64-bit doublewords + (Rev64) + ;; Floating-point convert to signed integer, rounding toward zero + (Fcvtzs) + ;; Floating-point convert to unsigned integer, rounding toward zero + (Fcvtzu) + ;; Signed integer convert to floating-point + (Scvtf) + ;; Unsigned integer convert to floating-point + (Ucvtf) + ;; Floating point round to integral, rounding towards nearest + (Frintn) + ;; Floating point round to integral, rounding towards zero + (Frintz) + ;; Floating point round to integral, rounding towards minus infinity + (Frintm) + ;; Floating point round to integral, rounding towards plus infinity + (Frintp) + ;; Population count per byte + (Cnt) + ;; Compare bitwise equal to 0 + (Cmeq0) +)) + +;; A vector widening operation with one argument. +(type VecRRLongOp + (enum + ;; Floating-point convert to higher precision long, 16-bit elements + (Fcvtl16) + ;; Floating-point convert to higher precision long, 32-bit elements + (Fcvtl32) + ;; Shift left long (by element size), 8-bit elements + (Shll8) + ;; Shift left long (by element size), 16-bit elements + (Shll16) + ;; Shift left long (by element size), 32-bit elements + (Shll32) +)) + +;; A vector narrowing operation with one argument. +(type VecRRNarrowOp + (enum + ;; Extract narrow, 16-bit elements + (Xtn16) + ;; Extract narrow, 32-bit elements + (Xtn32) + ;; Extract narrow, 64-bit elements + (Xtn64) + ;; Signed saturating extract narrow, 16-bit elements + (Sqxtn16) + ;; Signed saturating extract narrow, 32-bit elements + (Sqxtn32) + ;; Signed saturating extract narrow, 64-bit elements + (Sqxtn64) + ;; Signed saturating extract unsigned narrow, 16-bit elements + (Sqxtun16) + ;; Signed saturating extract unsigned narrow, 32-bit elements + (Sqxtun32) + ;; Signed saturating extract unsigned narrow, 64-bit elements + (Sqxtun64) + ;; Unsigned saturating extract narrow, 16-bit elements + (Uqxtn16) + ;; Unsigned saturating extract narrow, 32-bit elements + (Uqxtn32) + ;; Unsigned saturating extract narrow, 64-bit elements + (Uqxtn64) + ;; Floating-point convert to lower precision narrow, 32-bit elements + (Fcvtn32) + ;; Floating-point convert to lower precision narrow, 64-bit elements + (Fcvtn64) +)) + +(type VecRRRLongOp + (enum + ;; Signed multiply long. + (Smull8) + (Smull16) + (Smull32) + ;; Unsigned multiply long. + (Umull8) + (Umull16) + (Umull32) + ;; Unsigned multiply add long + (Umlal8) + (Umlal16) + (Umlal32) +)) + +;; A vector operation on a pair of elements with one register. +(type VecPairOp + (enum + ;; Add pair of elements + (Addp) +)) + +;; 1-operand vector instruction that extends elements of the input register +;; and operates on a pair of elements. +(type VecRRPairLongOp + (enum + ;; Sign extend and add pair of elements + (Saddlp8) + (Saddlp16) + ;; Unsigned extend and add pair of elements + (Uaddlp8) + (Uaddlp16) +)) + +;; An operation across the lanes of vectors. +(type VecLanesOp + (enum + ;; Integer addition across a vector + (Addv) + ;; Unsigned minimum across a vector + (Uminv) +)) + +;; A shift-by-immediate operation on each lane of a vector. +(type VecShiftImmOp + (enum + ;; Unsigned shift left + (Shl) + ;; Unsigned shift right + (Ushr) + ;; Signed shift right + (Sshr) +)) + +;; Atomic read-modify-write operations with acquire-release semantics +(type AtomicRMWOp + (enum + (Add) + (Clr) + (Eor) + (Set) + (Smax) + (Smin) + (Umax) + (Umin) +)) + +;; Extractor helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl move_wide_const_from_u64 (MoveWideConst) u64) +(extern extractor move_wide_const_from_u64 move_wide_const_from_u64) + +(decl move_wide_const_from_negated_u64 (MoveWideConst) u64) +(extern extractor move_wide_const_from_negated_u64 move_wide_const_from_negated_u64) + +(decl imm_logic_from_u64 (ImmLogic) u64) +(extern extractor imm_logic_from_u64 imm_logic_from_u64) + +(decl integral_ty (Type) Type) +(extern extractor integral_ty integral_ty) + +;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Emit an instruction. +;; +;; This is low-level and side-effectful; it should only be used as an +;; implementation detail by helpers that preserve the SSA facade themselves. +(decl emit (MInst) Unit) +(extern constructor emit emit) + +;; Helper for creating the zero register. +(decl zero_reg () Reg) +(extern constructor zero_reg zero_reg) + +;; Helper for emitting `MInst.MovZ` instructions. +(decl movz (MoveWideConst OperandSize) Reg) +(rule (movz imm size) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.MovZ dst imm size)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.MovN` instructions. +(decl movn (MoveWideConst OperandSize) Reg) +(rule (movn imm size) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.MovN dst imm size)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `MInst.AluRRImmLogic` instructions. +(decl alu_rr_imm_logic (ALUOp Reg ImmLogic) Reg) +(rule (alu_rr_imm_logic op src imm) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRImmLogic op dst src imm)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `orr64` instructions. +(decl orr64 (Reg ImmLogic) Reg) +(rule (orr64 src imm) (alu_rr_imm_logic (ALUOp.Orr64) src imm)) + +;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl imm (Type u64) Reg) + +;; 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ +(rule (imm (integral_ty _ty) (move_wide_const_from_u64 n)) + (movz n (OperandSize.Size64))) + +;; 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVN +(rule (imm (integral_ty _ty) (move_wide_const_from_negated_u64 n)) + (movn n (OperandSize.Size64))) + +;; Weird logical-instruction immediate in ORI using zero register +(rule (imm (integral_ty _ty) (imm_logic_from_u64 n)) + (orr64 (zero_reg) n)) + +(decl load_constant64_full (u64) Reg) +(extern constructor load_constant64_full load_constant64_full) + +;; Fallback for integral 64-bit constants that uses lots of `movk` +(rule (imm (integral_ty _ty) n) + (load_constant64_full n)) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 5b8fe374d8..915657c2f5 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -1,4 +1,5 @@ use crate::ir::types::*; +use crate::ir::TrapCode; use crate::isa::aarch64::inst::*; use crate::isa::test_utils; use crate::isa::CallConv; diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rs index 08fc75431d..79473375dd 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs @@ -327,7 +327,7 @@ impl Imm12 { } /// An immediate for logical instructions. -#[derive(Clone, Debug, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq)] pub struct ImmLogic { /// The actual value. value: u64, diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 98e3a85cff..83d878b746 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -7,16 +7,14 @@ use crate::binemit::{Addend, CodeOffset, Reloc}; use crate::ir::types::{ B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64, }; -use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; -use crate::isa::unwind::UnwindInst; +use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, Type, ValueLabel}; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; +use regalloc::RegUsageCollector; use regalloc::{PrettyPrint, RealRegUniverse, Reg, RegClass, SpillSlot, VirtualReg, Writable}; -use regalloc::{RegUsageCollector, RegUsageMapper}; -use alloc::boxed::Box; use alloc::vec::Vec; use core::convert::TryFrom; use smallvec::{smallvec, SmallVec}; @@ -40,118 +38,11 @@ mod emit_tests; //============================================================================= // Instructions (top level): definition -/// An ALU operation. This can be paired with several instruction formats -/// below (see `Inst`) in any combination. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum ALUOp { - Add32, - Add64, - Sub32, - Sub64, - Orr32, - Orr64, - OrrNot32, - OrrNot64, - And32, - And64, - AndS32, - AndS64, - AndNot32, - AndNot64, - /// XOR (AArch64 calls this "EOR") - Eor32, - /// XOR (AArch64 calls this "EOR") - Eor64, - /// XNOR (AArch64 calls this "EOR-NOT") - EorNot32, - /// XNOR (AArch64 calls this "EOR-NOT") - EorNot64, - /// Add, setting flags - AddS32, - /// Add, setting flags - AddS64, - /// Sub, setting flags - SubS32, - /// Sub, setting flags - SubS64, - /// Signed multiply, high-word result - SMulH, - /// Unsigned multiply, high-word result - UMulH, - SDiv64, - UDiv64, - RotR32, - RotR64, - Lsr32, - Lsr64, - Asr32, - Asr64, - Lsl32, - Lsl64, - /// Add with carry - Adc32, - Adc64, - /// Add with carry, settings flags - AdcS32, - AdcS64, - /// Subtract with carry - Sbc32, - Sbc64, - /// Subtract with carry, settings flags - SbcS32, - SbcS64, -} - -/// An ALU operation with three arguments. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum ALUOp3 { - /// Multiply-add - MAdd32, - /// Multiply-add - MAdd64, - /// Multiply-sub - MSub32, - /// Multiply-sub - MSub64, -} - -/// A floating-point unit (FPU) operation with one arg. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FPUOp1 { - Abs32, - Abs64, - Neg32, - Neg64, - Sqrt32, - Sqrt64, - Cvt32To64, - Cvt64To32, -} - -/// A floating-point unit (FPU) operation with two args. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FPUOp2 { - Add32, - Add64, - Sub32, - Sub64, - Mul32, - Mul64, - Div32, - Div64, - Max32, - Max64, - Min32, - Min64, - /// Signed saturating add - Sqadd64, - /// Unsigned saturating add - Uqadd64, - /// Signed saturating subtract - Sqsub64, - /// Unsigned saturating subtract - Uqsub64, -} +pub use crate::isa::aarch64::lower::isle::generated_code::{ + ALUOp, ALUOp3, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, + IntToFpuOp, MInst as Inst, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, + VecRRNarrowOp, VecRRPairLongOp, VecRRRLongOp, VecShiftImmOp, +}; /// A floating-point unit (FPU) operation with two args, a register and an immediate. #[derive(Copy, Clone, Debug)] @@ -166,318 +57,6 @@ pub enum FPUOpRI { Sli64(FPULeftShiftImm), } -/// A floating-point unit (FPU) operation with three args. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FPUOp3 { - MAdd32, - MAdd64, -} - -/// A conversion from an FP to an integer value. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FpuToIntOp { - F32ToU32, - F32ToI32, - F32ToU64, - F32ToI64, - F64ToU32, - F64ToI32, - F64ToU64, - F64ToI64, -} - -/// A conversion from an integer to an FP value. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum IntToFpuOp { - U32ToF32, - I32ToF32, - U32ToF64, - I32ToF64, - U64ToF32, - I64ToF32, - U64ToF64, - I64ToF64, -} - -/// Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to -/// nearest, and for 32- or 64-bit FP values. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum FpuRoundMode { - Minus32, - Minus64, - Plus32, - Plus64, - Zero32, - Zero64, - Nearest32, - Nearest64, -} - -/// Type of vector element extensions. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecExtendOp { - /// Signed extension of 8-bit elements - Sxtl8, - /// Signed extension of 16-bit elements - Sxtl16, - /// Signed extension of 32-bit elements - Sxtl32, - /// Unsigned extension of 8-bit elements - Uxtl8, - /// Unsigned extension of 16-bit elements - Uxtl16, - /// Unsigned extension of 32-bit elements - Uxtl32, -} - -/// A vector ALU operation. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecALUOp { - /// Signed saturating add - Sqadd, - /// Unsigned saturating add - Uqadd, - /// Signed saturating subtract - Sqsub, - /// Unsigned saturating subtract - Uqsub, - /// Compare bitwise equal - Cmeq, - /// Compare signed greater than or equal - Cmge, - /// Compare signed greater than - Cmgt, - /// Compare unsigned higher - Cmhs, - /// Compare unsigned higher or same - Cmhi, - /// Floating-point compare equal - Fcmeq, - /// Floating-point compare greater than - Fcmgt, - /// Floating-point compare greater than or equal - Fcmge, - /// Bitwise and - And, - /// Bitwise bit clear - Bic, - /// Bitwise inclusive or - Orr, - /// Bitwise exclusive or - Eor, - /// Bitwise select - Bsl, - /// Unsigned maximum pairwise - Umaxp, - /// Add - Add, - /// Subtract - Sub, - /// Multiply - Mul, - /// Signed shift left - Sshl, - /// Unsigned shift left - Ushl, - /// Unsigned minimum - Umin, - /// Signed minimum - Smin, - /// Unsigned maximum - Umax, - /// Signed maximum - Smax, - /// Unsigned rounding halving add - Urhadd, - /// Floating-point add - Fadd, - /// Floating-point subtract - Fsub, - /// Floating-point divide - Fdiv, - /// Floating-point maximum - Fmax, - /// Floating-point minimum - Fmin, - /// Floating-point multiply - Fmul, - /// Add pairwise - Addp, - /// Zip vectors (primary) [meaning, high halves] - Zip1, - /// Signed saturating rounding doubling multiply returning high half - Sqrdmulh, -} - -/// A Vector miscellaneous operation with two registers. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecMisc2 { - /// Bitwise NOT - Not, - /// Negate - Neg, - /// Absolute value - Abs, - /// Floating-point absolute value - Fabs, - /// Floating-point negate - Fneg, - /// Floating-point square root - Fsqrt, - /// Reverse elements in 64-bit doublewords - Rev64, - /// Floating-point convert to signed integer, rounding toward zero - Fcvtzs, - /// Floating-point convert to unsigned integer, rounding toward zero - Fcvtzu, - /// Signed integer convert to floating-point - Scvtf, - /// Unsigned integer convert to floating-point - Ucvtf, - /// Floating point round to integral, rounding towards nearest - Frintn, - /// Floating point round to integral, rounding towards zero - Frintz, - /// Floating point round to integral, rounding towards minus infinity - Frintm, - /// Floating point round to integral, rounding towards plus infinity - Frintp, - /// Population count per byte - Cnt, - /// Compare bitwise equal to 0 - Cmeq0, -} - -/// A vector widening operation with one argument. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRLongOp { - /// Floating-point convert to higher precision long, 16-bit elements - Fcvtl16, - /// Floating-point convert to higher precision long, 32-bit elements - Fcvtl32, - /// Shift left long (by element size), 8-bit elements - Shll8, - /// Shift left long (by element size), 16-bit elements - Shll16, - /// Shift left long (by element size), 32-bit elements - Shll32, -} - -/// A vector narrowing operation with one argument. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRNarrowOp { - /// Extract narrow, 16-bit elements - Xtn16, - /// Extract narrow, 32-bit elements - Xtn32, - /// Extract narrow, 64-bit elements - Xtn64, - /// Signed saturating extract narrow, 16-bit elements - Sqxtn16, - /// Signed saturating extract narrow, 32-bit elements - Sqxtn32, - /// Signed saturating extract narrow, 64-bit elements - Sqxtn64, - /// Signed saturating extract unsigned narrow, 16-bit elements - Sqxtun16, - /// Signed saturating extract unsigned narrow, 32-bit elements - Sqxtun32, - /// Signed saturating extract unsigned narrow, 64-bit elements - Sqxtun64, - /// Unsigned saturating extract narrow, 16-bit elements - Uqxtn16, - /// Unsigned saturating extract narrow, 32-bit elements - Uqxtn32, - /// Unsigned saturating extract narrow, 64-bit elements - Uqxtn64, - /// Floating-point convert to lower precision narrow, 32-bit elements - Fcvtn32, - /// Floating-point convert to lower precision narrow, 64-bit elements - Fcvtn64, -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRRLongOp { - /// Signed multiply long. - Smull8, - Smull16, - Smull32, - /// Unsigned multiply long. - Umull8, - Umull16, - Umull32, - /// Unsigned multiply add long - Umlal8, - Umlal16, - Umlal32, -} - -/// A vector operation on a pair of elements with one register. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecPairOp { - /// Add pair of elements - Addp, -} - -/// 1-operand vector instruction that extends elements of the input register -/// and operates on a pair of elements. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecRRPairLongOp { - /// Sign extend and add pair of elements - Saddlp8, - Saddlp16, - /// Unsigned extend and add pair of elements - Uaddlp8, - Uaddlp16, -} - -/// An operation across the lanes of vectors. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecLanesOp { - /// Integer addition across a vector - Addv, - /// Unsigned minimum across a vector - Uminv, -} - -/// A shift-by-immediate operation on each lane of a vector. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VecShiftImmOp { - // Unsigned shift left - Shl, - // Unsigned shift right - Ushr, - // Signed shift right - Sshr, -} - -/// Atomic read-modify-write operations with acquire-release semantics -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum AtomicRMWOp { - Add, - Clr, - Eor, - Set, - Smax, - Smin, - Umax, - Umin, -} - -/// An operation on the bits of a register. This can be paired with several instruction formats -/// below (see `Inst`) in any combination. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum BitOp { - /// Bit reverse - RBit32, - /// Bit reverse - RBit64, - Clz32, - Clz64, - Cls32, - Cls64, -} - impl BitOp { /// What is the opcode's native width? pub fn operand_size(&self) -> OperandSize { @@ -545,864 +124,6 @@ pub struct JTSequenceInfo { pub targets_for_term: Vec, // needed for MachTerminator. } -/// Instruction formats. -#[derive(Clone, Debug)] -pub enum Inst { - /// A no-op of zero size. - Nop0, - - /// A no-op that is one instruction large. - Nop4, - - /// An ALU operation with two register sources and a register destination. - AluRRR { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - }, - /// An ALU operation with three register sources and a register destination. - AluRRRR { - alu_op: ALUOp3, - rd: Writable, - rn: Reg, - rm: Reg, - ra: Reg, - }, - /// An ALU operation with a register source and an immediate-12 source, and a register - /// destination. - AluRRImm12 { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - imm12: Imm12, - }, - /// An ALU operation with a register source and an immediate-logic source, and a register destination. - AluRRImmLogic { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - imml: ImmLogic, - }, - /// An ALU operation with a register source and an immediate-shiftamt source, and a register destination. - AluRRImmShift { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - immshift: ImmShift, - }, - /// An ALU operation with two register sources, one of which can be shifted, and a register - /// destination. - AluRRRShift { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - shiftop: ShiftOpAndAmt, - }, - /// An ALU operation with two register sources, one of which can be {zero,sign}-extended and - /// shifted, and a register destination. - AluRRRExtend { - alu_op: ALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - extendop: ExtendOp, - }, - - /// A bit op instruction with a single register source. - BitRR { - op: BitOp, - rd: Writable, - rn: Reg, - }, - - /// An unsigned (zero-extending) 8-bit load. - ULoad8 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A signed (sign-extending) 8-bit load. - SLoad8 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// An unsigned (zero-extending) 16-bit load. - ULoad16 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A signed (sign-extending) 16-bit load. - SLoad16 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// An unsigned (zero-extending) 32-bit load. - ULoad32 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A signed (sign-extending) 32-bit load. - SLoad32 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// A 64-bit load. - ULoad64 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - - /// An 8-bit store. - Store8 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A 16-bit store. - Store16 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A 32-bit store. - Store32 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A 64-bit store. - Store64 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - - /// A store of a pair of registers. - StoreP64 { - rt: Reg, - rt2: Reg, - mem: PairAMode, - flags: MemFlags, - }, - /// A load of a pair of registers. - LoadP64 { - rt: Writable, - rt2: Writable, - mem: PairAMode, - flags: MemFlags, - }, - - /// A MOV instruction. These are encoded as ORR's (AluRRR form) but we - /// keep them separate at the `Inst` level for better pretty-printing - /// and faster `is_move()` logic. - Mov64 { - rd: Writable, - rm: Reg, - }, - - /// A 32-bit MOV. Zeroes the top 32 bits of the destination. This is - /// effectively an alias for an unsigned 32-to-64-bit extension. - Mov32 { - rd: Writable, - rm: Reg, - }, - - /// A MOVZ with a 16-bit immediate. - MovZ { - rd: Writable, - imm: MoveWideConst, - size: OperandSize, - }, - - /// A MOVN with a 16-bit immediate. - MovN { - rd: Writable, - imm: MoveWideConst, - size: OperandSize, - }, - - /// A MOVK with a 16-bit immediate. - MovK { - rd: Writable, - imm: MoveWideConst, - size: OperandSize, - }, - - /// A sign- or zero-extend operation. - Extend { - rd: Writable, - rn: Reg, - signed: bool, - from_bits: u8, - to_bits: u8, - }, - - /// A conditional-select operation. - CSel { - rd: Writable, - cond: Cond, - rn: Reg, - rm: Reg, - }, - - /// A conditional-set operation. - CSet { - rd: Writable, - cond: Cond, - }, - - /// A conditional-set-mask operation. - CSetm { - rd: Writable, - cond: Cond, - }, - - /// A conditional comparison with an immediate. - CCmpImm { - size: OperandSize, - rn: Reg, - imm: UImm5, - nzcv: NZCV, - cond: Cond, - }, - - /// A synthetic insn, which is a load-linked store-conditional loop, that has the overall - /// effect of atomically modifying a memory location in a particular way. Because we have - /// no way to explain to the regalloc about earlyclobber registers, this instruction has - /// completely fixed operand registers, and we rely on the RA's coalescing to remove copies - /// in the surrounding code to the extent it can. The sequence is both preceded and - /// followed by a fence which is at least as comprehensive as that of the `Fence` - /// instruction below. This instruction is sequentially consistent. The operand - /// conventions are: - /// - /// x25 (rd) address - /// x26 (rd) second operand for `op` - /// x27 (wr) old value - /// x24 (wr) scratch reg; value afterwards has no meaning - /// x28 (wr) scratch reg; value afterwards has no meaning - AtomicRMWLoop { - ty: Type, // I8, I16, I32 or I64 - op: inst_common::AtomicRmwOp, - }, - - /// An atomic read-modify-write operation. These instructions require the - /// Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have - /// acquire-release semantics. - AtomicRMW { - op: AtomicRMWOp, - rs: Reg, - rt: Writable, - rn: Reg, - ty: Type, - }, - - /// An atomic compare-and-swap operation. This instruction is sequentially consistent. - AtomicCAS { - rs: Writable, - rt: Reg, - rn: Reg, - ty: Type, - }, - - /// Similar to AtomicRMWLoop, a compare-and-swap operation implemented using a load-linked - /// store-conditional loop. - /// This instruction is sequentially consistent. - /// Note that the operand conventions, although very similar to AtomicRMWLoop, are different: - /// - /// x25 (rd) address - /// x26 (rd) expected value - /// x28 (rd) replacement value - /// x27 (wr) old value - /// x24 (wr) scratch reg; value afterwards has no meaning - AtomicCASLoop { - ty: Type, // I8, I16, I32 or I64 - }, - - /// Read `access_ty` bits from address `rt`, either 8, 16, 32 or 64-bits, and put - /// it in `rn`, optionally zero-extending to fill a word or double word result. - /// This instruction is sequentially consistent. - LoadAcquire { - access_ty: Type, // I8, I16, I32 or I64 - rt: Writable, - rn: Reg, - }, - - /// Write the lowest `ty` bits of `rt` to address `rn`. - /// This instruction is sequentially consistent. - StoreRelease { - access_ty: Type, // I8, I16, I32 or I64 - rt: Reg, - rn: Reg, - }, - - /// A memory fence. This must provide ordering to ensure that, at a minimum, neither loads - /// nor stores may move forwards or backwards across the fence. Currently emitted as "dmb - /// ish". This instruction is sequentially consistent. - Fence, - - /// FPU move. Note that this is distinct from a vector-register - /// move; moving just 64 bits seems to be significantly faster. - FpuMove64 { - rd: Writable, - rn: Reg, - }, - - /// Vector register move. - FpuMove128 { - rd: Writable, - rn: Reg, - }, - - /// Move to scalar from a vector element. - FpuMoveFromVec { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - }, - - /// Zero-extend a SIMD & FP scalar to the full width of a vector register. - /// 16-bit scalars require half-precision floating-point support (FEAT_FP16). - FpuExtend { - rd: Writable, - rn: Reg, - size: ScalarSize, - }, - - /// 1-op FPU instruction. - FpuRR { - fpu_op: FPUOp1, - rd: Writable, - rn: Reg, - }, - - /// 2-op FPU instruction. - FpuRRR { - fpu_op: FPUOp2, - rd: Writable, - rn: Reg, - rm: Reg, - }, - - FpuRRI { - fpu_op: FPUOpRI, - rd: Writable, - rn: Reg, - }, - - /// 3-op FPU instruction. - FpuRRRR { - fpu_op: FPUOp3, - rd: Writable, - rn: Reg, - rm: Reg, - ra: Reg, - }, - - /// FPU comparison, single-precision (32 bit). - FpuCmp32 { - rn: Reg, - rm: Reg, - }, - - /// FPU comparison, double-precision (64 bit). - FpuCmp64 { - rn: Reg, - rm: Reg, - }, - - /// Floating-point load, single-precision (32 bit). - FpuLoad32 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point store, single-precision (32 bit). - FpuStore32 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point load, double-precision (64 bit). - FpuLoad64 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point store, double-precision (64 bit). - FpuStore64 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point/vector load, 128 bit. - FpuLoad128 { - rd: Writable, - mem: AMode, - flags: MemFlags, - }, - /// Floating-point/vector store, 128 bit. - FpuStore128 { - rd: Reg, - mem: AMode, - flags: MemFlags, - }, - /// A load of a pair of floating-point registers, double precision (64-bit). - FpuLoadP64 { - rt: Writable, - rt2: Writable, - mem: PairAMode, - flags: MemFlags, - }, - /// A store of a pair of floating-point registers, double precision (64-bit). - FpuStoreP64 { - rt: Reg, - rt2: Reg, - mem: PairAMode, - flags: MemFlags, - }, - /// A load of a pair of floating-point registers, 128-bit. - FpuLoadP128 { - rt: Writable, - rt2: Writable, - mem: PairAMode, - flags: MemFlags, - }, - /// A store of a pair of floating-point registers, 128-bit. - FpuStoreP128 { - rt: Reg, - rt2: Reg, - mem: PairAMode, - flags: MemFlags, - }, - LoadFpuConst64 { - rd: Writable, - const_data: u64, - }, - - LoadFpuConst128 { - rd: Writable, - const_data: u128, - }, - - /// Conversion: FP -> integer. - FpuToInt { - op: FpuToIntOp, - rd: Writable, - rn: Reg, - }, - - /// Conversion: integer -> FP. - IntToFpu { - op: IntToFpuOp, - rd: Writable, - rn: Reg, - }, - - /// FP conditional select, 32 bit. - FpuCSel32 { - rd: Writable, - rn: Reg, - rm: Reg, - cond: Cond, - }, - /// FP conditional select, 64 bit. - FpuCSel64 { - rd: Writable, - rn: Reg, - rm: Reg, - cond: Cond, - }, - - /// Round to integer. - FpuRound { - op: FpuRoundMode, - rd: Writable, - rn: Reg, - }, - - /// Move from a GPR to a vector register. The scalar value is parked in the lowest lane - /// of the destination, and all other lanes are zeroed out. Currently only 32- and 64-bit - /// transactions are supported. - MovToFpu { - rd: Writable, - rn: Reg, - size: ScalarSize, - }, - - /// Move to a vector element from a GPR. - MovToVec { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - }, - - /// Unsigned move from a vector element to a GPR. - MovFromVec { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - }, - - /// Signed move from a vector element to a GPR. - MovFromVecSigned { - rd: Writable, - rn: Reg, - idx: u8, - size: VectorSize, - scalar_size: OperandSize, - }, - - /// Duplicate general-purpose register to vector. - VecDup { - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Duplicate scalar to vector. - VecDupFromFpu { - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Duplicate FP immediate to vector. - VecDupFPImm { - rd: Writable, - imm: ASIMDFPModImm, - size: VectorSize, - }, - - /// Duplicate immediate to vector. - VecDupImm { - rd: Writable, - imm: ASIMDMovModImm, - invert: bool, - size: VectorSize, - }, - - /// Vector extend. - VecExtend { - t: VecExtendOp, - rd: Writable, - rn: Reg, - high_half: bool, - }, - - /// Move vector element to another vector element. - VecMovElement { - rd: Writable, - rn: Reg, - dest_idx: u8, - src_idx: u8, - size: VectorSize, - }, - - /// Vector widening operation. - VecRRLong { - op: VecRRLongOp, - rd: Writable, - rn: Reg, - high_half: bool, - }, - - /// Vector narrowing operation. - VecRRNarrow { - op: VecRRNarrowOp, - rd: Writable, - rn: Reg, - high_half: bool, - }, - - /// 1-operand vector instruction that operates on a pair of elements. - VecRRPair { - op: VecPairOp, - rd: Writable, - rn: Reg, - }, - - /// 2-operand vector instruction that produces a result with twice the - /// lane width and half the number of lanes. - VecRRRLong { - alu_op: VecRRRLongOp, - rd: Writable, - rn: Reg, - rm: Reg, - high_half: bool, - }, - - /// 1-operand vector instruction that extends elements of the input - /// register and operates on a pair of elements. The output lane width - /// is double that of the input. - VecRRPairLong { - op: VecRRPairLongOp, - rd: Writable, - rn: Reg, - }, - - /// A vector ALU op. - VecRRR { - alu_op: VecALUOp, - rd: Writable, - rn: Reg, - rm: Reg, - size: VectorSize, - }, - - /// Vector two register miscellaneous instruction. - VecMisc { - op: VecMisc2, - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Vector instruction across lanes. - VecLanes { - op: VecLanesOp, - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Vector shift by immediate: Shift Left (immediate), Unsigned Shift Right (immediate), - /// Signed Shift Right (immediate). These are somewhat unusual in that, for right shifts, - /// the allowed range of `imm` values is 1 to lane-size-in-bits, inclusive. A zero - /// right-shift cannot be encoded. Left shifts are "normal", though, having valid `imm` - /// values from 0 to lane-size-in-bits - 1 inclusive. - VecShiftImm { - op: VecShiftImmOp, - rd: Writable, - rn: Reg, - size: VectorSize, - imm: u8, - }, - - /// Vector extract - create a new vector, being the concatenation of the lowest `imm4` bytes - /// of `rm` followed by the uppermost `16 - imm4` bytes of `rn`. - VecExtract { - rd: Writable, - rn: Reg, - rm: Reg, - imm4: u8, - }, - - /// Table vector lookup - single register table. The table consists of 8-bit elements and is - /// stored in `rn`, while `rm` contains 8-bit element indices. `is_extension` specifies whether - /// to emit a TBX or a TBL instruction, i.e. whether to leave the elements in the destination - /// vector that correspond to out-of-range indices (greater than 15) unmodified or to set them - /// to 0. - VecTbl { - rd: Writable, - rn: Reg, - rm: Reg, - is_extension: bool, - }, - - /// Table vector lookup - two register table. The table consists of 8-bit elements and is - /// stored in `rn` and `rn2`, while `rm` contains 8-bit element indices. `is_extension` - /// specifies whether to emit a TBX or a TBL instruction, i.e. whether to leave the elements in - /// the destination vector that correspond to out-of-range indices (greater than 31) unmodified - /// or to set them to 0. The table registers `rn` and `rn2` must have consecutive numbers - /// modulo 32, that is v31 and v0 (in that order) are consecutive registers. - VecTbl2 { - rd: Writable, - rn: Reg, - rn2: Reg, - rm: Reg, - is_extension: bool, - }, - - /// Load an element and replicate to all lanes of a vector. - VecLoadReplicate { - rd: Writable, - rn: Reg, - size: VectorSize, - }, - - /// Vector conditional select, 128 bit. A synthetic instruction, which generates a 4-insn - /// control-flow diamond. - VecCSel { - rd: Writable, - rn: Reg, - rm: Reg, - cond: Cond, - }, - - /// Move to the NZCV flags (actually a `MSR NZCV, Xn` insn). - MovToNZCV { - rn: Reg, - }, - - /// Move from the NZCV flags (actually a `MRS Xn, NZCV` insn). - MovFromNZCV { - rd: Writable, - }, - - /// A machine call instruction. N.B.: this allows only a +/- 128MB offset (it uses a relocation - /// of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the - /// code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit - /// target. - Call { - info: Box, - }, - /// A machine indirect-call instruction. - CallInd { - info: Box, - }, - - // ---- branches (exactly one must appear at end of BB) ---- - /// A machine return instruction. - Ret, - - /// A placeholder instruction, generating no code, meaning that a function epilogue must be - /// inserted there. - EpiloguePlaceholder, - - /// An unconditional branch. - Jump { - dest: BranchTarget, - }, - - /// A conditional branch. Contains two targets; at emission time, both are emitted, but - /// the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the - /// choice of taken/not_taken (inverting the branch polarity as needed) based on the - /// fallthrough at the time of lowering. - CondBr { - taken: BranchTarget, - not_taken: BranchTarget, - kind: CondBrKind, - }, - - /// A conditional trap: execute a `udf` if the condition is true. This is - /// one VCode instruction because it uses embedded control flow; it is - /// logically a single-in, single-out region, but needs to appear as one - /// unit to the register allocator. - /// - /// The `CondBrKind` gives the conditional-branch condition that will - /// *execute* the embedded `Inst`. (In the emitted code, we use the inverse - /// of this condition in a branch that skips the trap instruction.) - TrapIf { - kind: CondBrKind, - trap_code: TrapCode, - }, - - /// An indirect branch through a register, augmented with set of all - /// possible successors. - IndirectBr { - rn: Reg, - targets: Vec, - }, - - /// A "break" instruction, used for e.g. traps and debug breakpoints. - Brk, - - /// An instruction guaranteed to always be undefined and to trigger an illegal instruction at - /// runtime. - Udf { - trap_code: TrapCode, - }, - - /// Compute the address (using a PC-relative offset) of a memory location, using the `ADR` - /// instruction. Note that we take a simple offset, not a `MemLabel`, here, because `Adr` is - /// only used for now in fixed lowering sequences with hardcoded offsets. In the future we may - /// need full `MemLabel` support. - Adr { - rd: Writable, - /// Offset in range -2^20 .. 2^20. - off: i32, - }, - - /// Raw 32-bit word, used for inline constants and jump-table entries. - Word4 { - data: u32, - }, - - /// Raw 64-bit word, used for inline constants. - Word8 { - data: u64, - }, - - /// Jump-table sequence, as one compound instruction (see note in lower_inst.rs for rationale). - JTSequence { - info: Box, - ridx: Reg, - rtmp1: Writable, - rtmp2: Writable, - }, - - /// Load an inline symbol reference. - LoadExtName { - rd: Writable, - name: Box, - offset: i64, - }, - - /// Load address referenced by `mem` into `rd`. - LoadAddr { - rd: Writable, - mem: AMode, - }, - - /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This - /// controls how AMode::NominalSPOffset args are lowered. - VirtualSPOffsetAdj { - offset: i64, - }, - - /// Meta-insn, no-op in generated code: emit constant/branch veneer island - /// at this point (with a guard jump around it) if less than the needed - /// space is available before the next branch deadline. See the `MachBuffer` - /// implementation in `machinst/buffer.rs` for the overall algorithm. In - /// brief, we retain a set of "pending/unresolved label references" from - /// branches as we scan forward through instructions to emit machine code; - /// if we notice we're about to go out of range on an unresolved reference, - /// we stop, emit a bunch of "veneers" (branches in a form that has a longer - /// range, e.g. a 26-bit-offset unconditional jump), and point the original - /// label references to those. This is an "island" because it comes in the - /// middle of the code. - /// - /// This meta-instruction is a necessary part of the logic that determines - /// where to place islands. Ordinarily, we want to place them between basic - /// blocks, so we compute the worst-case size of each block, and emit the - /// island before starting a block if we would exceed a deadline before the - /// end of the block. However, some sequences (such as an inline jumptable) - /// are variable-length and not accounted for by this logic; so these - /// lowered sequences include an `EmitIsland` to trigger island generation - /// where necessary. - EmitIsland { - /// The needed space before the next deadline. - needed_space: CodeOffset, - }, - - /// A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol in x0. - ElfTlsGetAddr { - symbol: ExternalName, - }, - - /// A definition of a value label. - ValueLabelMarker { - reg: Reg, - label: ValueLabel, - }, - - /// An unwind pseudo-instruction. - Unwind { - inst: UnwindInst, - }, -} - fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize { let mut count = 0; for _ in 0..num_half_words { @@ -1426,6 +147,10 @@ impl Inst { /// Create an instruction that loads a constant, using one of serveral options (MOVZ, MOVN, /// logical immediate, or constant pool). pub fn load_constant(rd: Writable, value: u64) -> SmallVec<[Inst; 4]> { + // NB: this is duplicated in `lower/isle.rs` and `inst.isle` right now, + // if modifications are made here before this is deleted after moving to + // ISLE then those locations should be updated as well. + if let Some(imm) = MoveWideConst::maybe_from_u64(value) { // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ smallvec![Inst::MovZ { @@ -2304,65 +1029,44 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { //============================================================================= // Instructions: map_regs -fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { - fn map_use(m: &RUM, r: &mut Reg) { - if r.is_virtual() { - let new = m.get_use(r.to_virtual_reg()).unwrap().to_reg(); - *r = new; - } - } - - fn map_def(m: &RUM, r: &mut Writable) { - if r.to_reg().is_virtual() { - let new = m.get_def(r.to_reg().to_virtual_reg()).unwrap().to_reg(); - *r = Writable::from_reg(new); - } - } - - fn map_mod(m: &RUM, r: &mut Writable) { - if r.to_reg().is_virtual() { - let new = m.get_mod(r.to_reg().to_virtual_reg()).unwrap().to_reg(); - *r = Writable::from_reg(new); - } - } - - fn map_mem(m: &RUM, mem: &mut AMode) { +pub fn aarch64_map_regs(inst: &mut Inst, mapper: &RM) { + fn map_mem(m: &RM, mem: &mut AMode) { // N.B.: we take only the pre-map here, but this is OK because the // only addressing modes that update registers (pre/post-increment on // AArch64) both read and write registers, so they are "mods" rather // than "defs", so must be the same in both the pre- and post-map. match mem { - &mut AMode::Unscaled(ref mut reg, ..) => map_use(m, reg), - &mut AMode::UnsignedOffset(ref mut reg, ..) => map_use(m, reg), + &mut AMode::Unscaled(ref mut reg, ..) => m.map_use(reg), + &mut AMode::UnsignedOffset(ref mut reg, ..) => m.map_use(reg), &mut AMode::RegReg(ref mut r1, ref mut r2) | &mut AMode::RegScaled(ref mut r1, ref mut r2, ..) | &mut AMode::RegScaledExtended(ref mut r1, ref mut r2, ..) | &mut AMode::RegExtended(ref mut r1, ref mut r2, ..) => { - map_use(m, r1); - map_use(m, r2); + m.map_use(r1); + m.map_use(r2); } &mut AMode::Label(..) => {} - &mut AMode::PreIndexed(ref mut r, ..) => map_mod(m, r), - &mut AMode::PostIndexed(ref mut r, ..) => map_mod(m, r), + &mut AMode::PreIndexed(ref mut r, ..) => m.map_mod(r), + &mut AMode::PostIndexed(ref mut r, ..) => m.map_mod(r), &mut AMode::FPOffset(..) | &mut AMode::SPOffset(..) | &mut AMode::NominalSPOffset(..) => {} - &mut AMode::RegOffset(ref mut r, ..) => map_use(m, r), + &mut AMode::RegOffset(ref mut r, ..) => m.map_use(r), }; } - fn map_pairmem(m: &RUM, mem: &mut PairAMode) { + fn map_pairmem(m: &RM, mem: &mut PairAMode) { match mem { - &mut PairAMode::SignedOffset(ref mut reg, ..) => map_use(m, reg), - &mut PairAMode::PreIndexed(ref mut reg, ..) => map_def(m, reg), - &mut PairAMode::PostIndexed(ref mut reg, ..) => map_def(m, reg), + &mut PairAMode::SignedOffset(ref mut reg, ..) => m.map_use(reg), + &mut PairAMode::PreIndexed(ref mut reg, ..) => m.map_def(reg), + &mut PairAMode::PostIndexed(ref mut reg, ..) => m.map_def(reg), } } - fn map_br(m: &RUM, br: &mut CondBrKind) { + fn map_br(m: &RM, br: &mut CondBrKind) { match br { - &mut CondBrKind::Zero(ref mut reg) => map_use(m, reg), - &mut CondBrKind::NotZero(ref mut reg) => map_use(m, reg), + &mut CondBrKind::Zero(ref mut reg) => m.map_use(reg), + &mut CondBrKind::NotZero(ref mut reg) => m.map_use(reg), &mut CondBrKind::Cond(..) => {} }; } @@ -2374,9 +1078,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::AluRRRR { ref mut rd, @@ -2385,34 +1089,34 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut ra, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); - map_use(mapper, ra); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); + mapper.map_use(ra); } &mut Inst::AluRRImm12 { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::AluRRImmLogic { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::AluRRImmShift { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::AluRRRShift { ref mut rd, @@ -2420,9 +1124,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::AluRRRExtend { ref mut rd, @@ -2430,24 +1134,24 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::BitRR { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::ULoad8 { ref mut rd, ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::SLoad8 { @@ -2455,7 +1159,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::ULoad16 { @@ -2463,7 +1167,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::SLoad16 { @@ -2471,7 +1175,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::ULoad32 { @@ -2479,7 +1183,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::SLoad32 { @@ -2487,7 +1191,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } @@ -2496,7 +1200,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::Store8 { @@ -2504,7 +1208,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::Store16 { @@ -2512,7 +1216,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::Store32 { @@ -2520,7 +1224,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::Store64 { @@ -2528,7 +1232,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } @@ -2538,8 +1242,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rt); - map_use(mapper, rt2); + mapper.map_use(rt); + mapper.map_use(rt2); map_pairmem(mapper, mem); } &mut Inst::LoadP64 { @@ -2548,32 +1252,32 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rt); - map_def(mapper, rt2); + mapper.map_def(rt); + mapper.map_def(rt2); map_pairmem(mapper, mem); } &mut Inst::Mov64 { ref mut rd, ref mut rm, } => { - map_def(mapper, rd); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rm); } &mut Inst::Mov32 { ref mut rd, ref mut rm, } => { - map_def(mapper, rd); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rm); } &mut Inst::MovZ { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::MovN { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::MovK { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::CSel { ref mut rd, @@ -2581,15 +1285,15 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::CSet { ref mut rd, .. } | &mut Inst::CSetm { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::CCmpImm { ref mut rn, .. } => { - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::AtomicRMWLoop { .. } => { // There are no vregs to map in this insn. @@ -2600,9 +1304,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_use(mapper, rs); - map_def(mapper, rt); - map_use(mapper, rn); + mapper.map_use(rs); + mapper.map_def(rt); + mapper.map_use(rn); } &mut Inst::AtomicCAS { ref mut rs, @@ -2610,9 +1314,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_mod(mapper, rs); - map_use(mapper, rt); - map_use(mapper, rn); + mapper.map_mod(rs); + mapper.map_use(rt); + mapper.map_use(rn); } &mut Inst::AtomicCASLoop { .. } => { // There are no vregs to map in this insn. @@ -2622,55 +1326,55 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rt); - map_use(mapper, rn); + mapper.map_def(rt); + mapper.map_use(rn); } &mut Inst::StoreRelease { ref mut rt, ref mut rn, .. } => { - map_use(mapper, rt); - map_use(mapper, rn); + mapper.map_use(rt); + mapper.map_use(rn); } &mut Inst::Fence {} => {} &mut Inst::FpuMove64 { ref mut rd, ref mut rn, } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuMove128 { ref mut rd, ref mut rn, } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuMoveFromVec { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuExtend { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuRR { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuRRR { ref mut rd, @@ -2678,9 +1382,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuRRI { fpu_op, @@ -2689,10 +1393,10 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { .. } => { match fpu_op { - FPUOpRI::UShr32(..) | FPUOpRI::UShr64(..) => map_def(mapper, rd), - FPUOpRI::Sli32(..) | FPUOpRI::Sli64(..) => map_mod(mapper, rd), + FPUOpRI::UShr32(..) | FPUOpRI::UShr64(..) => mapper.map_def(rd), + FPUOpRI::Sli32(..) | FPUOpRI::Sli64(..) => mapper.map_mod(rd), } - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::FpuRRRR { ref mut rd, @@ -2701,34 +1405,34 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut ra, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); - map_use(mapper, ra); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); + mapper.map_use(ra); } &mut Inst::VecMisc { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecLanes { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecShiftImm { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecExtract { ref mut rd, @@ -2736,9 +1440,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::VecTbl { ref mut rd, @@ -2746,13 +1450,13 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, is_extension, } => { - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); if is_extension { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } } &mut Inst::VecTbl2 { @@ -2762,14 +1466,14 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, is_extension, } => { - map_use(mapper, rn); - map_use(mapper, rn2); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rn2); + mapper.map_use(rm); if is_extension { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } } &mut Inst::VecLoadReplicate { @@ -2777,8 +1481,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecCSel { ref mut rd, @@ -2786,30 +1490,30 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuCmp32 { ref mut rn, ref mut rm, } => { - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuCmp64 { ref mut rn, ref mut rm, } => { - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuLoad32 { ref mut rd, ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::FpuLoad64 { @@ -2817,7 +1521,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::FpuLoad128 { @@ -2825,7 +1529,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::FpuStore32 { @@ -2833,7 +1537,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::FpuStore64 { @@ -2841,7 +1545,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::FpuStore128 { @@ -2849,7 +1553,7 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rd); + mapper.map_use(rd); map_mem(mapper, mem); } &mut Inst::FpuLoadP64 { @@ -2858,8 +1562,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rt); - map_def(mapper, rt2); + mapper.map_def(rt); + mapper.map_def(rt2); map_pairmem(mapper, mem); } &mut Inst::FpuStoreP64 { @@ -2868,8 +1572,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rt); - map_use(mapper, rt2); + mapper.map_use(rt); + mapper.map_use(rt2); map_pairmem(mapper, mem); } &mut Inst::FpuLoadP128 { @@ -2878,8 +1582,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_def(mapper, rt); - map_def(mapper, rt2); + mapper.map_def(rt); + mapper.map_def(rt2); map_pairmem(mapper, mem); } &mut Inst::FpuStoreP128 { @@ -2888,31 +1592,31 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut mem, .. } => { - map_use(mapper, rt); - map_use(mapper, rt2); + mapper.map_use(rt); + mapper.map_use(rt2); map_pairmem(mapper, mem); } &mut Inst::LoadFpuConst64 { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::LoadFpuConst128 { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::FpuToInt { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::IntToFpu { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::FpuCSel32 { ref mut rd, @@ -2920,9 +1624,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuCSel64 { ref mut rd, @@ -2930,33 +1634,33 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rm, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_def(rd); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::FpuRound { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::MovToFpu { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::MovToVec { ref mut rd, ref mut rn, .. } => { - map_mod(mapper, rd); - map_use(mapper, rn); + mapper.map_mod(rd); + mapper.map_use(rn); } &mut Inst::MovFromVec { ref mut rd, @@ -2968,54 +1672,54 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecDup { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecDupFromFpu { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecDupFPImm { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::VecDupImm { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::VecExtend { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecMovElement { ref mut rd, ref mut rn, .. } => { - map_mod(mapper, rd); - map_use(mapper, rn); + mapper.map_mod(rd); + mapper.map_use(rn); } &mut Inst::VecRRLong { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecRRNarrow { ref mut rd, @@ -3023,12 +1727,12 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { high_half, .. } => { - map_use(mapper, rn); + mapper.map_use(rn); if high_half { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } } &mut Inst::VecRRPair { @@ -3036,8 +1740,8 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecRRRLong { alu_op, @@ -3048,20 +1752,20 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { } => { match alu_op { VecRRRLongOp::Umlal8 | VecRRRLongOp::Umlal16 | VecRRRLongOp::Umlal32 => { - map_mod(mapper, rd) + mapper.map_mod(rd) } - _ => map_def(mapper, rd), + _ => mapper.map_def(rd), }; - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::VecRRPairLong { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::VecRRR { alu_op, @@ -3071,58 +1775,58 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { .. } => { if alu_op == VecALUOp::Bsl { - map_mod(mapper, rd); + mapper.map_mod(rd); } else { - map_def(mapper, rd); + mapper.map_def(rd); } - map_use(mapper, rn); - map_use(mapper, rm); + mapper.map_use(rn); + mapper.map_use(rm); } &mut Inst::MovToNZCV { ref mut rn } => { - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::MovFromNZCV { ref mut rd } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::Extend { ref mut rd, ref mut rn, .. } => { - map_def(mapper, rd); - map_use(mapper, rn); + mapper.map_def(rd); + mapper.map_use(rn); } &mut Inst::Jump { .. } => {} &mut Inst::Call { ref mut info } => { for r in info.uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in info.defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } } &mut Inst::Ret | &mut Inst::EpiloguePlaceholder => {} &mut Inst::CallInd { ref mut info, .. } => { for r in info.uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in info.defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } - map_use(mapper, &mut info.rn); + mapper.map_use(&mut info.rn); } &mut Inst::CondBr { ref mut kind, .. } => { map_br(mapper, kind); } &mut Inst::IndirectBr { ref mut rn, .. } => { - map_use(mapper, rn); + mapper.map_use(rn); } &mut Inst::Nop0 | &mut Inst::Nop4 | &mut Inst::Brk | &mut Inst::Udf { .. } => {} &mut Inst::TrapIf { ref mut kind, .. } => { map_br(mapper, kind); } &mut Inst::Adr { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::Word4 { .. } | &mut Inst::Word8 { .. } => {} &mut Inst::JTSequence { @@ -3131,25 +1835,25 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut rtmp2, .. } => { - map_use(mapper, ridx); - map_def(mapper, rtmp1); - map_def(mapper, rtmp2); + mapper.map_use(ridx); + mapper.map_def(rtmp1); + mapper.map_def(rtmp2); } &mut Inst::LoadExtName { ref mut rd, .. } => { - map_def(mapper, rd); + mapper.map_def(rd); } &mut Inst::LoadAddr { ref mut rd, ref mut mem, } => { - map_def(mapper, rd); + mapper.map_def(rd); map_mem(mapper, mem); } &mut Inst::VirtualSPOffsetAdj { .. } => {} &mut Inst::EmitIsland { .. } => {} &mut Inst::ElfTlsGetAddr { .. } => {} &mut Inst::ValueLabelMarker { ref mut reg, .. } => { - map_use(mapper, reg); + mapper.map_use(reg); } &mut Inst::Unwind { .. } => {} } @@ -3165,7 +1869,7 @@ impl MachInst for Inst { aarch64_get_regs(self, collector) } - fn map_regs(&mut self, mapper: &RUM) { + fn map_regs(&mut self, mapper: &RM) { aarch64_map_regs(self, mapper); } diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle new file mode 100644 index 0000000000..7b793dc271 --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -0,0 +1,23 @@ +;; aarch64 instruction selection and CLIF-to-MachInst lowering. + +;; The main lowering constructor term: takes a clif `Inst` and returns the +;; register(s) within which the lowered instruction's result values live. +(decl lower (Inst) ValueRegs) + +;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty (iconst (u64_from_imm64 n)))) + (value_reg (imm ty n))) + +;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty (bconst $false))) + (value_reg (imm ty 0))) + +(rule (lower (has_type ty (bconst $true))) + (value_reg (imm ty 1))) + +;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty (null))) + (value_reg (imm ty 0))) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 34a627dfba..4697411afa 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -25,6 +25,8 @@ use regalloc::{Reg, Writable}; use smallvec::SmallVec; use std::cmp; +pub mod isle; + //============================================================================ // Result enum types. // diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs new file mode 100644 index 0000000000..6708431eb9 --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -0,0 +1,187 @@ +//! ISLE integration glue code for aarch64 lowering. + +// Pull in the ISLE generated code. +pub mod generated_code; + +// Types that the generated ISLE code uses via `use super::*`. +use super::{ + zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, AtomicRmwOp, BranchTarget, CallIndInfo, + CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, Imm12, ImmLogic, ImmShift, Inst as MInst, + JTSequenceInfo, MachLabel, MoveWideConst, Opcode, OperandSize, PairAMode, Reg, ScalarSize, + ShiftOpAndAmt, UImm5, VectorSize, NZCV, +}; +use crate::isa::aarch64::settings as aarch64_settings; +use crate::machinst::isle::*; +use crate::{ + binemit::CodeOffset, + ir::{ + immediates::*, types::*, ExternalName, Inst, InstructionData, MemFlags, TrapCode, Value, + ValueLabel, ValueList, + }, + isa::aarch64::inst::aarch64_map_regs, + isa::unwind::UnwindInst, + machinst::{get_output_reg, InsnOutput, LowerCtx, RegRenamer}, +}; +use smallvec::SmallVec; +use std::boxed::Box; +use std::vec::Vec; + +type BoxCallInfo = Box; +type BoxCallIndInfo = Box; +type VecMachLabel = Vec; +type BoxJTSequenceInfo = Box; +type BoxExternalName = Box; + +/// The main entry point for lowering with ISLE. +pub(crate) fn lower( + lower_ctx: &mut C, + isa_flags: &aarch64_settings::Flags, + outputs: &[InsnOutput], + inst: Inst, +) -> Result<(), ()> +where + C: LowerCtx, +{ + // TODO: reuse the ISLE context across lowerings so we can reuse its + // internal heap allocations. + let mut isle_ctx = IsleContext::new(lower_ctx, isa_flags); + + let temp_regs = generated_code::constructor_lower(&mut isle_ctx, inst).ok_or(())?; + let mut temp_regs = temp_regs.regs().iter(); + + // The ISLE generated code emits its own registers to define the + // instruction's lowered values in. We rename those registers to the + // registers they were assigned when their value was used as an operand in + // earlier lowerings. + let mut renamer = RegRenamer::default(); + for output in outputs { + let dsts = get_output_reg(isle_ctx.lower_ctx, *output); + for (temp, dst) in temp_regs.by_ref().zip(dsts.regs()) { + renamer.add_rename(*temp, dst.to_reg()); + } + } + + for mut inst in isle_ctx.into_emitted_insts() { + aarch64_map_regs(&mut inst, &renamer); + lower_ctx.emit(inst); + } + + Ok(()) +} + +pub struct IsleContext<'a, C> { + lower_ctx: &'a mut C, + #[allow(dead_code)] // dead for now, but probably not for long + isa_flags: &'a aarch64_settings::Flags, + emitted_insts: SmallVec<[MInst; 6]>, +} + +impl<'a, C> IsleContext<'a, C> { + pub fn new(lower_ctx: &'a mut C, isa_flags: &'a aarch64_settings::Flags) -> Self { + IsleContext { + lower_ctx, + isa_flags, + emitted_insts: SmallVec::new(), + } + } + + pub fn into_emitted_insts(self) -> SmallVec<[MInst; 6]> { + self.emitted_insts + } +} + +impl<'a, C> generated_code::Context for IsleContext<'a, C> +where + C: LowerCtx, +{ + isle_prelude_methods!(); + + fn move_wide_const_from_u64(&mut self, n: u64) -> Option { + MoveWideConst::maybe_from_u64(n) + } + + fn move_wide_const_from_negated_u64(&mut self, n: u64) -> Option { + MoveWideConst::maybe_from_u64(!n) + } + + fn imm_logic_from_u64(&mut self, n: u64) -> Option { + ImmLogic::maybe_from_u64(n, I64) + } + + fn integral_ty(&mut self, ty: Type) -> Option { + match ty { + I8 | I16 | I32 | I64 | R64 => Some(ty), + ty if ty.is_bool() => Some(ty), + _ => None, + } + } + + /// This is the fallback case for loading a 64-bit integral constant into a + /// register. + /// + /// The logic here is nontrivial enough that it's not really worth porting + /// this over to ISLE. + fn load_constant64_full(&mut self, value: u64) -> Reg { + // If the top 32 bits are zero, use 32-bit `mov` operations. + let (num_half_words, size, negated) = if value >> 32 == 0 { + (2, OperandSize::Size32, (!value << 32) >> 32) + } else { + (4, OperandSize::Size64, !value) + }; + // If the number of 0xffff half words is greater than the number of 0x0000 half words + // it is more efficient to use `movn` for the first instruction. + let first_is_inverted = count_zero_half_words(negated, num_half_words) + > count_zero_half_words(value, num_half_words); + // Either 0xffff or 0x0000 half words can be skipped, depending on the first + // instruction used. + let ignored_halfword = if first_is_inverted { 0xffff } else { 0 }; + let mut first_mov_emitted = false; + + let rd = self.temp_writable_reg(I64); + + for i in 0..num_half_words { + let imm16 = (value >> (16 * i)) & 0xffff; + if imm16 != ignored_halfword { + if !first_mov_emitted { + first_mov_emitted = true; + if first_is_inverted { + let imm = + MoveWideConst::maybe_with_shift(((!imm16) & 0xffff) as u16, i * 16) + .unwrap(); + self.emitted_insts.push(MInst::MovN { rd, imm, size }); + } else { + let imm = MoveWideConst::maybe_with_shift(imm16 as u16, i * 16).unwrap(); + self.emitted_insts.push(MInst::MovZ { rd, imm, size }); + } + } else { + let imm = MoveWideConst::maybe_with_shift(imm16 as u16, i * 16).unwrap(); + self.emitted_insts.push(MInst::MovK { rd, imm, size }); + } + } + } + + assert!(first_mov_emitted); + + return self.writable_reg_to_reg(rd); + + fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize { + let mut count = 0; + for _ in 0..num_half_words { + if value & 0xffff == 0 { + count += 1; + } + value >>= 16; + } + + count + } + } + + fn zero_reg(&mut self) -> Reg { + zero_reg() + } + + fn emit(&mut self, inst: &MInst) -> Unit { + self.emitted_insts.push(inst.clone()); + } +} diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest new file mode 100644 index 0000000000..f02a0fefe3 --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -0,0 +1,4 @@ +src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb +src/prelude.isle a069d14321601afc63959af23086709d67d189dafcdc7d1fc8534b32d89d49008acb8368b7b5a7bc51a353736a378197ac352ccce2bb3be89d93afb6979e480a +src/isa/aarch64/inst.isle 841748c9c5900821b7086a09a41c6dcdb2172eb47a45293b6ef10f2e1f1389620bf6a2c75152af807d8bc8929029a357af5191f5d87bac2c9ec54bf63a9a2a8f +src/isa/aarch64/lower.isle b3cd0834484e543f39d477d47ee66042276e99955c21fb8c9340a5f27ac317936acb2907a30f758bf596066e36db801a179fda6dbcecaee758a0187a5a5f1412 diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs new file mode 100644 index 0000000000..137cff579a --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -0,0 +1,1108 @@ +// GENERATED BY ISLE. DO NOT EDIT! +// +// Generated automatically from the instruction-selection DSL code in: +// - src/clif.isle +// - src/prelude.isle +// - src/isa/aarch64/inst.isle +// - src/isa/aarch64/lower.isle + +#![allow(dead_code, unreachable_code, unreachable_patterns)] +#![allow(unused_imports, unused_variables, non_snake_case)] +#![allow(irrefutable_let_patterns)] + +use super::*; // Pulls in all external types. + +/// Context during lowering: an implementation of this trait +/// must be provided with all external constructors and extractors. +/// A mutable borrow is passed along through all lowering logic. +pub trait Context { + fn unpack_value_array_2(&mut self, arg0: &ValueArray2) -> (Value, Value); + fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> ValueArray2; + fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value); + fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3; + fn value_reg(&mut self, arg0: Reg) -> ValueRegs; + fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; + fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; + fn invalid_reg(&mut self) -> Reg; + fn put_in_reg(&mut self, arg0: Value) -> Reg; + fn put_in_regs(&mut self, arg0: Value) -> ValueRegs; + fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg; + fn u8_as_u64(&mut self, arg0: u8) -> u64; + fn u16_as_u64(&mut self, arg0: u16) -> u64; + fn u32_as_u64(&mut self, arg0: u32) -> u64; + fn ty_bits(&mut self, arg0: Type) -> u16; + fn fits_in_64(&mut self, arg0: Type) -> Option; + fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; + fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); + fn unwrap_head_value_list_2(&mut self, arg0: ValueList) -> (Value, Value, ValueSlice); + fn writable_reg_to_reg(&mut self, arg0: WritableReg) -> Reg; + fn u64_from_imm64(&mut self, arg0: Imm64) -> u64; + fn u64_from_ieee32(&mut self, arg0: Ieee32) -> u64; + fn u64_from_ieee64(&mut self, arg0: Ieee64) -> u64; + fn inst_results(&mut self, arg0: Inst) -> ValueSlice; + fn first_result(&mut self, arg0: Inst) -> Option; + fn inst_data(&mut self, arg0: Inst) -> InstructionData; + fn value_type(&mut self, arg0: Value) -> Type; + fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>; + fn def_inst(&mut self, arg0: Value) -> Option; + fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option; + fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option; + fn imm_logic_from_u64(&mut self, arg0: u64) -> Option; + fn integral_ty(&mut self, arg0: Type) -> Option; + fn emit(&mut self, arg0: &MInst) -> Unit; + fn zero_reg(&mut self) -> Reg; + fn load_constant64_full(&mut self, arg0: u64) -> Reg; +} + +/// Internal type MInst: defined at src/isa/aarch64/inst.isle line 2. +#[derive(Clone, Debug)] +pub enum MInst { + Nop0, + Nop4, + AluRRR { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + }, + AluRRRR { + alu_op: ALUOp3, + rd: WritableReg, + rn: Reg, + rm: Reg, + ra: Reg, + }, + AluRRImm12 { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + imm12: Imm12, + }, + AluRRImmLogic { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + imml: ImmLogic, + }, + AluRRImmShift { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + immshift: ImmShift, + }, + AluRRRShift { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + shiftop: ShiftOpAndAmt, + }, + AluRRRExtend { + alu_op: ALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + extendop: ExtendOp, + }, + BitRR { + op: BitOp, + rd: WritableReg, + rn: Reg, + }, + ULoad8 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + SLoad8 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + ULoad16 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + SLoad16 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + ULoad32 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + SLoad32 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + ULoad64 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + Store8 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + Store16 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + Store32 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + Store64 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + StoreP64 { + rt: Reg, + rt2: Reg, + mem: PairAMode, + flags: MemFlags, + }, + LoadP64 { + rt: WritableReg, + rt2: WritableReg, + mem: PairAMode, + flags: MemFlags, + }, + Mov64 { + rd: WritableReg, + rm: Reg, + }, + Mov32 { + rd: WritableReg, + rm: Reg, + }, + MovZ { + rd: WritableReg, + imm: MoveWideConst, + size: OperandSize, + }, + MovN { + rd: WritableReg, + imm: MoveWideConst, + size: OperandSize, + }, + MovK { + rd: WritableReg, + imm: MoveWideConst, + size: OperandSize, + }, + Extend { + rd: WritableReg, + rn: Reg, + signed: bool, + from_bits: u8, + to_bits: u8, + }, + CSel { + rd: WritableReg, + cond: Cond, + rn: Reg, + rm: Reg, + }, + CSet { + rd: WritableReg, + cond: Cond, + }, + CSetm { + rd: WritableReg, + cond: Cond, + }, + CCmpImm { + size: OperandSize, + rn: Reg, + imm: UImm5, + nzcv: NZCV, + cond: Cond, + }, + AtomicRMWLoop { + ty: Type, + op: AtomicRmwOp, + }, + AtomicRMW { + op: AtomicRMWOp, + rs: Reg, + rt: WritableReg, + rn: Reg, + ty: Type, + }, + AtomicCAS { + rs: WritableReg, + rt: Reg, + rn: Reg, + ty: Type, + }, + AtomicCASLoop { + ty: Type, + }, + LoadAcquire { + access_ty: Type, + rt: WritableReg, + rn: Reg, + }, + StoreRelease { + access_ty: Type, + rt: Reg, + rn: Reg, + }, + Fence, + FpuMove64 { + rd: WritableReg, + rn: Reg, + }, + FpuMove128 { + rd: WritableReg, + rn: Reg, + }, + FpuMoveFromVec { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + }, + FpuExtend { + rd: WritableReg, + rn: Reg, + size: ScalarSize, + }, + FpuRR { + fpu_op: FPUOp1, + rd: WritableReg, + rn: Reg, + }, + FpuRRR { + fpu_op: FPUOp2, + rd: WritableReg, + rn: Reg, + rm: Reg, + }, + FpuRRI { + fpu_op: FPUOpRI, + rd: WritableReg, + rn: Reg, + }, + FpuRRRR { + fpu_op: FPUOp3, + rd: WritableReg, + rn: Reg, + rm: Reg, + ra: Reg, + }, + FpuCmp32 { + rn: Reg, + rm: Reg, + }, + FpuCmp64 { + rn: Reg, + rm: Reg, + }, + FpuLoad32 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + FpuStore32 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + FpuLoad64 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + FpuStore64 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + FpuLoad128 { + rd: WritableReg, + mem: AMode, + flags: MemFlags, + }, + FpuStore128 { + rd: Reg, + mem: AMode, + flags: MemFlags, + }, + FpuLoadP64 { + rt: WritableReg, + rt2: WritableReg, + mem: PairAMode, + flags: MemFlags, + }, + FpuStoreP64 { + rt: Reg, + rt2: Reg, + mem: PairAMode, + flags: MemFlags, + }, + FpuLoadP128 { + rt: WritableReg, + rt2: WritableReg, + mem: PairAMode, + flags: MemFlags, + }, + FpuStoreP128 { + rt: Reg, + rt2: Reg, + mem: PairAMode, + flags: MemFlags, + }, + LoadFpuConst64 { + rd: WritableReg, + const_data: u64, + }, + LoadFpuConst128 { + rd: WritableReg, + const_data: u128, + }, + FpuToInt { + op: FpuToIntOp, + rd: WritableReg, + rn: Reg, + }, + IntToFpu { + op: IntToFpuOp, + rd: WritableReg, + rn: Reg, + }, + FpuCSel32 { + rd: WritableReg, + rn: Reg, + rm: Reg, + cond: Cond, + }, + FpuCSel64 { + rd: WritableReg, + rn: Reg, + rm: Reg, + cond: Cond, + }, + FpuRound { + op: FpuRoundMode, + rd: WritableReg, + rn: Reg, + }, + MovToFpu { + rd: WritableReg, + rn: Reg, + size: ScalarSize, + }, + MovToVec { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + }, + MovFromVec { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + }, + MovFromVecSigned { + rd: WritableReg, + rn: Reg, + idx: u8, + size: VectorSize, + scalar_size: OperandSize, + }, + VecDup { + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecDupFromFpu { + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecDupFPImm { + rd: WritableReg, + imm: ASIMDFPModImm, + size: VectorSize, + }, + VecDupImm { + rd: WritableReg, + imm: ASIMDMovModImm, + invert: bool, + size: VectorSize, + }, + VecExtend { + t: VecExtendOp, + rd: WritableReg, + rn: Reg, + high_half: bool, + }, + VecMovElement { + rd: WritableReg, + rn: Reg, + dest_idx: u8, + src_idx: u8, + size: VectorSize, + }, + VecRRLong { + op: VecRRLongOp, + rd: WritableReg, + rn: Reg, + high_half: bool, + }, + VecRRNarrow { + op: VecRRNarrowOp, + rd: WritableReg, + rn: Reg, + high_half: bool, + }, + VecRRPair { + op: VecPairOp, + rd: WritableReg, + rn: Reg, + }, + VecRRRLong { + alu_op: VecRRRLongOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + high_half: bool, + }, + VecRRPairLong { + op: VecRRPairLongOp, + rd: WritableReg, + rn: Reg, + }, + VecRRR { + alu_op: VecALUOp, + rd: WritableReg, + rn: Reg, + rm: Reg, + size: VectorSize, + }, + VecMisc { + op: VecMisc2, + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecLanes { + op: VecLanesOp, + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecShiftImm { + op: VecShiftImmOp, + rd: WritableReg, + rn: Reg, + size: VectorSize, + imm: u8, + }, + VecExtract { + rd: WritableReg, + rn: Reg, + rm: Reg, + imm4: u8, + }, + VecTbl { + rd: WritableReg, + rn: Reg, + rm: Reg, + is_extension: bool, + }, + VecTbl2 { + rd: WritableReg, + rn: Reg, + rn2: Reg, + rm: Reg, + is_extension: bool, + }, + VecLoadReplicate { + rd: WritableReg, + rn: Reg, + size: VectorSize, + }, + VecCSel { + rd: WritableReg, + rn: Reg, + rm: Reg, + cond: Cond, + }, + MovToNZCV { + rn: Reg, + }, + MovFromNZCV { + rd: WritableReg, + }, + Call { + info: BoxCallInfo, + }, + CallInd { + info: BoxCallIndInfo, + }, + Ret, + EpiloguePlaceholder, + Jump { + dest: BranchTarget, + }, + CondBr { + taken: BranchTarget, + not_taken: BranchTarget, + kind: CondBrKind, + }, + TrapIf { + kind: CondBrKind, + trap_code: TrapCode, + }, + IndirectBr { + rn: Reg, + targets: VecMachLabel, + }, + Brk, + Udf { + trap_code: TrapCode, + }, + Adr { + rd: WritableReg, + off: i32, + }, + Word4 { + data: u32, + }, + Word8 { + data: u64, + }, + JTSequence { + info: BoxJTSequenceInfo, + ridx: Reg, + rtmp1: WritableReg, + rtmp2: WritableReg, + }, + LoadExtName { + rd: WritableReg, + name: BoxExternalName, + offset: i64, + }, + LoadAddr { + rd: WritableReg, + mem: AMode, + }, + VirtualSPOffsetAdj { + offset: i64, + }, + EmitIsland { + needed_space: CodeOffset, + }, + ElfTlsGetAddr { + symbol: ExternalName, + }, + ValueLabelMarker { + reg: Reg, + label: ValueLabel, + }, + Unwind { + inst: UnwindInst, + }, +} + +/// Internal type ALUOp: defined at src/isa/aarch64/inst.isle line 783. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ALUOp { + Add32, + Add64, + Sub32, + Sub64, + Orr32, + Orr64, + OrrNot32, + OrrNot64, + And32, + And64, + AndS32, + AndS64, + AndNot32, + AndNot64, + Eor32, + Eor64, + EorNot32, + EorNot64, + AddS32, + AddS64, + SubS32, + SubS64, + SMulH, + UMulH, + SDiv64, + UDiv64, + RotR32, + RotR64, + Lsr32, + Lsr64, + Asr32, + Asr64, + Lsl32, + Lsl64, + Adc32, + Adc64, + AdcS32, + AdcS64, + Sbc32, + Sbc64, + SbcS32, + SbcS64, +} + +/// Internal type ALUOp3: defined at src/isa/aarch64/inst.isle line 844. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ALUOp3 { + MAdd32, + MAdd64, + MSub32, + MSub64, +} + +/// Internal type BitOp: defined at src/isa/aarch64/inst.isle line 892. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum BitOp { + RBit32, + RBit64, + Clz32, + Clz64, + Cls32, + Cls64, +} + +/// Internal type FPUOp1: defined at src/isa/aarch64/inst.isle line 951. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FPUOp1 { + Abs32, + Abs64, + Neg32, + Neg64, + Sqrt32, + Sqrt64, + Cvt32To64, + Cvt64To32, +} + +/// Internal type FPUOp2: defined at src/isa/aarch64/inst.isle line 964. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FPUOp2 { + Add32, + Add64, + Sub32, + Sub64, + Mul32, + Mul64, + Div32, + Div64, + Max32, + Max64, + Min32, + Min64, + Sqadd64, + Uqadd64, + Sqsub64, + Uqsub64, +} + +/// Internal type FPUOp3: defined at src/isa/aarch64/inst.isle line 989. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FPUOp3 { + MAdd32, + MAdd64, +} + +/// Internal type FpuToIntOp: defined at src/isa/aarch64/inst.isle line 996. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FpuToIntOp { + F32ToU32, + F32ToI32, + F32ToU64, + F32ToI64, + F64ToU32, + F64ToI32, + F64ToU64, + F64ToI64, +} + +/// Internal type IntToFpuOp: defined at src/isa/aarch64/inst.isle line 1009. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum IntToFpuOp { + U32ToF32, + I32ToF32, + U32ToF64, + I32ToF64, + U64ToF32, + I64ToF32, + U64ToF64, + I64ToF64, +} + +/// Internal type FpuRoundMode: defined at src/isa/aarch64/inst.isle line 1023. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FpuRoundMode { + Minus32, + Minus64, + Plus32, + Plus64, + Zero32, + Zero64, + Nearest32, + Nearest64, +} + +/// Internal type VecExtendOp: defined at src/isa/aarch64/inst.isle line 1036. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecExtendOp { + Sxtl8, + Sxtl16, + Sxtl32, + Uxtl8, + Uxtl16, + Uxtl32, +} + +/// Internal type VecALUOp: defined at src/isa/aarch64/inst.isle line 1053. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecALUOp { + Sqadd, + Uqadd, + Sqsub, + Uqsub, + Cmeq, + Cmge, + Cmgt, + Cmhs, + Cmhi, + Fcmeq, + Fcmgt, + Fcmge, + And, + Bic, + Orr, + Eor, + Bsl, + Umaxp, + Add, + Sub, + Mul, + Sshl, + Ushl, + Umin, + Smin, + Umax, + Smax, + Urhadd, + Fadd, + Fsub, + Fdiv, + Fmax, + Fmin, + Fmul, + Addp, + Zip1, + Sqrdmulh, +} + +/// Internal type VecMisc2: defined at src/isa/aarch64/inst.isle line 1132. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecMisc2 { + Not, + Neg, + Abs, + Fabs, + Fneg, + Fsqrt, + Rev64, + Fcvtzs, + Fcvtzu, + Scvtf, + Ucvtf, + Frintn, + Frintz, + Frintm, + Frintp, + Cnt, + Cmeq0, +} + +/// Internal type VecRRLongOp: defined at src/isa/aarch64/inst.isle line 1171. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRLongOp { + Fcvtl16, + Fcvtl32, + Shll8, + Shll16, + Shll32, +} + +/// Internal type VecRRNarrowOp: defined at src/isa/aarch64/inst.isle line 1186. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRNarrowOp { + Xtn16, + Xtn32, + Xtn64, + Sqxtn16, + Sqxtn32, + Sqxtn64, + Sqxtun16, + Sqxtun32, + Sqxtun64, + Uqxtn16, + Uqxtn32, + Uqxtn64, + Fcvtn32, + Fcvtn64, +} + +/// Internal type VecRRRLongOp: defined at src/isa/aarch64/inst.isle line 1218. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRRLongOp { + Smull8, + Smull16, + Smull32, + Umull8, + Umull16, + Umull32, + Umlal8, + Umlal16, + Umlal32, +} + +/// Internal type VecPairOp: defined at src/isa/aarch64/inst.isle line 1235. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecPairOp { + Addp, +} + +/// Internal type VecRRPairLongOp: defined at src/isa/aarch64/inst.isle line 1243. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecRRPairLongOp { + Saddlp8, + Saddlp16, + Uaddlp8, + Uaddlp16, +} + +/// Internal type VecLanesOp: defined at src/isa/aarch64/inst.isle line 1254. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecLanesOp { + Addv, + Uminv, +} + +/// Internal type VecShiftImmOp: defined at src/isa/aarch64/inst.isle line 1263. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VecShiftImmOp { + Shl, + Ushr, + Sshr, +} + +/// Internal type AtomicRMWOp: defined at src/isa/aarch64/inst.isle line 1274. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum AtomicRMWOp { + Add, + Clr, + Eor, + Set, + Smax, + Smin, + Umax, + Umin, +} + +// Generated as internal constructor for term temp_reg. +pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + // Rule at src/prelude.isle line 60. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term lo_reg. +pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/prelude.isle line 95. + let expr0_0 = C::put_in_regs(ctx, pattern0_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term movz. +pub fn constructor_movz( + ctx: &mut C, + arg0: MoveWideConst, + arg1: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1315. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::MovZ { + rd: expr1_0, + imm: pattern0_0, + size: pattern1_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term movn. +pub fn constructor_movn( + ctx: &mut C, + arg0: MoveWideConst, + arg1: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1322. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::MovN { + rd: expr1_0, + imm: pattern0_0, + size: pattern1_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term alu_rr_imm_logic. +pub fn constructor_alu_rr_imm_logic( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: ImmLogic, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1329. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRImmLogic { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + imml: pattern2_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term orr64. +pub fn constructor_orr64(ctx: &mut C, arg0: Reg, arg1: ImmLogic) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1336. + let expr0_0 = ALUOp::Orr64; + let expr1_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term imm. +pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) { + let pattern2_0 = arg1; + if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) { + // Rule at src/isa/aarch64/inst.isle line 1351. + let expr0_0 = C::zero_reg(ctx); + let expr1_0 = constructor_orr64(ctx, expr0_0, pattern3_0)?; + return Some(expr1_0); + } + if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { + // Rule at src/isa/aarch64/inst.isle line 1343. + let expr0_0 = OperandSize::Size64; + let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; + return Some(expr1_0); + } + if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { + // Rule at src/isa/aarch64/inst.isle line 1347. + let expr0_0 = OperandSize::Size64; + let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; + return Some(expr1_0); + } + // Rule at src/isa/aarch64/inst.isle line 1358. + let expr0_0 = C::load_constant64_full(ctx, pattern2_0); + return Some(expr0_0); + } + return None; +} + +// Generated as internal constructor for term lower. +pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) { + let pattern2_0 = C::value_type(ctx, pattern1_0); + let pattern3_0 = C::inst_data(ctx, pattern0_0); + match &pattern3_0 { + &InstructionData::NullAry { + opcode: ref pattern4_0, + } => { + if let &Opcode::Null = &pattern4_0 { + // Rule at src/isa/aarch64/lower.isle line 22. + let expr0_0: u64 = 0; + let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + } + &InstructionData::UnaryImm { + opcode: ref pattern4_0, + imm: pattern4_1, + } => { + if let &Opcode::Iconst = &pattern4_0 { + let pattern6_0 = C::u64_from_imm64(ctx, pattern4_1); + // Rule at src/isa/aarch64/lower.isle line 9. + let expr0_0 = constructor_imm(ctx, pattern2_0, pattern6_0)?; + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); + } + } + &InstructionData::UnaryBool { + opcode: ref pattern4_0, + imm: pattern4_1, + } => { + if let &Opcode::Bconst = &pattern4_0 { + if pattern4_1 == true { + // Rule at src/isa/aarch64/lower.isle line 17. + let expr0_0: u64 = 1; + let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if pattern4_1 == false { + // Rule at src/isa/aarch64/lower.isle line 14. + let expr0_0: u64 = 0; + let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + } + } + _ => {} + } + } + return None; +} diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index c2ae72055e..74d1506c74 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -38,22 +38,21 @@ pub(crate) fn lower_insn_to_regs>( None }; + if let Ok(()) = super::lower::isle::lower(ctx, isa_flags, &outputs, insn) { + return Ok(()); + } + + let implemented_in_isle = || { + unreachable!( + "implemented in ISLE: inst = `{}`, type = `{:?}`", + ctx.dfg().display_inst(insn), + ty + ); + }; + match op { - Opcode::Iconst | Opcode::Bconst | Opcode::Null => { - let value = ctx.get_constant(insn).unwrap(); - match ty.unwrap() { - I8 | I16 | I32 | I64 | R64 => {} - ty if ty.is_bool() => {} - ty => { - return Err(CodegenError::Unsupported(format!( - "{}: Unsupported type: {:?}", - op, ty - ))); - } - } - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - lower_constant_u64(ctx, rd, value); - } + Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(), + Opcode::F32const => { let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32); let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 74e3373c6c..9b9cf2fd0c 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -1,7 +1,7 @@ //! Instruction operand sub-components (aka "parts"): definitions and printing. use super::regs::{self, show_ireg_sized}; -use super::{EmitState, RegMapper}; +use super::EmitState; use crate::ir::condcodes::{FloatCC, IntCC}; use crate::ir::{MemFlags, Type}; use crate::isa::x64::inst::Inst; diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index db1b48226f..c65c3c7523 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2565,64 +2565,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { //============================================================================= // Instructions and subcomponents: map_regs -// Define our own register-mapping trait so we can do arbitrary register -// renaming that are more free form than what `regalloc` constrains us to with -// its `RegUsageMapper` trait definition. -pub trait RegMapper { - fn get_use(&self, reg: Reg) -> Option; - fn get_def(&self, reg: Reg) -> Option; - fn get_mod(&self, reg: Reg) -> Option; -} - -impl RegMapper for T -where - T: regalloc::RegUsageMapper, -{ - fn get_use(&self, reg: Reg) -> Option { - let v = reg.as_virtual_reg()?; - self.get_use(v).map(|r| r.to_reg()) - } - - fn get_def(&self, reg: Reg) -> Option { - let v = reg.as_virtual_reg()?; - self.get_def(v).map(|r| r.to_reg()) - } - - fn get_mod(&self, reg: Reg) -> Option { - let v = reg.as_virtual_reg()?; - self.get_mod(v).map(|r| r.to_reg()) - } -} - -fn map_use(m: &RM, r: &mut Reg) { - if let Some(new) = m.get_use(*r) { - *r = new; - } -} - -fn map_def(m: &RM, r: &mut Writable) { - if let Some(new) = m.get_def(r.to_reg()) { - *r = Writable::from_reg(new); - } -} - -fn map_mod(m: &RM, r: &mut Writable) { - if let Some(new) = m.get_mod(r.to_reg()) { - *r = Writable::from_reg(new); - } -} - impl Amode { fn map_uses(&mut self, map: &RM) { match self { - Amode::ImmReg { ref mut base, .. } => map_use(map, base), + Amode::ImmReg { ref mut base, .. } => map.map_use(base), Amode::ImmRegRegShift { ref mut base, ref mut index, .. } => { - map_use(map, base); - map_use(map, index); + map.map_use(base); + map.map_use(index); } Amode::RipRelative { .. } => { // RIP isn't involved in regalloc. @@ -2645,7 +2598,7 @@ impl Amode { impl RegMemImm { fn map_uses(&mut self, map: &RM) { match self { - RegMemImm::Reg { ref mut reg } => map_use(map, reg), + RegMemImm::Reg { ref mut reg } => map.map_use(reg), RegMemImm::Mem { ref mut addr } => addr.map_uses(map), RegMemImm::Imm { .. } => {} } @@ -2655,7 +2608,7 @@ impl RegMemImm { match self { Self::Reg { reg } => { let mut writable_src = Writable::from_reg(*reg); - map_def(mapper, &mut writable_src); + mapper.map_def(&mut writable_src); *self = Self::reg(writable_src.to_reg()); } _ => panic!("unexpected RegMemImm kind in map_src_reg_as_def"), @@ -2666,7 +2619,7 @@ impl RegMemImm { impl RegMem { fn map_uses(&mut self, map: &RM) { match self { - RegMem::Reg { ref mut reg } => map_use(map, reg), + RegMem::Reg { ref mut reg } => map.map_use(reg), RegMem::Mem { ref mut addr, .. } => addr.map_uses(map), } } @@ -2675,7 +2628,7 @@ impl RegMem { match self { Self::Reg { reg } => { let mut writable_src = Writable::from_reg(*reg); - map_def(mapper, &mut writable_src); + mapper.map_def(&mut writable_src); *self = Self::reg(writable_src.to_reg()); } _ => panic!("unexpected RegMem kind in map_src_reg_as_def"), @@ -2698,25 +2651,25 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } } Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { debug_assert_eq!(*src, dst.to_reg()); - map_mod(mapper, dst); + mapper.map_mod(dst); *src = dst.to_reg(); } Inst::Div { divisor, .. } => divisor.map_uses(mapper), Inst::MulHi { src2, .. } => src2.map_uses(mapper), Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => { - map_mod(mapper, divisor); + mapper.map_mod(divisor); if let Some(tmp) = tmp { - map_def(mapper, tmp) + mapper.map_def(tmp) } } Inst::SignExtendData { .. } => {} @@ -2736,7 +2689,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::XmmRmRImm { ref op, @@ -2748,7 +2701,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else if *op == SseOpcode::Pextrb || *op == SseOpcode::Pextrw @@ -2760,11 +2713,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { || *op == SseOpcode::Roundpd { src2.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } } @@ -2777,11 +2730,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { debug_assert_eq!(*src1, dst.to_reg()); if produces_const { src2.map_as_def(mapper); - map_def(mapper, dst); + mapper.map_def(dst); *src1 = dst.to_reg(); } else { src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } } @@ -2793,10 +2746,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src1.map_uses(mapper); - map_use(mapper, src2); + mapper.map_use(src2); match *op { - Avx512Opcode::Vpermi2b => map_mod(mapper, dst), - _ => map_def(mapper, dst), + Avx512Opcode::Vpermi2b => mapper.map_mod(dst), + _ => mapper.map_def(dst), } } Inst::XmmRmiReg { @@ -2807,29 +2760,29 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { } => { debug_assert_eq!(*src1, dst.to_reg()); src2.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *src1 = dst.to_reg(); } Inst::XmmUninitializedValue { ref mut dst, .. } => { - map_def(mapper, dst); + mapper.map_def(dst); } Inst::XmmLoadConst { ref mut dst, .. } => { - map_def(mapper, dst); + mapper.map_def(dst); } Inst::XmmMinMaxSeq { ref mut lhs, ref mut rhs_dst, .. } => { - map_use(mapper, lhs); - map_mod(mapper, rhs_dst); + mapper.map_use(lhs); + mapper.map_mod(rhs_dst); } Inst::XmmMovRM { ref mut src, ref mut dst, .. } => { - map_use(mapper, src); + mapper.map_use(src); dst.map_uses(mapper); } Inst::XmmCmpRmR { @@ -2838,9 +2791,9 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_use(mapper, dst); + mapper.map_use(dst); } - Inst::Imm { ref mut dst, .. } => map_def(mapper, dst), + Inst::Imm { ref mut dst, .. } => mapper.map_def(dst), Inst::MovRR { ref mut src, ref mut dst, @@ -2851,8 +2804,8 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut dst, .. } => { - map_use(mapper, src); - map_def(mapper, dst); + mapper.map_use(src); + mapper.map_def(dst); } Inst::GprToXmm { ref mut src, @@ -2860,7 +2813,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::CvtUint64ToFloatSeq { ref mut src, @@ -2869,10 +2822,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_gpr2, .. } => { - map_mod(mapper, src); - map_def(mapper, dst); - map_def(mapper, tmp_gpr1); - map_def(mapper, tmp_gpr2); + mapper.map_mod(src); + mapper.map_def(dst); + mapper.map_def(tmp_gpr1); + mapper.map_def(tmp_gpr2); } Inst::CvtFloatToSintSeq { ref mut src, @@ -2888,10 +2841,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp_xmm, .. } => { - map_mod(mapper, src); - map_def(mapper, dst); - map_def(mapper, tmp_gpr); - map_def(mapper, tmp_xmm); + mapper.map_mod(src); + mapper.map_def(dst); + mapper.map_def(tmp_gpr); + mapper.map_def(tmp_xmm); } Inst::MovzxRmR { ref mut src, @@ -2899,11 +2852,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::Mov64MR { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::MovsxRmR { ref mut src, @@ -2911,14 +2864,14 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_def(mapper, dst); + mapper.map_def(dst); } Inst::MovRM { ref mut src, ref mut dst, .. } => { - map_use(mapper, src); + mapper.map_use(src); dst.map_uses(mapper); } Inst::ShiftR { @@ -2927,7 +2880,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { debug_assert_eq!(*src, dst.to_reg()); - map_mod(mapper, dst); + mapper.map_mod(dst); *src = dst.to_reg(); } Inst::CmpRmiR { @@ -2936,9 +2889,9 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_use(mapper, dst); + mapper.map_use(dst); } - Inst::Setcc { ref mut dst, .. } => map_def(mapper, dst), + Inst::Setcc { ref mut dst, .. } => mapper.map_def(dst), Inst::Cmove { consequent: ref mut src, ref mut dst, @@ -2946,7 +2899,7 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); *alternative = dst.to_reg(); } Inst::XmmCmove { @@ -2955,11 +2908,11 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { src.map_uses(mapper); - map_mod(mapper, dst); + mapper.map_mod(dst); } Inst::Push64 { ref mut src } => src.map_uses(mapper), Inst::Pop64 { ref mut dst } => { - map_def(mapper, dst); + mapper.map_def(dst); } Inst::CallKnown { @@ -2968,10 +2921,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { for r in uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } } @@ -2982,10 +2935,10 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { .. } => { for r in uses.iter_mut() { - map_use(mapper, r); + mapper.map_use(r); } for r in defs.iter_mut() { - map_def(mapper, r); + mapper.map_def(r); } dest.map_uses(mapper); } @@ -2996,25 +2949,25 @@ pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { ref mut tmp2, .. } => { - map_use(mapper, idx); - map_def(mapper, tmp1); - map_def(mapper, tmp2); + mapper.map_use(idx); + mapper.map_def(tmp1); + mapper.map_def(tmp2); } Inst::JmpUnknown { ref mut target } => target.map_uses(mapper), - Inst::LoadExtName { ref mut dst, .. } => map_def(mapper, dst), + Inst::LoadExtName { ref mut dst, .. } => mapper.map_def(dst), Inst::LockCmpxchg { ref mut replacement, ref mut mem, .. } => { - map_use(mapper, replacement); + mapper.map_use(replacement); mem.map_uses(mapper); } - Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg), + Inst::ValueLabelMarker { ref mut reg, .. } => mapper.map_use(reg), Inst::Ret | Inst::EpiloguePlaceholder diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 118a0cdb25..179b95fa72 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -6,70 +6,29 @@ mod generated_code; // Types that the generated ISLE code uses via `use super::*`. use super::{ is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm, - Writable, }; use crate::isa::x64::inst::args::SyntheticAmode; use crate::isa::x64::settings as x64_settings; +use crate::machinst::isle::*; use crate::{ ir::{immediates::*, types::*, Inst, InstructionData, Opcode, Value, ValueList}, isa::x64::inst::{ args::{ Avx512Opcode, CmpOpcode, ExtMode, FcmpImm, Imm8Reg, RegMem, ShiftKind, SseOpcode, CC, }, - x64_map_regs, RegMapper, + x64_map_regs, }, - machinst::{get_output_reg, InsnInput, InsnOutput, LowerCtx}, + machinst::{get_output_reg, InsnInput, InsnOutput, LowerCtx, RegRenamer}, }; use smallvec::SmallVec; use std::convert::TryFrom; -type Unit = (); -type ValueSlice<'a> = &'a [Value]; -type ValueArray2 = [Value; 2]; -type ValueArray3 = [Value; 3]; -type WritableReg = Writable; -type ValueRegs = crate::machinst::ValueRegs; - pub struct SinkableLoad { inst: Inst, addr_input: InsnInput, offset: i32, } -#[derive(Default)] -struct RegRenamer { - // Map of `(old, new)` register names. Use a `SmallVec` because we typically - // only have one or two renamings. - renames: SmallVec<[(Reg, Reg); 2]>, -} - -impl RegRenamer { - fn add_rename(&mut self, old: Reg, new: Reg) { - self.renames.push((old, new)); - } - - fn get_rename(&self, reg: Reg) -> Option { - self.renames - .iter() - .find(|(old, _)| reg == *old) - .map(|(_, new)| *new) - } -} - -impl RegMapper for RegRenamer { - fn get_use(&self, reg: Reg) -> Option { - self.get_rename(reg) - } - - fn get_def(&self, reg: Reg) -> Option { - self.get_rename(reg) - } - - fn get_mod(&self, reg: Reg) -> Option { - self.get_rename(reg) - } -} - /// The main entry point for lowering with ISLE. pub(crate) fn lower( lower_ctx: &mut C, @@ -131,159 +90,7 @@ impl<'a, C> generated_code::Context for IsleContext<'a, C> where C: LowerCtx, { - #[inline] - fn unpack_value_array_2(&mut self, arr: &ValueArray2) -> (Value, Value) { - let [a, b] = *arr; - (a, b) - } - - #[inline] - fn pack_value_array_2(&mut self, a: Value, b: Value) -> ValueArray2 { - [a, b] - } - - #[inline] - fn unpack_value_array_3(&mut self, arr: &ValueArray3) -> (Value, Value, Value) { - let [a, b, c] = *arr; - (a, b, c) - } - - #[inline] - fn pack_value_array_3(&mut self, a: Value, b: Value, c: Value) -> ValueArray3 { - [a, b, c] - } - - #[inline] - fn value_reg(&mut self, reg: Reg) -> ValueRegs { - ValueRegs::one(reg) - } - - #[inline] - fn value_regs(&mut self, r1: Reg, r2: Reg) -> ValueRegs { - ValueRegs::two(r1, r2) - } - - #[inline] - fn temp_writable_reg(&mut self, ty: Type) -> WritableReg { - let value_regs = self.lower_ctx.alloc_tmp(ty); - value_regs.only_reg().unwrap() - } - - #[inline] - fn invalid_reg(&mut self) -> Reg { - Reg::invalid() - } - - #[inline] - fn put_in_reg(&mut self, val: Value) -> Reg { - self.lower_ctx.put_value_in_regs(val).only_reg().unwrap() - } - - #[inline] - fn put_in_regs(&mut self, val: Value) -> ValueRegs { - self.lower_ctx.put_value_in_regs(val) - } - - #[inline] - fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg { - regs.regs()[i] - } - - #[inline] - fn u8_as_u64(&mut self, x: u8) -> u64 { - x.into() - } - - #[inline] - fn u16_as_u64(&mut self, x: u16) -> u64 { - x.into() - } - - #[inline] - fn u32_as_u64(&mut self, x: u32) -> u64 { - x.into() - } - - #[inline] - fn ty_bits(&mut self, ty: Type) -> u16 { - ty.bits() - } - - #[inline] - fn fits_in_64(&mut self, ty: Type) -> Option { - if ty.bits() <= 64 { - Some(ty) - } else { - None - } - } - - #[inline] - fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { - list.as_slice(&self.lower_ctx.dfg().value_lists) - } - - #[inline] - fn unwrap_head_value_list_1(&mut self, list: ValueList) -> (Value, ValueSlice) { - match self.value_list_slice(list) { - [head, tail @ ..] => (*head, tail), - _ => out_of_line_panic("`unwrap_head_value_list_1` on empty `ValueList`"), - } - } - - #[inline] - fn unwrap_head_value_list_2(&mut self, list: ValueList) -> (Value, Value, ValueSlice) { - match self.value_list_slice(list) { - [head1, head2, tail @ ..] => (*head1, *head2, tail), - _ => out_of_line_panic( - "`unwrap_head_value_list_2` on list without at least two elements", - ), - } - } - - #[inline] - fn writable_reg_to_reg(&mut self, r: WritableReg) -> Reg { - r.to_reg() - } - - #[inline] - fn u64_from_imm64(&mut self, imm: Imm64) -> u64 { - imm.bits() as u64 - } - - #[inline] - fn inst_results(&mut self, inst: Inst) -> ValueSlice { - self.lower_ctx.dfg().inst_results(inst) - } - - #[inline] - fn first_result(&mut self, inst: Inst) -> Option { - self.lower_ctx.dfg().inst_results(inst).first().copied() - } - - #[inline] - fn inst_data(&mut self, inst: Inst) -> InstructionData { - self.lower_ctx.dfg()[inst].clone() - } - - #[inline] - fn value_type(&mut self, val: Value) -> Type { - self.lower_ctx.dfg().value_type(val) - } - - #[inline] - fn multi_lane(&mut self, ty: Type) -> Option<(u8, u16)> { - if ty.lane_count() > 1 { - Some((ty.lane_bits(), ty.lane_count())) - } else { - None - } - } - - #[inline] - fn def_inst(&mut self, val: Value) -> Option { - self.lower_ctx.dfg().value_def(val).inst() - } + isle_prelude_methods!(); #[inline] fn operand_size_of_type(&mut self, ty: Type) -> OperandSize { @@ -400,14 +207,6 @@ where None } } - - fn u64_from_ieee32(&mut self, val: Ieee32) -> u64 { - val.bits().into() - } - - fn u64_from_ieee64(&mut self, val: Ieee64) -> u64 { - val.bits() - } } #[inline] @@ -420,10 +219,3 @@ fn to_simm32(constant: i64) -> Option { None } } - -#[inline(never)] -#[cold] -#[track_caller] -fn out_of_line_panic(msg: &str) -> ! { - panic!("{}", msg); -} diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 2c10784cd3..89fd2808d2 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle 21143c73c97885afd9c34acedb8dbc653dca3b5a3e2e73754a5761762f7fe6ce2eefa520227e64c2b18f5626ca0ffa0a6aff54971099e619a464a23adf365bd2 +src/prelude.isle a069d14321601afc63959af23086709d67d189dafcdc7d1fc8534b32d89d49008acb8368b7b5a7bc51a353736a378197ac352ccce2bb3be89d93afb6979e480a src/isa/x64/inst.isle fdfbfc6dfad1fc5ed252e0a14ccc69baba51d0538e05cfb9916f6213e5a6fcfc9d22605a29bd684d6a66f6d5e1c8ec36a963660d52c2e8b3fb6e0758f7adb7b5 src/isa/x64/lower.isle 8555abdae385431c96aaabc392b7b3a8b1bbe733be08b007ef776850860cb77e85a140db02f427586c155c0b0129f9ffd531abd2e4a772c72667535cc015e609 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index ba6a6f36c8..61bac2a6b8 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -73,7 +73,7 @@ pub enum ConsumesFlags { } /// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 433. -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, Zero, @@ -82,7 +82,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 57. + // Rule at src/prelude.isle line 60. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -91,7 +91,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 92. + // Rule at src/prelude.isle line 95. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index a19d94407c..426a9c0c4d 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -68,6 +68,9 @@ pub use cranelift_entity as entity; #[cfg(feature = "unwind")] pub use gimli; +#[macro_use] +mod machinst; + pub mod binemit; pub mod cfg_printer; pub mod cursor; @@ -99,7 +102,6 @@ mod iterators; mod legalizer; mod licm; mod log; -mod machinst; mod nan_canonicalization; mod remove_constant_phis; mod result; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs new file mode 100644 index 0000000000..99e4c8f49f --- /dev/null +++ b/cranelift/codegen/src/machinst/isle.rs @@ -0,0 +1,185 @@ +use crate::ir::Value; +use regalloc::{Reg, Writable}; + +pub type Unit = (); +pub type ValueSlice<'a> = &'a [Value]; +pub type ValueArray2 = [Value; 2]; +pub type ValueArray3 = [Value; 3]; +pub type WritableReg = Writable; +pub type ValueRegs = crate::machinst::ValueRegs; + +#[macro_export] +#[doc(hidden)] +macro_rules! isle_prelude_methods { + () => { + #[inline] + fn unpack_value_array_2(&mut self, arr: &ValueArray2) -> (Value, Value) { + let [a, b] = *arr; + (a, b) + } + + #[inline] + fn pack_value_array_2(&mut self, a: Value, b: Value) -> ValueArray2 { + [a, b] + } + + #[inline] + fn unpack_value_array_3(&mut self, arr: &ValueArray3) -> (Value, Value, Value) { + let [a, b, c] = *arr; + (a, b, c) + } + + #[inline] + fn pack_value_array_3(&mut self, a: Value, b: Value, c: Value) -> ValueArray3 { + [a, b, c] + } + + #[inline] + fn value_reg(&mut self, reg: Reg) -> ValueRegs { + ValueRegs::one(reg) + } + + #[inline] + fn value_regs(&mut self, r1: Reg, r2: Reg) -> ValueRegs { + ValueRegs::two(r1, r2) + } + + #[inline] + fn temp_writable_reg(&mut self, ty: Type) -> WritableReg { + let value_regs = self.lower_ctx.alloc_tmp(ty); + value_regs.only_reg().unwrap() + } + + #[inline] + fn invalid_reg(&mut self) -> Reg { + Reg::invalid() + } + + #[inline] + fn put_in_reg(&mut self, val: Value) -> Reg { + self.lower_ctx.put_value_in_regs(val).only_reg().unwrap() + } + + #[inline] + fn put_in_regs(&mut self, val: Value) -> ValueRegs { + self.lower_ctx.put_value_in_regs(val) + } + + #[inline] + fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg { + regs.regs()[i] + } + + #[inline] + fn u8_as_u64(&mut self, x: u8) -> u64 { + x.into() + } + + #[inline] + fn u16_as_u64(&mut self, x: u16) -> u64 { + x.into() + } + + #[inline] + fn u32_as_u64(&mut self, x: u32) -> u64 { + x.into() + } + + #[inline] + fn ty_bits(&mut self, ty: Type) -> u16 { + ty.bits() + } + + #[inline] + fn fits_in_64(&mut self, ty: Type) -> Option { + if ty.bits() <= 64 { + Some(ty) + } else { + None + } + } + + #[inline] + fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { + list.as_slice(&self.lower_ctx.dfg().value_lists) + } + + #[inline] + fn unwrap_head_value_list_1(&mut self, list: ValueList) -> (Value, ValueSlice) { + match self.value_list_slice(list) { + [head, tail @ ..] => (*head, tail), + _ => crate::machinst::isle::out_of_line_panic( + "`unwrap_head_value_list_1` on empty `ValueList`", + ), + } + } + + #[inline] + fn unwrap_head_value_list_2(&mut self, list: ValueList) -> (Value, Value, ValueSlice) { + match self.value_list_slice(list) { + [head1, head2, tail @ ..] => (*head1, *head2, tail), + _ => crate::machinst::isle::out_of_line_panic( + "`unwrap_head_value_list_2` on list without at least two elements", + ), + } + } + + #[inline] + fn writable_reg_to_reg(&mut self, r: WritableReg) -> Reg { + r.to_reg() + } + + #[inline] + fn u64_from_imm64(&mut self, imm: Imm64) -> u64 { + imm.bits() as u64 + } + + #[inline] + fn inst_results(&mut self, inst: Inst) -> ValueSlice { + self.lower_ctx.dfg().inst_results(inst) + } + + #[inline] + fn first_result(&mut self, inst: Inst) -> Option { + self.lower_ctx.dfg().inst_results(inst).first().copied() + } + + #[inline] + fn inst_data(&mut self, inst: Inst) -> InstructionData { + self.lower_ctx.dfg()[inst].clone() + } + + #[inline] + fn value_type(&mut self, val: Value) -> Type { + self.lower_ctx.dfg().value_type(val) + } + + #[inline] + fn multi_lane(&mut self, ty: Type) -> Option<(u8, u16)> { + if ty.lane_count() > 1 { + Some((ty.lane_bits(), ty.lane_count())) + } else { + None + } + } + + #[inline] + fn def_inst(&mut self, val: Value) -> Option { + self.lower_ctx.dfg().value_def(val).inst() + } + + fn u64_from_ieee32(&mut self, val: Ieee32) -> u64 { + val.bits().into() + } + + fn u64_from_ieee64(&mut self, val: Ieee64) -> u64 { + val.bits() + } + }; +} + +#[inline(never)] +#[cold] +pub fn out_of_line_panic(msg: &str) -> ! { + panic!("{}", msg); +} diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index b0122dfe9d..20a78b6c87 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -81,6 +81,9 @@ use target_lexicon::Triple; #[cfg(feature = "unwind")] use crate::isa::unwind::systemv::RegisterMappingError; +#[macro_use] +pub mod isle; + pub mod lower; pub use lower::*; pub mod vcode; @@ -104,6 +107,8 @@ pub use inst_common::*; pub mod valueregs; pub use valueregs::*; pub mod debug; +pub use regmapping::*; +pub mod regmapping; /// A machine instruction. pub trait MachInst: Clone + Debug { diff --git a/cranelift/codegen/src/machinst/regmapping.rs b/cranelift/codegen/src/machinst/regmapping.rs new file mode 100644 index 0000000000..0f132e47b4 --- /dev/null +++ b/cranelift/codegen/src/machinst/regmapping.rs @@ -0,0 +1,83 @@ +use regalloc::{Reg, RegUsageMapper, Writable}; +use smallvec::SmallVec; + +// Define our own register-mapping trait so we can do arbitrary register +// renaming that are more free form than what `regalloc` constrains us to with +// its `RegUsageMapper` trait definition. +pub trait RegMapper { + fn get_use(&self, reg: Reg) -> Option; + fn get_def(&self, reg: Reg) -> Option; + fn get_mod(&self, reg: Reg) -> Option; + + fn map_use(&self, r: &mut Reg) { + if let Some(new) = self.get_use(*r) { + *r = new; + } + } + + fn map_def(&self, r: &mut Writable) { + if let Some(new) = self.get_def(r.to_reg()) { + *r = Writable::from_reg(new); + } + } + + fn map_mod(&self, r: &mut Writable) { + if let Some(new) = self.get_mod(r.to_reg()) { + *r = Writable::from_reg(new); + } + } +} + +impl RegMapper for T +where + T: RegUsageMapper, +{ + fn get_use(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_use(v).map(|r| r.to_reg()) + } + + fn get_def(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_def(v).map(|r| r.to_reg()) + } + + fn get_mod(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_mod(v).map(|r| r.to_reg()) + } +} + +#[derive(Default)] +pub struct RegRenamer { + // Map of `(old, new)` register names. Use a `SmallVec` because we typically + // only have one or two renamings. + renames: SmallVec<[(Reg, Reg); 2]>, +} + +impl RegRenamer { + pub fn add_rename(&mut self, old: Reg, new: Reg) { + self.renames.push((old, new)); + } + + fn get_rename(&self, reg: Reg) -> Option { + self.renames + .iter() + .find(|(old, _)| reg == *old) + .map(|(_, new)| *new) + } +} + +impl RegMapper for RegRenamer { + fn get_use(&self, reg: Reg) -> Option { + self.get_rename(reg) + } + + fn get_def(&self, reg: Reg) -> Option { + self.get_rename(reg) + } + + fn get_mod(&self, reg: Reg) -> Option { + self.get_rename(reg) + } +} diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 3f6f8ef7f7..c1367cc575 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -1,5 +1,8 @@ ;; This is a prelude of standard definitions for ISLE, the instruction-selector ;; DSL, as we use it bound to our interfaces. +;; +;; Note that all `extern` functions here are typically defined in the +;; `isle_prelude_methods` macro defined in `src/isa/isle.rs` ;;;; Primitive and External Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/isle/isle/src/codegen.rs b/cranelift/isle/isle/src/codegen.rs index 74874b6a1d..23a76d39e7 100644 --- a/cranelift/isle/isle/src/codegen.rs +++ b/cranelift/isle/isle/src/codegen.rs @@ -150,7 +150,11 @@ impl<'a> Codegen<'a> { pos.pretty_print_line(&self.typeenv.filenames[..]) ) .unwrap(); - writeln!(code, "#[derive(Clone, Debug)]").unwrap(); + if variants.iter().all(|v| v.fields.is_empty()) { + writeln!(code, "#[derive(Copy, Clone, Debug, PartialEq, Eq)]").unwrap(); + } else { + writeln!(code, "#[derive(Clone, Debug)]").unwrap(); + } writeln!(code, "pub enum {} {{", name).unwrap(); for variant in variants { let name = &self.typeenv.syms[variant.name.index()];