ca6d648e37
Ensure that constants generated for the memory case of XmmMem values are always 16 bytes, ensuring that we don't accidantally perform an unaligned load. Fixes #4761
979 lines
35 KiB
Common Lisp
979 lines
35 KiB
Common Lisp
;; This is a prelude of standard definitions for ISLE, the instruction-selector
|
|
;; DSL, as we use it bound to our interfaces.
|
|
;;
|
|
;; Note that all `extern` functions here are typically defined in the
|
|
;; `isle_prelude_methods` macro defined in `src/isa/isle.rs`
|
|
|
|
;;;; Primitive and External Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; `()`
|
|
(type Unit (primitive Unit))
|
|
|
|
;; `bool` is declared in `clif.isle`.
|
|
(extern const $true bool)
|
|
(extern const $false bool)
|
|
|
|
(type u8 (primitive u8))
|
|
(type u16 (primitive u16))
|
|
(type u32 (primitive u32))
|
|
(type u64 (primitive u64))
|
|
(type u128 (primitive u128))
|
|
(type usize (primitive usize))
|
|
|
|
(type i8 (primitive i8))
|
|
(type i16 (primitive i16))
|
|
(type i32 (primitive i32))
|
|
(type i64 (primitive i64))
|
|
(type i128 (primitive i128))
|
|
(type isize (primitive isize))
|
|
|
|
;; `cranelift-entity`-based identifiers.
|
|
(type Inst (primitive Inst))
|
|
(type Type (primitive Type))
|
|
(type Value (primitive Value))
|
|
|
|
;; ISLE representation of `&[Value]`.
|
|
(type ValueSlice (primitive ValueSlice))
|
|
|
|
;; ISLE representation of `Vec<u8>`
|
|
(type VecMask extern (enum))
|
|
|
|
(type ValueList (primitive ValueList))
|
|
(type ValueRegs (primitive ValueRegs))
|
|
(type WritableValueRegs (primitive WritableValueRegs))
|
|
|
|
;; Instruction lowering result: a vector of `ValueRegs`.
|
|
(type InstOutput (primitive InstOutput))
|
|
;; (Mutable) builder to incrementally construct an `InstOutput`.
|
|
(type InstOutputBuilder extern (enum))
|
|
|
|
(decl u32_add (u32 u32) u32)
|
|
(extern constructor u32_add u32_add)
|
|
|
|
;; Pure/fallible constructor that tries to add two `u32`s, interpreted
|
|
;; as signed values, and fails to match on overflow.
|
|
(decl pure s32_add_fallible (u32 u32) u32)
|
|
(extern constructor s32_add_fallible s32_add_fallible)
|
|
|
|
;; Extractor that matches a `u32` only if non-negative.
|
|
(decl u32_nonnegative (u32) u32)
|
|
(extern extractor u32_nonnegative u32_nonnegative)
|
|
|
|
;; Extractor that pulls apart an Offset32 into a u32 with the raw
|
|
;; signed-32-bit twos-complement bits.
|
|
(decl offset32 (u32) Offset32)
|
|
(extern extractor offset32 offset32)
|
|
|
|
;; Pure/fallible constructor that tests if one u32 is less than or
|
|
;; equal to another.
|
|
(decl pure u32_lteq (u32 u32) Unit)
|
|
(extern constructor u32_lteq u32_lteq)
|
|
|
|
;; Get a signed 32-bit immediate in an u32 from an Imm64, if possible.
|
|
(decl simm32 (u32) Imm64)
|
|
(extern extractor simm32 simm32)
|
|
|
|
;; Get an unsigned 8-bit immediate in a u8 from an Imm64, if possible.
|
|
(decl uimm8 (u8) Imm64)
|
|
(extern extractor uimm8 uimm8)
|
|
|
|
(decl u8_and (u8 u8) u8)
|
|
(extern constructor u8_and u8_and)
|
|
|
|
;;;; Registers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(type Reg (primitive Reg))
|
|
(type WritableReg (primitive WritableReg))
|
|
(type OptionWritableReg (primitive OptionWritableReg))
|
|
(type VecReg extern (enum))
|
|
(type VecWritableReg extern (enum))
|
|
(type PReg (primitive PReg))
|
|
|
|
;; Construct a `ValueRegs` of one register.
|
|
(decl value_reg (Reg) ValueRegs)
|
|
(extern constructor value_reg value_reg)
|
|
|
|
;; Construct a `ValueRegs` of two registers.
|
|
(decl value_regs (Reg Reg) ValueRegs)
|
|
(extern constructor value_regs value_regs)
|
|
|
|
;; Construct an empty `ValueRegs` containing only invalid register sentinels.
|
|
(decl value_regs_invalid () ValueRegs)
|
|
(extern constructor value_regs_invalid value_regs_invalid)
|
|
|
|
;; Construct an empty `InstOutput`.
|
|
(decl output_none () InstOutput)
|
|
(extern constructor output_none output_none)
|
|
|
|
;; Construct a single-element `InstOutput`.
|
|
(decl output (ValueRegs) InstOutput)
|
|
(extern constructor output output)
|
|
|
|
;; Construct a two-element `InstOutput`.
|
|
(decl output_pair (ValueRegs ValueRegs) InstOutput)
|
|
(extern constructor output_pair output_pair)
|
|
|
|
;; Construct a single-element `InstOutput` from a single register.
|
|
(decl output_reg (Reg) InstOutput)
|
|
(rule (output_reg reg) (output (value_reg reg)))
|
|
|
|
;; Construct a single-element `InstOutput` from a value.
|
|
(decl output_value (Value) InstOutput)
|
|
(rule (output_value val) (output (put_in_regs val)))
|
|
|
|
;; Initially empty `InstOutput` builder.
|
|
(decl output_builder_new () InstOutputBuilder)
|
|
(extern constructor output_builder_new output_builder_new)
|
|
|
|
;; Append a `ValueRegs` to an `InstOutput` under construction.
|
|
(decl output_builder_push (InstOutputBuilder ValueRegs) Unit)
|
|
(extern constructor output_builder_push output_builder_push)
|
|
|
|
;; Finish building an `InstOutput` incrementally.
|
|
(decl output_builder_finish (InstOutputBuilder) InstOutput)
|
|
(extern constructor output_builder_finish output_builder_finish)
|
|
|
|
;; Get a temporary register for writing.
|
|
(decl temp_writable_reg (Type) WritableReg)
|
|
(extern constructor temp_writable_reg temp_writable_reg)
|
|
|
|
;; Get a temporary register for reading.
|
|
(decl temp_reg (Type) Reg)
|
|
(rule (temp_reg ty)
|
|
(writable_reg_to_reg (temp_writable_reg ty)))
|
|
|
|
;; Get or match the invalid register.
|
|
(decl invalid_reg () Reg)
|
|
(extern constructor invalid_reg invalid_reg)
|
|
(extern extractor invalid_reg invalid_reg_etor)
|
|
|
|
;; Match any register but the invalid register.
|
|
(decl valid_reg () Reg)
|
|
(extern extractor valid_reg valid_reg)
|
|
|
|
;; Mark this value as used, to ensure that it gets lowered.
|
|
(decl mark_value_used (Value) Unit)
|
|
(extern constructor mark_value_used mark_value_used)
|
|
|
|
;; Put the given value into a register.
|
|
;;
|
|
;; Asserts that the value fits into a single register, and doesn't require
|
|
;; multiple registers for its representation (like `i128` on x64 for example).
|
|
;;
|
|
;; As a side effect, this marks the value as used.
|
|
(decl put_in_reg (Value) Reg)
|
|
(extern constructor put_in_reg put_in_reg)
|
|
|
|
;; Put the given value into one or more registers.
|
|
;;
|
|
;; As a side effect, this marks the value as used.
|
|
(decl put_in_regs (Value) ValueRegs)
|
|
(extern constructor put_in_regs put_in_regs)
|
|
|
|
;; If the given reg is a real register, cause the value in reg to be in a virtual
|
|
;; reg, by copying it into a new virtual reg.
|
|
(decl ensure_in_vreg (Reg Type) Reg)
|
|
(extern constructor ensure_in_vreg ensure_in_vreg)
|
|
|
|
;; Get the `n`th register inside a `ValueRegs`.
|
|
(decl value_regs_get (ValueRegs usize) Reg)
|
|
(extern constructor value_regs_get value_regs_get)
|
|
|
|
;; Get the number of registers in a `ValueRegs`.
|
|
(decl value_regs_len (ValueRegs) usize)
|
|
(extern constructor value_regs_len value_regs_len)
|
|
|
|
;; Get a range for the number of regs in a `ValueRegs`.
|
|
(decl value_regs_range (ValueRegs) Range)
|
|
(rule (value_regs_range regs) (range 0 (value_regs_len regs)))
|
|
|
|
;; Put the value into one or more registers and return the first register.
|
|
;;
|
|
;; Unlike `put_in_reg`, this does not assert that the value fits in a single
|
|
;; register. This is useful for things like a `i128` shift amount, where we mask
|
|
;; the shift amount to the bit width of the value being shifted, and so the high
|
|
;; half of the `i128` won't ever be used.
|
|
;;
|
|
;; As a side efect, this marks that value as used.
|
|
(decl lo_reg (Value) Reg)
|
|
(rule (lo_reg val)
|
|
(let ((regs ValueRegs (put_in_regs val)))
|
|
(value_regs_get regs 0)))
|
|
|
|
;; Convert a `PReg` into a `Reg`
|
|
(decl preg_to_reg (PReg) Reg)
|
|
(extern constructor preg_to_reg preg_to_reg)
|
|
|
|
;;;; Common Mach Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(type MachLabel (primitive MachLabel))
|
|
(type ValueLabel (primitive ValueLabel))
|
|
(type UnwindInst (primitive UnwindInst))
|
|
(type ExternalName (primitive ExternalName))
|
|
(type BoxExternalName (primitive BoxExternalName))
|
|
(type RelocDistance (primitive RelocDistance))
|
|
|
|
;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(decl pure u8_as_u32 (u8) u32)
|
|
(extern constructor u8_as_u32 u8_as_u32)
|
|
|
|
(decl pure u8_as_u64 (u8) u64)
|
|
(extern constructor u8_as_u64 u8_as_u64)
|
|
|
|
(decl pure u16_as_u64 (u16) u64)
|
|
(extern constructor u16_as_u64 u16_as_u64)
|
|
|
|
(decl pure u32_as_u64 (u32) u64)
|
|
(extern constructor u32_as_u64 u32_as_u64)
|
|
|
|
(decl pure i64_as_u64 (i64) u64)
|
|
(extern constructor i64_as_u64 i64_as_u64)
|
|
|
|
;;;; Primitive Arithmetic ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(decl pure u64_add (u64 u64) u64)
|
|
(extern constructor u64_add u64_add)
|
|
|
|
(decl pure u64_sub (u64 u64) u64)
|
|
(extern constructor u64_sub u64_sub)
|
|
|
|
(decl pure u64_and (u64 u64) u64)
|
|
(extern constructor u64_and u64_and)
|
|
|
|
;;;; `cranelift_codegen::ir::Type` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(extern const $B1 Type)
|
|
(extern const $B8 Type)
|
|
(extern const $B16 Type)
|
|
(extern const $B32 Type)
|
|
(extern const $B64 Type)
|
|
(extern const $B128 Type)
|
|
|
|
(extern const $I8 Type)
|
|
(extern const $I16 Type)
|
|
(extern const $I32 Type)
|
|
(extern const $I64 Type)
|
|
(extern const $I128 Type)
|
|
|
|
(extern const $R32 Type)
|
|
(extern const $R64 Type)
|
|
|
|
(extern const $F32 Type)
|
|
(extern const $F64 Type)
|
|
|
|
(extern const $B8X16 Type)
|
|
(extern const $B16X8 Type)
|
|
(extern const $B32X4 Type)
|
|
(extern const $B64X2 Type)
|
|
|
|
(extern const $I8X8 Type)
|
|
(extern const $I8X16 Type)
|
|
(extern const $I16X4 Type)
|
|
(extern const $I16X8 Type)
|
|
(extern const $I32X2 Type)
|
|
(extern const $I32X4 Type)
|
|
(extern const $I64X2 Type)
|
|
|
|
(extern const $F32X4 Type)
|
|
(extern const $F64X2 Type)
|
|
|
|
(extern const $I32X4XN Type)
|
|
|
|
;; Get the bit width of a given type.
|
|
(decl pure ty_bits (Type) u8)
|
|
(extern constructor ty_bits ty_bits)
|
|
|
|
;; Get the bit width of a given type.
|
|
(decl ty_bits_u16 (Type) u16)
|
|
(extern constructor ty_bits_u16 ty_bits_u16)
|
|
|
|
;; Get the bit width of a given type.
|
|
(decl ty_bits_u64 (Type) u64)
|
|
(extern constructor ty_bits_u64 ty_bits_u64)
|
|
|
|
;; Get a mask for the width of a given type.
|
|
(decl ty_mask (Type) u64)
|
|
(extern constructor ty_mask ty_mask)
|
|
|
|
;; Get the byte width of a given type.
|
|
(decl ty_bytes (Type) u16)
|
|
(extern constructor ty_bytes ty_bytes)
|
|
|
|
;; Get the type of each lane in the given type.
|
|
(decl lane_type (Type) Type)
|
|
(extern constructor lane_type lane_type)
|
|
|
|
;;;; `cranelift_codegen::ir::MemFlags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; `MemFlags::trusted`
|
|
(decl mem_flags_trusted () MemFlags)
|
|
(extern constructor mem_flags_trusted mem_flags_trusted)
|
|
|
|
;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; An extractor that only matches types that can fit in 16 bits.
|
|
(decl fits_in_16 (Type) Type)
|
|
(extern extractor fits_in_16 fits_in_16)
|
|
|
|
;; An extractor that only matches types that can fit in 32 bits.
|
|
(decl fits_in_32 (Type) Type)
|
|
(extern extractor fits_in_32 fits_in_32)
|
|
|
|
;; An extractor that only matches types that can fit in 32 bits.
|
|
(decl lane_fits_in_32 (Type) Type)
|
|
(extern extractor lane_fits_in_32 lane_fits_in_32)
|
|
|
|
;; An extractor that only matches types that can fit in 64 bits.
|
|
(decl fits_in_64 (Type) Type)
|
|
(extern extractor fits_in_64 fits_in_64)
|
|
|
|
;; An extractor that only matches types that fit in exactly 32 bits.
|
|
(decl ty_32 (Type) Type)
|
|
(extern extractor ty_32 ty_32)
|
|
|
|
;; An extractor that only matches types that fit in exactly 64 bits.
|
|
(decl ty_64 (Type) Type)
|
|
(extern extractor ty_64 ty_64)
|
|
|
|
;; A pure constructor that only matches scalar booleans, integers, and
|
|
;; references that can fit in 64 bits.
|
|
(decl pure ty_int_bool_ref_scalar_64 (Type) Type)
|
|
(extern constructor ty_int_bool_ref_scalar_64 ty_int_bool_ref_scalar_64)
|
|
|
|
;; An extractor that matches 32- and 64-bit types only.
|
|
(decl ty_32_or_64 (Type) Type)
|
|
(extern extractor ty_32_or_64 ty_32_or_64)
|
|
|
|
;; An extractor that matches 8- and 16-bit types only.
|
|
(decl ty_8_or_16 (Type) Type)
|
|
(extern extractor ty_8_or_16 ty_8_or_16)
|
|
|
|
;; An extractor that matches int and bool types that fit in 32 bits.
|
|
(decl int_bool_fits_in_32 (Type) Type)
|
|
(extern extractor int_bool_fits_in_32 int_bool_fits_in_32)
|
|
|
|
;; An extractor that matches I64 or B64.
|
|
(decl ty_int_bool_64 (Type) Type)
|
|
(extern extractor ty_int_bool_64 ty_int_bool_64)
|
|
|
|
;; An extractor that matches I64 or B64 or R64.
|
|
(decl ty_int_bool_ref_64 (Type) Type)
|
|
(extern extractor ty_int_bool_ref_64 ty_int_bool_ref_64)
|
|
|
|
;; An extractor that matches I128 or B128.
|
|
(decl ty_int_bool_128 (Type) Type)
|
|
(extern extractor ty_int_bool_128 ty_int_bool_128)
|
|
|
|
;; An extractor that only matches integers.
|
|
(decl ty_int (Type) Type)
|
|
(extern extractor ty_int ty_int)
|
|
|
|
;; An extractor that only matches scalar floating-point types--F32 or F64.
|
|
(decl ty_scalar_float (Type) Type)
|
|
(extern extractor ty_scalar_float ty_scalar_float)
|
|
|
|
;; A pure constructor that only matches 64-bit vector types.
|
|
(decl pure ty_vec64 (Type) Type)
|
|
(extern constructor ty_vec64 ty_vec64)
|
|
|
|
;; An extractor that only matches 128-bit vector types.
|
|
(decl ty_vec128 (Type) Type)
|
|
(extern extractor ty_vec128 ty_vec128)
|
|
|
|
;; An extractor that only matches 64-bit vector types with integer
|
|
;; lanes (I8X8, I16X4, I32X2)
|
|
(decl ty_vec64_int (Type) Type)
|
|
(extern extractor ty_vec64_int ty_vec64_int)
|
|
|
|
;; An extractor that only matches 128-bit vector types with integer
|
|
;; lanes (I8X16, I16X8, I32X4, I64X2).
|
|
(decl ty_vec128_int (Type) Type)
|
|
(extern extractor ty_vec128_int ty_vec128_int)
|
|
|
|
;; A pure constructor that matches everything except vectors with size 32X2.
|
|
(decl pure not_vec32x2 (Type) Type)
|
|
(extern constructor not_vec32x2 not_vec32x2)
|
|
|
|
;; An extractor that matches everything except I64X2
|
|
(decl not_i64x2 () Type)
|
|
(extern extractor not_i64x2 not_i64x2)
|
|
|
|
;; Extractor to get a `ValueSlice` out of a `ValueList`.
|
|
(decl value_list_slice (ValueSlice) ValueList)
|
|
(extern extractor infallible value_list_slice value_list_slice)
|
|
|
|
;; Extractor to test whether a `ValueSlice` is empty.
|
|
(decl value_slice_empty () ValueSlice)
|
|
(extern extractor value_slice_empty value_slice_empty)
|
|
|
|
;; Extractor to split a `ValueSlice` into its first element plus a tail.
|
|
(decl value_slice_unwrap (Value ValueSlice) ValueSlice)
|
|
(extern extractor value_slice_unwrap value_slice_unwrap)
|
|
|
|
;; Return the length of a `ValueSlice`.
|
|
(decl value_slice_len (ValueSlice) usize)
|
|
(extern constructor value_slice_len value_slice_len)
|
|
|
|
;; Return any element of a `ValueSlice`.
|
|
(decl value_slice_get (ValueSlice usize) Value)
|
|
(extern constructor value_slice_get value_slice_get)
|
|
|
|
;; Extractor to get the first element from a value list, along with its tail as
|
|
;; a `ValueSlice`.
|
|
(decl unwrap_head_value_list_1 (Value ValueSlice) ValueList)
|
|
(extractor (unwrap_head_value_list_1 head tail)
|
|
(value_list_slice (value_slice_unwrap head tail)))
|
|
|
|
;; Extractor to get the first two elements from a value list, along with its
|
|
;; tail as a `ValueSlice`.
|
|
(decl unwrap_head_value_list_2 (Value Value ValueSlice) ValueList)
|
|
(extractor (unwrap_head_value_list_2 head1 head2 tail)
|
|
(value_list_slice (value_slice_unwrap head1 (value_slice_unwrap head2 tail))))
|
|
|
|
;; Constructor to test whether two values are same.
|
|
(decl pure same_value (Value Value) Value)
|
|
(extern constructor same_value same_value)
|
|
|
|
;; Turn a `Writable<Reg>` into a `Reg` via `Writable::to_reg`.
|
|
(decl writable_reg_to_reg (WritableReg) Reg)
|
|
(extern constructor writable_reg_to_reg writable_reg_to_reg)
|
|
|
|
;; Extract a `u8` from an `Uimm8`.
|
|
(decl u8_from_uimm8 (u8) Uimm8)
|
|
(extern extractor infallible u8_from_uimm8 u8_from_uimm8)
|
|
|
|
;; Extract a `u64` from a `bool`.
|
|
(decl u64_from_bool (u64) bool)
|
|
(extern extractor infallible u64_from_bool u64_from_bool)
|
|
|
|
;; Extract a `u64` from an `Imm64`.
|
|
(decl u64_from_imm64 (u64) Imm64)
|
|
(extern extractor infallible u64_from_imm64 u64_from_imm64)
|
|
|
|
;; Extract a `u64` from an `Imm64` which is not zero.
|
|
(decl nonzero_u64_from_imm64 (u64) Imm64)
|
|
(extern extractor nonzero_u64_from_imm64 nonzero_u64_from_imm64)
|
|
|
|
;; Extract a `u64` from an `Ieee32`.
|
|
(decl u64_from_ieee32 (u64) Ieee32)
|
|
(extern extractor infallible u64_from_ieee32 u64_from_ieee32)
|
|
|
|
;; Extract a `u64` from an `Ieee64`.
|
|
(decl u64_from_ieee64 (u64) Ieee64)
|
|
(extern extractor infallible u64_from_ieee64 u64_from_ieee64)
|
|
|
|
;; Extract the result values for the given instruction.
|
|
(decl inst_results (ValueSlice) Inst)
|
|
(extern extractor infallible inst_results inst_results)
|
|
|
|
;; Extract the first result value of the given instruction.
|
|
(decl first_result (Value) Inst)
|
|
(extern extractor first_result first_result)
|
|
|
|
;; Extract the `InstructionData` for an `Inst`.
|
|
(decl inst_data (InstructionData) Inst)
|
|
(extern extractor infallible inst_data inst_data)
|
|
|
|
;; Extract the type of a `Value`.
|
|
(decl value_type (Type) Value)
|
|
(extern extractor infallible value_type value_type)
|
|
|
|
;; Extract the type of the instruction's first result.
|
|
(decl result_type (Type) Inst)
|
|
(extractor (result_type ty)
|
|
(first_result (value_type ty)))
|
|
|
|
;; Extract the type of the instruction's first result and pass along the
|
|
;; instruction as well.
|
|
(decl has_type (Type Inst) Inst)
|
|
(extractor (has_type ty inst)
|
|
(and (result_type ty)
|
|
inst))
|
|
|
|
;; Match a multi-lane type, extracting (# bits per lane, # lanes) from the given
|
|
;; type. Will only match when there is more than one lane.
|
|
(decl multi_lane (u32 u32) Type)
|
|
(extern extractor multi_lane multi_lane)
|
|
|
|
;; Match a dynamic-lane type, extracting (# bits per lane) from the given
|
|
;; type.
|
|
(decl dynamic_lane (u32 u32) Type)
|
|
(extern extractor dynamic_lane dynamic_lane)
|
|
|
|
;; Match a dynamic-lane integer type, extracting (# bits per lane) from the given
|
|
;; type.
|
|
(decl dynamic_int_lane (u32) Type)
|
|
(extern extractor dynamic_int_lane dynamic_int_lane)
|
|
|
|
;; Match a dynamic-lane floating point type, extracting (# bits per lane)
|
|
;; from the given type.
|
|
(decl dynamic_fp_lane (u32) Type)
|
|
(extern extractor dynamic_fp_lane dynamic_fp_lane)
|
|
|
|
;; An extractor that only matches 64-bit dynamic vector types with integer
|
|
;; lanes (I8X8XN, I16X4XN, I32X2XN)
|
|
(decl ty_dyn64_int (Type) Type)
|
|
(extern extractor ty_dyn64_int ty_dyn64_int)
|
|
|
|
;; An extractor that only matches 128-bit dynamic vector types with integer
|
|
;; lanes (I8X16XN, I16X8XN, I32X4XN, I64X2XN).
|
|
(decl ty_dyn128_int (Type) Type)
|
|
(extern extractor ty_dyn128_int ty_dyn128_int)
|
|
|
|
;; Match the instruction that defines the given value, if any.
|
|
(decl def_inst (Inst) Value)
|
|
(extern extractor def_inst def_inst)
|
|
|
|
;; Extract a constant `u64` from a value defined by an `iconst`.
|
|
(decl u64_from_iconst (u64) Value)
|
|
(extractor (u64_from_iconst x)
|
|
(def_inst (iconst (u64_from_imm64 x))))
|
|
|
|
;; Convert an `Offset32` to a primitive number.
|
|
(decl offset32_to_u32 (Offset32) u32)
|
|
(extern constructor offset32_to_u32 offset32_to_u32)
|
|
|
|
;; Match any zero value for iconst, fconst32, fconst64, vconst and splat.
|
|
(decl pure zero_value (Value) Value)
|
|
(extern constructor zero_value zero_value)
|
|
|
|
;; Match a sinkable instruction from a value operand.
|
|
(decl pure is_sinkable_inst (Value) Inst)
|
|
(extern constructor is_sinkable_inst is_sinkable_inst)
|
|
|
|
;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Emit an instruction.
|
|
;;
|
|
;; This is low-level and side-effectful; it should only be used as an
|
|
;; implementation detail by helpers that preserve the SSA facade themselves.
|
|
|
|
(decl emit (MInst) Unit)
|
|
(extern constructor emit emit)
|
|
|
|
;; Sink an instruction.
|
|
;;
|
|
;; This is a side-effectful operation that notifies the context that the
|
|
;; instruction has been sunk into another instruction, and no longer needs to
|
|
;; be lowered.
|
|
(decl sink_inst (Inst) Unit)
|
|
(extern constructor sink_inst sink_inst)
|
|
|
|
;; Constant pool emission.
|
|
|
|
(type VCodeConstant (primitive VCodeConstant))
|
|
|
|
;; Add a u64 little-endian constant to the in-memory constant pool and
|
|
;; return a VCodeConstant index that refers to it. This is
|
|
;; side-effecting but idempotent (constants are deduplicated).
|
|
(decl emit_u64_le_const (u64) VCodeConstant)
|
|
(extern constructor emit_u64_le_const emit_u64_le_const)
|
|
|
|
;; Add a u128 little-endian constant to the in-memory constant pool and
|
|
;; return a VCodeConstant index that refers to it. This is
|
|
;; side-effecting but idempotent (constants are deduplicated).
|
|
(decl emit_u128_le_const (u128) VCodeConstant)
|
|
(extern constructor emit_u128_le_const emit_u128_le_const)
|
|
|
|
;; Fetch the VCodeConstant associated with a Constant.
|
|
(decl const_to_vconst (Constant) VCodeConstant)
|
|
(extern constructor const_to_vconst const_to_vconst)
|
|
|
|
;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;;
|
|
|
|
(type SideEffectNoResult (enum
|
|
(Inst (inst MInst))
|
|
(Inst2 (inst1 MInst)
|
|
(inst2 MInst))
|
|
(Inst3 (inst1 MInst)
|
|
(inst2 MInst)
|
|
(inst3 MInst))))
|
|
|
|
;; Create an empty `InstOutput`, but do emit the given side-effectful
|
|
;; instruction.
|
|
(decl side_effect (SideEffectNoResult) InstOutput)
|
|
(rule (side_effect (SideEffectNoResult.Inst inst))
|
|
(let ((_ Unit (emit inst)))
|
|
(output_none)))
|
|
(rule (side_effect (SideEffectNoResult.Inst2 inst1 inst2))
|
|
(let ((_ Unit (emit inst1))
|
|
(_ Unit (emit inst2)))
|
|
(output_none)))
|
|
(rule (side_effect (SideEffectNoResult.Inst3 inst1 inst2 inst3))
|
|
(let ((_ Unit (emit inst1))
|
|
(_ Unit (emit inst2))
|
|
(_ Unit (emit inst3)))
|
|
(output_none)))
|
|
|
|
(decl side_effect_concat (SideEffectNoResult SideEffectNoResult) SideEffectNoResult)
|
|
(rule (side_effect_concat (SideEffectNoResult.Inst inst1) (SideEffectNoResult.Inst inst2))
|
|
(SideEffectNoResult.Inst2 inst1 inst2))
|
|
(rule (side_effect_concat (SideEffectNoResult.Inst inst1) (SideEffectNoResult.Inst2 inst2 inst3))
|
|
(SideEffectNoResult.Inst3 inst1 inst2 inst3))
|
|
(rule (side_effect_concat (SideEffectNoResult.Inst2 inst1 inst2) (SideEffectNoResult.Inst inst3))
|
|
(SideEffectNoResult.Inst3 inst1 inst2 inst3))
|
|
|
|
;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Newtype wrapper around `MInst` for instructions that are used for their
|
|
;; effect on flags.
|
|
;;
|
|
;; Variant determines how result is given when combined with a
|
|
;; ConsumesFlags. See `with_flags` below for more.
|
|
(type ProducesFlags (enum
|
|
;; For cases where the flags have been produced by another
|
|
;; instruction, and we have out-of-band reasons to know
|
|
;; that they won't be clobbered by the time we depend on
|
|
;; them.
|
|
(AlreadyExistingFlags)
|
|
(ProducesFlagsSideEffect (inst MInst))
|
|
;; Not directly combinable with a ConsumesFlags;
|
|
;; used in s390x and unwrapped directly by `trapif`.
|
|
(ProducesFlagsReturnsReg (inst MInst) (result Reg))
|
|
(ProducesFlagsReturnsResultWithConsumer (inst MInst) (result Reg))))
|
|
|
|
;; Newtype wrapper around `MInst` for instructions that consume flags.
|
|
;;
|
|
;; Variant determines how result is given when combined with a
|
|
;; ProducesFlags. See `with_flags` below for more.
|
|
(type ConsumesFlags (enum
|
|
(ConsumesFlagsSideEffect (inst MInst))
|
|
(ConsumesFlagsSideEffect2 (inst1 MInst) (inst2 MInst))
|
|
(ConsumesFlagsReturnsResultWithProducer (inst MInst) (result Reg))
|
|
(ConsumesFlagsReturnsReg (inst MInst) (result Reg))
|
|
(ConsumesFlagsTwiceReturnsValueRegs (inst1 MInst)
|
|
(inst2 MInst)
|
|
(result ValueRegs))
|
|
(ConsumesFlagsFourTimesReturnsValueRegs (inst1 MInst)
|
|
(inst2 MInst)
|
|
(inst3 MInst)
|
|
(inst4 MInst)
|
|
(result ValueRegs))))
|
|
|
|
|
|
|
|
;; Get the produced register out of a ProducesFlags.
|
|
(decl produces_flags_get_reg (ProducesFlags) Reg)
|
|
(rule (produces_flags_get_reg (ProducesFlags.ProducesFlagsReturnsReg _ reg)) reg)
|
|
|
|
;; Modify a ProducesFlags to use it only for its side-effect, ignoring
|
|
;; its result.
|
|
(decl produces_flags_ignore (ProducesFlags) ProducesFlags)
|
|
(rule (produces_flags_ignore (ProducesFlags.ProducesFlagsReturnsReg inst _))
|
|
(ProducesFlags.ProducesFlagsSideEffect inst))
|
|
(rule (produces_flags_ignore (ProducesFlags.ProducesFlagsReturnsResultWithConsumer inst _))
|
|
(ProducesFlags.ProducesFlagsSideEffect inst))
|
|
|
|
;; Helper for combining two flags-consumer instructions that return a
|
|
;; single Reg, giving a ConsumesFlags that returns both values in a
|
|
;; ValueRegs.
|
|
(decl consumes_flags_concat (ConsumesFlags ConsumesFlags) ConsumesFlags)
|
|
(rule (consumes_flags_concat (ConsumesFlags.ConsumesFlagsReturnsReg inst1 reg1)
|
|
(ConsumesFlags.ConsumesFlagsReturnsReg inst2 reg2))
|
|
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
|
|
inst1
|
|
inst2
|
|
(value_regs reg1 reg2)))
|
|
(rule (consumes_flags_concat
|
|
(ConsumesFlags.ConsumesFlagsSideEffect inst1)
|
|
(ConsumesFlags.ConsumesFlagsSideEffect inst2))
|
|
(ConsumesFlags.ConsumesFlagsSideEffect2 inst1 inst2))
|
|
|
|
;; Combine flags-producing and -consuming instructions together, ensuring that
|
|
;; they are emitted back-to-back and no other instructions can be emitted
|
|
;; between them and potentially clobber the flags.
|
|
;;
|
|
;; Returns a `ValueRegs` according to the specific combination of ProducesFlags and ConsumesFlags modes:
|
|
;; - SideEffect + ReturnsReg --> ValueReg with one Reg from consumer
|
|
;; - SideEffect + ReturnsValueRegs --> ValueReg as given from consumer
|
|
;; - ReturnsResultWithProducer + ReturnsResultWithConsumer --> ValueReg with low part from producer, high part from consumer
|
|
;;
|
|
;; See `with_flags_reg` below for a variant that extracts out just the lower Reg.
|
|
(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs)
|
|
|
|
(rule (with_flags (ProducesFlags.ProducesFlagsReturnsResultWithConsumer producer_inst producer_result)
|
|
(ConsumesFlags.ConsumesFlagsReturnsResultWithProducer consumer_inst consumer_result))
|
|
(let ((_x Unit (emit producer_inst))
|
|
(_y Unit (emit consumer_inst)))
|
|
(value_regs producer_result consumer_result)))
|
|
|
|
(rule (with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
|
|
(ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))
|
|
(let ((_x Unit (emit producer_inst))
|
|
(_y Unit (emit consumer_inst)))
|
|
(value_reg consumer_result)))
|
|
|
|
(rule (with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
|
|
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs consumer_inst_1
|
|
consumer_inst_2
|
|
consumer_result))
|
|
;; We must emit these instructions in order as the creator of
|
|
;; the ConsumesFlags may be relying on dataflow dependencies
|
|
;; amongst them.
|
|
(let ((_x Unit (emit producer_inst))
|
|
(_y Unit (emit consumer_inst_1))
|
|
(_z Unit (emit consumer_inst_2)))
|
|
consumer_result))
|
|
|
|
(rule (with_flags (ProducesFlags.ProducesFlagsSideEffect producer_inst)
|
|
(ConsumesFlags.ConsumesFlagsFourTimesReturnsValueRegs consumer_inst_1
|
|
consumer_inst_2
|
|
consumer_inst_3
|
|
consumer_inst_4
|
|
consumer_result))
|
|
;; We must emit these instructions in order as the creator of
|
|
;; the ConsumesFlags may be relying on dataflow dependencies
|
|
;; amongst them.
|
|
(let ((_x Unit (emit producer_inst))
|
|
(_y Unit (emit consumer_inst_1))
|
|
(_z Unit (emit consumer_inst_2))
|
|
(_w Unit (emit consumer_inst_3))
|
|
(_v Unit (emit consumer_inst_4)))
|
|
consumer_result))
|
|
|
|
(decl with_flags_reg (ProducesFlags ConsumesFlags) Reg)
|
|
(rule (with_flags_reg p c)
|
|
(let ((v ValueRegs (with_flags p c)))
|
|
(value_regs_get v 0)))
|
|
|
|
;; Indicate that the current state of the flags register from the instruction
|
|
;; that produces this Value is relied on.
|
|
(decl flags_to_producesflags (Value) ProducesFlags)
|
|
(rule (flags_to_producesflags val)
|
|
(let ((_ Unit (mark_value_used val)))
|
|
(ProducesFlags.AlreadyExistingFlags)))
|
|
|
|
;; Combine a flags-producing instruction and a flags-consuming instruction that
|
|
;; produces no results.
|
|
;;
|
|
;; This function handles the following case only:
|
|
;; - ProducesFlagsSideEffect + ConsumesFlagsSideEffect
|
|
(decl with_flags_side_effect (ProducesFlags ConsumesFlags) SideEffectNoResult)
|
|
|
|
(rule (with_flags_side_effect
|
|
(ProducesFlags.AlreadyExistingFlags)
|
|
(ConsumesFlags.ConsumesFlagsSideEffect c))
|
|
(SideEffectNoResult.Inst c))
|
|
|
|
(rule (with_flags_side_effect
|
|
(ProducesFlags.AlreadyExistingFlags)
|
|
(ConsumesFlags.ConsumesFlagsSideEffect2 c1 c2))
|
|
(SideEffectNoResult.Inst2 c1 c2))
|
|
|
|
(rule (with_flags_side_effect
|
|
(ProducesFlags.ProducesFlagsSideEffect p)
|
|
(ConsumesFlags.ConsumesFlagsSideEffect c))
|
|
(SideEffectNoResult.Inst2 p c))
|
|
|
|
(rule (with_flags_side_effect
|
|
(ProducesFlags.ProducesFlagsSideEffect p)
|
|
(ConsumesFlags.ConsumesFlagsSideEffect2 c1 c2))
|
|
(SideEffectNoResult.Inst3 p c1 c2))
|
|
|
|
;;;; Helpers for Working with TrapCode ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(decl trap_code_division_by_zero () TrapCode)
|
|
(extern constructor trap_code_division_by_zero trap_code_division_by_zero)
|
|
|
|
(decl trap_code_integer_overflow () TrapCode)
|
|
(extern constructor trap_code_integer_overflow trap_code_integer_overflow)
|
|
|
|
(decl trap_code_bad_conversion_to_integer () TrapCode)
|
|
(extern constructor trap_code_bad_conversion_to_integer trap_code_bad_conversion_to_integer)
|
|
|
|
;;;; Helpers for accessing compilation flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(decl avoid_div_traps () Type)
|
|
(extern extractor avoid_div_traps avoid_div_traps)
|
|
|
|
(decl pure tls_model_is_elf_gd () Unit)
|
|
(extern constructor tls_model_is_elf_gd tls_model_is_elf_gd)
|
|
|
|
;;;; Helpers for accessing instruction data ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Accessor for `FuncRef`.
|
|
|
|
(decl func_ref_data (SigRef ExternalName RelocDistance) FuncRef)
|
|
(extern extractor infallible func_ref_data func_ref_data)
|
|
|
|
;; Accessor for `GobalValue`.
|
|
|
|
(decl symbol_value_data (ExternalName RelocDistance i64) GlobalValue)
|
|
(extern extractor symbol_value_data symbol_value_data)
|
|
|
|
(decl box_external_name (ExternalName) BoxExternalName)
|
|
(extern constructor box_external_name box_external_name)
|
|
|
|
;; Accessor for `RelocDistance`.
|
|
|
|
(decl reloc_distance_near () RelocDistance)
|
|
(extern extractor reloc_distance_near reloc_distance_near)
|
|
|
|
;; Accessor for `Immediate` as u128.
|
|
|
|
(decl u128_from_immediate (u128) Immediate)
|
|
(extern extractor u128_from_immediate u128_from_immediate)
|
|
|
|
;; Accessor for `Immediate` as a vector of u8 values.
|
|
|
|
(decl vec_mask_from_immediate (VecMask) Immediate)
|
|
(extern extractor vec_mask_from_immediate vec_mask_from_immediate)
|
|
|
|
;; Accessor for `Constant` as u128.
|
|
|
|
(decl u128_from_constant (u128) Constant)
|
|
(extern extractor u128_from_constant u128_from_constant)
|
|
|
|
;; Accessor for `Constant` as u64.
|
|
|
|
(decl u64_from_constant (u64) Constant)
|
|
(extern extractor u64_from_constant u64_from_constant)
|
|
|
|
;;;; Helpers for tail recursion loops ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; A range of integers to loop through.
|
|
(type Range (primitive Range))
|
|
|
|
;; Create a new range from `start` through `end` (exclusive).
|
|
(decl range (usize usize) Range)
|
|
(extern constructor range range)
|
|
|
|
;; Extractor to test whether a range is empty.
|
|
(decl range_empty () Range)
|
|
(extern extractor range_empty range_empty)
|
|
|
|
;; Extractor to test whether a range has a single element in it
|
|
(decl range_singleton (usize) Range)
|
|
(extern extractor range_singleton range_singleton)
|
|
|
|
;; Extractor to return the first value in the range, and a sub-range
|
|
;; containing the remaining values.
|
|
(decl range_unwrap (usize Range) Range)
|
|
(extern extractor range_unwrap range_unwrap)
|
|
|
|
;;;; Helpers for generating returns ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; The (writable) register(s) that will contain the n'th return value.
|
|
(decl retval (usize) WritableValueRegs)
|
|
(extern constructor retval retval)
|
|
|
|
;; Extractor to check for the special case that a `WritableValueRegs`
|
|
;; contains only a single register.
|
|
(decl only_writable_reg (WritableReg) WritableValueRegs)
|
|
(extern extractor only_writable_reg only_writable_reg)
|
|
|
|
;; Get the `n`th register inside a `WritableValueRegs`.
|
|
(decl writable_regs_get (WritableValueRegs usize) WritableReg)
|
|
(extern constructor writable_regs_get writable_regs_get)
|
|
|
|
;;;; Helpers for generating calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Type to hold information about a function call signature.
|
|
(type ABISig extern (enum))
|
|
|
|
;; Information how to pass one argument or return value.
|
|
(type ABIArg extern (enum))
|
|
|
|
;; Information how to pass a single slot of one argument or return value.
|
|
(type ABIArgSlot extern
|
|
(enum
|
|
(Reg
|
|
(reg RealReg)
|
|
(ty Type)
|
|
(extension ArgumentExtension))
|
|
(Stack
|
|
(offset i64)
|
|
(ty Type)
|
|
(extension ArgumentExtension))
|
|
))
|
|
|
|
;; Physical register that may hold an argument or return value.
|
|
(type RealReg (primitive RealReg))
|
|
|
|
;; Instruction on whether and how to extend an argument value.
|
|
(type ArgumentExtension extern
|
|
(enum
|
|
(None)
|
|
(Uext)
|
|
(Sext)
|
|
))
|
|
|
|
;; Get the number of arguments expected.
|
|
(decl abi_num_args (ABISig) usize)
|
|
(extern constructor abi_num_args abi_num_args)
|
|
|
|
;; Get information specifying how to pass one argument.
|
|
(decl abi_get_arg (ABISig usize) ABIArg)
|
|
(extern constructor abi_get_arg abi_get_arg)
|
|
|
|
;; Get the number of return values expected.
|
|
(decl abi_num_rets (ABISig) usize)
|
|
(extern constructor abi_num_rets abi_num_rets)
|
|
|
|
;; Get information specifying how to pass one return value.
|
|
(decl abi_get_ret (ABISig usize) ABIArg)
|
|
(extern constructor abi_get_ret abi_get_ret)
|
|
|
|
;; Get information specifying how to pass the implicit pointer
|
|
;; to the return-value area on the stack, if required.
|
|
(decl abi_ret_arg (ABIArg) ABISig)
|
|
(extern extractor abi_ret_arg abi_ret_arg)
|
|
|
|
;; Succeeds if no implicit return-value area pointer is required.
|
|
(decl abi_no_ret_arg () ABISig)
|
|
(extern extractor abi_no_ret_arg abi_no_ret_arg)
|
|
|
|
;; Size of the argument area.
|
|
(decl abi_sized_stack_arg_space (ABISig) i64)
|
|
(extern constructor abi_sized_stack_arg_space abi_sized_stack_arg_space)
|
|
|
|
;; Size of the return-value area.
|
|
(decl abi_sized_stack_ret_space (ABISig) i64)
|
|
(extern constructor abi_sized_stack_ret_space abi_sized_stack_ret_space)
|
|
|
|
;; StackSlot addr
|
|
(decl abi_stackslot_addr (WritableReg StackSlot Offset32) MInst)
|
|
(extern constructor abi_stackslot_addr abi_stackslot_addr)
|
|
|
|
;; DynamicStackSlot addr
|
|
(decl abi_dynamic_stackslot_addr (WritableReg DynamicStackSlot) MInst)
|
|
(extern constructor abi_dynamic_stackslot_addr abi_dynamic_stackslot_addr)
|
|
|
|
;; Extractor to detect the special case where an argument or
|
|
;; return value only requires a single slot to be passed.
|
|
(decl abi_arg_only_slot (ABIArgSlot) ABIArg)
|
|
(extern extractor abi_arg_only_slot abi_arg_only_slot)
|
|
|
|
;; Extractor to detect the special case where a struct argument
|
|
;; is explicitly passed by reference using a hidden pointer.
|
|
(decl abi_arg_struct_pointer (ABIArgSlot i64 u64) ABIArg)
|
|
(extern extractor abi_arg_struct_pointer abi_arg_struct_pointer)
|
|
|
|
;; Extractor to detect the special case where a non-struct argument
|
|
;; is implicitly passed by reference using a hidden pointer.
|
|
(decl abi_arg_implicit_pointer (ABIArgSlot i64 Type) ABIArg)
|
|
(extern extractor abi_arg_implicit_pointer abi_arg_implicit_pointer)
|
|
|
|
;; Convert a real register number into a virtual register.
|
|
(decl real_reg_to_reg (RealReg) Reg)
|
|
(extern constructor real_reg_to_reg real_reg_to_reg)
|
|
|
|
;; Convert a real register number into a writable virtual register.
|
|
(decl real_reg_to_writable_reg (RealReg) WritableReg)
|
|
(extern constructor real_reg_to_writable_reg real_reg_to_writable_reg)
|
|
|
|
;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(convert Inst Value def_inst)
|
|
(convert Reg ValueRegs value_reg)
|
|
(convert Value Reg put_in_reg)
|
|
(convert Value ValueRegs put_in_regs)
|
|
(convert WritableReg Reg writable_reg_to_reg)
|
|
(convert ValueRegs InstOutput output)
|
|
(convert Reg InstOutput output_reg)
|
|
(convert Value InstOutput output_value)
|
|
(convert Offset32 u32 offset32_to_u32)
|
|
(convert ExternalName BoxExternalName box_external_name)
|
|
(convert PReg Reg preg_to_reg)
|