T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

aarch64: Migrate ishl/ushr/sshr to ISLE (#3608)

Browse Source 
* aarch64: Migrate ishl/ushr/sshr to ISLE

This commit migrates the `ishl`, `ushr`, and `sshr` instructions to
ISLE. These involve special cases for almost all types of integers
(including vectors) and helper functions for the i128 lowerings since
the i128 lowerings look to be used for other instructions as well. This
doesn't delete the i128 lowerings in the Rust code just yet because
they're still used by Rust lowerings, but they should be deletable in
due time once those lowerings are translated to ISLE.

* Use more descriptive names for i128 lowerings

* Use a with_flags-lookalike for csel

* Use existing `with_flags_*`

* Coment backwards order

* Update generated code
This commit is contained in:
Alex Crichton
2021-12-16 17:37:53 -06:00
committed by GitHub
parent e1e2f3ca15
commit d8974ce6bc
13 changed files with 895 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
cranelift/codegen/src/isa/aarch64/inst.isle
View File

@@ -1304,6 +1304,9 @@
(decl imm_logic_from_imm64 (Type ImmLogic) Imm64)
(extern extractor imm_logic_from_imm64 imm_logic_from_imm64 (in out))
(decl imm_shift_from_imm64 (Type ImmShift) Imm64)
(extern extractor imm_shift_from_imm64 imm_shift_from_imm64 (in out))
(decl imm_shift_from_u8 (u8) ImmShift)
(extern constructor imm_shift_from_u8 imm_shift_from_u8)
@@ -1316,6 +1319,9 @@
(decl u8_into_imm12 (u8) Imm12)
(extern constructor u8_into_imm12 u8_into_imm12)
(decl u64_into_imm_logic (Type u64) ImmLogic)
(extern constructor u64_into_imm_logic u64_into_imm_logic)
(decl imm12_from_negated_u64 (Imm12) u64)
(extern extractor imm12_from_negated_u64 imm12_from_negated_u64)
@@ -1415,6 +1421,13 @@
(_ Unit (emit (MInst.VecRRR op dst src1 src2 size))))
(writable_reg_to_reg dst)))
;; Helper for emitting `MInst.VecDup` instructions.
(decl vec_dup (Reg VectorSize) Reg)
(rule (vec_dup src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecDup dst src size))))
(writable_reg_to_reg dst)))
;; Helper for emitting `MInst.AluRRImm12` instructions.
(decl alu_rr_imm12 (ALUOp Reg Imm12) Reg)
(rule (alu_rr_imm12 op src imm)
@@ -1546,6 +1559,30 @@
(_ Unit (emit (MInst.LoadAcquire ty dst addr))))
(writable_reg_to_reg dst)))
;; Helper for generating a `tst` instruction.
;;
;; Produces a `ProducesFlags` rather than a register or emitted instruction
;; which must be paired with `with_flags*` helpers.
(decl tst64_imm (Reg ImmLogic) ProducesFlags)
(rule (tst64_imm reg imm)
(ProducesFlags.ProducesFlags (MInst.AluRRImmLogic (ALUOp.AndS64)
(writable_zero_reg)
reg
imm)
(invalid_reg)))
;; Helper for generating a `CSel` instruction.
;;
;; Note that this doesn't actually emit anything, instead it produces a
;; `ConsumesFlags` instruction which must be consumed with `with_flags*`
;; helpers.
(decl csel (Cond Reg Reg) ConsumesFlags)
(rule (csel cond if_true if_false)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlags (MInst.CSel dst cond if_true if_false)
(writable_reg_to_reg dst))))
;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl imm (Type u64) Reg)
@@ -1571,6 +1608,24 @@
;; Sign extension helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Place a `Value` into a register, sign extending it to 32-bits
(decl put_in_reg_sext32 (Value) Reg)
(rule (put_in_reg_sext32 val @ (value_type (fits_in_32 ty)))
(extend (put_in_reg val) $true (ty_bits ty) 32))
;; 32/64-bit passthrough.
(rule (put_in_reg_sext32 val @ (value_type $I32)) (put_in_reg val))
(rule (put_in_reg_sext32 val @ (value_type $I64)) (put_in_reg val))
;; Place a `Value` into a register, zero extending it to 32-bits
(decl put_in_reg_zext32 (Value) Reg)
(rule (put_in_reg_zext32 val @ (value_type (fits_in_32 ty)))
(extend (put_in_reg val) $false (ty_bits ty) 32))
;; 32/64-bit passthrough.
(rule (put_in_reg_zext32 val @ (value_type $I32)) (put_in_reg val))
(rule (put_in_reg_zext32 val @ (value_type $I64)) (put_in_reg val))
;; Place a `Value` into a register, sign extending it to 64-bits
(decl put_in_reg_sext64 (Value) Reg)
(rule (put_in_reg_sext64 val @ (value_type (fits_in_32 ty)))

199
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -695,3 +695,202 @@
(value_reg (alu_rs_imm_logic (ALUOp.EorNot64) $I64 x y)))
(rule (lower (has_type $I128 (bxor_not x y))) (i128_alu_bitop (ALUOp.EorNot64) x y))
;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Shift for i8/i16/i32.
(rule (lower (has_type (fits_in_32 ty) (ishl x y)))
(value_reg (do_shift (ALUOp.Lsl32) ty (put_in_reg x) y)))
;; Shift for i64.
(rule (lower (has_type $I64 (ishl x y)))
(value_reg (do_shift (ALUOp.Lsl64) $I64 (put_in_reg x) y)))
;; Shift for i128.
(rule (lower (has_type $I128 (ishl x y)))
(lower_shl128 (put_in_regs x) (value_regs_get (put_in_regs y) 0)))
;; lsl lo_lshift, src_lo, amt
;; lsl hi_lshift, src_hi, amt
;; mvn inv_amt, amt
;; lsr lo_rshift, src_lo, #1
;; lsr lo_rshift, lo_rshift, inv_amt
;; orr maybe_hi, hi_lshift, lo_rshift
;; tst amt, #0x40
;; csel dst_hi, lo_lshift, maybe_hi, ne
;; csel dst_lo, xzr, lo_lshift, ne
(decl lower_shl128 (ValueRegs Reg) ValueRegs)
(rule (lower_shl128 src amt)
(let (
(src_lo Reg (value_regs_get src 0))
(src_hi Reg (value_regs_get src 1))
(lo_lshift Reg (alu_rrr (ALUOp.Lsl64) src_lo amt))
(hi_lshift Reg (alu_rrr (ALUOp.Lsl64) src_hi amt))
(inv_amt Reg (alu_rrr (ALUOp.OrrNot32) (zero_reg) amt))
(lo_rshift Reg (alu_rrr (ALUOp.Lsr64)
(alu_rr_imm_shift (ALUOp.Lsr64)
src_lo
(imm_shift_from_u8 1))
inv_amt))
(maybe_hi Reg (alu_rrr (ALUOp.Orr64) hi_lshift lo_rshift))
)
(with_flags_2
(tst64_imm amt (u64_into_imm_logic $I64 64))
(csel (Cond.Ne) (zero_reg) lo_lshift)
(csel (Cond.Ne) lo_lshift maybe_hi))))
;; Shift for vector types.
(rule (lower (has_type (vec128 ty) (ishl x y)))
(let (
(size VectorSize (vector_size ty))
(shift Reg (vec_dup (put_in_reg y) size))
)
(value_reg (vec_rrr (VecALUOp.Sshl) (put_in_reg x) shift size))))
;; Helper function to emit a shift operation with the opcode specified and
;; the output type specified. The `Reg` provided is shifted by the `Value`
;; given.
;;
;; Note that this automatically handles the clif semantics of masking the
;; shift amount where necessary.
(decl do_shift (ALUOp Type Reg Value) Reg)
;; 8/16-bit shift base case.
;;
;; When shifting for amounts larger than the size of the type, the CLIF shift
;; instructions implement a "wrapping" behaviour, such that an i8 << 8 is
;; equivalent to i8 << 0
;;
;; On i32 and i64 types this matches what the aarch64 spec does, but on smaller
;; types (i16, i8) we need to do this manually, so we wrap the shift amount
;; with an AND instruction
(rule (do_shift op (fits_in_16 ty) x y)
(let (
(shift_amt Reg (value_regs_get (put_in_regs y) 0))
(masked_shift_amt Reg (alu_rr_imm_logic (ALUOp.And32) shift_amt (shift_mask ty)))
)
(alu_rrr op x masked_shift_amt)))
(decl shift_mask (Type) ImmLogic)
(extern constructor shift_mask shift_mask)
;; 32/64-bit shift base cases.
(rule (do_shift op $I32 x y) (alu_rrr op x (value_regs_get (put_in_regs y) 0)))
(rule (do_shift op $I64 x y) (alu_rrr op x (value_regs_get (put_in_regs y) 0)))
;; Special case for shifting by a constant value where the value can fit into an
;; `ImmShift`.
;;
;; Note that this rule explicitly has a higher priority than the others
;; to ensure it's attempted first, otherwise the type-based filters on the
;; previous rules seem to take priority over this rule.
(rule 1 (do_shift op ty x (def_inst (iconst (imm_shift_from_imm64 <ty shift))))
(alu_rr_imm_shift op x shift))
;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Shift for i8/i16/i32.
(rule (lower (has_type (fits_in_32 ty) (ushr x y)))
(value_reg (do_shift (ALUOp.Lsr32) ty (put_in_reg_zext32 x) y)))
;; Shift for i64.
(rule (lower (has_type $I64 (ushr x y)))
(value_reg (do_shift (ALUOp.Lsr64) $I64 (put_in_reg_zext64 x) y)))
;; Shift for i128.
(rule (lower (has_type $I128 (ushr x y)))
(lower_ushr128 (put_in_regs x) (value_regs_get (put_in_regs y) 0)))
;; Vector shifts.
(rule (lower (has_type (vec128 ty) (ushr x y)))
(let (
(size VectorSize (vector_size ty))
(shift Reg (vec_dup (alu_rrr (ALUOp.Sub32) (zero_reg) (put_in_reg y)) size))
)
(value_reg (vec_rrr (VecALUOp.Ushl) (put_in_reg x) shift size))))
;; lsr lo_rshift, src_lo, amt
;; lsr hi_rshift, src_hi, amt
;; mvn inv_amt, amt
;; lsl hi_lshift, src_hi, #1
;; lsl hi_lshift, hi_lshift, inv_amt
;; tst amt, #0x40
;; orr maybe_lo, lo_rshift, hi_lshift
;; csel dst_hi, xzr, hi_rshift, ne
;; csel dst_lo, hi_rshift, maybe_lo, ne
(decl lower_ushr128 (ValueRegs Reg) ValueRegs)
(rule (lower_ushr128 src amt)
(let (
(src_lo Reg (value_regs_get src 0))
(src_hi Reg (value_regs_get src 1))
(lo_rshift Reg (alu_rrr (ALUOp.Lsr64) src_lo amt))
(hi_rshift Reg (alu_rrr (ALUOp.Lsr64) src_hi amt))
(inv_amt Reg (alu_rrr (ALUOp.OrrNot32) (zero_reg) amt))
(hi_lshift Reg (alu_rrr (ALUOp.Lsl64)
(alu_rr_imm_shift (ALUOp.Lsl64)
src_hi
(imm_shift_from_u8 1))
inv_amt))
(maybe_lo Reg (alu_rrr (ALUOp.Orr64) lo_rshift hi_lshift))
)
(with_flags_2
(tst64_imm amt (u64_into_imm_logic $I64 64))
(csel (Cond.Ne) hi_rshift maybe_lo)
(csel (Cond.Ne) (zero_reg) hi_rshift))))
;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Shift for i8/i16/i32.
(rule (lower (has_type (fits_in_32 ty) (sshr x y)))
(value_reg (do_shift (ALUOp.Asr32) ty (put_in_reg_sext32 x) y)))
;; Shift for i64.
(rule (lower (has_type $I64 (sshr x y)))
(value_reg (do_shift (ALUOp.Asr64) $I64 (put_in_reg_sext64 x) y)))
;; Shift for i128.
(rule (lower (has_type $I128 (sshr x y)))
(lower_sshr128 (put_in_regs x) (value_regs_get (put_in_regs y) 0)))
;; Vector shifts.
;;
;; Note that right shifts are implemented with a negative left shift.
(rule (lower (has_type (vec128 ty) (sshr x y)))
(let (
(size VectorSize (vector_size ty))
(shift Reg (vec_dup (alu_rrr (ALUOp.Sub32) (zero_reg) (put_in_reg y)) size))
)
(value_reg (vec_rrr (VecALUOp.Sshl) (put_in_reg x) shift size))))
;; lsr lo_rshift, src_lo, amt
;; asr hi_rshift, src_hi, amt
;; mvn inv_amt, amt
;; lsl hi_lshift, src_hi, #1
;; lsl hi_lshift, hi_lshift, inv_amt
;; asr hi_sign, src_hi, #63
;; orr maybe_lo, lo_rshift, hi_lshift
;; tst amt, #0x40
;; csel dst_hi, hi_sign, hi_rshift, ne
;; csel dst_lo, hi_rshift, maybe_lo, ne
(decl lower_sshr128 (ValueRegs Reg) ValueRegs)
(rule (lower_sshr128 src amt)
(let (
(src_lo Reg (value_regs_get src 0))
(src_hi Reg (value_regs_get src 1))
(lo_rshift Reg (alu_rrr (ALUOp.Lsr64) src_lo amt))
(hi_rshift Reg (alu_rrr (ALUOp.Asr64) src_hi amt))
(inv_amt Reg (alu_rrr (ALUOp.OrrNot32) (zero_reg) amt))
(hi_lshift Reg (alu_rrr (ALUOp.Lsl64)
(alu_rr_imm_shift (ALUOp.Lsl64)
src_hi
(imm_shift_from_u8 1))
inv_amt))
(hi_sign Reg (alu_rr_imm_shift (ALUOp.Asr64) src_hi (imm_shift_from_u8 63)))
(maybe_lo Reg (alu_rrr (ALUOp.Orr64) lo_rshift hi_lshift))
)
(with_flags_2
(tst64_imm amt (u64_into_imm_logic $I64 64))
(csel (Cond.Ne) hi_rshift maybe_lo)
(csel (Cond.Ne) hi_sign hi_rshift))))

68
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -89,15 +89,6 @@ pub(crate) enum ResultRegImmShift {
ImmShift(ImmShift),
}
impl ResultRegImmShift {
pub fn unwrap_reg(self) -> Reg {
match self {
ResultRegImmShift::Reg(r) => r,
_ => panic!("Unwrapped ResultRegImmShift, expected reg, got: {:?}", self),
}
}
}
//============================================================================
// Lowering: convert instruction inputs to forms that we can use.
@@ -518,28 +509,6 @@ pub(crate) fn alu_inst_imm12(op: ALUOp, rd: Writable<Reg>, rn: Reg, rm: ResultRS
}
}
pub(crate) fn alu_inst_immshift(
op: ALUOp,
rd: Writable<Reg>,
rn: Reg,
rm: ResultRegImmShift,
) -> Inst {
match rm {
ResultRegImmShift::ImmShift(immshift) => Inst::AluRRImmShift {
alu_op: op,
rd,
rn,
immshift,
},
ResultRegImmShift::Reg(rm) => Inst::AluRRR {
alu_op: op,
rd,
rn,
rm,
},
}
}
//============================================================================
// Lowering: addressing mode support. Takes instruction directly, rather
// than an `InsnInput`, to do more introspection.
@@ -1516,43 +1485,6 @@ pub(crate) fn materialize_bool_result<C: LowerCtx<I = Inst>>(
}
}
pub(crate) fn lower_shift_amt<C: LowerCtx<I = Inst>>(
ctx: &mut C,
amt_input: InsnInput,
dst_ty: Type,
tmp_reg: Writable<Reg>,
) -> ResultRegImmShift {
let amt_ty = ctx.input_ty(amt_input.insn, amt_input.input);
match (dst_ty, amt_ty) {
// When shifting for amounts larger than the size of the type, the CLIF shift
// instructions implement a "wrapping" behaviour, such that an i8 << 8 is
// equivalent to i8 << 0
//
// On i32 and i64 types this matches what the aarch64 spec does, but on smaller
// types (i16, i8) we need to do this manually, so we wrap the shift amount
// with an AND instruction
(I16 | I8, _) => {
// We can ignore the top half of the shift amount register if the type is I128
let amt_reg = put_input_in_regs(ctx, amt_input).regs()[0];
let mask = (ty_bits(dst_ty) - 1) as u64;
ctx.emit(Inst::AluRRImmLogic {
alu_op: ALUOp::And32,
rd: tmp_reg,
rn: amt_reg,
imml: ImmLogic::maybe_from_u64(mask, I32).unwrap(),
});
ResultRegImmShift::Reg(tmp_reg.to_reg())
}
// TODO: We can use immlogic for i128 types here
(I128, _) | (_, I128) => {
// For I128 shifts, we need a register without immlogic
ResultRegImmShift::Reg(put_input_in_regs(ctx, amt_input).regs()[0])
}
_ => put_input_in_reg_immshift(ctx, amt_input, ty_bits(dst_ty)),
}
}
/// This is target-word-size dependent. And it excludes booleans and reftypes.
pub(crate) fn is_valid_atomic_transaction_ty(ty: Type) -> bool {
match ty {

14
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -248,4 +248,18 @@ where
self.lower_ctx.sink_inst(load.atomic_load);
self.put_in_reg(load.atomic_addr)
}
fn shift_mask(&mut self, ty: Type) -> ImmLogic {
let mask = (ty.bits() - 1) as u64;
ImmLogic::maybe_from_u64(mask, I32).unwrap()
}
fn imm_shift_from_imm64(&mut self, val: Imm64, ty: Type) -> Option<ImmShift> {
let imm_value = (val.bits() as u64) & ((ty.bits() - 1) as u64);
ImmShift::maybe_from_u64(imm_value)
}
fn u64_into_imm_logic(&mut self, ty: Type, val: u64) -> ImmLogic {
ImmLogic::maybe_from_u64(val, ty).unwrap()
}
}

6
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602
src/prelude.isle 75a46b97817ad6a4c34e618b81e60876eec6fd1c83ac3ee174851e42045c951644663b2cbc31f1749ce2bc3ad9eb94fb0b877eb2c3bc4885cab7d7e87e9df1d6
src/isa/aarch64/inst.isle 8e4b8e452cf06a368c2e1d930042027a5d3bd690ab46d498d959257e9b4461d17abf244838395cd80da1fe5e2e86fc43855fb5753ca4f1643538c2ae4b3b4a1e
src/isa/aarch64/lower.isle bc3db9c1e6ac186b918cc04f4d26af398f99ec36c8cdc20ec4d02d18dd57dba12e3184fea031b4ac97051c5e194a69666afb5e204807c818e6688c177f9c1b91
src/prelude.isle 15c8dd937171bd0f619179e219422d43af0eb0ef9a6e88f23b2aa55776712e27342309dd3a4441876b2dfec7f16ce7fe13b3a926ace89b25cfc9577e7b1d1578
src/isa/aarch64/inst.isle a6921329a85a252b8059657056ee0c4477304dff461bd58c39133a2870666b23a34c911be55e25e7887074cb54b640ff09998730af09b3c1ba792f434309af24
src/isa/aarch64/lower.isle 1fb105feeabfd582e680900a5c3dd82950045761dc82ff30e92107cdac479ff046ce10853489d158c6b98961c7eab5d274ede3b0181c06f4ae67bf4556d6130d

575
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

@@ -32,6 +32,7 @@ pub trait Context {
fn u32_as_u64(&mut self, arg0: u32) -> u64;
fn ty_bits(&mut self, arg0: Type) -> u8;
fn ty_bits_u16(&mut self, arg0: Type) -> u16;
fn fits_in_16(&mut self, arg0: Type) -> Option<Type>;
fn fits_in_32(&mut self, arg0: Type) -> Option<Type>;
fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
@@ -60,10 +61,12 @@ pub trait Context {
fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option<MoveWideConst>;
fn imm_logic_from_u64(&mut self, arg0: u64, arg1: Type) -> Option<ImmLogic>;
fn imm_logic_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option<ImmLogic>;
fn imm_shift_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option<ImmShift>;
fn imm_shift_from_u8(&mut self, arg0: u8) -> ImmShift;
fn imm12_from_u64(&mut self, arg0: u64) -> Option<Imm12>;
fn u8_into_uimm5(&mut self, arg0: u8) -> UImm5;
fn u8_into_imm12(&mut self, arg0: u8) -> Imm12;
fn u64_into_imm_logic(&mut self, arg0: Type, arg1: u64) -> ImmLogic;
fn imm12_from_negated_u64(&mut self, arg0: u64) -> Option<Imm12>;
fn lshl_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option<ShiftOpAndAmt>;
fn integral_ty(&mut self, arg0: Type) -> Option<Type>;
@@ -80,15 +83,16 @@ pub trait Context {
fn sinkable_atomic_load(&mut self, arg0: Value) -> Option<SinkableAtomicLoad>;
fn sink_atomic_load(&mut self, arg0: &SinkableAtomicLoad) -> Reg;
fn safe_divisor_from_imm64(&mut self, arg0: Imm64) -> Option<u64>;
fn shift_mask(&mut self, arg0: Type) -> ImmLogic;
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 259.
/// Internal type ProducesFlags: defined at src/prelude.isle line 263.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlags { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 262.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 266.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlags { inst: MInst, result: Reg },
@@ -1002,7 +1006,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 272.
// Rule at src/prelude.isle line 276.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1030,7 +1034,7 @@ pub fn constructor_with_flags_1<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 280.
// Rule at src/prelude.isle line 284.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
return Some(pattern3_1);
@@ -1064,10 +1068,10 @@ pub fn constructor_with_flags_2<C: Context>(
result: pattern5_1,
} = pattern4_0
{
// Rule at src/prelude.isle line 290.
// Rule at src/prelude.isle line 294.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
let expr2_0 = C::emit(ctx, &pattern5_0);
let expr1_0 = C::emit(ctx, &pattern5_0);
let expr2_0 = C::emit(ctx, &pattern3_0);
let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1);
return Some(expr3_0);
}
@@ -1120,7 +1124,7 @@ pub fn constructor_movz<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1378.
// Rule at src/isa/aarch64/inst.isle line 1384.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::MovZ {
@@ -1141,7 +1145,7 @@ pub fn constructor_movn<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1385.
// Rule at src/isa/aarch64/inst.isle line 1391.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::MovN {
@@ -1164,7 +1168,7 @@ pub fn constructor_alu_rr_imm_logic<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1392.
// Rule at src/isa/aarch64/inst.isle line 1398.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::AluRRImmLogic {
@@ -1188,7 +1192,7 @@ pub fn constructor_alu_rr_imm_shift<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1399.
// Rule at src/isa/aarch64/inst.isle line 1405.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::AluRRImmShift {
@@ -1212,7 +1216,7 @@ pub fn constructor_alu_rrr<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1406.
// Rule at src/isa/aarch64/inst.isle line 1412.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::AluRRR {
@@ -1238,7 +1242,7 @@ pub fn constructor_vec_rrr<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1413.
// Rule at src/isa/aarch64/inst.isle line 1419.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::VecRRR {
@@ -1253,6 +1257,23 @@ pub fn constructor_vec_rrr<C: Context>(
return Some(expr4_0);
}
// Generated as internal constructor for term vec_dup.
pub fn constructor_vec_dup<C: Context>(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1426.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::VecDup {
rd: expr1_0,
rn: pattern0_0,
size: pattern1_0.clone(),
};
let expr3_0 = C::emit(ctx, &expr2_0);
let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
return Some(expr4_0);
}
// Generated as internal constructor for term alu_rr_imm12.
pub fn constructor_alu_rr_imm12<C: Context>(
ctx: &mut C,
@@ -1263,7 +1284,7 @@ pub fn constructor_alu_rr_imm12<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1420.
// Rule at src/isa/aarch64/inst.isle line 1433.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::AluRRImm12 {
@@ -1289,7 +1310,7 @@ pub fn constructor_alu_rrr_shift<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1427.
// Rule at src/isa/aarch64/inst.isle line 1440.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::AluRRRShift {
@@ -1316,7 +1337,7 @@ pub fn constructor_alu_rrr_extend<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1434.
// Rule at src/isa/aarch64/inst.isle line 1447.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::AluRRRExtend {
@@ -1341,7 +1362,7 @@ pub fn constructor_alu_rr_extend_reg<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1442.
// Rule at src/isa/aarch64/inst.isle line 1455.
let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0);
let expr1_0 = C::get_extended_op(ctx, pattern2_0);
let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?;
@@ -1360,7 +1381,7 @@ pub fn constructor_alu_rrrr<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1449.
// Rule at src/isa/aarch64/inst.isle line 1462.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::AluRRRR {
@@ -1383,7 +1404,7 @@ pub fn constructor_add64_with_flags<C: Context>(
) -> Option<ProducesFlags> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1456.
// Rule at src/isa/aarch64/inst.isle line 1469.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = ALUOp::AddS64;
@@ -1405,7 +1426,7 @@ pub fn constructor_add64_with_flags<C: Context>(
pub fn constructor_adc64<C: Context>(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option<ConsumesFlags> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1463.
// Rule at src/isa/aarch64/inst.isle line 1476.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = ALUOp::Adc64;
@@ -1431,7 +1452,7 @@ pub fn constructor_sub64_with_flags<C: Context>(
) -> Option<ProducesFlags> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1470.
// Rule at src/isa/aarch64/inst.isle line 1483.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = ALUOp::SubS64;
@@ -1453,7 +1474,7 @@ pub fn constructor_sub64_with_flags<C: Context>(
pub fn constructor_sbc64<C: Context>(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option<ConsumesFlags> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1477.
// Rule at src/isa/aarch64/inst.isle line 1490.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = ALUOp::Sbc64;
@@ -1481,7 +1502,7 @@ pub fn constructor_vec_misc<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1484.
// Rule at src/isa/aarch64/inst.isle line 1497.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::VecMisc {
@@ -1507,7 +1528,7 @@ pub fn constructor_vec_rrr_long<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1491.
// Rule at src/isa/aarch64/inst.isle line 1504.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::VecRRRLong {
@@ -1536,7 +1557,7 @@ pub fn constructor_vec_rrrr_long<C: Context>(
let pattern2_0 = arg2;
let pattern3_0 = arg3;
let pattern4_0 = arg4;
// Rule at src/isa/aarch64/inst.isle line 1501.
// Rule at src/isa/aarch64/inst.isle line 1514.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::FpuMove128 {
@@ -1566,7 +1587,7 @@ pub fn constructor_vec_rr_narrow<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1509.
// Rule at src/isa/aarch64/inst.isle line 1522.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::VecRRNarrow {
@@ -1590,7 +1611,7 @@ pub fn constructor_vec_rr_long<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1516.
// Rule at src/isa/aarch64/inst.isle line 1529.
let expr0_0: Type = I8X16;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::VecRRLong {
@@ -1614,7 +1635,7 @@ pub fn constructor_mov_from_vec<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1523.
// Rule at src/isa/aarch64/inst.isle line 1536.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::MovFromVec {
@@ -1640,7 +1661,7 @@ pub fn constructor_mov_from_vec_signed<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1530.
// Rule at src/isa/aarch64/inst.isle line 1543.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::MovFromVecSigned {
@@ -1667,7 +1688,7 @@ pub fn constructor_extend<C: Context>(
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1537.
// Rule at src/isa/aarch64/inst.isle line 1550.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::Extend {
@@ -1686,7 +1707,7 @@ pub fn constructor_extend<C: Context>(
pub fn constructor_load_acquire<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1544.
// Rule at src/isa/aarch64/inst.isle line 1557.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::LoadAcquire {
@@ -1699,6 +1720,58 @@ pub fn constructor_load_acquire<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg)
return Some(expr4_0);
}
// Generated as internal constructor for term tst64_imm.
pub fn constructor_tst64_imm<C: Context>(
ctx: &mut C,
arg0: Reg,
arg1: ImmLogic,
) -> Option<ProducesFlags> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/inst.isle line 1567.
let expr0_0 = ALUOp::AndS64;
let expr1_0 = C::writable_zero_reg(ctx);
let expr2_0 = MInst::AluRRImmLogic {
alu_op: expr0_0,
rd: expr1_0,
rn: pattern0_0,
imml: pattern1_0,
};
let expr3_0 = C::invalid_reg(ctx);
let expr4_0 = ProducesFlags::ProducesFlags {
inst: expr2_0,
result: expr3_0,
};
return Some(expr4_0);
}
// Generated as internal constructor for term csel.
pub fn constructor_csel<C: Context>(
ctx: &mut C,
arg0: &Cond,
arg1: Reg,
arg2: Reg,
) -> Option<ConsumesFlags> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1580.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = MInst::CSel {
rd: expr1_0,
cond: pattern0_0.clone(),
rn: pattern1_0,
rm: pattern2_0,
};
let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
let expr4_0 = ConsumesFlags::ConsumesFlags {
inst: expr2_0,
result: expr3_0,
};
return Some(expr4_0);
}
// Generated as internal constructor for term imm.
pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option<Reg> {
let pattern0_0 = arg0;
@@ -1710,7 +1783,7 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
};
if let Some(pattern3_0) = closure3() {
if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) {
// Rule at src/isa/aarch64/inst.isle line 1562.
// Rule at src/isa/aarch64/inst.isle line 1599.
let expr0_0 = ALUOp::Orr64;
let expr1_0 = C::zero_reg(ctx);
let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern4_0)?;
@@ -1718,35 +1791,87 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
}
}
if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) {
// Rule at src/isa/aarch64/inst.isle line 1554.
// Rule at src/isa/aarch64/inst.isle line 1591.
let expr0_0 = OperandSize::Size64;
let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?;
return Some(expr1_0);
}
if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) {
// Rule at src/isa/aarch64/inst.isle line 1558.
// Rule at src/isa/aarch64/inst.isle line 1595.
let expr0_0 = OperandSize::Size64;
let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?;
return Some(expr1_0);
}
// Rule at src/isa/aarch64/inst.isle line 1569.
// Rule at src/isa/aarch64/inst.isle line 1606.
let expr0_0 = C::load_constant64_full(ctx, pattern2_0);
return Some(expr0_0);
}
return None;
}
// Generated as internal constructor for term put_in_reg_sext32.
pub fn constructor_put_in_reg_sext32<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = C::value_type(ctx, pattern0_0);
if pattern1_0 == I32 {
// Rule at src/isa/aarch64/inst.isle line 1617.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
return Some(expr0_0);
}
if pattern1_0 == I64 {
// Rule at src/isa/aarch64/inst.isle line 1618.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
return Some(expr0_0);
}
if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
// Rule at src/isa/aarch64/inst.isle line 1613.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = true;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
let expr3_0: u8 = 32;
let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
return Some(expr4_0);
}
return None;
}
// Generated as internal constructor for term put_in_reg_zext32.
pub fn constructor_put_in_reg_zext32<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = C::value_type(ctx, pattern0_0);
if pattern1_0 == I32 {
// Rule at src/isa/aarch64/inst.isle line 1626.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
return Some(expr0_0);
}
if pattern1_0 == I64 {
// Rule at src/isa/aarch64/inst.isle line 1627.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
return Some(expr0_0);
}
if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
// Rule at src/isa/aarch64/inst.isle line 1622.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = false;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
let expr3_0: u8 = 32;
let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
return Some(expr4_0);
}
return None;
}
// Generated as internal constructor for term put_in_reg_sext64.
pub fn constructor_put_in_reg_sext64<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = C::value_type(ctx, pattern0_0);
if pattern1_0 == I64 {
// Rule at src/isa/aarch64/inst.isle line 1580.
// Rule at src/isa/aarch64/inst.isle line 1635.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
return Some(expr0_0);
}
if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
// Rule at src/isa/aarch64/inst.isle line 1576.
// Rule at src/isa/aarch64/inst.isle line 1631.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = true;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -1762,12 +1887,12 @@ pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
let pattern0_0 = arg0;
let pattern1_0 = C::value_type(ctx, pattern0_0);
if pattern1_0 == I64 {
// Rule at src/isa/aarch64/inst.isle line 1588.
// Rule at src/isa/aarch64/inst.isle line 1643.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
return Some(expr0_0);
}
if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
// Rule at src/isa/aarch64/inst.isle line 1584.
// Rule at src/isa/aarch64/inst.isle line 1639.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = false;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -1781,7 +1906,7 @@ pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
// Generated as internal constructor for term trap_if_zero_divisor.
pub fn constructor_trap_if_zero_divisor<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/isa/aarch64/inst.isle line 1593.
// Rule at src/isa/aarch64/inst.isle line 1648.
let expr0_0 = C::cond_br_zero(ctx, pattern0_0);
let expr1_0 = C::trap_code_division_by_zero(ctx);
let expr2_0 = MInst::TrapIf {
@@ -1796,12 +1921,12 @@ pub fn constructor_trap_if_zero_divisor<C: Context>(ctx: &mut C, arg0: Reg) -> O
pub fn constructor_size_from_ty<C: Context>(ctx: &mut C, arg0: Type) -> Option<OperandSize> {
let pattern0_0 = arg0;
if pattern0_0 == I64 {
// Rule at src/isa/aarch64/inst.isle line 1599.
// Rule at src/isa/aarch64/inst.isle line 1654.
let expr0_0 = OperandSize::Size64;
return Some(expr0_0);
}
if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
// Rule at src/isa/aarch64/inst.isle line 1598.
// Rule at src/isa/aarch64/inst.isle line 1653.
let expr0_0 = OperandSize::Size32;
return Some(expr0_0);
}
@@ -1818,7 +1943,7 @@ pub fn constructor_trap_if_div_overflow<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1605.
// Rule at src/isa/aarch64/inst.isle line 1660.
let expr0_0 = constructor_adds_op(ctx, pattern0_0)?;
let expr1_0 = C::writable_zero_reg(ctx);
let expr2_0: u8 = 1;
@@ -1862,12 +1987,12 @@ pub fn constructor_trap_if_div_overflow<C: Context>(
pub fn constructor_adds_op<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
let pattern0_0 = arg0;
if pattern0_0 == I64 {
// Rule at src/isa/aarch64/inst.isle line 1625.
// Rule at src/isa/aarch64/inst.isle line 1680.
let expr0_0 = ALUOp::AddS64;
return Some(expr0_0);
}
if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
// Rule at src/isa/aarch64/inst.isle line 1624.
// Rule at src/isa/aarch64/inst.isle line 1679.
let expr0_0 = ALUOp::AddS32;
return Some(expr0_0);
}
@@ -1901,7 +2026,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0)
{
let pattern9_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1655.
// Rule at src/isa/aarch64/inst.isle line 1710.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 =
constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern8_0)?;
@@ -1932,7 +2057,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
C::lshl_from_imm64(ctx, pattern10_1, pattern12_0)
{
let pattern14_0 = arg3;
// Rule at src/isa/aarch64/inst.isle line 1661.
// Rule at src/isa/aarch64/inst.isle line 1716.
let expr0_0 = C::put_in_reg(ctx, pattern14_0);
let expr1_0 = C::put_in_reg(ctx, pattern7_0);
let expr2_0 = constructor_alu_rrr_shift(
@@ -1969,7 +2094,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
if let Some(pattern9_0) =
C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0)
{
// Rule at src/isa/aarch64/inst.isle line 1653.
// Rule at src/isa/aarch64/inst.isle line 1708.
let expr0_0 = C::put_in_reg(ctx, pattern2_0);
let expr1_0 =
constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?;
@@ -1999,7 +2124,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
if let Some(pattern14_0) =
C::lshl_from_imm64(ctx, pattern11_1, pattern13_0)
{
// Rule at src/isa/aarch64/inst.isle line 1659.
// Rule at src/isa/aarch64/inst.isle line 1714.
let expr0_0 = C::put_in_reg(ctx, pattern2_0);
let expr1_0 = C::put_in_reg(ctx, pattern8_0);
let expr2_0 = constructor_alu_rrr_shift(
@@ -2020,7 +2145,7 @@ pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
_ => {}
}
}
// Rule at src/isa/aarch64/inst.isle line 1649.
// Rule at src/isa/aarch64/inst.isle line 1704.
let expr0_0 = C::put_in_reg(ctx, pattern2_0);
let expr1_0 = C::put_in_reg(ctx, pattern3_0);
let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?;
@@ -2054,7 +2179,7 @@ pub fn constructor_alu_rs_imm_logic<C: Context>(
if let Some(pattern9_0) =
C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0)
{
// Rule at src/isa/aarch64/inst.isle line 1669.
// Rule at src/isa/aarch64/inst.isle line 1724.
let expr0_0 = C::put_in_reg(ctx, pattern2_0);
let expr1_0 =
constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?;
@@ -2084,7 +2209,7 @@ pub fn constructor_alu_rs_imm_logic<C: Context>(
if let Some(pattern14_0) =
C::lshl_from_imm64(ctx, pattern11_1, pattern13_0)
{
// Rule at src/isa/aarch64/inst.isle line 1671.
// Rule at src/isa/aarch64/inst.isle line 1726.
let expr0_0 = C::put_in_reg(ctx, pattern2_0);
let expr1_0 = C::put_in_reg(ctx, pattern8_0);
let expr2_0 = constructor_alu_rrr_shift(
@@ -2105,7 +2230,7 @@ pub fn constructor_alu_rs_imm_logic<C: Context>(
_ => {}
}
}
// Rule at src/isa/aarch64/inst.isle line 1667.
// Rule at src/isa/aarch64/inst.isle line 1722.
let expr0_0 = C::put_in_reg(ctx, pattern2_0);
let expr1_0 = C::put_in_reg(ctx, pattern3_0);
let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?;
@@ -2122,7 +2247,7 @@ pub fn constructor_i128_alu_bitop<C: Context>(
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
// Rule at src/isa/aarch64/inst.isle line 1679.
// Rule at src/isa/aarch64/inst.isle line 1734.
let expr0_0 = C::put_in_regs(ctx, pattern1_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2238,6 +2363,39 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
&Opcode::Ishl => {
let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 706.
let expr0_0 = ALUOp::Lsl64;
let expr1_0: Type = I64;
let expr2_0 = C::put_in_reg(ctx, pattern7_0);
let expr3_0 =
constructor_do_shift(ctx, &expr0_0, expr1_0, expr2_0, pattern7_1)?;
let expr4_0 = C::value_reg(ctx, expr3_0);
return Some(expr4_0);
}
&Opcode::Ushr => {
let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 797.
let expr0_0 = ALUOp::Lsr64;
let expr1_0: Type = I64;
let expr2_0 = constructor_put_in_reg_zext64(ctx, pattern7_0)?;
let expr3_0 =
constructor_do_shift(ctx, &expr0_0, expr1_0, expr2_0, pattern7_1)?;
let expr4_0 = C::value_reg(ctx, expr3_0);
return Some(expr4_0);
}
&Opcode::Sshr => {
let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 849.
let expr0_0 = ALUOp::Asr64;
let expr1_0: Type = I64;
let expr2_0 = constructor_put_in_reg_sext64(ctx, pattern7_0)?;
let expr3_0 =
constructor_do_shift(ctx, &expr0_0, expr1_0, expr2_0, pattern7_1)?;
let expr4_0 = C::value_reg(ctx, expr3_0);
return Some(expr4_0);
}
_ => {}
}
}
@@ -2371,6 +2529,39 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
constructor_i128_alu_bitop(ctx, &expr0_0, pattern7_0, pattern7_1)?;
return Some(expr1_0);
}
&Opcode::Ishl => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 710.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0 = C::put_in_regs(ctx, pattern7_1);
let expr2_0: usize = 0;
let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0);
let expr4_0 = constructor_lower_shl128(ctx, expr0_0, expr3_0)?;
return Some(expr4_0);
}
&Opcode::Ushr => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 801.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0 = C::put_in_regs(ctx, pattern7_1);
let expr2_0: usize = 0;
let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0);
let expr4_0 = constructor_lower_ushr128(ctx, expr0_0, expr3_0)?;
return Some(expr4_0);
}
&Opcode::Sshr => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 853.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0 = C::put_in_regs(ctx, pattern7_1);
let expr2_0: usize = 0;
let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0);
let expr4_0 = constructor_lower_sshr128(ctx, expr0_0, expr3_0)?;
return Some(expr4_0);
}
_ => {}
}
}
@@ -3189,6 +3380,36 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let expr2_0 = C::value_reg(ctx, expr1_0);
return Some(expr2_0);
}
&Opcode::Ishl => {
let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 702.
let expr0_0 = ALUOp::Lsl32;
let expr1_0 = C::put_in_reg(ctx, pattern7_0);
let expr2_0 =
constructor_do_shift(ctx, &expr0_0, pattern3_0, expr1_0, pattern7_1)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
&Opcode::Ushr => {
let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 793.
let expr0_0 = ALUOp::Lsr32;
let expr1_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
let expr2_0 =
constructor_do_shift(ctx, &expr0_0, pattern3_0, expr1_0, pattern7_1)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
&Opcode::Sshr => {
let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 845.
let expr0_0 = ALUOp::Asr32;
let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern7_0)?;
let expr2_0 =
constructor_do_shift(ctx, &expr0_0, pattern3_0, expr1_0, pattern7_1)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
_ => {}
}
}
@@ -3976,6 +4197,54 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let expr5_0 = C::value_reg(ctx, expr4_0);
return Some(expr5_0);
}
&Opcode::Ishl => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 743.
let expr0_0 = constructor_vector_size(ctx, pattern3_0)?;
let expr1_0 = C::put_in_reg(ctx, pattern7_1);
let expr2_0 = constructor_vec_dup(ctx, expr1_0, &expr0_0)?;
let expr3_0 = VecALUOp::Sshl;
let expr4_0 = C::put_in_reg(ctx, pattern7_0);
let expr5_0 =
constructor_vec_rrr(ctx, &expr3_0, expr4_0, expr2_0, &expr0_0)?;
let expr6_0 = C::value_reg(ctx, expr5_0);
return Some(expr6_0);
}
&Opcode::Ushr => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 805.
let expr0_0 = constructor_vector_size(ctx, pattern3_0)?;
let expr1_0 = ALUOp::Sub32;
let expr2_0 = C::zero_reg(ctx);
let expr3_0 = C::put_in_reg(ctx, pattern7_1);
let expr4_0 = constructor_alu_rrr(ctx, &expr1_0, expr2_0, expr3_0)?;
let expr5_0 = constructor_vec_dup(ctx, expr4_0, &expr0_0)?;
let expr6_0 = VecALUOp::Ushl;
let expr7_0 = C::put_in_reg(ctx, pattern7_0);
let expr8_0 =
constructor_vec_rrr(ctx, &expr6_0, expr7_0, expr5_0, &expr0_0)?;
let expr9_0 = C::value_reg(ctx, expr8_0);
return Some(expr9_0);
}
&Opcode::Sshr => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/aarch64/lower.isle line 859.
let expr0_0 = constructor_vector_size(ctx, pattern3_0)?;
let expr1_0 = ALUOp::Sub32;
let expr2_0 = C::zero_reg(ctx);
let expr3_0 = C::put_in_reg(ctx, pattern7_1);
let expr4_0 = constructor_alu_rrr(ctx, &expr1_0, expr2_0, expr3_0)?;
let expr5_0 = constructor_vec_dup(ctx, expr4_0, &expr0_0)?;
let expr6_0 = VecALUOp::Sshl;
let expr7_0 = C::put_in_reg(ctx, pattern7_0);
let expr8_0 =
constructor_vec_rrr(ctx, &expr6_0, expr7_0, expr5_0, &expr0_0)?;
let expr9_0 = C::value_reg(ctx, expr8_0);
return Some(expr9_0);
}
_ => {}
}
}
@@ -4148,3 +4417,201 @@ pub fn constructor_orr_not_op<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALU
}
return None;
}
// Generated as internal constructor for term lower_shl128.
pub fn constructor_lower_shl128<C: Context>(
ctx: &mut C,
arg0: ValueRegs,
arg1: Reg,
) -> Option<ValueRegs> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/lower.isle line 723.
let expr0_0: usize = 0;
let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
let expr2_0: usize = 1;
let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0);
let expr4_0 = ALUOp::Lsl64;
let expr5_0 = constructor_alu_rrr(ctx, &expr4_0, expr1_0, pattern1_0)?;
let expr6_0 = ALUOp::Lsl64;
let expr7_0 = constructor_alu_rrr(ctx, &expr6_0, expr3_0, pattern1_0)?;
let expr8_0 = ALUOp::OrrNot32;
let expr9_0 = C::zero_reg(ctx);
let expr10_0 = constructor_alu_rrr(ctx, &expr8_0, expr9_0, pattern1_0)?;
let expr11_0 = ALUOp::Lsr64;
let expr12_0 = ALUOp::Lsr64;
let expr13_0: u8 = 1;
let expr14_0 = C::imm_shift_from_u8(ctx, expr13_0);
let expr15_0 = constructor_alu_rr_imm_shift(ctx, &expr12_0, expr1_0, expr14_0)?;
let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr15_0, expr10_0)?;
let expr17_0 = ALUOp::Orr64;
let expr18_0 = constructor_alu_rrr(ctx, &expr17_0, expr7_0, expr16_0)?;
let expr19_0: Type = I64;
let expr20_0: u64 = 64;
let expr21_0 = C::u64_into_imm_logic(ctx, expr19_0, expr20_0);
let expr22_0 = constructor_tst64_imm(ctx, pattern1_0, expr21_0)?;
let expr23_0 = Cond::Ne;
let expr24_0 = C::zero_reg(ctx);
let expr25_0 = constructor_csel(ctx, &expr23_0, expr24_0, expr5_0)?;
let expr26_0 = Cond::Ne;
let expr27_0 = constructor_csel(ctx, &expr26_0, expr5_0, expr18_0)?;
let expr28_0 = constructor_with_flags_2(ctx, &expr22_0, &expr25_0, &expr27_0)?;
return Some(expr28_0);
}
// Generated as internal constructor for term do_shift.
pub fn constructor_do_shift<C: Context>(
ctx: &mut C,
arg0: &ALUOp,
arg1: Type,
arg2: Reg,
arg3: Value,
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
let pattern2_0 = arg2;
let pattern3_0 = arg3;
if let Some(pattern4_0) = C::def_inst(ctx, pattern3_0) {
let pattern5_0 = C::inst_data(ctx, pattern4_0);
if let &InstructionData::UnaryImm {
opcode: ref pattern6_0,
imm: pattern6_1,
} = &pattern5_0
{
if let &Opcode::Iconst = &pattern6_0 {
let closure8 = || {
return Some(pattern1_0);
};
if let Some(pattern8_0) = closure8() {
if let Some(pattern9_0) = C::imm_shift_from_imm64(ctx, pattern6_1, pattern8_0) {
// Rule at src/isa/aarch64/lower.isle line 787.
let expr0_0 =
constructor_alu_rr_imm_shift(ctx, pattern0_0, pattern2_0, pattern9_0)?;
return Some(expr0_0);
}
}
}
}
}
let pattern0_0 = arg0;
let pattern1_0 = arg1;
if pattern1_0 == I32 {
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/aarch64/lower.isle line 778.
let expr0_0 = C::put_in_regs(ctx, pattern4_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
let expr3_0 = constructor_alu_rrr(ctx, pattern0_0, pattern3_0, expr2_0)?;
return Some(expr3_0);
}
if pattern1_0 == I64 {
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/aarch64/lower.isle line 779.
let expr0_0 = C::put_in_regs(ctx, pattern4_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
let expr3_0 = constructor_alu_rrr(ctx, pattern0_0, pattern3_0, expr2_0)?;
return Some(expr3_0);
}
if let Some(pattern2_0) = C::fits_in_16(ctx, pattern1_0) {
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/aarch64/lower.isle line 767.
let expr0_0 = C::put_in_regs(ctx, pattern4_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
let expr3_0 = ALUOp::And32;
let expr4_0 = C::shift_mask(ctx, pattern2_0);
let expr5_0 = constructor_alu_rr_imm_logic(ctx, &expr3_0, expr2_0, expr4_0)?;
let expr6_0 = constructor_alu_rrr(ctx, pattern0_0, pattern3_0, expr5_0)?;
return Some(expr6_0);
}
return None;
}
// Generated as internal constructor for term lower_ushr128.
pub fn constructor_lower_ushr128<C: Context>(
ctx: &mut C,
arg0: ValueRegs,
arg1: Reg,
) -> Option<ValueRegs> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/lower.isle line 822.
let expr0_0: usize = 0;
let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
let expr2_0: usize = 1;
let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0);
let expr4_0 = ALUOp::Lsr64;
let expr5_0 = constructor_alu_rrr(ctx, &expr4_0, expr1_0, pattern1_0)?;
let expr6_0 = ALUOp::Lsr64;
let expr7_0 = constructor_alu_rrr(ctx, &expr6_0, expr3_0, pattern1_0)?;
let expr8_0 = ALUOp::OrrNot32;
let expr9_0 = C::zero_reg(ctx);
let expr10_0 = constructor_alu_rrr(ctx, &expr8_0, expr9_0, pattern1_0)?;
let expr11_0 = ALUOp::Lsl64;
let expr12_0 = ALUOp::Lsl64;
let expr13_0: u8 = 1;
let expr14_0 = C::imm_shift_from_u8(ctx, expr13_0);
let expr15_0 = constructor_alu_rr_imm_shift(ctx, &expr12_0, expr3_0, expr14_0)?;
let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr15_0, expr10_0)?;
let expr17_0 = ALUOp::Orr64;
let expr18_0 = constructor_alu_rrr(ctx, &expr17_0, expr5_0, expr16_0)?;
let expr19_0: Type = I64;
let expr20_0: u64 = 64;
let expr21_0 = C::u64_into_imm_logic(ctx, expr19_0, expr20_0);
let expr22_0 = constructor_tst64_imm(ctx, pattern1_0, expr21_0)?;
let expr23_0 = Cond::Ne;
let expr24_0 = constructor_csel(ctx, &expr23_0, expr7_0, expr18_0)?;
let expr25_0 = Cond::Ne;
let expr26_0 = C::zero_reg(ctx);
let expr27_0 = constructor_csel(ctx, &expr25_0, expr26_0, expr7_0)?;
let expr28_0 = constructor_with_flags_2(ctx, &expr22_0, &expr24_0, &expr27_0)?;
return Some(expr28_0);
}
// Generated as internal constructor for term lower_sshr128.
pub fn constructor_lower_sshr128<C: Context>(
ctx: &mut C,
arg0: ValueRegs,
arg1: Reg,
) -> Option<ValueRegs> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/aarch64/lower.isle line 877.
let expr0_0: usize = 0;
let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
let expr2_0: usize = 1;
let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0);
let expr4_0 = ALUOp::Lsr64;
let expr5_0 = constructor_alu_rrr(ctx, &expr4_0, expr1_0, pattern1_0)?;
let expr6_0 = ALUOp::Asr64;
let expr7_0 = constructor_alu_rrr(ctx, &expr6_0, expr3_0, pattern1_0)?;
let expr8_0 = ALUOp::OrrNot32;
let expr9_0 = C::zero_reg(ctx);
let expr10_0 = constructor_alu_rrr(ctx, &expr8_0, expr9_0, pattern1_0)?;
let expr11_0 = ALUOp::Lsl64;
let expr12_0 = ALUOp::Lsl64;
let expr13_0: u8 = 1;
let expr14_0 = C::imm_shift_from_u8(ctx, expr13_0);
let expr15_0 = constructor_alu_rr_imm_shift(ctx, &expr12_0, expr3_0, expr14_0)?;
let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr15_0, expr10_0)?;
let expr17_0 = ALUOp::Asr64;
let expr18_0: u8 = 63;
let expr19_0 = C::imm_shift_from_u8(ctx, expr18_0);
let expr20_0 = constructor_alu_rr_imm_shift(ctx, &expr17_0, expr3_0, expr19_0)?;
let expr21_0 = ALUOp::Orr64;
let expr22_0 = constructor_alu_rrr(ctx, &expr21_0, expr5_0, expr16_0)?;
let expr23_0: Type = I64;
let expr24_0: u64 = 64;
let expr25_0 = C::u64_into_imm_logic(ctx, expr23_0, expr24_0);
let expr26_0 = constructor_tst64_imm(ctx, pattern1_0, expr25_0)?;
let expr27_0 = Cond::Ne;
let expr28_0 = constructor_csel(ctx, &expr27_0, expr7_0, expr22_0)?;
let expr29_0 = Cond::Ne;
let expr30_0 = constructor_csel(ctx, &expr29_0, expr20_0, expr7_0)?;
let expr31_0 = constructor_with_flags_2(ctx, &expr26_0, &expr28_0, &expr30_0)?;
return Some(expr31_0);
}

78
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -42,7 +42,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
return Ok(());
}
let implemented_in_isle = |ctx: &mut C| {
let implemented_in_isle = |ctx: &mut C| -> ! {
unreachable!(
"implemented in ISLE: inst = `{}`, type = `{:?}`",
ctx.dfg().display_inst(insn),
@@ -88,81 +88,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::BorNot
| Opcode::BxorNot => implemented_in_isle(ctx),
Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => {
let out_regs = get_output_reg(ctx, outputs[0]);
let ty = ty.unwrap();
if ty == I128 {
let src = put_input_in_regs(ctx, inputs[0]);
let amt = lower_shift_amt(ctx, inputs[1], ty, out_regs.regs()[0]).unwrap_reg();
match op {
Opcode::Ishl => emit_shl_i128(ctx, src, out_regs, amt),
Opcode::Ushr => {
emit_shr_i128(ctx, src, out_regs, amt, /* is_signed = */ false)
}
Opcode::Sshr => {
emit_shr_i128(ctx, src, out_regs, amt, /* is_signed = */ true)
}
_ => unreachable!(),
};
} else if !ty.is_vector() {
let rd = out_regs.only_reg().unwrap();
let size = OperandSize::from_bits(ty_bits(ty));
let narrow_mode = match (op, size) {
(Opcode::Ishl, _) => NarrowValueMode::None,
(Opcode::Ushr, OperandSize::Size64) => NarrowValueMode::ZeroExtend64,
(Opcode::Ushr, OperandSize::Size32) => NarrowValueMode::ZeroExtend32,
(Opcode::Sshr, OperandSize::Size64) => NarrowValueMode::SignExtend64,
(Opcode::Sshr, OperandSize::Size32) => NarrowValueMode::SignExtend32,
_ => unreachable!(),
};
let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
let rm = lower_shift_amt(ctx, inputs[1], ty, out_regs.regs()[0]);
let alu_op = match op {
Opcode::Ishl => choose_32_64(ty, ALUOp::Lsl32, ALUOp::Lsl64),
Opcode::Ushr => choose_32_64(ty, ALUOp::Lsr32, ALUOp::Lsr64),
Opcode::Sshr => choose_32_64(ty, ALUOp::Asr32, ALUOp::Asr64),
_ => unreachable!(),
};
ctx.emit(alu_inst_immshift(alu_op, rd, rn, rm));
} else {
let rd = out_regs.only_reg().unwrap();
let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let size = VectorSize::from_ty(ty);
let (alu_op, is_right_shift) = match op {
Opcode::Ishl => (VecALUOp::Sshl, false),
Opcode::Ushr => (VecALUOp::Ushl, true),
Opcode::Sshr => (VecALUOp::Sshl, true),
_ => unreachable!(),
};
let rm = if is_right_shift {
// Right shifts are implemented with a negative left shift.
let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
let rn = zero_reg();
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::Sub32,
rd: tmp,
rn,
rm,
});
tmp.to_reg()
} else {
put_input_in_reg(ctx, inputs[1], NarrowValueMode::None)
};
ctx.emit(Inst::VecDup { rd, rn: rm, size });
ctx.emit(Inst::VecRRR {
alu_op,
rd,
rn,
rm: rd.to_reg(),
size,
});
}
}
Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => implemented_in_isle(ctx),
Opcode::Rotr | Opcode::Rotl => {
// aarch64 doesn't have a left-rotate instruction, but a left rotation of K places is

2
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602
src/prelude.isle 75a46b97817ad6a4c34e618b81e60876eec6fd1c83ac3ee174851e42045c951644663b2cbc31f1749ce2bc3ad9eb94fb0b877eb2c3bc4885cab7d7e87e9df1d6
src/prelude.isle 15c8dd937171bd0f619179e219422d43af0eb0ef9a6e88f23b2aa55776712e27342309dd3a4441876b2dfec7f16ce7fe13b3a926ace89b25cfc9577e7b1d1578
src/isa/x64/inst.isle 1a44ccc0c2cad90447762848461fcae714216ef058d42bdba89330a6008061526e92bbf1c17055c465b20fc75d98d1faa34feda8b22fa7ae0504a0f808798b41
src/isa/x64/lower.isle c7943201b32e9eb9726466e8cc417f7e84c4c4052de31e05ab6e0ad7502a587cf1d7d9835703c4ff5a506390f7a0668741e7f3feaa1edda6396571a425949fc9

15
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

@@ -32,6 +32,7 @@ pub trait Context {
fn u32_as_u64(&mut self, arg0: u32) -> u64;
fn ty_bits(&mut self, arg0: Type) -> u8;
fn ty_bits_u16(&mut self, arg0: Type) -> u16;
fn fits_in_16(&mut self, arg0: Type) -> Option<Type>;
fn fits_in_32(&mut self, arg0: Type) -> Option<Type>;
fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
@@ -76,13 +77,13 @@ pub trait Context {
fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8;
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 259.
/// Internal type ProducesFlags: defined at src/prelude.isle line 263.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlags { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 262.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 266.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlags { inst: MInst, result: Reg },
@@ -132,7 +133,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 272.
// Rule at src/prelude.isle line 276.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -160,7 +161,7 @@ pub fn constructor_with_flags_1<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 280.
// Rule at src/prelude.isle line 284.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
return Some(pattern3_1);
@@ -194,10 +195,10 @@ pub fn constructor_with_flags_2<C: Context>(
result: pattern5_1,
} = pattern4_0
{
// Rule at src/prelude.isle line 290.
// Rule at src/prelude.isle line 294.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
let expr2_0 = C::emit(ctx, &pattern5_0);
let expr1_0 = C::emit(ctx, &pattern5_0);
let expr2_0 = C::emit(ctx, &pattern3_0);
let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1);
return Some(expr3_0);
}

8
cranelift/codegen/src/machinst/isle.rs
View File

@@ -105,6 +105,14 @@ macro_rules! isle_prelude_methods {
ty.bits()
}
fn fits_in_16(&mut self, ty: Type) -> Option<Type> {
if ty.bits() <= 16 {
Some(ty)
} else {
None
}
}
#[inline]
fn fits_in_32(&mut self, ty: Type) -> Option<Type> {
if ty.bits() <= 32 {

13
cranelift/codegen/src/prelude.isle
View File

@@ -148,6 +148,10 @@
;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; An extractor that only matches types that can fit in 16 bits.
(decl fits_in_16 (Type) Type)
(extern extractor fits_in_16 fits_in_16)
;; An extractor that only matches types that can fit in 32 bits.
(decl fits_in_32 (Type) Type)
(extern extractor fits_in_32 fits_in_32)
@@ -287,12 +291,15 @@
;; `ValueRegs` containing the first consumer's result and then the second
;; consumer's result.
(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs)
(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result)
(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst _producer_result)
(ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1)
(ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2))
(let ((_x Unit (emit producer_inst))
(_y Unit (emit consumer_inst_1))
(_z Unit (emit consumer_inst_2)))
;; Note that the order of emission here is swapped, as this seems
;; to generate better register allocation for now with fewer
;; `mov` instructions.
(_y Unit (emit consumer_inst_2))
(_z Unit (emit consumer_inst_1)))
(value_regs consumer_result_1 consumer_result_2)))
;;;; Helpers for Working with TrapCode ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

100
cranelift/filetests/filetests/isa/aarch64/bitops.clif
View File

@@ -742,15 +742,15 @@ block0(v0: i128, v1: i8):
return v2
}
; check: orn w3, wzr, w2
; nextln: lsr x4, x0, #1
; nextln: lsl x1, x1, x2
; nextln: lsr x3, x4, x3
; nextln: lsl x0, x0, x2
; check: lsl x4, x0, x2
; nextln: lsl x3, x1, x2
; nextln: orn w1, wzr, w2
; nextln: lsr x0, x0, #1
; nextln: lsr x0, x0, x1
; nextln: orr x0, x3, x0
; nextln: ands xzr, x2, #64
; nextln: orr x1, x1, x3
; nextln: csel x1, x0, x1, ne
; nextln: csel x0, xzr, x0, ne
; nextln: csel x1, x4, x0, ne
; nextln: csel x0, xzr, x4, ne
; nextln: ret
@@ -760,15 +760,15 @@ block0(v0: i128, v1: i128):
return v2
}
; check: orn w3, wzr, w2
; nextln: lsr x4, x0, #1
; check: lsl x3, x0, x2
; nextln: lsl x1, x1, x2
; nextln: lsr x3, x4, x3
; nextln: lsl x0, x0, x2
; nextln: orn w4, wzr, w2
; nextln: lsr x0, x0, #1
; nextln: lsr x0, x0, x4
; nextln: orr x0, x1, x0
; nextln: ands xzr, x2, #64
; nextln: orr x1, x1, x3
; nextln: csel x1, x0, x1, ne
; nextln: csel x0, xzr, x0, ne
; nextln: csel x1, x3, x0, ne
; nextln: csel x0, xzr, x3, ne
; nextln: ret
@@ -778,16 +778,15 @@ block0(v0: i128, v1: i8):
return v2
}
; check: orn w3, wzr, w2
; nextln: lsl x4, x1, #1
; nextln: lsr x0, x0, x2
; nextln: lsl x3, x4, x3
; nextln: lsr x1, x1, x2
; check: lsr x3, x0, x2
; nextln: lsr x0, x1, x2
; nextln: orn w4, wzr, w2
; nextln: lsl x1, x1, #1
; nextln: lsl x1, x1, x4
; nextln: orr x3, x3, x1
; nextln: ands xzr, x2, #64
; nextln: orr x0, x0, x3
; nextln: csel x2, xzr, x1, ne
; nextln: csel x0, x1, x0, ne
; nextln: mov x1, x2
; nextln: csel x1, xzr, x0, ne
; nextln: csel x0, x0, x3, ne
; nextln: ret
@@ -797,16 +796,15 @@ block0(v0: i128, v1: i128):
return v2
}
; check: orn w3, wzr, w2
; nextln: lsl x4, x1, #1
; nextln: lsr x0, x0, x2
; nextln: lsl x3, x4, x3
; nextln: lsr x1, x1, x2
; check: lsr x3, x0, x2
; nextln: lsr x0, x1, x2
; nextln: orn w4, wzr, w2
; nextln: lsl x1, x1, #1
; nextln: lsl x1, x1, x4
; nextln: orr x3, x3, x1
; nextln: ands xzr, x2, #64
; nextln: orr x0, x0, x3
; nextln: csel x2, xzr, x1, ne
; nextln: csel x0, x1, x0, ne
; nextln: mov x1, x2
; nextln: csel x1, xzr, x0, ne
; nextln: csel x0, x0, x3, ne
; nextln: ret
@@ -816,16 +814,16 @@ block0(v0: i128, v1: i8):
return v2
}
; check: orn w3, wzr, w2
; nextln: lsl x4, x1, #1
; nextln: lsr x0, x0, x2
; nextln: lsl x4, x4, x3
; nextln: asr x3, x1, x2
; nextln: ands xzr, x2, #64
; check: lsr x3, x0, x2
; nextln: asr x0, x1, x2
; nextln: orn w4, wzr, w2
; nextln: lsl x5, x1, #1
; nextln: lsl x4, x5, x4
; nextln: asr x1, x1, #63
; nextln: orr x0, x0, x4
; nextln: csel x1, x1, x3, ne
; nextln: csel x0, x3, x0, ne
; nextln: orr x3, x3, x4
; nextln: ands xzr, x2, #64
; nextln: csel x1, x1, x0, ne
; nextln: csel x0, x0, x3, ne
; nextln: ret
@@ -835,14 +833,14 @@ block0(v0: i128, v1: i128):
return v2
}
; check: orn w3, wzr, w2
; nextln: lsl x4, x1, #1
; nextln: lsr x0, x0, x2
; nextln: lsl x4, x4, x3
; nextln: asr x3, x1, x2
; nextln: ands xzr, x2, #64
; check: lsr x3, x0, x2
; nextln: asr x0, x1, x2
; nextln: orn w4, wzr, w2
; nextln: lsl x5, x1, #1
; nextln: lsl x4, x5, x4
; nextln: asr x1, x1, #63
; nextln: orr x0, x0, x4
; nextln: csel x1, x1, x3, ne
; nextln: csel x0, x3, x0, ne
; nextln: orr x3, x3, x4
; nextln: ands xzr, x2, #64
; nextln: csel x1, x1, x0, ne
; nextln: csel x0, x0, x3, ne
; nextln: ret

52
cranelift/filetests/filetests/isa/x64/i128.clif
View File

@@ -917,10 +917,11 @@ block0(v0: i128, v1: i128):
; nextln: cmovzq %rcx, %rax
; nextln: orq %rdi, %rax
; nextln: testq $$64, %rdx
; nextln: movq %rsi, %rdi
; nextln: cmovzq %rax, %rdi
; nextln: cmovzq %rsi, %rcx
; nextln: cmovzq %rax, %rsi
; nextln: movq %rcx, %rax
; nextln: movq %rsi, %rdx
; nextln: movq %rdi, %rdx
; nextln: movq %rbp, %rsp
; nextln: popq %rbp
; nextln: ret
@@ -946,13 +947,12 @@ block0(v0: i128, v1: i128):
; nextln: testq $$127, %rdx
; nextln: cmovzq %rcx, %rax
; nextln: orq %rdi, %rax
; nextln: xorq %rcx, %rcx
; nextln: xorq %rdi, %rdi
; nextln: testq $$64, %rdx
; nextln: movq %rsi, %rdi
; nextln: cmovzq %rax, %rdi
; nextln: cmovzq %rsi, %rcx
; nextln: movq %rdi, %rax
; nextln: movq %rcx, %rdx
; nextln: cmovzq %rsi, %rdi
; nextln: cmovzq %rax, %rsi
; nextln: movq %rsi, %rax
; nextln: movq %rdi, %rdx
; nextln: movq %rbp, %rsp
; nextln: popq %rbp
; nextln: ret
@@ -1000,24 +1000,24 @@ block0(v0: i128, v1: i128):
; check: pushq %rbp
; nextln: movq %rsp, %rbp
; nextln: movq %rdi, %rax
; nextln: movq %rdi, %r9
; nextln: movq %rdx, %rcx
; nextln: shlq %cl, %r9
; nextln: movq %rsi, %rax
; nextln: movq %rdx, %rcx
; nextln: shlq %cl, %rax
; nextln: movq %rsi, %r8
; nextln: movq %rdx, %rcx
; nextln: shlq %cl, %r8
; nextln: movl $$64, %ecx
; nextln: subq %rdx, %rcx
; nextln: movq %rdi, %r9
; nextln: shrq %cl, %r9
; nextln: xorq %rcx, %rcx
; nextln: movq %rdi, %r10
; nextln: shrq %cl, %r10
; nextln: xorq %r8, %r8
; nextln: testq $$127, %rdx
; nextln: cmovzq %rcx, %r9
; nextln: orq %r8, %r9
; nextln: cmovzq %r8, %r10
; nextln: orq %rax, %r10
; nextln: testq $$64, %rdx
; nextln: movq %rcx, %r8
; nextln: cmovzq %rax, %r8
; nextln: cmovzq %r9, %rax
; nextln: movq %r9, %rax
; nextln: cmovzq %r10, %rax
; nextln: cmovzq %r9, %r8
; nextln: movl $$128, %r9d
; nextln: subq %rdx, %r9
; nextln: movq %rdi, %rdx
@@ -1033,14 +1033,12 @@ block0(v0: i128, v1: i128):
; nextln: testq $$127, %r9
; nextln: cmovzq %rcx, %rsi
; nextln: orq %rdx, %rsi
; nextln: xorq %rdx, %rdx
; nextln: xorq %rcx, %rcx
; nextln: testq $$64, %r9
; nextln: movq %rdi, %rcx
; nextln: cmovzq %rsi, %rcx
; nextln: movq %rdx, %rsi
; nextln: cmovzq %rdi, %rsi
; nextln: orq %rcx, %r8
; nextln: orq %rsi, %rax
; nextln: cmovzq %rdi, %rcx
; nextln: cmovzq %rsi, %rdi
; nextln: orq %rdi, %r8
; nextln: orq %rcx, %rax
; nextln: movq %rax, %rdx
; nextln: movq %r8, %rax
; nextln: movq %rbp, %rsp