diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 14c074cc23..e712cc3212 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -1304,6 +1304,9 @@ (decl imm_logic_from_imm64 (Type ImmLogic) Imm64) (extern extractor imm_logic_from_imm64 imm_logic_from_imm64 (in out)) +(decl imm_shift_from_imm64 (Type ImmShift) Imm64) +(extern extractor imm_shift_from_imm64 imm_shift_from_imm64 (in out)) + (decl imm_shift_from_u8 (u8) ImmShift) (extern constructor imm_shift_from_u8 imm_shift_from_u8) @@ -1316,6 +1319,9 @@ (decl u8_into_imm12 (u8) Imm12) (extern constructor u8_into_imm12 u8_into_imm12) +(decl u64_into_imm_logic (Type u64) ImmLogic) +(extern constructor u64_into_imm_logic u64_into_imm_logic) + (decl imm12_from_negated_u64 (Imm12) u64) (extern extractor imm12_from_negated_u64 imm12_from_negated_u64) @@ -1415,6 +1421,13 @@ (_ Unit (emit (MInst.VecRRR op dst src1 src2 size)))) (writable_reg_to_reg dst))) +;; Helper for emitting `MInst.VecDup` instructions. +(decl vec_dup (Reg VectorSize) Reg) +(rule (vec_dup src size) + (let ((dst WritableReg (temp_writable_reg $I8X16)) + (_ Unit (emit (MInst.VecDup dst src size)))) + (writable_reg_to_reg dst))) + ;; Helper for emitting `MInst.AluRRImm12` instructions. (decl alu_rr_imm12 (ALUOp Reg Imm12) Reg) (rule (alu_rr_imm12 op src imm) @@ -1546,6 +1559,30 @@ (_ Unit (emit (MInst.LoadAcquire ty dst addr)))) (writable_reg_to_reg dst))) +;; Helper for generating a `tst` instruction. +;; +;; Produces a `ProducesFlags` rather than a register or emitted instruction +;; which must be paired with `with_flags*` helpers. +(decl tst64_imm (Reg ImmLogic) ProducesFlags) +(rule (tst64_imm reg imm) + (ProducesFlags.ProducesFlags (MInst.AluRRImmLogic (ALUOp.AndS64) + (writable_zero_reg) + reg + imm) + (invalid_reg))) + +;; Helper for generating a `CSel` instruction. +;; +;; Note that this doesn't actually emit anything, instead it produces a +;; `ConsumesFlags` instruction which must be consumed with `with_flags*` +;; helpers. +(decl csel (Cond Reg Reg) ConsumesFlags) +(rule (csel cond if_true if_false) + (let ((dst WritableReg (temp_writable_reg $I64))) + (ConsumesFlags.ConsumesFlags (MInst.CSel dst cond if_true if_false) + (writable_reg_to_reg dst)))) + + ;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl imm (Type u64) Reg) @@ -1571,6 +1608,24 @@ ;; Sign extension helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Place a `Value` into a register, sign extending it to 32-bits +(decl put_in_reg_sext32 (Value) Reg) +(rule (put_in_reg_sext32 val @ (value_type (fits_in_32 ty))) + (extend (put_in_reg val) $true (ty_bits ty) 32)) + +;; 32/64-bit passthrough. +(rule (put_in_reg_sext32 val @ (value_type $I32)) (put_in_reg val)) +(rule (put_in_reg_sext32 val @ (value_type $I64)) (put_in_reg val)) + +;; Place a `Value` into a register, zero extending it to 32-bits +(decl put_in_reg_zext32 (Value) Reg) +(rule (put_in_reg_zext32 val @ (value_type (fits_in_32 ty))) + (extend (put_in_reg val) $false (ty_bits ty) 32)) + +;; 32/64-bit passthrough. +(rule (put_in_reg_zext32 val @ (value_type $I32)) (put_in_reg val)) +(rule (put_in_reg_zext32 val @ (value_type $I64)) (put_in_reg val)) + ;; Place a `Value` into a register, sign extending it to 64-bits (decl put_in_reg_sext64 (Value) Reg) (rule (put_in_reg_sext64 val @ (value_type (fits_in_32 ty))) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index a3e1b165c6..64ae824c91 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -695,3 +695,202 @@ (value_reg (alu_rs_imm_logic (ALUOp.EorNot64) $I64 x y))) (rule (lower (has_type $I128 (bxor_not x y))) (i128_alu_bitop (ALUOp.EorNot64) x y)) + +;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Shift for i8/i16/i32. +(rule (lower (has_type (fits_in_32 ty) (ishl x y))) + (value_reg (do_shift (ALUOp.Lsl32) ty (put_in_reg x) y))) + +;; Shift for i64. +(rule (lower (has_type $I64 (ishl x y))) + (value_reg (do_shift (ALUOp.Lsl64) $I64 (put_in_reg x) y))) + +;; Shift for i128. +(rule (lower (has_type $I128 (ishl x y))) + (lower_shl128 (put_in_regs x) (value_regs_get (put_in_regs y) 0))) + +;; lsl lo_lshift, src_lo, amt +;; lsl hi_lshift, src_hi, amt +;; mvn inv_amt, amt +;; lsr lo_rshift, src_lo, #1 +;; lsr lo_rshift, lo_rshift, inv_amt +;; orr maybe_hi, hi_lshift, lo_rshift +;; tst amt, #0x40 +;; csel dst_hi, lo_lshift, maybe_hi, ne +;; csel dst_lo, xzr, lo_lshift, ne +(decl lower_shl128 (ValueRegs Reg) ValueRegs) +(rule (lower_shl128 src amt) + (let ( + (src_lo Reg (value_regs_get src 0)) + (src_hi Reg (value_regs_get src 1)) + (lo_lshift Reg (alu_rrr (ALUOp.Lsl64) src_lo amt)) + (hi_lshift Reg (alu_rrr (ALUOp.Lsl64) src_hi amt)) + (inv_amt Reg (alu_rrr (ALUOp.OrrNot32) (zero_reg) amt)) + (lo_rshift Reg (alu_rrr (ALUOp.Lsr64) + (alu_rr_imm_shift (ALUOp.Lsr64) + src_lo + (imm_shift_from_u8 1)) + inv_amt)) + (maybe_hi Reg (alu_rrr (ALUOp.Orr64) hi_lshift lo_rshift)) + ) + (with_flags_2 + (tst64_imm amt (u64_into_imm_logic $I64 64)) + (csel (Cond.Ne) (zero_reg) lo_lshift) + (csel (Cond.Ne) lo_lshift maybe_hi)))) + +;; Shift for vector types. +(rule (lower (has_type (vec128 ty) (ishl x y))) + (let ( + (size VectorSize (vector_size ty)) + (shift Reg (vec_dup (put_in_reg y) size)) + ) + (value_reg (vec_rrr (VecALUOp.Sshl) (put_in_reg x) shift size)))) + +;; Helper function to emit a shift operation with the opcode specified and +;; the output type specified. The `Reg` provided is shifted by the `Value` +;; given. +;; +;; Note that this automatically handles the clif semantics of masking the +;; shift amount where necessary. +(decl do_shift (ALUOp Type Reg Value) Reg) + +;; 8/16-bit shift base case. +;; +;; When shifting for amounts larger than the size of the type, the CLIF shift +;; instructions implement a "wrapping" behaviour, such that an i8 << 8 is +;; equivalent to i8 << 0 +;; +;; On i32 and i64 types this matches what the aarch64 spec does, but on smaller +;; types (i16, i8) we need to do this manually, so we wrap the shift amount +;; with an AND instruction +(rule (do_shift op (fits_in_16 ty) x y) + (let ( + (shift_amt Reg (value_regs_get (put_in_regs y) 0)) + (masked_shift_amt Reg (alu_rr_imm_logic (ALUOp.And32) shift_amt (shift_mask ty))) + ) + (alu_rrr op x masked_shift_amt))) + +(decl shift_mask (Type) ImmLogic) +(extern constructor shift_mask shift_mask) + +;; 32/64-bit shift base cases. +(rule (do_shift op $I32 x y) (alu_rrr op x (value_regs_get (put_in_regs y) 0))) +(rule (do_shift op $I64 x y) (alu_rrr op x (value_regs_get (put_in_regs y) 0))) + +;; Special case for shifting by a constant value where the value can fit into an +;; `ImmShift`. +;; +;; Note that this rule explicitly has a higher priority than the others +;; to ensure it's attempted first, otherwise the type-based filters on the +;; previous rules seem to take priority over this rule. +(rule 1 (do_shift op ty x (def_inst (iconst (imm_shift_from_imm64 Reg { - match self { - ResultRegImmShift::Reg(r) => r, - _ => panic!("Unwrapped ResultRegImmShift, expected reg, got: {:?}", self), - } - } -} - //============================================================================ // Lowering: convert instruction inputs to forms that we can use. @@ -518,28 +509,6 @@ pub(crate) fn alu_inst_imm12(op: ALUOp, rd: Writable, rn: Reg, rm: ResultRS } } -pub(crate) fn alu_inst_immshift( - op: ALUOp, - rd: Writable, - rn: Reg, - rm: ResultRegImmShift, -) -> Inst { - match rm { - ResultRegImmShift::ImmShift(immshift) => Inst::AluRRImmShift { - alu_op: op, - rd, - rn, - immshift, - }, - ResultRegImmShift::Reg(rm) => Inst::AluRRR { - alu_op: op, - rd, - rn, - rm, - }, - } -} - //============================================================================ // Lowering: addressing mode support. Takes instruction directly, rather // than an `InsnInput`, to do more introspection. @@ -1516,43 +1485,6 @@ pub(crate) fn materialize_bool_result>( } } -pub(crate) fn lower_shift_amt>( - ctx: &mut C, - amt_input: InsnInput, - dst_ty: Type, - tmp_reg: Writable, -) -> ResultRegImmShift { - let amt_ty = ctx.input_ty(amt_input.insn, amt_input.input); - - match (dst_ty, amt_ty) { - // When shifting for amounts larger than the size of the type, the CLIF shift - // instructions implement a "wrapping" behaviour, such that an i8 << 8 is - // equivalent to i8 << 0 - // - // On i32 and i64 types this matches what the aarch64 spec does, but on smaller - // types (i16, i8) we need to do this manually, so we wrap the shift amount - // with an AND instruction - (I16 | I8, _) => { - // We can ignore the top half of the shift amount register if the type is I128 - let amt_reg = put_input_in_regs(ctx, amt_input).regs()[0]; - let mask = (ty_bits(dst_ty) - 1) as u64; - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::And32, - rd: tmp_reg, - rn: amt_reg, - imml: ImmLogic::maybe_from_u64(mask, I32).unwrap(), - }); - ResultRegImmShift::Reg(tmp_reg.to_reg()) - } - // TODO: We can use immlogic for i128 types here - (I128, _) | (_, I128) => { - // For I128 shifts, we need a register without immlogic - ResultRegImmShift::Reg(put_input_in_regs(ctx, amt_input).regs()[0]) - } - _ => put_input_in_reg_immshift(ctx, amt_input, ty_bits(dst_ty)), - } -} - /// This is target-word-size dependent. And it excludes booleans and reftypes. pub(crate) fn is_valid_atomic_transaction_ty(ty: Type) -> bool { match ty { diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 75fb31f9d2..84b7a6074e 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -248,4 +248,18 @@ where self.lower_ctx.sink_inst(load.atomic_load); self.put_in_reg(load.atomic_addr) } + + fn shift_mask(&mut self, ty: Type) -> ImmLogic { + let mask = (ty.bits() - 1) as u64; + ImmLogic::maybe_from_u64(mask, I32).unwrap() + } + + fn imm_shift_from_imm64(&mut self, val: Imm64, ty: Type) -> Option { + let imm_value = (val.bits() as u64) & ((ty.bits() - 1) as u64); + ImmShift::maybe_from_u64(imm_value) + } + + fn u64_into_imm_logic(&mut self, ty: Type, val: u64) -> ImmLogic { + ImmLogic::maybe_from_u64(val, ty).unwrap() + } } diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 9f20d1897c..a56afee6e1 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602 -src/prelude.isle 75a46b97817ad6a4c34e618b81e60876eec6fd1c83ac3ee174851e42045c951644663b2cbc31f1749ce2bc3ad9eb94fb0b877eb2c3bc4885cab7d7e87e9df1d6 -src/isa/aarch64/inst.isle 8e4b8e452cf06a368c2e1d930042027a5d3bd690ab46d498d959257e9b4461d17abf244838395cd80da1fe5e2e86fc43855fb5753ca4f1643538c2ae4b3b4a1e -src/isa/aarch64/lower.isle bc3db9c1e6ac186b918cc04f4d26af398f99ec36c8cdc20ec4d02d18dd57dba12e3184fea031b4ac97051c5e194a69666afb5e204807c818e6688c177f9c1b91 +src/prelude.isle 15c8dd937171bd0f619179e219422d43af0eb0ef9a6e88f23b2aa55776712e27342309dd3a4441876b2dfec7f16ce7fe13b3a926ace89b25cfc9577e7b1d1578 +src/isa/aarch64/inst.isle a6921329a85a252b8059657056ee0c4477304dff461bd58c39133a2870666b23a34c911be55e25e7887074cb54b640ff09998730af09b3c1ba792f434309af24 +src/isa/aarch64/lower.isle 1fb105feeabfd582e680900a5c3dd82950045761dc82ff30e92107cdac479ff046ce10853489d158c6b98961c7eab5d274ede3b0181c06f4ae67bf4556d6130d diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index ed4f7e2a0d..e4aac7e1e7 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -32,6 +32,7 @@ pub trait Context { fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u8; fn ty_bits_u16(&mut self, arg0: Type) -> u16; + fn fits_in_16(&mut self, arg0: Type) -> Option; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn ty_32_or_64(&mut self, arg0: Type) -> Option; @@ -60,10 +61,12 @@ pub trait Context { fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option; fn imm_logic_from_u64(&mut self, arg0: u64, arg1: Type) -> Option; fn imm_logic_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option; + fn imm_shift_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option; fn imm_shift_from_u8(&mut self, arg0: u8) -> ImmShift; fn imm12_from_u64(&mut self, arg0: u64) -> Option; fn u8_into_uimm5(&mut self, arg0: u8) -> UImm5; fn u8_into_imm12(&mut self, arg0: u8) -> Imm12; + fn u64_into_imm_logic(&mut self, arg0: Type, arg1: u64) -> ImmLogic; fn imm12_from_negated_u64(&mut self, arg0: u64) -> Option; fn lshl_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option; fn integral_ty(&mut self, arg0: Type) -> Option; @@ -80,15 +83,16 @@ pub trait Context { fn sinkable_atomic_load(&mut self, arg0: Value) -> Option; fn sink_atomic_load(&mut self, arg0: &SinkableAtomicLoad) -> Reg; fn safe_divisor_from_imm64(&mut self, arg0: Imm64) -> Option; + fn shift_mask(&mut self, arg0: Type) -> ImmLogic; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 259. +/// Internal type ProducesFlags: defined at src/prelude.isle line 263. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 262. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 266. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -1002,7 +1006,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 272. + // Rule at src/prelude.isle line 276. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1030,7 +1034,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 280. + // Rule at src/prelude.isle line 284. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1064,10 +1068,10 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 290. + // Rule at src/prelude.isle line 294. let expr0_0 = C::emit(ctx, &pattern1_0); - let expr1_0 = C::emit(ctx, &pattern3_0); - let expr2_0 = C::emit(ctx, &pattern5_0); + let expr1_0 = C::emit(ctx, &pattern5_0); + let expr2_0 = C::emit(ctx, &pattern3_0); let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1); return Some(expr3_0); } @@ -1120,7 +1124,7 @@ pub fn constructor_movz( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1378. + // Rule at src/isa/aarch64/inst.isle line 1384. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -1141,7 +1145,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1385. + // Rule at src/isa/aarch64/inst.isle line 1391. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -1164,7 +1168,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1392. + // Rule at src/isa/aarch64/inst.isle line 1398. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1188,7 +1192,7 @@ pub fn constructor_alu_rr_imm_shift( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1399. + // Rule at src/isa/aarch64/inst.isle line 1405. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmShift { @@ -1212,7 +1216,7 @@ pub fn constructor_alu_rrr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1406. + // Rule at src/isa/aarch64/inst.isle line 1412. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRR { @@ -1238,7 +1242,7 @@ pub fn constructor_vec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1413. + // Rule at src/isa/aarch64/inst.isle line 1419. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRR { @@ -1253,6 +1257,23 @@ pub fn constructor_vec_rrr( return Some(expr4_0); } +// Generated as internal constructor for term vec_dup. +pub fn constructor_vec_dup(ctx: &mut C, arg0: Reg, arg1: &VectorSize) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1426. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::VecDup { + rd: expr1_0, + rn: pattern0_0, + size: pattern1_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + // Generated as internal constructor for term alu_rr_imm12. pub fn constructor_alu_rr_imm12( ctx: &mut C, @@ -1263,7 +1284,7 @@ pub fn constructor_alu_rr_imm12( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1420. + // Rule at src/isa/aarch64/inst.isle line 1433. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImm12 { @@ -1289,7 +1310,7 @@ pub fn constructor_alu_rrr_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1427. + // Rule at src/isa/aarch64/inst.isle line 1440. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRShift { @@ -1316,7 +1337,7 @@ pub fn constructor_alu_rrr_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1434. + // Rule at src/isa/aarch64/inst.isle line 1447. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRExtend { @@ -1341,7 +1362,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1442. + // Rule at src/isa/aarch64/inst.isle line 1455. let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); let expr1_0 = C::get_extended_op(ctx, pattern2_0); let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; @@ -1360,7 +1381,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1449. + // Rule at src/isa/aarch64/inst.isle line 1462. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1383,7 +1404,7 @@ pub fn constructor_add64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1456. + // Rule at src/isa/aarch64/inst.isle line 1469. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::AddS64; @@ -1405,7 +1426,7 @@ pub fn constructor_add64_with_flags( pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1463. + // Rule at src/isa/aarch64/inst.isle line 1476. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Adc64; @@ -1431,7 +1452,7 @@ pub fn constructor_sub64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1470. + // Rule at src/isa/aarch64/inst.isle line 1483. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::SubS64; @@ -1453,7 +1474,7 @@ pub fn constructor_sub64_with_flags( pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1477. + // Rule at src/isa/aarch64/inst.isle line 1490. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Sbc64; @@ -1481,7 +1502,7 @@ pub fn constructor_vec_misc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1484. + // Rule at src/isa/aarch64/inst.isle line 1497. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecMisc { @@ -1507,7 +1528,7 @@ pub fn constructor_vec_rrr_long( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1491. + // Rule at src/isa/aarch64/inst.isle line 1504. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRRLong { @@ -1536,7 +1557,7 @@ pub fn constructor_vec_rrrr_long( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1501. + // Rule at src/isa/aarch64/inst.isle line 1514. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1566,7 +1587,7 @@ pub fn constructor_vec_rr_narrow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1509. + // Rule at src/isa/aarch64/inst.isle line 1522. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRNarrow { @@ -1590,7 +1611,7 @@ pub fn constructor_vec_rr_long( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1516. + // Rule at src/isa/aarch64/inst.isle line 1529. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRLong { @@ -1614,7 +1635,7 @@ pub fn constructor_mov_from_vec( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1523. + // Rule at src/isa/aarch64/inst.isle line 1536. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVec { @@ -1640,7 +1661,7 @@ pub fn constructor_mov_from_vec_signed( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1530. + // Rule at src/isa/aarch64/inst.isle line 1543. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovFromVecSigned { @@ -1667,7 +1688,7 @@ pub fn constructor_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1537. + // Rule at src/isa/aarch64/inst.isle line 1550. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::Extend { @@ -1686,7 +1707,7 @@ pub fn constructor_extend( pub fn constructor_load_acquire(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1544. + // Rule at src/isa/aarch64/inst.isle line 1557. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::LoadAcquire { @@ -1699,6 +1720,58 @@ pub fn constructor_load_acquire(ctx: &mut C, arg0: Type, arg1: Reg) return Some(expr4_0); } +// Generated as internal constructor for term tst64_imm. +pub fn constructor_tst64_imm( + ctx: &mut C, + arg0: Reg, + arg1: ImmLogic, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/inst.isle line 1567. + let expr0_0 = ALUOp::AndS64; + let expr1_0 = C::writable_zero_reg(ctx); + let expr2_0 = MInst::AluRRImmLogic { + alu_op: expr0_0, + rd: expr1_0, + rn: pattern0_0, + imml: pattern1_0, + }; + let expr3_0 = C::invalid_reg(ctx); + let expr4_0 = ProducesFlags::ProducesFlags { + inst: expr2_0, + result: expr3_0, + }; + return Some(expr4_0); +} + +// Generated as internal constructor for term csel. +pub fn constructor_csel( + ctx: &mut C, + arg0: &Cond, + arg1: Reg, + arg2: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1580. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::CSel { + rd: expr1_0, + cond: pattern0_0.clone(), + rn: pattern1_0, + rm: pattern2_0, + }; + let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr4_0 = ConsumesFlags::ConsumesFlags { + inst: expr2_0, + result: expr3_0, + }; + return Some(expr4_0); +} + // Generated as internal constructor for term imm. pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { let pattern0_0 = arg0; @@ -1710,7 +1783,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option }; if let Some(pattern3_0) = closure3() { if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) { - // Rule at src/isa/aarch64/inst.isle line 1562. + // Rule at src/isa/aarch64/inst.isle line 1599. let expr0_0 = ALUOp::Orr64; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern4_0)?; @@ -1718,35 +1791,87 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1554. + // Rule at src/isa/aarch64/inst.isle line 1591. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1558. + // Rule at src/isa/aarch64/inst.isle line 1595. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1569. + // Rule at src/isa/aarch64/inst.isle line 1606. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } return None; } +// Generated as internal constructor for term put_in_reg_sext32. +pub fn constructor_put_in_reg_sext32(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = C::value_type(ctx, pattern0_0); + if pattern1_0 == I32 { + // Rule at src/isa/aarch64/inst.isle line 1617. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + return Some(expr0_0); + } + if pattern1_0 == I64 { + // Rule at src/isa/aarch64/inst.isle line 1618. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + return Some(expr0_0); + } + if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { + // Rule at src/isa/aarch64/inst.isle line 1613. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0: bool = true; + let expr2_0 = C::ty_bits(ctx, pattern2_0); + let expr3_0: u8 = 32; + let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?; + return Some(expr4_0); + } + return None; +} + +// Generated as internal constructor for term put_in_reg_zext32. +pub fn constructor_put_in_reg_zext32(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = C::value_type(ctx, pattern0_0); + if pattern1_0 == I32 { + // Rule at src/isa/aarch64/inst.isle line 1626. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + return Some(expr0_0); + } + if pattern1_0 == I64 { + // Rule at src/isa/aarch64/inst.isle line 1627. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + return Some(expr0_0); + } + if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { + // Rule at src/isa/aarch64/inst.isle line 1622. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0: bool = false; + let expr2_0 = C::ty_bits(ctx, pattern2_0); + let expr3_0: u8 = 32; + let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?; + return Some(expr4_0); + } + return None; +} + // Generated as internal constructor for term put_in_reg_sext64. pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1580. + // Rule at src/isa/aarch64/inst.isle line 1635. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1576. + // Rule at src/isa/aarch64/inst.isle line 1631. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = true; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -1762,12 +1887,12 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op let pattern0_0 = arg0; let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1588. + // Rule at src/isa/aarch64/inst.isle line 1643. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { - // Rule at src/isa/aarch64/inst.isle line 1584. + // Rule at src/isa/aarch64/inst.isle line 1639. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0: bool = false; let expr2_0 = C::ty_bits(ctx, pattern2_0); @@ -1781,7 +1906,7 @@ pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Op // Generated as internal constructor for term trap_if_zero_divisor. pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/aarch64/inst.isle line 1593. + // Rule at src/isa/aarch64/inst.isle line 1648. let expr0_0 = C::cond_br_zero(ctx, pattern0_0); let expr1_0 = C::trap_code_division_by_zero(ctx); let expr2_0 = MInst::TrapIf { @@ -1796,12 +1921,12 @@ pub fn constructor_trap_if_zero_divisor(ctx: &mut C, arg0: Reg) -> O pub fn constructor_size_from_ty(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1599. + // Rule at src/isa/aarch64/inst.isle line 1654. let expr0_0 = OperandSize::Size64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 1598. + // Rule at src/isa/aarch64/inst.isle line 1653. let expr0_0 = OperandSize::Size32; return Some(expr0_0); } @@ -1818,7 +1943,7 @@ pub fn constructor_trap_if_div_overflow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1605. + // Rule at src/isa/aarch64/inst.isle line 1660. let expr0_0 = constructor_adds_op(ctx, pattern0_0)?; let expr1_0 = C::writable_zero_reg(ctx); let expr2_0: u8 = 1; @@ -1862,12 +1987,12 @@ pub fn constructor_trap_if_div_overflow( pub fn constructor_adds_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == I64 { - // Rule at src/isa/aarch64/inst.isle line 1625. + // Rule at src/isa/aarch64/inst.isle line 1680. let expr0_0 = ALUOp::AddS64; return Some(expr0_0); } if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) { - // Rule at src/isa/aarch64/inst.isle line 1624. + // Rule at src/isa/aarch64/inst.isle line 1679. let expr0_0 = ALUOp::AddS32; return Some(expr0_0); } @@ -1901,7 +2026,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0) { let pattern9_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1655. + // Rule at src/isa/aarch64/inst.isle line 1710. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern8_0)?; @@ -1932,7 +2057,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( C::lshl_from_imm64(ctx, pattern10_1, pattern12_0) { let pattern14_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1661. + // Rule at src/isa/aarch64/inst.isle line 1716. let expr0_0 = C::put_in_reg(ctx, pattern14_0); let expr1_0 = C::put_in_reg(ctx, pattern7_0); let expr2_0 = constructor_alu_rrr_shift( @@ -1969,7 +2094,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 1653. + // Rule at src/isa/aarch64/inst.isle line 1708. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -1999,7 +2124,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 1659. + // Rule at src/isa/aarch64/inst.isle line 1714. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -2020,7 +2145,7 @@ pub fn constructor_alu_rs_imm_logic_commutative( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 1649. + // Rule at src/isa/aarch64/inst.isle line 1704. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -2054,7 +2179,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern9_0) = C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0) { - // Rule at src/isa/aarch64/inst.isle line 1669. + // Rule at src/isa/aarch64/inst.isle line 1724. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?; @@ -2084,7 +2209,7 @@ pub fn constructor_alu_rs_imm_logic( if let Some(pattern14_0) = C::lshl_from_imm64(ctx, pattern11_1, pattern13_0) { - // Rule at src/isa/aarch64/inst.isle line 1671. + // Rule at src/isa/aarch64/inst.isle line 1726. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern8_0); let expr2_0 = constructor_alu_rrr_shift( @@ -2105,7 +2230,7 @@ pub fn constructor_alu_rs_imm_logic( _ => {} } } - // Rule at src/isa/aarch64/inst.isle line 1667. + // Rule at src/isa/aarch64/inst.isle line 1722. let expr0_0 = C::put_in_reg(ctx, pattern2_0); let expr1_0 = C::put_in_reg(ctx, pattern3_0); let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?; @@ -2122,7 +2247,7 @@ pub fn constructor_i128_alu_bitop( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1679. + // Rule at src/isa/aarch64/inst.isle line 1734. let expr0_0 = C::put_in_regs(ctx, pattern1_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2238,6 +2363,39 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 706. + let expr0_0 = ALUOp::Lsl64; + let expr1_0: Type = I64; + let expr2_0 = C::put_in_reg(ctx, pattern7_0); + let expr3_0 = + constructor_do_shift(ctx, &expr0_0, expr1_0, expr2_0, pattern7_1)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Ushr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 797. + let expr0_0 = ALUOp::Lsr64; + let expr1_0: Type = I64; + let expr2_0 = constructor_put_in_reg_zext64(ctx, pattern7_0)?; + let expr3_0 = + constructor_do_shift(ctx, &expr0_0, expr1_0, expr2_0, pattern7_1)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Sshr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 849. + let expr0_0 = ALUOp::Asr64; + let expr1_0: Type = I64; + let expr2_0 = constructor_put_in_reg_sext64(ctx, pattern7_0)?; + let expr3_0 = + constructor_do_shift(ctx, &expr0_0, expr1_0, expr2_0, pattern7_1)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } _ => {} } } @@ -2371,6 +2529,39 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 710. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = C::put_in_regs(ctx, pattern7_1); + let expr2_0: usize = 0; + let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0); + let expr4_0 = constructor_lower_shl128(ctx, expr0_0, expr3_0)?; + return Some(expr4_0); + } + &Opcode::Ushr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 801. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = C::put_in_regs(ctx, pattern7_1); + let expr2_0: usize = 0; + let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0); + let expr4_0 = constructor_lower_ushr128(ctx, expr0_0, expr3_0)?; + return Some(expr4_0); + } + &Opcode::Sshr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 853. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = C::put_in_regs(ctx, pattern7_1); + let expr2_0: usize = 0; + let expr3_0 = C::value_regs_get(ctx, expr1_0, expr2_0); + let expr4_0 = constructor_lower_sshr128(ctx, expr0_0, expr3_0)?; + return Some(expr4_0); + } _ => {} } } @@ -3189,6 +3380,36 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 702. + let expr0_0 = ALUOp::Lsl32; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = + constructor_do_shift(ctx, &expr0_0, pattern3_0, expr1_0, pattern7_1)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Ushr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 793. + let expr0_0 = ALUOp::Lsr32; + let expr1_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?; + let expr2_0 = + constructor_do_shift(ctx, &expr0_0, pattern3_0, expr1_0, pattern7_1)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Sshr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 845. + let expr0_0 = ALUOp::Asr32; + let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern7_0)?; + let expr2_0 = + constructor_do_shift(ctx, &expr0_0, pattern3_0, expr1_0, pattern7_1)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } _ => {} } } @@ -3976,6 +4197,54 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 743. + let expr0_0 = constructor_vector_size(ctx, pattern3_0)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = constructor_vec_dup(ctx, expr1_0, &expr0_0)?; + let expr3_0 = VecALUOp::Sshl; + let expr4_0 = C::put_in_reg(ctx, pattern7_0); + let expr5_0 = + constructor_vec_rrr(ctx, &expr3_0, expr4_0, expr2_0, &expr0_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some(expr6_0); + } + &Opcode::Ushr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 805. + let expr0_0 = constructor_vector_size(ctx, pattern3_0)?; + let expr1_0 = ALUOp::Sub32; + let expr2_0 = C::zero_reg(ctx); + let expr3_0 = C::put_in_reg(ctx, pattern7_1); + let expr4_0 = constructor_alu_rrr(ctx, &expr1_0, expr2_0, expr3_0)?; + let expr5_0 = constructor_vec_dup(ctx, expr4_0, &expr0_0)?; + let expr6_0 = VecALUOp::Ushl; + let expr7_0 = C::put_in_reg(ctx, pattern7_0); + let expr8_0 = + constructor_vec_rrr(ctx, &expr6_0, expr7_0, expr5_0, &expr0_0)?; + let expr9_0 = C::value_reg(ctx, expr8_0); + return Some(expr9_0); + } + &Opcode::Sshr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 859. + let expr0_0 = constructor_vector_size(ctx, pattern3_0)?; + let expr1_0 = ALUOp::Sub32; + let expr2_0 = C::zero_reg(ctx); + let expr3_0 = C::put_in_reg(ctx, pattern7_1); + let expr4_0 = constructor_alu_rrr(ctx, &expr1_0, expr2_0, expr3_0)?; + let expr5_0 = constructor_vec_dup(ctx, expr4_0, &expr0_0)?; + let expr6_0 = VecALUOp::Sshl; + let expr7_0 = C::put_in_reg(ctx, pattern7_0); + let expr8_0 = + constructor_vec_rrr(ctx, &expr6_0, expr7_0, expr5_0, &expr0_0)?; + let expr9_0 = C::value_reg(ctx, expr8_0); + return Some(expr9_0); + } _ => {} } } @@ -4148,3 +4417,201 @@ pub fn constructor_orr_not_op(ctx: &mut C, arg0: Type) -> Option( + ctx: &mut C, + arg0: ValueRegs, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/lower.isle line 723. + let expr0_0: usize = 0; + let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr2_0: usize = 1; + let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr4_0 = ALUOp::Lsl64; + let expr5_0 = constructor_alu_rrr(ctx, &expr4_0, expr1_0, pattern1_0)?; + let expr6_0 = ALUOp::Lsl64; + let expr7_0 = constructor_alu_rrr(ctx, &expr6_0, expr3_0, pattern1_0)?; + let expr8_0 = ALUOp::OrrNot32; + let expr9_0 = C::zero_reg(ctx); + let expr10_0 = constructor_alu_rrr(ctx, &expr8_0, expr9_0, pattern1_0)?; + let expr11_0 = ALUOp::Lsr64; + let expr12_0 = ALUOp::Lsr64; + let expr13_0: u8 = 1; + let expr14_0 = C::imm_shift_from_u8(ctx, expr13_0); + let expr15_0 = constructor_alu_rr_imm_shift(ctx, &expr12_0, expr1_0, expr14_0)?; + let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr15_0, expr10_0)?; + let expr17_0 = ALUOp::Orr64; + let expr18_0 = constructor_alu_rrr(ctx, &expr17_0, expr7_0, expr16_0)?; + let expr19_0: Type = I64; + let expr20_0: u64 = 64; + let expr21_0 = C::u64_into_imm_logic(ctx, expr19_0, expr20_0); + let expr22_0 = constructor_tst64_imm(ctx, pattern1_0, expr21_0)?; + let expr23_0 = Cond::Ne; + let expr24_0 = C::zero_reg(ctx); + let expr25_0 = constructor_csel(ctx, &expr23_0, expr24_0, expr5_0)?; + let expr26_0 = Cond::Ne; + let expr27_0 = constructor_csel(ctx, &expr26_0, expr5_0, expr18_0)?; + let expr28_0 = constructor_with_flags_2(ctx, &expr22_0, &expr25_0, &expr27_0)?; + return Some(expr28_0); +} + +// Generated as internal constructor for term do_shift. +pub fn constructor_do_shift( + ctx: &mut C, + arg0: &ALUOp, + arg1: Type, + arg2: Reg, + arg3: Value, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + if let Some(pattern4_0) = C::def_inst(ctx, pattern3_0) { + let pattern5_0 = C::inst_data(ctx, pattern4_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern6_0, + imm: pattern6_1, + } = &pattern5_0 + { + if let &Opcode::Iconst = &pattern6_0 { + let closure8 = || { + return Some(pattern1_0); + }; + if let Some(pattern8_0) = closure8() { + if let Some(pattern9_0) = C::imm_shift_from_imm64(ctx, pattern6_1, pattern8_0) { + // Rule at src/isa/aarch64/lower.isle line 787. + let expr0_0 = + constructor_alu_rr_imm_shift(ctx, pattern0_0, pattern2_0, pattern9_0)?; + return Some(expr0_0); + } + } + } + } + } + let pattern0_0 = arg0; + let pattern1_0 = arg1; + if pattern1_0 == I32 { + let pattern3_0 = arg2; + let pattern4_0 = arg3; + // Rule at src/isa/aarch64/lower.isle line 778. + let expr0_0 = C::put_in_regs(ctx, pattern4_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0 = constructor_alu_rrr(ctx, pattern0_0, pattern3_0, expr2_0)?; + return Some(expr3_0); + } + if pattern1_0 == I64 { + let pattern3_0 = arg2; + let pattern4_0 = arg3; + // Rule at src/isa/aarch64/lower.isle line 779. + let expr0_0 = C::put_in_regs(ctx, pattern4_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0 = constructor_alu_rrr(ctx, pattern0_0, pattern3_0, expr2_0)?; + return Some(expr3_0); + } + if let Some(pattern2_0) = C::fits_in_16(ctx, pattern1_0) { + let pattern3_0 = arg2; + let pattern4_0 = arg3; + // Rule at src/isa/aarch64/lower.isle line 767. + let expr0_0 = C::put_in_regs(ctx, pattern4_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0 = ALUOp::And32; + let expr4_0 = C::shift_mask(ctx, pattern2_0); + let expr5_0 = constructor_alu_rr_imm_logic(ctx, &expr3_0, expr2_0, expr4_0)?; + let expr6_0 = constructor_alu_rrr(ctx, pattern0_0, pattern3_0, expr5_0)?; + return Some(expr6_0); + } + return None; +} + +// Generated as internal constructor for term lower_ushr128. +pub fn constructor_lower_ushr128( + ctx: &mut C, + arg0: ValueRegs, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/lower.isle line 822. + let expr0_0: usize = 0; + let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr2_0: usize = 1; + let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr4_0 = ALUOp::Lsr64; + let expr5_0 = constructor_alu_rrr(ctx, &expr4_0, expr1_0, pattern1_0)?; + let expr6_0 = ALUOp::Lsr64; + let expr7_0 = constructor_alu_rrr(ctx, &expr6_0, expr3_0, pattern1_0)?; + let expr8_0 = ALUOp::OrrNot32; + let expr9_0 = C::zero_reg(ctx); + let expr10_0 = constructor_alu_rrr(ctx, &expr8_0, expr9_0, pattern1_0)?; + let expr11_0 = ALUOp::Lsl64; + let expr12_0 = ALUOp::Lsl64; + let expr13_0: u8 = 1; + let expr14_0 = C::imm_shift_from_u8(ctx, expr13_0); + let expr15_0 = constructor_alu_rr_imm_shift(ctx, &expr12_0, expr3_0, expr14_0)?; + let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr15_0, expr10_0)?; + let expr17_0 = ALUOp::Orr64; + let expr18_0 = constructor_alu_rrr(ctx, &expr17_0, expr5_0, expr16_0)?; + let expr19_0: Type = I64; + let expr20_0: u64 = 64; + let expr21_0 = C::u64_into_imm_logic(ctx, expr19_0, expr20_0); + let expr22_0 = constructor_tst64_imm(ctx, pattern1_0, expr21_0)?; + let expr23_0 = Cond::Ne; + let expr24_0 = constructor_csel(ctx, &expr23_0, expr7_0, expr18_0)?; + let expr25_0 = Cond::Ne; + let expr26_0 = C::zero_reg(ctx); + let expr27_0 = constructor_csel(ctx, &expr25_0, expr26_0, expr7_0)?; + let expr28_0 = constructor_with_flags_2(ctx, &expr22_0, &expr24_0, &expr27_0)?; + return Some(expr28_0); +} + +// Generated as internal constructor for term lower_sshr128. +pub fn constructor_lower_sshr128( + ctx: &mut C, + arg0: ValueRegs, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/aarch64/lower.isle line 877. + let expr0_0: usize = 0; + let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr2_0: usize = 1; + let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr4_0 = ALUOp::Lsr64; + let expr5_0 = constructor_alu_rrr(ctx, &expr4_0, expr1_0, pattern1_0)?; + let expr6_0 = ALUOp::Asr64; + let expr7_0 = constructor_alu_rrr(ctx, &expr6_0, expr3_0, pattern1_0)?; + let expr8_0 = ALUOp::OrrNot32; + let expr9_0 = C::zero_reg(ctx); + let expr10_0 = constructor_alu_rrr(ctx, &expr8_0, expr9_0, pattern1_0)?; + let expr11_0 = ALUOp::Lsl64; + let expr12_0 = ALUOp::Lsl64; + let expr13_0: u8 = 1; + let expr14_0 = C::imm_shift_from_u8(ctx, expr13_0); + let expr15_0 = constructor_alu_rr_imm_shift(ctx, &expr12_0, expr3_0, expr14_0)?; + let expr16_0 = constructor_alu_rrr(ctx, &expr11_0, expr15_0, expr10_0)?; + let expr17_0 = ALUOp::Asr64; + let expr18_0: u8 = 63; + let expr19_0 = C::imm_shift_from_u8(ctx, expr18_0); + let expr20_0 = constructor_alu_rr_imm_shift(ctx, &expr17_0, expr3_0, expr19_0)?; + let expr21_0 = ALUOp::Orr64; + let expr22_0 = constructor_alu_rrr(ctx, &expr21_0, expr5_0, expr16_0)?; + let expr23_0: Type = I64; + let expr24_0: u64 = 64; + let expr25_0 = C::u64_into_imm_logic(ctx, expr23_0, expr24_0); + let expr26_0 = constructor_tst64_imm(ctx, pattern1_0, expr25_0)?; + let expr27_0 = Cond::Ne; + let expr28_0 = constructor_csel(ctx, &expr27_0, expr7_0, expr22_0)?; + let expr29_0 = Cond::Ne; + let expr30_0 = constructor_csel(ctx, &expr29_0, expr20_0, expr7_0)?; + let expr31_0 = constructor_with_flags_2(ctx, &expr26_0, &expr28_0, &expr30_0)?; + return Some(expr31_0); +} diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 133d722e04..1d143b7316 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -42,7 +42,7 @@ pub(crate) fn lower_insn_to_regs>( return Ok(()); } - let implemented_in_isle = |ctx: &mut C| { + let implemented_in_isle = |ctx: &mut C| -> ! { unreachable!( "implemented in ISLE: inst = `{}`, type = `{:?}`", ctx.dfg().display_inst(insn), @@ -88,81 +88,7 @@ pub(crate) fn lower_insn_to_regs>( | Opcode::BorNot | Opcode::BxorNot => implemented_in_isle(ctx), - Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => { - let out_regs = get_output_reg(ctx, outputs[0]); - let ty = ty.unwrap(); - if ty == I128 { - let src = put_input_in_regs(ctx, inputs[0]); - let amt = lower_shift_amt(ctx, inputs[1], ty, out_regs.regs()[0]).unwrap_reg(); - - match op { - Opcode::Ishl => emit_shl_i128(ctx, src, out_regs, amt), - Opcode::Ushr => { - emit_shr_i128(ctx, src, out_regs, amt, /* is_signed = */ false) - } - Opcode::Sshr => { - emit_shr_i128(ctx, src, out_regs, amt, /* is_signed = */ true) - } - _ => unreachable!(), - }; - } else if !ty.is_vector() { - let rd = out_regs.only_reg().unwrap(); - let size = OperandSize::from_bits(ty_bits(ty)); - let narrow_mode = match (op, size) { - (Opcode::Ishl, _) => NarrowValueMode::None, - (Opcode::Ushr, OperandSize::Size64) => NarrowValueMode::ZeroExtend64, - (Opcode::Ushr, OperandSize::Size32) => NarrowValueMode::ZeroExtend32, - (Opcode::Sshr, OperandSize::Size64) => NarrowValueMode::SignExtend64, - (Opcode::Sshr, OperandSize::Size32) => NarrowValueMode::SignExtend32, - _ => unreachable!(), - }; - let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); - let rm = lower_shift_amt(ctx, inputs[1], ty, out_regs.regs()[0]); - let alu_op = match op { - Opcode::Ishl => choose_32_64(ty, ALUOp::Lsl32, ALUOp::Lsl64), - Opcode::Ushr => choose_32_64(ty, ALUOp::Lsr32, ALUOp::Lsr64), - Opcode::Sshr => choose_32_64(ty, ALUOp::Asr32, ALUOp::Asr64), - _ => unreachable!(), - }; - ctx.emit(alu_inst_immshift(alu_op, rd, rn, rm)); - } else { - let rd = out_regs.only_reg().unwrap(); - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - let size = VectorSize::from_ty(ty); - let (alu_op, is_right_shift) = match op { - Opcode::Ishl => (VecALUOp::Sshl, false), - Opcode::Ushr => (VecALUOp::Ushl, true), - Opcode::Sshr => (VecALUOp::Sshl, true), - _ => unreachable!(), - }; - - let rm = if is_right_shift { - // Right shifts are implemented with a negative left shift. - let tmp = ctx.alloc_tmp(I32).only_reg().unwrap(); - let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - let rn = zero_reg(); - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Sub32, - rd: tmp, - rn, - rm, - }); - tmp.to_reg() - } else { - put_input_in_reg(ctx, inputs[1], NarrowValueMode::None) - }; - - ctx.emit(Inst::VecDup { rd, rn: rm, size }); - - ctx.emit(Inst::VecRRR { - alu_op, - rd, - rn, - rm: rd.to_reg(), - size, - }); - } - } + Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => implemented_in_isle(ctx), Opcode::Rotr | Opcode::Rotl => { // aarch64 doesn't have a left-rotate instruction, but a left rotation of K places is diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 58a8cbe7fe..72f336c4ea 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602 -src/prelude.isle 75a46b97817ad6a4c34e618b81e60876eec6fd1c83ac3ee174851e42045c951644663b2cbc31f1749ce2bc3ad9eb94fb0b877eb2c3bc4885cab7d7e87e9df1d6 +src/prelude.isle 15c8dd937171bd0f619179e219422d43af0eb0ef9a6e88f23b2aa55776712e27342309dd3a4441876b2dfec7f16ce7fe13b3a926ace89b25cfc9577e7b1d1578 src/isa/x64/inst.isle 1a44ccc0c2cad90447762848461fcae714216ef058d42bdba89330a6008061526e92bbf1c17055c465b20fc75d98d1faa34feda8b22fa7ae0504a0f808798b41 src/isa/x64/lower.isle c7943201b32e9eb9726466e8cc417f7e84c4c4052de31e05ab6e0ad7502a587cf1d7d9835703c4ff5a506390f7a0668741e7f3feaa1edda6396571a425949fc9 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 6da0edeff3..bbfa55047f 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -32,6 +32,7 @@ pub trait Context { fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u8; fn ty_bits_u16(&mut self, arg0: Type) -> u16; + fn fits_in_16(&mut self, arg0: Type) -> Option; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn ty_32_or_64(&mut self, arg0: Type) -> Option; @@ -76,13 +77,13 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 259. +/// Internal type ProducesFlags: defined at src/prelude.isle line 263. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 262. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 266. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -132,7 +133,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 272. + // Rule at src/prelude.isle line 276. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -160,7 +161,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 280. + // Rule at src/prelude.isle line 284. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -194,10 +195,10 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 290. + // Rule at src/prelude.isle line 294. let expr0_0 = C::emit(ctx, &pattern1_0); - let expr1_0 = C::emit(ctx, &pattern3_0); - let expr2_0 = C::emit(ctx, &pattern5_0); + let expr1_0 = C::emit(ctx, &pattern5_0); + let expr2_0 = C::emit(ctx, &pattern3_0); let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1); return Some(expr3_0); } diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index b3354d807e..1fe9b067fb 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -105,6 +105,14 @@ macro_rules! isle_prelude_methods { ty.bits() } + fn fits_in_16(&mut self, ty: Type) -> Option { + if ty.bits() <= 16 { + Some(ty) + } else { + None + } + } + #[inline] fn fits_in_32(&mut self, ty: Type) -> Option { if ty.bits() <= 32 { diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 1d8c5ad87c..9c1d5b72be 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -148,6 +148,10 @@ ;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; An extractor that only matches types that can fit in 16 bits. +(decl fits_in_16 (Type) Type) +(extern extractor fits_in_16 fits_in_16) + ;; An extractor that only matches types that can fit in 32 bits. (decl fits_in_32 (Type) Type) (extern extractor fits_in_32 fits_in_32) @@ -287,12 +291,15 @@ ;; `ValueRegs` containing the first consumer's result and then the second ;; consumer's result. (decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs) -(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result) +(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst _producer_result) (ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1) (ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2)) (let ((_x Unit (emit producer_inst)) - (_y Unit (emit consumer_inst_1)) - (_z Unit (emit consumer_inst_2))) + ;; Note that the order of emission here is swapped, as this seems + ;; to generate better register allocation for now with fewer + ;; `mov` instructions. + (_y Unit (emit consumer_inst_2)) + (_z Unit (emit consumer_inst_1))) (value_regs consumer_result_1 consumer_result_2))) ;;;; Helpers for Working with TrapCode ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/filetests/filetests/isa/aarch64/bitops.clif b/cranelift/filetests/filetests/isa/aarch64/bitops.clif index c3b1820794..1156aa83ca 100644 --- a/cranelift/filetests/filetests/isa/aarch64/bitops.clif +++ b/cranelift/filetests/filetests/isa/aarch64/bitops.clif @@ -742,15 +742,15 @@ block0(v0: i128, v1: i8): return v2 } -; check: orn w3, wzr, w2 -; nextln: lsr x4, x0, #1 -; nextln: lsl x1, x1, x2 -; nextln: lsr x3, x4, x3 -; nextln: lsl x0, x0, x2 +; check: lsl x4, x0, x2 +; nextln: lsl x3, x1, x2 +; nextln: orn w1, wzr, w2 +; nextln: lsr x0, x0, #1 +; nextln: lsr x0, x0, x1 +; nextln: orr x0, x3, x0 ; nextln: ands xzr, x2, #64 -; nextln: orr x1, x1, x3 -; nextln: csel x1, x0, x1, ne -; nextln: csel x0, xzr, x0, ne +; nextln: csel x1, x4, x0, ne +; nextln: csel x0, xzr, x4, ne ; nextln: ret @@ -760,15 +760,15 @@ block0(v0: i128, v1: i128): return v2 } -; check: orn w3, wzr, w2 -; nextln: lsr x4, x0, #1 +; check: lsl x3, x0, x2 ; nextln: lsl x1, x1, x2 -; nextln: lsr x3, x4, x3 -; nextln: lsl x0, x0, x2 +; nextln: orn w4, wzr, w2 +; nextln: lsr x0, x0, #1 +; nextln: lsr x0, x0, x4 +; nextln: orr x0, x1, x0 ; nextln: ands xzr, x2, #64 -; nextln: orr x1, x1, x3 -; nextln: csel x1, x0, x1, ne -; nextln: csel x0, xzr, x0, ne +; nextln: csel x1, x3, x0, ne +; nextln: csel x0, xzr, x3, ne ; nextln: ret @@ -778,16 +778,15 @@ block0(v0: i128, v1: i8): return v2 } -; check: orn w3, wzr, w2 -; nextln: lsl x4, x1, #1 -; nextln: lsr x0, x0, x2 -; nextln: lsl x3, x4, x3 -; nextln: lsr x1, x1, x2 +; check: lsr x3, x0, x2 +; nextln: lsr x0, x1, x2 +; nextln: orn w4, wzr, w2 +; nextln: lsl x1, x1, #1 +; nextln: lsl x1, x1, x4 +; nextln: orr x3, x3, x1 ; nextln: ands xzr, x2, #64 -; nextln: orr x0, x0, x3 -; nextln: csel x2, xzr, x1, ne -; nextln: csel x0, x1, x0, ne -; nextln: mov x1, x2 +; nextln: csel x1, xzr, x0, ne +; nextln: csel x0, x0, x3, ne ; nextln: ret @@ -797,16 +796,15 @@ block0(v0: i128, v1: i128): return v2 } -; check: orn w3, wzr, w2 -; nextln: lsl x4, x1, #1 -; nextln: lsr x0, x0, x2 -; nextln: lsl x3, x4, x3 -; nextln: lsr x1, x1, x2 +; check: lsr x3, x0, x2 +; nextln: lsr x0, x1, x2 +; nextln: orn w4, wzr, w2 +; nextln: lsl x1, x1, #1 +; nextln: lsl x1, x1, x4 +; nextln: orr x3, x3, x1 ; nextln: ands xzr, x2, #64 -; nextln: orr x0, x0, x3 -; nextln: csel x2, xzr, x1, ne -; nextln: csel x0, x1, x0, ne -; nextln: mov x1, x2 +; nextln: csel x1, xzr, x0, ne +; nextln: csel x0, x0, x3, ne ; nextln: ret @@ -816,16 +814,16 @@ block0(v0: i128, v1: i8): return v2 } -; check: orn w3, wzr, w2 -; nextln: lsl x4, x1, #1 -; nextln: lsr x0, x0, x2 -; nextln: lsl x4, x4, x3 -; nextln: asr x3, x1, x2 -; nextln: ands xzr, x2, #64 +; check: lsr x3, x0, x2 +; nextln: asr x0, x1, x2 +; nextln: orn w4, wzr, w2 +; nextln: lsl x5, x1, #1 +; nextln: lsl x4, x5, x4 ; nextln: asr x1, x1, #63 -; nextln: orr x0, x0, x4 -; nextln: csel x1, x1, x3, ne -; nextln: csel x0, x3, x0, ne +; nextln: orr x3, x3, x4 +; nextln: ands xzr, x2, #64 +; nextln: csel x1, x1, x0, ne +; nextln: csel x0, x0, x3, ne ; nextln: ret @@ -835,14 +833,14 @@ block0(v0: i128, v1: i128): return v2 } -; check: orn w3, wzr, w2 -; nextln: lsl x4, x1, #1 -; nextln: lsr x0, x0, x2 -; nextln: lsl x4, x4, x3 -; nextln: asr x3, x1, x2 -; nextln: ands xzr, x2, #64 +; check: lsr x3, x0, x2 +; nextln: asr x0, x1, x2 +; nextln: orn w4, wzr, w2 +; nextln: lsl x5, x1, #1 +; nextln: lsl x4, x5, x4 ; nextln: asr x1, x1, #63 -; nextln: orr x0, x0, x4 -; nextln: csel x1, x1, x3, ne -; nextln: csel x0, x3, x0, ne +; nextln: orr x3, x3, x4 +; nextln: ands xzr, x2, #64 +; nextln: csel x1, x1, x0, ne +; nextln: csel x0, x0, x3, ne ; nextln: ret diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index 55a2f48b4d..57791f92fd 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -917,10 +917,11 @@ block0(v0: i128, v1: i128): ; nextln: cmovzq %rcx, %rax ; nextln: orq %rdi, %rax ; nextln: testq $$64, %rdx +; nextln: movq %rsi, %rdi +; nextln: cmovzq %rax, %rdi ; nextln: cmovzq %rsi, %rcx -; nextln: cmovzq %rax, %rsi ; nextln: movq %rcx, %rax -; nextln: movq %rsi, %rdx +; nextln: movq %rdi, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret @@ -946,13 +947,12 @@ block0(v0: i128, v1: i128): ; nextln: testq $$127, %rdx ; nextln: cmovzq %rcx, %rax ; nextln: orq %rdi, %rax -; nextln: xorq %rcx, %rcx +; nextln: xorq %rdi, %rdi ; nextln: testq $$64, %rdx -; nextln: movq %rsi, %rdi -; nextln: cmovzq %rax, %rdi -; nextln: cmovzq %rsi, %rcx -; nextln: movq %rdi, %rax -; nextln: movq %rcx, %rdx +; nextln: cmovzq %rsi, %rdi +; nextln: cmovzq %rax, %rsi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret @@ -1000,24 +1000,24 @@ block0(v0: i128, v1: i128): ; check: pushq %rbp ; nextln: movq %rsp, %rbp -; nextln: movq %rdi, %rax +; nextln: movq %rdi, %r9 +; nextln: movq %rdx, %rcx +; nextln: shlq %cl, %r9 +; nextln: movq %rsi, %rax ; nextln: movq %rdx, %rcx ; nextln: shlq %cl, %rax -; nextln: movq %rsi, %r8 -; nextln: movq %rdx, %rcx -; nextln: shlq %cl, %r8 ; nextln: movl $$64, %ecx ; nextln: subq %rdx, %rcx -; nextln: movq %rdi, %r9 -; nextln: shrq %cl, %r9 -; nextln: xorq %rcx, %rcx +; nextln: movq %rdi, %r10 +; nextln: shrq %cl, %r10 +; nextln: xorq %r8, %r8 ; nextln: testq $$127, %rdx -; nextln: cmovzq %rcx, %r9 -; nextln: orq %r8, %r9 +; nextln: cmovzq %r8, %r10 +; nextln: orq %rax, %r10 ; nextln: testq $$64, %rdx -; nextln: movq %rcx, %r8 -; nextln: cmovzq %rax, %r8 -; nextln: cmovzq %r9, %rax +; nextln: movq %r9, %rax +; nextln: cmovzq %r10, %rax +; nextln: cmovzq %r9, %r8 ; nextln: movl $$128, %r9d ; nextln: subq %rdx, %r9 ; nextln: movq %rdi, %rdx @@ -1033,14 +1033,12 @@ block0(v0: i128, v1: i128): ; nextln: testq $$127, %r9 ; nextln: cmovzq %rcx, %rsi ; nextln: orq %rdx, %rsi -; nextln: xorq %rdx, %rdx +; nextln: xorq %rcx, %rcx ; nextln: testq $$64, %r9 -; nextln: movq %rdi, %rcx -; nextln: cmovzq %rsi, %rcx -; nextln: movq %rdx, %rsi -; nextln: cmovzq %rdi, %rsi -; nextln: orq %rcx, %r8 -; nextln: orq %rsi, %rax +; nextln: cmovzq %rdi, %rcx +; nextln: cmovzq %rsi, %rdi +; nextln: orq %rdi, %r8 +; nextln: orq %rcx, %rax ; nextln: movq %rax, %rdx ; nextln: movq %r8, %rax ; nextln: movq %rbp, %rsp