From 25b380d5fcfc106987ece06b7cd0030db27ddac0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 19 Nov 2021 11:34:00 -0800 Subject: [PATCH] aarch64: Migrate `{s,u}mulhi` to ISLE This starts moving over some sign/zero-extend helpers also present in lowering in Rust. Otherwise this is a relatively unsurprising transition with the various cases of the instructions mapping well to ISLE utilities. --- cranelift/codegen/src/isa/aarch64/inst.isle | 34 +++ cranelift/codegen/src/isa/aarch64/lower.isle | 29 +++ .../codegen/src/isa/aarch64/lower/isle.rs | 4 + .../lower/isle/generated_code.manifest | 6 +- .../isa/aarch64/lower/isle/generated_code.rs | 224 +++++++++++++++--- .../codegen/src/isa/aarch64/lower_inst.rs | 61 +---- cranelift/codegen/src/isa/x64/inst.isle | 2 +- .../x64/lower/isle/generated_code.manifest | 4 +- .../src/isa/x64/lower/isle/generated_code.rs | 15 +- cranelift/codegen/src/machinst/isle.rs | 8 +- cranelift/codegen/src/prelude.isle | 6 +- 11 files changed, 289 insertions(+), 104 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 12198d8db0..e202f798c6 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -1301,6 +1301,9 @@ (decl imm_logic_from_u64 (ImmLogic) u64) (extern extractor imm_logic_from_u64 imm_logic_from_u64) +(decl imm_shift_from_u8 (u8) ImmShift) +(extern constructor imm_shift_from_u8 imm_shift_from_u8) + (decl imm12_from_u64 (Imm12) u64) (extern extractor imm12_from_u64 imm12_from_u64) @@ -1370,6 +1373,13 @@ (_ Unit (emit (MInst.AluRRImmLogic op dst src imm)))) (writable_reg_to_reg dst))) +;; Helper for emitting `MInst.AluRRImmShift` instructions. +(decl alu_rr_imm_shift (ALUOp Reg ImmShift) Reg) +(rule (alu_rr_imm_shift op src imm) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.AluRRImmShift op dst src imm)))) + (writable_reg_to_reg dst))) + ;; Helper for emitting `MInst.AluRRR` instructions. (decl alu_rrr (ALUOp Reg Reg) Reg) (rule (alu_rrr op src1 src2) @@ -1509,3 +1519,27 @@ ;; Fallback for integral 64-bit constants that uses lots of `movk` (rule (imm (integral_ty _ty) n) (load_constant64_full n)) + +;; Sign extension helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Place a `Value` into a register, sign extending it to 64-bits +(decl put_in_reg_sext64 (Value) Reg) +(rule (put_in_reg_sext64 val @ (value_type (fits_in_32 ty))) + (let ((dst WritableReg (temp_writable_reg $I32)) + (src Reg (put_in_reg val)) + (_ Unit (emit (MInst.Extend dst src $true (ty_bits ty) 64)))) + (writable_reg_to_reg dst))) + +;; 64-bit passthrough. +(rule (put_in_reg_sext64 val @ (value_type $I64)) (put_in_reg val)) + +;; Place a `Value` into a register, zero extending it to 64-bits +(decl put_in_reg_zext64 (Value) Reg) +(rule (put_in_reg_zext64 val @ (value_type (fits_in_32 ty))) + (let ((dst WritableReg (temp_writable_reg $I32)) + (src Reg (put_in_reg val)) + (_ Unit (emit (MInst.Extend dst src $false (ty_bits ty) 64)))) + (writable_reg_to_reg dst))) + +;; 64-bit passthrough. +(rule (put_in_reg_zext64 val @ (value_type $I64)) (put_in_reg val)) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 5dce2a3741..4114a1be9e 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -369,3 +369,32 @@ (imul (def_inst (uwiden_high x @ (value_type $I32X4))) (def_inst (uwiden_high y @ (value_type $I32X4)))))) (value_reg (vec_rrr_long (VecRRRLongOp.Umull32) (put_in_reg x) (put_in_reg y) $true))) + +;;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type $I64 (smulhi x y))) + (value_reg (alu_rrr (ALUOp.SMulH) (put_in_reg x) (put_in_reg y)))) + +(rule (lower (has_type (fits_in_32 ty) (smulhi x y))) + (let ( + (x64 Reg (put_in_reg_sext64 x)) + (y64 Reg (put_in_reg_sext64 y)) + (mul Reg (alu_rrrr (ALUOp3.MAdd64) x64 y64 (zero_reg))) + (result Reg (alu_rr_imm_shift (ALUOp.Asr64) mul (imm_shift_from_u8 (ty_bits ty)))) + ) + (value_reg result))) + +;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type $I64 (umulhi x y))) + (value_reg (alu_rrr (ALUOp.UMulH) (put_in_reg x) (put_in_reg y)))) + +(rule (lower (has_type (fits_in_32 ty) (umulhi x y))) + (let ( + (x64 Reg (put_in_reg_zext64 x)) + (y64 Reg (put_in_reg_zext64 y)) + (mul Reg (alu_rrrr (ALUOp3.MAdd64) x64 y64 (zero_reg))) + (result Reg (alu_rr_imm_shift (ALUOp.Lsr64) mul (imm_shift_from_u8 (ty_bits ty)))) + ) + (value_reg result))) + diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 96bd32d5c8..221c7362e1 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -122,6 +122,10 @@ where Imm12::maybe_from_u64((n as i64).wrapping_neg() as u64) } + fn imm_shift_from_u8(&mut self, n: u8) -> ImmShift { + ImmShift::maybe_from_u64(n.into()).unwrap() + } + fn lshl_from_imm64(&mut self, n: Imm64, ty: Type) -> Option { let shiftimm = ShiftOpShiftImm::maybe_from_shift(n.bits() as u64)?; let shiftee_bits = ty_bits(ty); diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 39e1f0fc04..a5f3198e22 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle fc3ca134da0df8e7309db0f6969c8f1db85ca7b7590d2e43552ef3134b9a55bd358a93e3aadf79d5c31d3fc95ce5c9c52f8313183c688259c027ee494913869c -src/isa/aarch64/inst.isle 30c88514c23dfda849aa4a98b981b52b569994cdf3424a93d77429246ebce8c45575a76387ae2f3e4901ba6b21c846a419231da413f2df6c5dcea681eab6bf0c -src/isa/aarch64/lower.isle 97392236095b99e93c97732b2af0778aba409f81da22b6879cf6e8f2513f0de5b3017bfa072dc60e7f6bf21aac91a5153133c01b041fb174fab0680d2fd4886c +src/prelude.isle 9bd1fcb6a3604a24cf2e05e6b7eb04dcb3b9dc8fa9a2f1c8f29c25b6e3bf7f679b3b1b72dff87501497b72bc30fc92fd755b898db7e03f380235fae931b6a74b +src/isa/aarch64/inst.isle 6e042ec14166fceae4b7133f681fdf604e20a2997e1d60f797e40acd683ccb34e33376189f6b7ed2f5eb441dc61d592cad2592256dfea51296330752181b9403 +src/isa/aarch64/lower.isle 64a725771537f69c445f44c728e04bffd8a715d6a4d87a5a2bf2e89714ee290b7497c5ca8b335bdddd775f6734be03318ff9aa67e2e4068949ebae06b0902b3f diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 682d2a98e2..7bd58b5c39 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -30,7 +30,8 @@ pub trait Context { fn u8_as_u64(&mut self, arg0: u8) -> u64; fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; - fn ty_bits(&mut self, arg0: Type) -> u16; + fn ty_bits(&mut self, arg0: Type) -> u8; + fn ty_bits_u16(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; @@ -52,6 +53,7 @@ pub trait Context { fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option; fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option; fn imm_logic_from_u64(&mut self, arg0: u64) -> Option; + fn imm_shift_from_u8(&mut self, arg0: u8) -> ImmShift; fn imm12_from_u64(&mut self, arg0: u64) -> Option; fn imm12_from_negated_u64(&mut self, arg0: u64) -> Option; fn lshl_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option; @@ -64,13 +66,13 @@ pub trait Context { fn load_constant64_full(&mut self, arg0: u64) -> Reg; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 238. +/// Internal type ProducesFlags: defined at src/prelude.isle line 242. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 241. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 245. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -984,7 +986,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 251. + // Rule at src/prelude.isle line 255. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1012,7 +1014,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 259. + // Rule at src/prelude.isle line 263. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1046,7 +1048,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 269. + // Rule at src/prelude.isle line 273. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -1102,7 +1104,7 @@ pub fn constructor_movz( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1354. + // Rule at src/isa/aarch64/inst.isle line 1357. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovZ { @@ -1123,7 +1125,7 @@ pub fn constructor_movn( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1361. + // Rule at src/isa/aarch64/inst.isle line 1364. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::MovN { @@ -1146,7 +1148,7 @@ pub fn constructor_alu_rr_imm_logic( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1368. + // Rule at src/isa/aarch64/inst.isle line 1371. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImmLogic { @@ -1160,6 +1162,30 @@ pub fn constructor_alu_rr_imm_logic( return Some(expr4_0); } +// Generated as internal constructor for term alu_rr_imm_shift. +pub fn constructor_alu_rr_imm_shift( + ctx: &mut C, + arg0: &ALUOp, + arg1: Reg, + arg2: ImmShift, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/aarch64/inst.isle line 1378. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::AluRRImmShift { + alu_op: pattern0_0.clone(), + rd: expr1_0, + rn: pattern1_0, + immshift: pattern2_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + // Generated as internal constructor for term alu_rrr. pub fn constructor_alu_rrr( ctx: &mut C, @@ -1170,7 +1196,7 @@ pub fn constructor_alu_rrr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1375. + // Rule at src/isa/aarch64/inst.isle line 1385. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRR { @@ -1196,7 +1222,7 @@ pub fn constructor_vec_rrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1382. + // Rule at src/isa/aarch64/inst.isle line 1392. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRR { @@ -1221,7 +1247,7 @@ pub fn constructor_alu_rr_imm12( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1389. + // Rule at src/isa/aarch64/inst.isle line 1399. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRImm12 { @@ -1247,7 +1273,7 @@ pub fn constructor_alu_rrr_shift( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1396. + // Rule at src/isa/aarch64/inst.isle line 1406. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRShift { @@ -1274,7 +1300,7 @@ pub fn constructor_alu_rrr_extend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1403. + // Rule at src/isa/aarch64/inst.isle line 1413. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRExtend { @@ -1299,7 +1325,7 @@ pub fn constructor_alu_rr_extend_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1411. + // Rule at src/isa/aarch64/inst.isle line 1421. let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0); let expr1_0 = C::get_extended_op(ctx, pattern2_0); let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?; @@ -1318,7 +1344,7 @@ pub fn constructor_alu_rrrr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1418. + // Rule at src/isa/aarch64/inst.isle line 1428. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::AluRRRR { @@ -1341,7 +1367,7 @@ pub fn constructor_add64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1425. + // Rule at src/isa/aarch64/inst.isle line 1435. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::AddS64; @@ -1363,7 +1389,7 @@ pub fn constructor_add64_with_flags( pub fn constructor_adc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1432. + // Rule at src/isa/aarch64/inst.isle line 1442. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Adc64; @@ -1389,7 +1415,7 @@ pub fn constructor_sub64_with_flags( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1439. + // Rule at src/isa/aarch64/inst.isle line 1449. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::SubS64; @@ -1411,7 +1437,7 @@ pub fn constructor_sub64_with_flags( pub fn constructor_sbc64(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/aarch64/inst.isle line 1446. + // Rule at src/isa/aarch64/inst.isle line 1456. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = ALUOp::Sbc64; @@ -1439,7 +1465,7 @@ pub fn constructor_vec_misc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1453. + // Rule at src/isa/aarch64/inst.isle line 1463. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecMisc { @@ -1465,7 +1491,7 @@ pub fn constructor_vec_rrr_long( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/aarch64/inst.isle line 1460. + // Rule at src/isa/aarch64/inst.isle line 1470. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRRLong { @@ -1494,7 +1520,7 @@ pub fn constructor_vec_rrrr_long( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/aarch64/inst.isle line 1470. + // Rule at src/isa/aarch64/inst.isle line 1480. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::FpuMove128 { @@ -1524,7 +1550,7 @@ pub fn constructor_vec_rr_narrow( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1478. + // Rule at src/isa/aarch64/inst.isle line 1488. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRNarrow { @@ -1548,7 +1574,7 @@ pub fn constructor_vec_rr_long( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/aarch64/inst.isle line 1485. + // Rule at src/isa/aarch64/inst.isle line 1495. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::VecRRLong { @@ -1568,36 +1594,130 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) { let pattern2_0 = arg1; if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1503. + // Rule at src/isa/aarch64/inst.isle line 1513. let expr0_0 = ALUOp::Orr64; let expr1_0 = C::zero_reg(ctx); let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern3_0)?; return Some(expr2_0); } if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1495. + // Rule at src/isa/aarch64/inst.isle line 1505. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) { - // Rule at src/isa/aarch64/inst.isle line 1499. + // Rule at src/isa/aarch64/inst.isle line 1509. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?; return Some(expr1_0); } - // Rule at src/isa/aarch64/inst.isle line 1510. + // Rule at src/isa/aarch64/inst.isle line 1520. let expr0_0 = C::load_constant64_full(ctx, pattern2_0); return Some(expr0_0); } return None; } +// Generated as internal constructor for term put_in_reg_sext64. +pub fn constructor_put_in_reg_sext64(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = C::value_type(ctx, pattern0_0); + if pattern1_0 == I64 { + // Rule at src/isa/aarch64/inst.isle line 1534. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + return Some(expr0_0); + } + if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { + // Rule at src/isa/aarch64/inst.isle line 1527. + let expr0_0: Type = I32; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = C::put_in_reg(ctx, pattern0_0); + let expr3_0: bool = true; + let expr4_0 = C::ty_bits(ctx, pattern2_0); + let expr5_0: u8 = 64; + let expr6_0 = MInst::Extend { + rd: expr1_0, + rn: expr2_0, + signed: expr3_0, + from_bits: expr4_0, + to_bits: expr5_0, + }; + let expr7_0 = C::emit(ctx, &expr6_0); + let expr8_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr8_0); + } + return None; +} + +// Generated as internal constructor for term put_in_reg_zext64. +pub fn constructor_put_in_reg_zext64(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = C::value_type(ctx, pattern0_0); + if pattern1_0 == I64 { + // Rule at src/isa/aarch64/inst.isle line 1545. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + return Some(expr0_0); + } + if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { + // Rule at src/isa/aarch64/inst.isle line 1538. + let expr0_0: Type = I32; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = C::put_in_reg(ctx, pattern0_0); + let expr3_0: bool = false; + let expr4_0 = C::ty_bits(ctx, pattern2_0); + let expr5_0: u8 = 64; + let expr6_0 = MInst::Extend { + rd: expr1_0, + rn: expr2_0, + signed: expr3_0, + from_bits: expr4_0, + to_bits: expr5_0, + }; + let expr7_0 = C::emit(ctx, &expr6_0); + let expr8_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr8_0); + } + return None; +} + // Generated as internal constructor for term lower. pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern0_0 = arg0; if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) { let pattern2_0 = C::value_type(ctx, pattern1_0); + if pattern2_0 == I64 { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Umulhi => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 389. + let expr0_0 = ALUOp::UMulH; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Smulhi => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 375. + let expr0_0 = ALUOp::SMulH; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + _ => {} + } + } + } if pattern2_0 == I128 { let pattern4_0 = C::inst_data(ctx, pattern0_0); if let &InstructionData::Binary { @@ -2255,6 +2375,52 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 392. + let expr0_0 = constructor_put_in_reg_zext64(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_reg_zext64(ctx, pattern7_1)?; + let expr2_0 = ALUOp3::MAdd64; + let expr3_0 = C::zero_reg(ctx); + let expr4_0 = + constructor_alu_rrrr(ctx, &expr2_0, expr0_0, expr1_0, expr3_0)?; + let expr5_0 = ALUOp::Lsr64; + let expr6_0 = C::ty_bits(ctx, pattern3_0); + let expr7_0 = C::imm_shift_from_u8(ctx, expr6_0); + let expr8_0 = + constructor_alu_rr_imm_shift(ctx, &expr5_0, expr4_0, expr7_0)?; + let expr9_0 = C::value_reg(ctx, expr8_0); + return Some(expr9_0); + } + &Opcode::Smulhi => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 378. + let expr0_0 = constructor_put_in_reg_sext64(ctx, pattern7_0)?; + let expr1_0 = constructor_put_in_reg_sext64(ctx, pattern7_1)?; + let expr2_0 = ALUOp3::MAdd64; + let expr3_0 = C::zero_reg(ctx); + let expr4_0 = + constructor_alu_rrrr(ctx, &expr2_0, expr0_0, expr1_0, expr3_0)?; + let expr5_0 = ALUOp::Asr64; + let expr6_0 = C::ty_bits(ctx, pattern3_0); + let expr7_0 = C::imm_shift_from_u8(ctx, expr6_0); + let expr8_0 = + constructor_alu_rr_imm_shift(ctx, &expr5_0, expr4_0, expr7_0)?; + let expr9_0 = C::value_reg(ctx, expr8_0); + return Some(expr9_0); + } + _ => {} + } + } + } if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) { let pattern4_0 = C::inst_data(ctx, pattern0_0); match &pattern4_0 { diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index b46acd59de..38ad44590d 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -73,66 +73,7 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Imul => implemented_in_isle(ctx), - Opcode::Umulhi | Opcode::Smulhi => { - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let is_signed = op == Opcode::Smulhi; - let input_ty = ctx.input_ty(insn, 0); - assert!(ctx.input_ty(insn, 1) == input_ty); - assert!(ctx.output_ty(insn, 0) == input_ty); - - match input_ty { - I64 => { - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - let alu_op = if is_signed { - ALUOp::SMulH - } else { - ALUOp::UMulH - }; - ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm }); - } - I32 | I16 | I8 => { - let narrow_mode = if is_signed { - NarrowValueMode::SignExtend64 - } else { - NarrowValueMode::ZeroExtend64 - }; - let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); - let rm = put_input_in_reg(ctx, inputs[1], narrow_mode); - let ra = zero_reg(); - ctx.emit(Inst::AluRRRR { - alu_op: ALUOp3::MAdd64, - rd, - rn, - rm, - ra, - }); - let shift_op = if is_signed { - ALUOp::Asr64 - } else { - ALUOp::Lsr64 - }; - let shift_amt = match input_ty { - I32 => 32, - I16 => 16, - I8 => 8, - _ => unreachable!(), - }; - ctx.emit(Inst::AluRRImmShift { - alu_op: shift_op, - rd, - rn: rd.to_reg(), - immshift: ImmShift::maybe_from_u64(shift_amt).unwrap(), - }); - } - _ => { - return Err(CodegenError::Unsupported(format!( - "{}: Unsupported type: {:?}", - op, input_ty - ))); - } - } - } + Opcode::Umulhi | Opcode::Smulhi => implemented_in_isle(ctx), Opcode::Udiv | Opcode::Sdiv | Opcode::Urem | Opcode::Srem => { let ty = ty.unwrap(); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index a39b7fbca4..cd9a08c068 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -420,7 +420,7 @@ (rule (extend_to_reg (and val (value_type from_ty)) to_ty kind) - (let ((from_bits u16 (ty_bits from_ty)) + (let ((from_bits u16 (ty_bits_u16 from_ty)) ;; Use `operand_size_of_type` so that the we clamp the output to 32- ;; or 64-bit width types. (to_bits u16 (operand_size_bits (operand_size_of_type to_ty)))) diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index f2c580d485..85f349a988 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle fc3ca134da0df8e7309db0f6969c8f1db85ca7b7590d2e43552ef3134b9a55bd358a93e3aadf79d5c31d3fc95ce5c9c52f8313183c688259c027ee494913869c -src/isa/x64/inst.isle 12dc8fa43cbba6e9c5cf46a2472e2754abfe33b7fd38f80e271afa3f6c002efad7a4202c8f00ff27d5e6176de8fec97e1887d382cbd4ef06eaac177a0b5992e3 +src/prelude.isle 9bd1fcb6a3604a24cf2e05e6b7eb04dcb3b9dc8fa9a2f1c8f29c25b6e3bf7f679b3b1b72dff87501497b72bc30fc92fd755b898db7e03f380235fae931b6a74b +src/isa/x64/inst.isle c93db8fe2819e952af629c4f4c31163545efcf5158f0747987eca30b5e42f2b6e75290bbccc72d7f44c2b43aab67fcbc339ef5a687918f796dd9317bad77a695 src/isa/x64/lower.isle 333e1be62f602bb835a3cebc3299290a3d386438e9190d2db219263d974e097bfc3f1afdaac9401853806d21d548cad70bab2ffbc3b1cf5c3bebdd971a961f70 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index d5273dae90..3ee85cdcda 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -30,7 +30,8 @@ pub trait Context { fn u8_as_u64(&mut self, arg0: u8) -> u64; fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; - fn ty_bits(&mut self, arg0: Type) -> u16; + fn ty_bits(&mut self, arg0: Type) -> u8; + fn ty_bits_u16(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; @@ -67,13 +68,13 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 238. +/// Internal type ProducesFlags: defined at src/prelude.isle line 242. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 241. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 245. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -123,7 +124,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 251. + // Rule at src/prelude.isle line 255. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -151,7 +152,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 259. + // Rule at src/prelude.isle line 263. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -185,7 +186,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 269. + // Rule at src/prelude.isle line 273. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -244,7 +245,7 @@ pub fn constructor_extend_to_reg( } let pattern3_0 = arg2; // Rule at src/isa/x64/inst.isle line 420. - let expr0_0 = C::ty_bits(ctx, pattern1_0); + let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; let expr3_0 = C::ext_mode(ctx, expr0_0, expr2_0); diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index a93b2aff45..c969ac0106 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -86,7 +86,13 @@ macro_rules! isle_prelude_methods { } #[inline] - fn ty_bits(&mut self, ty: Type) -> u16 { + fn ty_bits(&mut self, ty: Type) -> u8 { + use std::convert::TryInto; + ty.bits().try_into().unwrap() + } + + #[inline] + fn ty_bits_u16(&mut self, ty: Type) -> u16 { ty.bits() } diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index dd4ef9530b..8b17e53f4f 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -139,9 +139,13 @@ (extern const $F64X2 Type) ;; Get the bit width of a given type. -(decl ty_bits (Type) u16) +(decl ty_bits (Type) u8) (extern constructor ty_bits ty_bits) +;; Get the bit width of a given type. +(decl ty_bits_u16 (Type) u16) +(extern constructor ty_bits_u16 ty_bits_u16) + ;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; An extractor that only matches types that can fit in 32 bits.