From ef8ea644f4a5abbaabeb8cd1200e68d13575c4ea Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 19 Nov 2021 12:59:06 -0600 Subject: [PATCH] aarch64: Migrate {s,u}{sub,add}_sat to ISLE (#3551) These were pretty straightforward! Only needed a single `rule` per instruction with a new 128-bit vector type matcher. --- cranelift/codegen/src/isa/aarch64/lower.isle | 20 ++++++ .../lower/isle/generated_code.manifest | 4 +- .../isa/aarch64/lower/isle/generated_code.rs | 71 +++++++++++++++++-- .../codegen/src/isa/aarch64/lower_inst.rs | 29 +------- .../x64/lower/isle/generated_code.manifest | 2 +- .../src/isa/x64/lower/isle/generated_code.rs | 11 +-- cranelift/codegen/src/machinst/isle.rs | 8 +++ cranelift/codegen/src/prelude.isle | 4 ++ 8 files changed, 108 insertions(+), 41 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 5e0f9076f3..2c8c8a8ea0 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -159,3 +159,23 @@ (with_flags (sub64_with_flags x_lo y_lo) (sbc64 x_hi y_hi)))) + +;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (vec128 ty) (uadd_sat x y))) + (value_reg (vec_rrr (VecALUOp.Uqadd) (put_in_reg x) (put_in_reg y) (vector_size ty)))) + +;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (vec128 ty) (sadd_sat x y))) + (value_reg (vec_rrr (VecALUOp.Sqadd) (put_in_reg x) (put_in_reg y) (vector_size ty)))) + +;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (vec128 ty) (usub_sat x y))) + (value_reg (vec_rrr (VecALUOp.Uqsub) (put_in_reg x) (put_in_reg y) (vector_size ty)))) + +;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (vec128 ty) (ssub_sat x y))) + (value_reg (vec_rrr (VecALUOp.Sqsub) (put_in_reg x) (put_in_reg y) (vector_size ty)))) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 040c81e0bb..202f798ec0 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c +src/prelude.isle e4933f2bcb6cd9e00cb6dc0c47c43d096d0c4e37468af17a38fad8906b864d975e0a8b98d15c6a5e2bccf255ec2ced2466991c3405533e9cafefbf4d9ac46823 src/isa/aarch64/inst.isle 67a43022bb2e0b8ae06b71c7c49f9b9997a9c6ca109e35f5018b9cd64105a0fe8b103943fb34ca7da45cea9db7327e00954e88606845d5ebc370bc6c3045a04f -src/isa/aarch64/lower.isle 56c3c3f709d2ca18d91e14d8c4cec804990cfa3d85f9ce4fac4eef4861c4c159acee1b5fe6deff7d89337092b35ecf67418da7ce29e6217fa1ab47b0deaf1759 +src/isa/aarch64/lower.isle efb81eb92d8ab0a408e0de222825f632b40d7c91b0bf79e3144f7d76f0d1c743bb505ab991723363cab504a0f95b18232811f1519ea1d8c15d2113be166f590c diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index ace3b1ecd5..f842978241 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -33,6 +33,7 @@ pub trait Context { fn ty_bits(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; + fn vec128(&mut self, arg0: Type) -> Option; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); fn unwrap_head_value_list_2(&mut self, arg0: ValueList) -> (Value, Value, ValueSlice); @@ -62,13 +63,13 @@ pub trait Context { fn load_constant64_full(&mut self, arg0: u64) -> Reg; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 230. +/// Internal type ProducesFlags: defined at src/prelude.isle line 234. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 233. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 237. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -982,7 +983,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 243. + // Rule at src/prelude.isle line 247. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1010,7 +1011,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 251. + // Rule at src/prelude.isle line 255. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1044,7 +1045,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 261. + // Rule at src/prelude.isle line 265. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); @@ -1978,6 +1979,66 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 165. + let expr0_0 = VecALUOp::Uqadd; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern3_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::SaddSat => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 170. + let expr0_0 = VecALUOp::Sqadd; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern3_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::UsubSat => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 175. + let expr0_0 = VecALUOp::Uqsub; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern3_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + &Opcode::SsubSat => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/aarch64/lower.isle line 180. + let expr0_0 = VecALUOp::Sqsub; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = C::put_in_reg(ctx, pattern7_1); + let expr3_0 = constructor_vector_size(ctx, pattern3_0)?; + let expr4_0 = + constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + _ => {} + } + } + } } return None; } diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index b42792bd39..f66b3275bf 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -66,34 +66,7 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Iadd => implemented_in_isle(ctx), Opcode::Isub => implemented_in_isle(ctx), Opcode::UaddSat | Opcode::SaddSat | Opcode::UsubSat | Opcode::SsubSat => { - let ty = ty.unwrap(); - - if !ty.is_vector() || ty_bits(ty) > 128 { - return Err(CodegenError::Unsupported(format!( - "{}: Unsupported type: {:?}", - op, ty - ))); - } - - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); - - let alu_op = match op { - Opcode::UaddSat => VecALUOp::Uqadd, - Opcode::SaddSat => VecALUOp::Sqadd, - Opcode::UsubSat => VecALUOp::Uqsub, - Opcode::SsubSat => VecALUOp::Sqsub, - _ => unreachable!(), - }; - - ctx.emit(Inst::VecRRR { - rd, - rn, - rm, - alu_op, - size: VectorSize::from_ty(ty), - }); + implemented_in_isle(ctx) } Opcode::Ineg => { diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index cacbaa17fb..117e3aec14 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb -src/prelude.isle 07dff11e5630f23a6a6b3ba54513097754d8d9feed49e8eb4ccc1e874f32c9ab053dc69e49cb39a82a2f49320515741a2e544f51816c5c71fd2df21957cc0c7c +src/prelude.isle e4933f2bcb6cd9e00cb6dc0c47c43d096d0c4e37468af17a38fad8906b864d975e0a8b98d15c6a5e2bccf255ec2ced2466991c3405533e9cafefbf4d9ac46823 src/isa/x64/inst.isle e4a0657406056a4cf116fe125e91d16377b602e0b41edd6628cbb7259b0fc2aa6b0482ffd33f00d63d68cf3546f188705877309d43eba5e75abd0f38a52a79b2 src/isa/x64/lower.isle e51b7a67343dba342a43b3c9e4b9ed7df9b2c66a677018acf7054ba48c27e4e93a4421fd892b9bf7c0e5b790bcfafab7cb3e93ce2b8206c04d456918d2ad0b5a diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index c147f914db..a89f8148b7 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -33,6 +33,7 @@ pub trait Context { fn ty_bits(&mut self, arg0: Type) -> u16; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; + fn vec128(&mut self, arg0: Type) -> Option; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); fn unwrap_head_value_list_2(&mut self, arg0: ValueList) -> (Value, Value, ValueSlice); @@ -63,13 +64,13 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 230. +/// Internal type ProducesFlags: defined at src/prelude.isle line 234. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 233. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 237. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -119,7 +120,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 243. + // Rule at src/prelude.isle line 247. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -147,7 +148,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 251. + // Rule at src/prelude.isle line 255. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -181,7 +182,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 261. + // Rule at src/prelude.isle line 265. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::emit(ctx, &pattern5_0); diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index dfab0f35d1..9098dc5df2 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -108,6 +108,14 @@ macro_rules! isle_prelude_methods { } } + fn vec128(&mut self, ty: Type) -> Option { + if ty.is_vector() && ty.bits() == 128 { + Some(ty) + } else { + None + } + } + #[inline] fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { list.as_slice(&self.lower_ctx.dfg().value_lists) diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 2a1d0b8227..6c2af0b581 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -152,6 +152,10 @@ (decl fits_in_64 (Type) Type) (extern extractor fits_in_64 fits_in_64) +;; An extractor that only matches 128-bit vector types. +(decl vec128 (Type) Type) +(extern extractor vec128 vec128) + ;; Extractor to get a `ValueSlice` out of a `ValueList`. (decl value_list_slice (ValueSlice) ValueList) (extern extractor infallible value_list_slice value_list_slice)