diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 84cca94422..27e90d2ae7 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1746,6 +1746,43 @@ (rule (lower (has_type ty (select (fcmp (FloatCC.NotEqual) a b) x y))) (with_flags (x64_ucomis a b) (cmove_or_from_values ty (CC.NZ) (CC.P) x y))) +;; We also can lower `select`s that depend on an `icmp` test, but more simply +;; than the `fcmp` variants above. In these cases, we lower to a `CMP` +;; instruction plus a `CMOV`; recall that `cmove_from_values` here may emit more +;; than one instruction for certain types (e.g., XMM-held, I128). + +(rule (lower (has_type ty (select (icmp cc a @ (value_type (fits_in_64 a_ty)) b) x y))) + ;; N.B.: we force the comparison operators into registers, and disallow + ;; load-op fusion, because we do not have a transitive guarantee that this + ;; cmp-site will be the sole user of the value. Consider: the `icmp` might + ;; be the only user of a load, but there may be multiple users of the + ;; `icmp` (e.g., `select` or `bint` instructions) that each invoke emit a + ;; comparison. If we were to allow a load to sink to the *latest* one, but + ;; other sites did not permit sinking, then we would be missing the load + ;; for other cmp-sites. TODO: + ;; https://github.com/bytecodealliance/wasmtime/issues/3953. + (let ((gpr_a Gpr (put_in_gpr a)) + (gpr_b Gpr (put_in_gpr b)) + (size OperandSize (raw_operand_size_of_type a_ty))) + (with_flags (x64_cmp size gpr_b gpr_a) (cmove_from_values ty cc x y)))) + +;; Finally, we lower `select` from a condition value `c`. These rules are meant +;; to be the final, default lowerings if no other patterns matched above. + +(rule (lower (has_type ty (select c @ (value_type $B1) x y))) + (let ((size OperandSize (raw_operand_size_of_type $B1)) + ;; N.B.: disallow load-op fusion, see above. TODO: + ;; https://github.com/bytecodealliance/wasmtime/issues/3953. + (gpr_c Gpr (put_in_gpr c))) + (with_flags (x64_test size (RegMemImm.Imm 1) gpr_c) (cmove_from_values ty (CC.NZ) x y)))) + +(rule (lower (has_type ty (select c @ (value_type (fits_in_64 a_ty)) x y))) + (let ((size OperandSize (raw_operand_size_of_type a_ty)) + ;; N.B.: disallow load-op fusion, see above. TODO: + ;; https://github.com/bytecodealliance/wasmtime/issues/3953. + (gpr_c Gpr (put_in_gpr c))) + (with_flags (x64_test size gpr_c gpr_c) (cmove_from_values ty (CC.NZ) x y)))) + ;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; If available, we can use a plain lzcnt instruction here. Note no diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 207dd1c06a..b599079e97 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2569,67 +2569,7 @@ fn lower_insn_to_regs>( } Opcode::Select => { - let flag_input = inputs[0]; - if let Some(_) = matches_input(ctx, flag_input, Opcode::Fcmp) { - implemented_in_isle(ctx); - } else { - let ty = ty.unwrap(); - - let size = ty.bytes() as u8; - let lhs = put_input_in_regs(ctx, inputs[1]); - let rhs = put_input_in_regs(ctx, inputs[2]); - let dst = get_output_reg(ctx, outputs[0]); - - let cc = if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { - let cond_code = ctx.data(icmp).cond_code().unwrap(); - let cond_code = emit_cmp(ctx, icmp, cond_code); - CC::from_intcc(cond_code) - } else { - let sel_ty = ctx.input_ty(insn, 0); - let size = OperandSize::from_ty(ctx.input_ty(insn, 0)); - let test = put_input_in_reg(ctx, flag_input); - let test_input = if sel_ty == types::B1 { - // The input is a boolean value; test the LSB for nonzero with: - // test reg, 1 - RegMemImm::imm(1) - } else { - // The input is an integer; test the whole value for - // nonzero with: - // test reg, reg - // - // (It doesn't make sense to have a boolean wider than - // one bit here -- which bit would cause us to select an - // input?) - assert!(!is_bool_ty(sel_ty)); - RegMemImm::reg(test) - }; - ctx.emit(Inst::test_rmi_r(size, test_input, test)); - CC::NZ - }; - - // This doesn't affect the flags. - emit_moves(ctx, dst, rhs, ty); - - if is_int_or_ref_ty(ty) || ty == types::I128 { - emit_cmoves(ctx, size, cc, lhs, dst); - } else { - debug_assert!( - ty == types::F32 - || ty == types::F64 - || (ty.is_vector() && ty.bits() == 128) - ); - ctx.emit(Inst::xmm_cmove( - if ty == types::F64 { - OperandSize::Size64 - } else { - OperandSize::Size32 - }, - cc, - RegMem::reg(lhs.only_reg().unwrap()), - dst.only_reg().unwrap(), - )); - } - } + implemented_in_isle(ctx); } Opcode::Selectif | Opcode::SelectifSpectreGuard => { diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index f208a2d444..505b62432e 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 src/prelude.isle 74d9514ac948e163 src/isa/x64/inst.isle a002d62dcfce285 -src/isa/x64/lower.isle d8facef52a4e2ac6 +src/isa/x64/lower.isle 8f3e1ed2929fd07e diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index d4727c6fc4..534e038a8e 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -4619,7 +4619,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Debugtrap = pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2225. + // Rule at src/isa/x64/lower.isle line 2262. let expr0_0 = constructor_x64_hlt(ctx)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); @@ -5384,7 +5384,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 2068. + // Rule at src/isa/x64/lower.isle line 2105. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 0; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -5399,7 +5399,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 2076. + // Rule at src/isa/x64/lower.isle line 2113. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 4294967295; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -5530,7 +5530,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1999. + // Rule at src/isa/x64/lower.isle line 2036. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0 = constructor_do_bitrev64(ctx, expr0_0, expr1_0)?; @@ -5604,7 +5604,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2115. + // Rule at src/isa/x64/lower.isle line 2152. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5612,7 +5612,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2121. + // Rule at src/isa/x64/lower.isle line 2158. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5620,7 +5620,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2124. + // Rule at src/isa/x64/lower.isle line 2161. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5628,7 +5628,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2118. + // Rule at src/isa/x64/lower.isle line 2155. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5636,7 +5636,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2127. + // Rule at src/isa/x64/lower.isle line 2164. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5644,7 +5644,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2130. + // Rule at src/isa/x64/lower.isle line 2167. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5652,7 +5652,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2133. + // Rule at src/isa/x64/lower.isle line 2170. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5660,7 +5660,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2136. + // Rule at src/isa/x64/lower.isle line 2173. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5668,7 +5668,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2139. + // Rule at src/isa/x64/lower.isle line 2176. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5683,7 +5683,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Uload32 = pattern12_0 { - // Rule at src/isa/x64/lower.isle line 2142. + // Rule at src/isa/x64/lower.isle line 2179. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -5696,7 +5696,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2002. + // Rule at src/isa/x64/lower.isle line 2039. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 1; @@ -5973,7 +5973,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1772. + // Rule at src/isa/x64/lower.isle line 1809. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -6011,7 +6011,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1814. + // Rule at src/isa/x64/lower.isle line 1851. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -6049,7 +6049,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1866. + // Rule at src/isa/x64/lower.isle line 1903. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 0; @@ -6072,7 +6072,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2217. + // Rule at src/isa/x64/lower.isle line 2254. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0: u32 = 1; @@ -6090,7 +6090,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern7_0 = C::value_type(ctx, pattern5_1); if pattern7_0 == I64 { - // Rule at src/isa/x64/lower.isle line 2089. + // Rule at src/isa/x64/lower.isle line 2126. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = I64; let expr2_0: u64 = 0; @@ -6100,7 +6100,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2240. + // Rule at src/isa/x64/lower.isle line 2277. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addss(ctx, expr0_0, &expr1_0)?; @@ -6170,7 +6170,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2251. + // Rule at src/isa/x64/lower.isle line 2288. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subss(ctx, expr0_0, &expr1_0)?; @@ -6179,7 +6179,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2262. + // Rule at src/isa/x64/lower.isle line 2299. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulss(ctx, expr0_0, &expr1_0)?; @@ -6188,7 +6188,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2273. + // Rule at src/isa/x64/lower.isle line 2310. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divss(ctx, expr0_0, &expr1_0)?; @@ -6197,7 +6197,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2284. + // Rule at src/isa/x64/lower.isle line 2321. let expr0_0: Type = F32; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6209,7 +6209,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2453. + // Rule at src/isa/x64/lower.isle line 2490. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minss(ctx, expr0_0, expr1_0)?; @@ -6218,7 +6218,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2356. + // Rule at src/isa/x64/lower.isle line 2393. let expr0_0: Type = F32; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6230,7 +6230,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2464. + // Rule at src/isa/x64/lower.isle line 2501. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxss(ctx, expr0_0, expr1_0)?; @@ -6251,7 +6251,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2242. + // Rule at src/isa/x64/lower.isle line 2279. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addsd(ctx, expr0_0, &expr1_0)?; @@ -6260,7 +6260,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2253. + // Rule at src/isa/x64/lower.isle line 2290. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subsd(ctx, expr0_0, &expr1_0)?; @@ -6269,7 +6269,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2264. + // Rule at src/isa/x64/lower.isle line 2301. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulsd(ctx, expr0_0, &expr1_0)?; @@ -6278,7 +6278,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2275. + // Rule at src/isa/x64/lower.isle line 2312. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divsd(ctx, expr0_0, &expr1_0)?; @@ -6287,7 +6287,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2286. + // Rule at src/isa/x64/lower.isle line 2323. let expr0_0: Type = F64; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6299,7 +6299,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2455. + // Rule at src/isa/x64/lower.isle line 2492. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minsd(ctx, expr0_0, expr1_0)?; @@ -6308,7 +6308,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2358. + // Rule at src/isa/x64/lower.isle line 2395. let expr0_0: Type = F64; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6320,7 +6320,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2466. + // Rule at src/isa/x64/lower.isle line 2503. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxsd(ctx, expr0_0, expr1_0)?; @@ -6459,7 +6459,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1973. + // Rule at src/isa/x64/lower.isle line 2010. let expr0_0 = C::popcount_4bit_table(ctx); let expr1_0: Type = I8X16; let expr2_0 = C::popcount_low_mask(ctx); @@ -6676,7 +6676,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2230. + // Rule at src/isa/x64/lower.isle line 2267. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaddwd(ctx, expr0_0, &expr1_0)?; @@ -6816,7 +6816,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2244. + // Rule at src/isa/x64/lower.isle line 2281. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addps(ctx, expr0_0, &expr1_0)?; @@ -6825,7 +6825,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2255. + // Rule at src/isa/x64/lower.isle line 2292. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subps(ctx, expr0_0, &expr1_0)?; @@ -6834,7 +6834,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2266. + // Rule at src/isa/x64/lower.isle line 2303. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulps(ctx, expr0_0, &expr1_0)?; @@ -6843,7 +6843,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2277. + // Rule at src/isa/x64/lower.isle line 2314. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divps(ctx, expr0_0, &expr1_0)?; @@ -6852,7 +6852,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2301. + // Rule at src/isa/x64/lower.isle line 2338. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -6877,7 +6877,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2457. + // Rule at src/isa/x64/lower.isle line 2494. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -6886,7 +6886,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2364. + // Rule at src/isa/x64/lower.isle line 2401. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -6914,7 +6914,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2468. + // Rule at src/isa/x64/lower.isle line 2505. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -6956,7 +6956,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2246. + // Rule at src/isa/x64/lower.isle line 2283. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addpd(ctx, expr0_0, &expr1_0)?; @@ -6965,7 +6965,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2257. + // Rule at src/isa/x64/lower.isle line 2294. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subpd(ctx, expr0_0, &expr1_0)?; @@ -6974,7 +6974,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2268. + // Rule at src/isa/x64/lower.isle line 2305. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulpd(ctx, expr0_0, &expr1_0)?; @@ -6983,7 +6983,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2279. + // Rule at src/isa/x64/lower.isle line 2316. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divpd(ctx, expr0_0, &expr1_0)?; @@ -6992,7 +6992,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2344. + // Rule at src/isa/x64/lower.isle line 2381. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -7017,7 +7017,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2459. + // Rule at src/isa/x64/lower.isle line 2496. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -7026,7 +7026,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2407. + // Rule at src/isa/x64/lower.isle line 2444. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -7054,7 +7054,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2470. + // Rule at src/isa/x64/lower.isle line 2507. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -7122,181 +7122,279 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1743. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; - let expr1_0 = CC::NZ; - let expr2_0 = CC::P; - let expr3_0 = constructor_cmove_or_from_values( - ctx, pattern2_0, &expr1_0, &expr2_0, pattern6_2, - pattern6_1, - )?; - let expr4_0 = - constructor_with_flags(ctx, &expr0_0, &expr3_0)?; - let expr5_0 = C::output(ctx, expr4_0); - return Some(expr5_0); + match &pattern8_0 { + &InstructionData::FloatCompare { + opcode: ref pattern9_0, + args: ref pattern9_1, + cond: ref pattern9_2, + } => { + if let &Opcode::Fcmp = pattern9_0 { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + match pattern9_2 { + &FloatCC::Equal => { + // Rule at src/isa/x64/lower.isle line 1743. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_0, + pattern11_1, + )?; + let expr1_0 = CC::NZ; + let expr2_0 = CC::P; + let expr3_0 = constructor_cmove_or_from_values( + ctx, pattern2_0, &expr1_0, &expr2_0, pattern6_2, + pattern6_1, + )?; + let expr4_0 = + constructor_with_flags(ctx, &expr0_0, &expr3_0)?; + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + &FloatCC::GreaterThan => { + // Rule at src/isa/x64/lower.isle line 1695. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_1, + pattern11_0, + )?; + let expr1_0 = CC::NBE; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::GreaterThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1698. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_1, + pattern11_0, + )?; + let expr1_0 = CC::NB; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::LessThan => { + // Rule at src/isa/x64/lower.isle line 1718. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_0, + pattern11_1, + )?; + let expr1_0 = CC::NBE; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::LessThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1721. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_0, + pattern11_1, + )?; + let expr1_0 = CC::NB; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::NotEqual => { + // Rule at src/isa/x64/lower.isle line 1746. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_0, + pattern11_1, + )?; + let expr1_0 = CC::NZ; + let expr2_0 = CC::P; + let expr3_0 = constructor_cmove_or_from_values( + ctx, pattern2_0, &expr1_0, &expr2_0, pattern6_1, + pattern6_2, + )?; + let expr4_0 = + constructor_with_flags(ctx, &expr0_0, &expr3_0)?; + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + &FloatCC::Ordered => { + // Rule at src/isa/x64/lower.isle line 1689. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_1, + pattern11_0, + )?; + let expr1_0 = CC::NP; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::Unordered => { + // Rule at src/isa/x64/lower.isle line 1692. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_1, + pattern11_0, + )?; + let expr1_0 = CC::P; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrGreaterThan => { + // Rule at src/isa/x64/lower.isle line 1724. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_0, + pattern11_1, + )?; + let expr1_0 = CC::B; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrGreaterThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1727. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_0, + pattern11_1, + )?; + let expr1_0 = CC::BE; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrLessThan => { + // Rule at src/isa/x64/lower.isle line 1701. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_1, + pattern11_0, + )?; + let expr1_0 = CC::B; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + &FloatCC::UnorderedOrLessThanOrEqual => { + // Rule at src/isa/x64/lower.isle line 1704. + let expr0_0 = constructor_x64_ucomis( + ctx, + pattern11_1, + pattern11_0, + )?; + let expr1_0 = CC::BE; + let expr2_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, + )?; + let expr3_0 = + constructor_with_flags(ctx, &expr0_0, &expr2_0)?; + let expr4_0 = C::output(ctx, expr3_0); + return Some(expr4_0); + } + _ => {} } - &FloatCC::GreaterThan => { - // Rule at src/isa/x64/lower.isle line 1695. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; - let expr1_0 = CC::NBE; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::GreaterThanOrEqual => { - // Rule at src/isa/x64/lower.isle line 1698. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; - let expr1_0 = CC::NB; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::LessThan => { - // Rule at src/isa/x64/lower.isle line 1718. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; - let expr1_0 = CC::NBE; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::LessThanOrEqual => { - // Rule at src/isa/x64/lower.isle line 1721. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; - let expr1_0 = CC::NB; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::NotEqual => { - // Rule at src/isa/x64/lower.isle line 1746. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; - let expr1_0 = CC::NZ; - let expr2_0 = CC::P; - let expr3_0 = constructor_cmove_or_from_values( - ctx, pattern2_0, &expr1_0, &expr2_0, pattern6_1, - pattern6_2, - )?; - let expr4_0 = - constructor_with_flags(ctx, &expr0_0, &expr3_0)?; - let expr5_0 = C::output(ctx, expr4_0); - return Some(expr5_0); - } - &FloatCC::Ordered => { - // Rule at src/isa/x64/lower.isle line 1689. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; - let expr1_0 = CC::NP; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::Unordered => { - // Rule at src/isa/x64/lower.isle line 1692. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; - let expr1_0 = CC::P; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::UnorderedOrGreaterThan => { - // Rule at src/isa/x64/lower.isle line 1724. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; - let expr1_0 = CC::B; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::UnorderedOrGreaterThanOrEqual => { - // Rule at src/isa/x64/lower.isle line 1727. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_0, pattern11_1)?; - let expr1_0 = CC::BE; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::UnorderedOrLessThan => { - // Rule at src/isa/x64/lower.isle line 1701. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; - let expr1_0 = CC::B; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - &FloatCC::UnorderedOrLessThanOrEqual => { - // Rule at src/isa/x64/lower.isle line 1704. - let expr0_0 = - constructor_x64_ucomis(ctx, pattern11_1, pattern11_0)?; - let expr1_0 = CC::BE; - let expr2_0 = constructor_cmove_from_values( - ctx, pattern2_0, &expr1_0, pattern6_1, pattern6_2, - )?; - let expr3_0 = - constructor_with_flags(ctx, &expr0_0, &expr2_0)?; - let expr4_0 = C::output(ctx, expr3_0); - return Some(expr4_0); - } - _ => {} } } + &InstructionData::IntCompare { + opcode: ref pattern9_0, + args: ref pattern9_1, + cond: ref pattern9_2, + } => { + if let &Opcode::Icmp = pattern9_0 { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + let pattern12_0 = C::value_type(ctx, pattern11_0); + if let Some(pattern13_0) = C::fits_in_64(ctx, pattern12_0) { + // Rule at src/isa/x64/lower.isle line 1754. + let expr0_0 = constructor_put_in_gpr(ctx, pattern11_0)?; + let expr1_0 = constructor_put_in_gpr(ctx, pattern11_1)?; + let expr2_0 = C::raw_operand_size_of_type(ctx, pattern13_0); + let expr3_0 = C::gpr_to_gpr_mem_imm(ctx, expr1_0); + let expr4_0 = + constructor_x64_cmp(ctx, &expr2_0, &expr3_0, expr0_0)?; + let expr5_0 = C::intcc_to_cc(ctx, pattern9_2); + let expr6_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr5_0, pattern6_1, pattern6_2, + )?; + let expr7_0 = + constructor_with_flags(ctx, &expr4_0, &expr6_0)?; + let expr8_0 = C::output(ctx, expr7_0); + return Some(expr8_0); + } + } + } + _ => {} } } + let pattern7_0 = C::value_type(ctx, pattern6_0); + if pattern7_0 == B1 { + // Rule at src/isa/x64/lower.isle line 1772. + let expr0_0: Type = B1; + let expr1_0 = C::raw_operand_size_of_type(ctx, expr0_0); + let expr2_0 = constructor_put_in_gpr(ctx, pattern6_0)?; + let expr3_0: u32 = 1; + let expr4_0 = RegMemImm::Imm { simm32: expr3_0 }; + let expr5_0 = C::gpr_mem_imm_new(ctx, &expr4_0); + let expr6_0 = constructor_x64_test(ctx, &expr1_0, &expr5_0, expr2_0)?; + let expr7_0 = CC::NZ; + let expr8_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr7_0, pattern6_1, pattern6_2, + )?; + let expr9_0 = constructor_with_flags(ctx, &expr6_0, &expr8_0)?; + let expr10_0 = C::output(ctx, expr9_0); + return Some(expr10_0); + } + if let Some(pattern8_0) = C::fits_in_64(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 1779. + let expr0_0 = C::raw_operand_size_of_type(ctx, pattern8_0); + let expr1_0 = constructor_put_in_gpr(ctx, pattern6_0)?; + let expr2_0 = C::gpr_to_gpr_mem_imm(ctx, expr1_0); + let expr3_0 = constructor_x64_test(ctx, &expr0_0, &expr2_0, expr1_0)?; + let expr4_0 = CC::NZ; + let expr5_0 = constructor_cmove_from_values( + ctx, pattern2_0, &expr4_0, pattern6_1, pattern6_2, + )?; + let expr6_0 = constructor_with_flags(ctx, &expr3_0, &expr5_0)?; + let expr7_0 = C::output(ctx, expr6_0); + return Some(expr7_0); + } } } &InstructionData::Unary { @@ -7307,14 +7405,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2203. + // Rule at src/isa/x64/lower.isle line 2240. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Bextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2184. + // Rule at src/isa/x64/lower.isle line 2221. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -7322,7 +7420,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2192. + // Rule at src/isa/x64/lower.isle line 2229. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } @@ -7330,14 +7428,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2085. + // Rule at src/isa/x64/lower.isle line 2122. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Sextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2178. + // Rule at src/isa/x64/lower.isle line 2215. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -8533,7 +8631,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2206. + // Rule at src/isa/x64/lower.isle line 2243. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -8989,7 +9087,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2214. + // Rule at src/isa/x64/lower.isle line 2251. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0: u32 = 1; let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; @@ -8999,7 +9097,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2198. + // Rule at src/isa/x64/lower.isle line 2235. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -9113,21 +9211,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1762. + // Rule at src/isa/x64/lower.isle line 1799. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_clz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Ctz => { - // Rule at src/isa/x64/lower.isle line 1804. + // Rule at src/isa/x64/lower.isle line 1841. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_ctz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Popcnt => { - // Rule at src/isa/x64/lower.isle line 1856. + // Rule at src/isa/x64/lower.isle line 1893. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_popcnt(ctx, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; @@ -9230,7 +9328,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1767. + // Rule at src/isa/x64/lower.isle line 1804. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -9241,7 +9339,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1809. + // Rule at src/isa/x64/lower.isle line 1846. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -9252,7 +9350,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1861. + // Rule at src/isa/x64/lower.isle line 1898. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -9884,7 +9982,7 @@ pub fn constructor_do_clz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1787. + // Rule at src/isa/x64/lower.isle line 1824. let expr0_0: Type = I64; let expr1_0: i64 = -1; let expr2_0 = constructor_imm_i64(ctx, expr0_0, expr1_0)?; @@ -9911,7 +10009,7 @@ pub fn constructor_do_ctz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1828. + // Rule at src/isa/x64/lower.isle line 1865. let expr0_0: Type = I64; let expr1_0 = C::ty_bits_u64(ctx, pattern1_0); let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -9925,7 +10023,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> let pattern0_0 = arg0; if pattern0_0 == I32 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1919. + // Rule at src/isa/x64/lower.isle line 1956. let expr0_0: Type = I32; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -9990,7 +10088,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> } if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1876. + // Rule at src/isa/x64/lower.isle line 1913. let expr0_0: Type = I64; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -10064,7 +10162,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2008. + // Rule at src/isa/x64/lower.isle line 2045. let expr0_0 = C::ty_mask(ctx, pattern0_0); let expr1_0: u64 = 6148914691236517205; let expr2_0 = C::u64_and(ctx, expr0_0, expr1_0); @@ -10127,7 +10225,7 @@ pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2031. + // Rule at src/isa/x64/lower.isle line 2068. let expr0_0 = constructor_do_bitrev8(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 71777214294589695; @@ -10155,7 +10253,7 @@ pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) - pub fn constructor_do_bitrev32(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2043. + // Rule at src/isa/x64/lower.isle line 2080. let expr0_0 = constructor_do_bitrev16(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 281470681808895; @@ -10184,7 +10282,7 @@ pub fn constructor_do_bitrev64(ctx: &mut C, arg0: Type, arg1: Gpr) - let pattern0_0 = arg0; if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2055. + // Rule at src/isa/x64/lower.isle line 2092. let expr0_0 = constructor_do_bitrev32(ctx, pattern0_0, pattern2_0)?; let expr1_0: u64 = 4294967295; let expr2_0 = constructor_imm(ctx, pattern0_0, expr1_0)?; @@ -10218,7 +10316,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::fits_in_32(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2175. + // Rule at src/isa/x64/lower.isle line 2212. let expr0_0: Type = I32; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -10226,7 +10324,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_64(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2171. + // Rule at src/isa/x64/lower.isle line 2208. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -10234,7 +10332,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2165. + // Rule at src/isa/x64/lower.isle line 2202. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -10249,7 +10347,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::ty_int_bool_64(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2161. + // Rule at src/isa/x64/lower.isle line 2198. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_put_in_gpr(ctx, pattern0_0)?; let expr2_0 = constructor_spread_sign_bit(ctx, expr1_0)?; @@ -10261,7 +10359,7 @@ pub fn constructor_generic_sextend( } let pattern2_0 = arg2; if pattern2_0 == pattern1_0 { - // Rule at src/isa/x64/lower.isle line 2151. + // Rule at src/isa/x64/lower.isle line 2188. let expr0_0 = constructor_output_value(ctx, pattern0_0)?; return Some(expr0_0); } @@ -10271,7 +10369,7 @@ pub fn constructor_generic_sextend( // Generated as internal constructor for term spread_sign_bit. pub fn constructor_spread_sign_bit(ctx: &mut C, arg0: Gpr) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 2157. + // Rule at src/isa/x64/lower.isle line 2194. let expr0_0: Type = I64; let expr1_0: u8 = 63; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; diff --git a/cranelift/filetests/filetests/isa/x64/b1.clif b/cranelift/filetests/filetests/isa/x64/b1.clif index 89325852c6..9d69d70e1c 100644 --- a/cranelift/filetests/filetests/isa/x64/b1.clif +++ b/cranelift/filetests/filetests/isa/x64/b1.clif @@ -11,15 +11,16 @@ block0(v0: b1, v1: i32, v2: i32): ; Entry block: 0 ; Block 0: ; (original IR block: block0) -; (instruction range: 0 .. 8) +; (instruction range: 0 .. 9) ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp ; Inst 2: testb $1, %dil -; Inst 3: cmovnzl %esi, %edx -; Inst 4: movq %rdx, %rax -; Inst 5: movq %rbp, %rsp -; Inst 6: popq %rbp -; Inst 7: ret +; Inst 3: movl %edx, %edi +; Inst 4: cmovnzl %esi, %edi +; Inst 5: movq %rdi, %rax +; Inst 6: movq %rbp, %rsp +; Inst 7: popq %rbp +; Inst 8: ret ; }} function %f1(b1) -> i32 {