From e4b7c8a7376c0b4196262b176373b606c7b4c760 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Fri, 22 Apr 2022 18:00:48 -0700 Subject: [PATCH] Cranelift: fix #3953: rework single/multiple-use logic in lowering. (#4061) * Cranelift: fix #3953: rework single/multiple-use logic in lowering. This PR addresses the longstanding issue with loads trying to merge into compares on x86-64, and more generally, with the lowering framework falsely recognizing "single uses" of one op by another (which would normally allow merging of side-effecting ops like loads) when there is *indirect* duplication. To fix this, we replace the direct `value_uses` count with a transitive notion of uniqueness (not unlike Rust's `&`/`&mut` and how a `&mut` downgrades to `&` when accessed through another `&`!). A value is used multiple times transitively if it has multiple direct uses, or is used by another op that is used multiple times transitively. The canonical example of badness is: ``` v1 := load v2 := ifcmp v1, ... v3 := selectif v2, ... v4 := selectif v2, ... ``` both `v3` and `v4` effectively merge the `ifcmp` (`v2`), so even though the use of `v1` is "unique", it is codegenned twice. This is why we ~~can't have nice things~~ can't merge loads into compares (#3953). There is quite a subtle and interesting design space around this problem and how we might solve it. See the long doc-comment on `ValueUseState` in this PR for more justification for the particular design here. In particular, this design deliberately simplifies a bit relative to an "optimal" solution: some uses can *become* unique depending on merging, but we don't design our data structures for such updates because that would require significant extra costly tracking (some sort of transitive refcounting). For example, in the above, if `selectif` somehow did not merge `ifcmp`, then we would only codegen the `ifcmp` once into its result register (and use that register twice); then the load *is* uniquely used, and could be merged. But that requires transitioning from "multiple use" back to "unique use" with careful tracking as we do pattern-matching, which I've chosen to make out-of-scope here for now. In practice, I don't think it will matter too much (and we can always improve later). With this PR, we can now re-enable load-op merging for compares. A subsequent commit does this. * Update x64 backend to allow load-op merging for `cmp`. * Update filetests. * Add test for cmp-mem merging on x64. * Comment fixes. * Rework ValueUseState analysis for better performance. * Update s390x filetest: iadd_ifcout cannot merge loads anymore because it has multiple outputs (ValueUseState limitation) * Address review comments. --- cranelift/codegen/src/isa/aarch64/lower.rs | 11 +- .../codegen/src/isa/aarch64/lower/isle.rs | 4 +- cranelift/codegen/src/isa/s390x/lower/isle.rs | 2 +- cranelift/codegen/src/isa/x64/lower.isle | 23 +- cranelift/codegen/src/isa/x64/lower.rs | 26 +- cranelift/codegen/src/isa/x64/lower/isle.rs | 8 +- .../x64/lower/isle/generated_code.manifest | 2 +- .../src/isa/x64/lower/isle/generated_code.rs | 424 +++++++++--------- cranelift/codegen/src/machinst/lower.rs | 307 ++++++++++++- .../filetests/isa/s390x/arithmetic.clif | 12 +- .../filetests/isa/s390x/heap_addr.clif | 12 +- .../filetests/filetests/isa/x64/load-op.clif | 11 + .../filetests/isa/x64/select-i128.clif | 3 +- 13 files changed, 547 insertions(+), 298 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index c0b0822343..a69d7ee24b 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -306,7 +306,8 @@ fn put_input_in_rs>( narrow_mode: NarrowValueMode, ) -> ResultRS { let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); - if let Some((insn, 0)) = inputs.inst { + // Unique or non-unique use is fine for merging here. + if let Some((insn, 0)) = inputs.inst.as_inst() { let op = ctx.data(insn).opcode(); if op == Opcode::Ishl { @@ -353,7 +354,7 @@ fn get_as_extended_value>( narrow_mode: NarrowValueMode, ) -> Option<(Value, ExtendOp)> { let inputs = ctx.get_value_as_source_or_const(val); - let (insn, n) = inputs.inst?; + let (insn, n) = inputs.inst.as_inst()?; if n != 0 { return None; } @@ -1125,7 +1126,7 @@ pub(crate) fn maybe_input_insn>( inputs, op ); - if let Some((src_inst, _)) = inputs.inst { + if let Some((src_inst, _)) = inputs.inst.as_inst() { let data = c.data(src_inst); log::trace!(" -> input inst {:?}", data); if data.opcode() == op { @@ -1161,14 +1162,14 @@ pub(crate) fn maybe_input_insn_via_conv>( conv: Opcode, ) -> Option { let inputs = c.get_input_as_source_or_const(input.insn, input.input); - if let Some((src_inst, _)) = inputs.inst { + if let Some((src_inst, _)) = inputs.inst.as_inst() { let data = c.data(src_inst); if data.opcode() == op { return Some(src_inst); } if data.opcode() == conv { let inputs = c.get_input_as_source_or_const(src_inst, 0); - if let Some((src_inst, _)) = inputs.inst { + if let Some((src_inst, _)) = inputs.inst.as_inst() { let data = c.data(src_inst); if data.opcode() == op { return Some(src_inst); diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 7dda0dac37..73497dda54 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -12,7 +12,7 @@ use super::{ NZCV, }; use crate::isa::aarch64::settings::Flags as IsaFlags; -use crate::machinst::isle::*; +use crate::machinst::{isle::*, InputSourceInst}; use crate::settings::Flags; use crate::{ binemit::CodeOffset, @@ -245,7 +245,7 @@ where fn sinkable_atomic_load(&mut self, val: Value) -> Option { let input = self.lower_ctx.get_value_as_source_or_const(val); - if let Some((atomic_load, 0)) = input.inst { + if let InputSourceInst::UniqueUse(atomic_load, 0) = input.inst { if self.lower_ctx.data(atomic_load).opcode() == Opcode::AtomicLoad { let atomic_addr = self.lower_ctx.input_as_value(atomic_load, 0); return Some(SinkableAtomicLoad { diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index a9424d88c9..8ad35571fc 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -497,7 +497,7 @@ where #[inline] fn sinkable_inst(&mut self, val: Value) -> Option { let input = self.lower_ctx.get_value_as_source_or_const(val); - if let Some((inst, 0)) = input.inst { + if let Some((inst, 0)) = input.inst.as_inst() { return Some(inst); } None diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 544266fda3..d064e7a5f7 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1367,11 +1367,9 @@ (decl cmp_and_choose (Type CC Value Value) ValueRegs) (rule (cmp_and_choose (fits_in_64 ty) cc x y) - (let ((x_reg Gpr x) - (y_reg Gpr y) - (size OperandSize (raw_operand_size_of_type ty))) - (with_flags_reg (x64_cmp size x_reg y_reg) - (cmove ty cc y_reg x_reg)))) + (let ((size OperandSize (raw_operand_size_of_type ty))) + (with_flags_reg (x64_cmp size x y) + (cmove ty cc y x)))) (rule (lower (has_type (fits_in_64 ty) (umin x y))) (cmp_and_choose ty (CC.B) x y)) @@ -1751,19 +1749,8 @@ ;; than one instruction for certain types (e.g., XMM-held, I128). (rule (lower (has_type ty (select (icmp cc a @ (value_type (fits_in_64 a_ty)) b) x y))) - ;; N.B.: we force the comparison operators into registers, and disallow - ;; load-op fusion, because we do not have a transitive guarantee that this - ;; cmp-site will be the sole user of the value. Consider: the `icmp` might - ;; be the only user of a load, but there may be multiple users of the - ;; `icmp` (e.g., `select` or `bint` instructions) that each invoke emit a - ;; comparison. If we were to allow a load to sink to the *latest* one, but - ;; other sites did not permit sinking, then we would be missing the load - ;; for other cmp-sites. TODO: - ;; https://github.com/bytecodealliance/wasmtime/issues/3953. - (let ((gpr_a Gpr (put_in_gpr a)) - (gpr_b Gpr (put_in_gpr b)) - (size OperandSize (raw_operand_size_of_type a_ty))) - (with_flags (x64_cmp size gpr_b gpr_a) (cmove_from_values ty cc x y)))) + (let ((size OperandSize (raw_operand_size_of_type a_ty))) + (with_flags (x64_cmp size b a) (cmove_from_values ty cc x y)))) ;; Finally, we lower `select` from a condition value `c`. These rules are meant ;; to be the final, default lowerings if no other patterns matched above. diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d1aedd7b17..ac852160a7 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -61,7 +61,7 @@ fn matches_input>( op: Opcode, ) -> Option { let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); - inputs.inst.and_then(|(src_inst, _)| { + inputs.inst.as_inst().and_then(|(src_inst, _)| { let data = ctx.data(src_inst); if data.opcode() == op { return Some(src_inst); @@ -172,7 +172,7 @@ fn input_to_reg_mem>(ctx: &mut C, spec: InsnInput) -> RegM return RegMem::reg(generate_constant(ctx, ty, c).only_reg().unwrap()); } - if let Some((src_insn, 0)) = inputs.inst { + if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let Some((addr_input, offset)) = is_mergeable_load(ctx, src_insn) { ctx.sink_inst(src_insn); let amode = lower_to_amode(ctx, addr_input, offset); @@ -479,22 +479,11 @@ fn emit_cmp>(ctx: &mut C, insn: IRInst, cc: IntCC) -> IntC } else { // TODO Try to commute the operands (and invert the condition) if one is an immediate. let lhs = put_input_in_reg(ctx, inputs[0]); - // We force the RHS into a register, and disallow load-op fusion, because we - // do not have a transitive guarantee that this cmp-site will be the sole - // user of the value. Consider: the icmp might be the only user of a load, - // but there may be multiple users of the icmp (e.g. select or bint - // instructions) that each invoke `emit_cmp()`. If we were to allow a load - // to sink to the *latest* one, but other sites did not permit sinking, then - // we would be missing the load for other cmp-sites. - let rhs = put_input_in_reg(ctx, inputs[1]); + let rhs = input_to_reg_mem_imm(ctx, inputs[1]); // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives // us dst - src at the machine instruction level, so invert operands. - ctx.emit(Inst::cmp_rmi_r( - OperandSize::from_ty(ty), - RegMemImm::reg(rhs), - lhs, - )); + ctx.emit(Inst::cmp_rmi_r(OperandSize::from_ty(ty), rhs, lhs)); cc } } @@ -578,10 +567,8 @@ fn emit_fcmp>( (inputs[0], inputs[1]) }; let lhs = put_input_in_reg(ctx, lhs_input); - // See above in `emit_cmp()`. We must only use the reg/reg form of the - // comparison in order to avoid issues with merged loads. - let rhs = put_input_in_reg(ctx, rhs_input); - ctx.emit(Inst::xmm_cmp_rm_r(op, RegMem::reg(rhs), lhs)); + let rhs = input_to_reg_mem(ctx, rhs_input); + ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs)); let cond_result = match cond_code { FloatCC::Equal => FcmpCondResult::AndConditions(CC::NP, CC::Z), @@ -2406,6 +2393,7 @@ fn lower_insn_to_regs>( let cmp_insn = ctx .get_input_as_source_or_const(inputs[0].insn, inputs[0].input) .inst + .as_inst() .unwrap() .0; debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp); diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index c8032ecde9..733ef1f7a1 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -2,7 +2,7 @@ // Pull in the ISLE generated code. pub(crate) mod generated_code; -use crate::machinst::{Reg, Writable}; +use crate::machinst::{InputSourceInst, Reg, Writable}; use generated_code::MInst; // Types that the generated ISLE code uses via `use super::*`. @@ -84,7 +84,7 @@ where return RegMemImm::reg(generated_code::constructor_imm(self, ty, c).unwrap()); } - if let Some((src_insn, 0)) = inputs.inst { + if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) { self.lower_ctx.sink_inst(src_insn); let amode = lower_to_amode(self.lower_ctx, addr_input, offset); @@ -105,7 +105,7 @@ where return RegMem::reg(generated_code::constructor_imm(self, ty, c).unwrap()); } - if let Some((src_insn, 0)) = inputs.inst { + if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) { self.lower_ctx.sink_inst(src_insn); let amode = lower_to_amode(self.lower_ctx, addr_input, offset); @@ -237,7 +237,7 @@ where fn sinkable_load(&mut self, val: Value) -> Option { let input = self.lower_ctx.get_value_as_source_or_const(val); - if let Some((inst, 0)) = input.inst { + if let InputSourceInst::UniqueUse(inst, 0) = input.inst { if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, inst) { return Some(SinkableLoad { inst, diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 687ad644d3..4636672ed1 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 443b34b797fc8ace src/prelude.isle afd037c4d91c875c src/isa/x64/inst.isle cad03431447aca1b -src/isa/x64/lower.isle 42bd3982a6132a2f +src/isa/x64/lower.isle 803aac716f6f4c39 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 4d723587a8..d42160c20e 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -5115,7 +5115,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Debugtrap = pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2261. + // Rule at src/isa/x64/lower.isle line 2248. let expr0_0 = constructor_x64_hlt(ctx)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); @@ -5278,13 +5278,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern2_0 { &Opcode::Trap => { - // Rule at src/isa/x64/lower.isle line 1434. + // Rule at src/isa/x64/lower.isle line 1432. let expr0_0 = constructor_x64_ud2(ctx, pattern2_1)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); } &Opcode::ResumableTrap => { - // Rule at src/isa/x64/lower.isle line 1439. + // Rule at src/isa/x64/lower.isle line 1437. let expr0_0 = constructor_x64_ud2(ctx, pattern2_1)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); @@ -5321,7 +5321,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1571. + // Rule at src/isa/x64/lower.isle line 1569. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NP; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5341,7 +5341,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1599. + // Rule at src/isa/x64/lower.isle line 1597. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NBE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5350,7 +5350,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1601. + // Rule at src/isa/x64/lower.isle line 1599. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NB; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5359,7 +5359,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1611. + // Rule at src/isa/x64/lower.isle line 1609. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::NBE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5368,7 +5368,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1614. + // Rule at src/isa/x64/lower.isle line 1612. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::NB; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5377,7 +5377,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1580. + // Rule at src/isa/x64/lower.isle line 1578. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::P; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5397,7 +5397,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1591. + // Rule at src/isa/x64/lower.isle line 1589. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NP; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5406,7 +5406,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1595. + // Rule at src/isa/x64/lower.isle line 1593. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NZ; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5415,7 +5415,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1593. + // Rule at src/isa/x64/lower.isle line 1591. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::P; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5424,7 +5424,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1597. + // Rule at src/isa/x64/lower.isle line 1595. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::Z; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5433,7 +5433,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1617. + // Rule at src/isa/x64/lower.isle line 1615. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::B; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5442,7 +5442,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1620. + // Rule at src/isa/x64/lower.isle line 1618. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::BE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5451,7 +5451,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1603. + // Rule at src/isa/x64/lower.isle line 1601. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::B; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5460,7 +5460,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1605. + // Rule at src/isa/x64/lower.isle line 1603. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::BE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5474,7 +5474,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1628. + // Rule at src/isa/x64/lower.isle line 1626. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::Equal; @@ -5484,7 +5484,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1648. + // Rule at src/isa/x64/lower.isle line 1646. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::LessThan; @@ -5494,7 +5494,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1650. + // Rule at src/isa/x64/lower.isle line 1648. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::LessThanOrEqual; @@ -5504,7 +5504,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1632. + // Rule at src/isa/x64/lower.isle line 1630. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::LessThan; @@ -5514,7 +5514,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1634. + // Rule at src/isa/x64/lower.isle line 1632. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::LessThanOrEqual; @@ -5524,7 +5524,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1630. + // Rule at src/isa/x64/lower.isle line 1628. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::NotEqual; @@ -5534,7 +5534,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1636. + // Rule at src/isa/x64/lower.isle line 1634. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::Ordered; @@ -5544,7 +5544,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1638. + // Rule at src/isa/x64/lower.isle line 1636. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::Unordered; @@ -5554,7 +5554,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1640. + // Rule at src/isa/x64/lower.isle line 1638. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThan; @@ -5564,7 +5564,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1642. + // Rule at src/isa/x64/lower.isle line 1640. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThanOrEqual; @@ -5574,7 +5574,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1652. + // Rule at src/isa/x64/lower.isle line 1650. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThan; @@ -5584,7 +5584,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1654. + // Rule at src/isa/x64/lower.isle line 1652. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThanOrEqual; @@ -5609,7 +5609,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1522. + // Rule at src/isa/x64/lower.isle line 1520. let expr0_0 = C::put_in_regs(ctx, pattern4_0); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -5653,7 +5653,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1541. + // Rule at src/isa/x64/lower.isle line 1539. let expr0_0 = C::put_in_regs(ctx, pattern4_0); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -5702,7 +5702,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1507. + // Rule at src/isa/x64/lower.isle line 1505. let expr0_0: Type = I64X2; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr2_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; @@ -5715,7 +5715,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1511. + // Rule at src/isa/x64/lower.isle line 1509. let expr0_0: Type = I64X2; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr2_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; @@ -5731,7 +5731,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1456. + // Rule at src/isa/x64/lower.isle line 1454. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5754,7 +5754,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1460. + // Rule at src/isa/x64/lower.isle line 1458. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5766,7 +5766,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1467. + // Rule at src/isa/x64/lower.isle line 1465. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5775,7 +5775,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1492. + // Rule at src/isa/x64/lower.isle line 1490. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5788,7 +5788,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1469. + // Rule at src/isa/x64/lower.isle line 1467. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = @@ -5797,7 +5797,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1495. + // Rule at src/isa/x64/lower.isle line 1493. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5810,7 +5810,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1471. + // Rule at src/isa/x64/lower.isle line 1469. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); @@ -5826,7 +5826,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1498. + // Rule at src/isa/x64/lower.isle line 1496. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5839,7 +5839,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1481. + // Rule at src/isa/x64/lower.isle line 1479. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); @@ -5855,7 +5855,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1501. + // Rule at src/isa/x64/lower.isle line 1499. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5880,7 +5880,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 2104. + // Rule at src/isa/x64/lower.isle line 2091. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 0; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -5895,7 +5895,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 2112. + // Rule at src/isa/x64/lower.isle line 2099. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 4294967295; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -6026,7 +6026,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2035. + // Rule at src/isa/x64/lower.isle line 2022. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0 = constructor_do_bitrev64(ctx, expr0_0, expr1_0)?; @@ -6100,7 +6100,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2151. + // Rule at src/isa/x64/lower.isle line 2138. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6108,7 +6108,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2157. + // Rule at src/isa/x64/lower.isle line 2144. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6116,7 +6116,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2160. + // Rule at src/isa/x64/lower.isle line 2147. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6124,7 +6124,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2154. + // Rule at src/isa/x64/lower.isle line 2141. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6132,7 +6132,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2163. + // Rule at src/isa/x64/lower.isle line 2150. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6140,7 +6140,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2166. + // Rule at src/isa/x64/lower.isle line 2153. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6148,7 +6148,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2169. + // Rule at src/isa/x64/lower.isle line 2156. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6156,7 +6156,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2172. + // Rule at src/isa/x64/lower.isle line 2159. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6164,7 +6164,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2175. + // Rule at src/isa/x64/lower.isle line 2162. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6179,7 +6179,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Uload32 = pattern12_0 { - // Rule at src/isa/x64/lower.isle line 2178. + // Rule at src/isa/x64/lower.isle line 2165. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6192,7 +6192,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2038. + // Rule at src/isa/x64/lower.isle line 2025. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 1; @@ -6469,7 +6469,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1808. + // Rule at src/isa/x64/lower.isle line 1795. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -6507,7 +6507,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1850. + // Rule at src/isa/x64/lower.isle line 1837. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -6545,7 +6545,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1902. + // Rule at src/isa/x64/lower.isle line 1889. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 0; @@ -6568,7 +6568,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2253. + // Rule at src/isa/x64/lower.isle line 2240. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0: u32 = 1; @@ -6586,7 +6586,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern7_0 = C::value_type(ctx, pattern5_1); if pattern7_0 == I64 { - // Rule at src/isa/x64/lower.isle line 2125. + // Rule at src/isa/x64/lower.isle line 2112. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = I64; let expr2_0: u64 = 0; @@ -6596,7 +6596,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2276. + // Rule at src/isa/x64/lower.isle line 2263. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addss(ctx, expr0_0, &expr1_0)?; @@ -6666,7 +6666,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2287. + // Rule at src/isa/x64/lower.isle line 2274. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subss(ctx, expr0_0, &expr1_0)?; @@ -6675,7 +6675,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2298. + // Rule at src/isa/x64/lower.isle line 2285. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulss(ctx, expr0_0, &expr1_0)?; @@ -6684,7 +6684,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2309. + // Rule at src/isa/x64/lower.isle line 2296. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divss(ctx, expr0_0, &expr1_0)?; @@ -6693,7 +6693,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2330. + // Rule at src/isa/x64/lower.isle line 2317. let expr0_0: Type = F32; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6706,7 +6706,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2499. + // Rule at src/isa/x64/lower.isle line 2486. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minss(ctx, expr0_0, expr1_0)?; @@ -6715,7 +6715,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2402. + // Rule at src/isa/x64/lower.isle line 2389. let expr0_0: Type = F32; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6728,7 +6728,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2510. + // Rule at src/isa/x64/lower.isle line 2497. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxss(ctx, expr0_0, expr1_0)?; @@ -6743,7 +6743,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Sqrt = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2319. + // Rule at src/isa/x64/lower.isle line 2306. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtss(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -6757,7 +6757,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2553. + // Rule at src/isa/x64/lower.isle line 2540. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -6779,7 +6779,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2278. + // Rule at src/isa/x64/lower.isle line 2265. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addsd(ctx, expr0_0, &expr1_0)?; @@ -6788,7 +6788,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2289. + // Rule at src/isa/x64/lower.isle line 2276. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subsd(ctx, expr0_0, &expr1_0)?; @@ -6797,7 +6797,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2300. + // Rule at src/isa/x64/lower.isle line 2287. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulsd(ctx, expr0_0, &expr1_0)?; @@ -6806,7 +6806,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2311. + // Rule at src/isa/x64/lower.isle line 2298. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divsd(ctx, expr0_0, &expr1_0)?; @@ -6815,7 +6815,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2332. + // Rule at src/isa/x64/lower.isle line 2319. let expr0_0: Type = F64; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6828,7 +6828,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2501. + // Rule at src/isa/x64/lower.isle line 2488. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minsd(ctx, expr0_0, expr1_0)?; @@ -6837,7 +6837,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2404. + // Rule at src/isa/x64/lower.isle line 2391. let expr0_0: Type = F64; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6850,7 +6850,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2512. + // Rule at src/isa/x64/lower.isle line 2499. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxsd(ctx, expr0_0, expr1_0)?; @@ -6865,7 +6865,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Sqrt = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2321. + // Rule at src/isa/x64/lower.isle line 2308. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtsd(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -6879,7 +6879,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2555. + // Rule at src/isa/x64/lower.isle line 2542. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -6901,7 +6901,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1401. + // Rule at src/isa/x64/lower.isle line 1399. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminsb(ctx, expr0_0, &expr1_0)?; @@ -6910,7 +6910,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1423. + // Rule at src/isa/x64/lower.isle line 1421. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminub(ctx, expr0_0, &expr1_0)?; @@ -6919,7 +6919,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1390. + // Rule at src/isa/x64/lower.isle line 1388. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -6928,7 +6928,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1412. + // Rule at src/isa/x64/lower.isle line 1410. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -7019,7 +7019,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2009. + // Rule at src/isa/x64/lower.isle line 1996. let expr0_0 = C::popcount_4bit_table(ctx); let expr1_0: Type = I8X16; let expr2_0 = C::popcount_low_mask(ctx); @@ -7064,7 +7064,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1404. + // Rule at src/isa/x64/lower.isle line 1402. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminsw(ctx, expr0_0, &expr1_0)?; @@ -7073,7 +7073,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1426. + // Rule at src/isa/x64/lower.isle line 1424. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminuw(ctx, expr0_0, &expr1_0)?; @@ -7082,7 +7082,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1393. + // Rule at src/isa/x64/lower.isle line 1391. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -7091,7 +7091,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1415. + // Rule at src/isa/x64/lower.isle line 1413. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -7165,7 +7165,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload8x8 => { - // Rule at src/isa/x64/lower.isle line 2568. + // Rule at src/isa/x64/lower.isle line 2555. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7174,7 +7174,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2566. + // Rule at src/isa/x64/lower.isle line 2553. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7198,7 +7198,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1407. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminsd(ctx, expr0_0, &expr1_0)?; @@ -7207,7 +7207,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1429. + // Rule at src/isa/x64/lower.isle line 1427. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminud(ctx, expr0_0, &expr1_0)?; @@ -7216,7 +7216,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1396. + // Rule at src/isa/x64/lower.isle line 1394. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -7225,7 +7225,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1418. + // Rule at src/isa/x64/lower.isle line 1416. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -7264,7 +7264,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2266. + // Rule at src/isa/x64/lower.isle line 2253. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaddwd(ctx, expr0_0, &expr1_0)?; @@ -7308,7 +7308,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload16x4 => { - // Rule at src/isa/x64/lower.isle line 2572. + // Rule at src/isa/x64/lower.isle line 2559. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7317,7 +7317,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2570. + // Rule at src/isa/x64/lower.isle line 2557. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7427,7 +7427,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload32x2 => { - // Rule at src/isa/x64/lower.isle line 2576. + // Rule at src/isa/x64/lower.isle line 2563. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7436,7 +7436,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2574. + // Rule at src/isa/x64/lower.isle line 2561. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7460,7 +7460,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2280. + // Rule at src/isa/x64/lower.isle line 2267. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addps(ctx, expr0_0, &expr1_0)?; @@ -7469,7 +7469,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2291. + // Rule at src/isa/x64/lower.isle line 2278. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subps(ctx, expr0_0, &expr1_0)?; @@ -7478,7 +7478,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2302. + // Rule at src/isa/x64/lower.isle line 2289. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulps(ctx, expr0_0, &expr1_0)?; @@ -7487,7 +7487,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2313. + // Rule at src/isa/x64/lower.isle line 2300. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divps(ctx, expr0_0, &expr1_0)?; @@ -7496,7 +7496,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2347. + // Rule at src/isa/x64/lower.isle line 2334. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -7521,7 +7521,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2503. + // Rule at src/isa/x64/lower.isle line 2490. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -7530,7 +7530,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2410. + // Rule at src/isa/x64/lower.isle line 2397. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -7558,7 +7558,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2514. + // Rule at src/isa/x64/lower.isle line 2501. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -7574,7 +7574,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Sqrt => { - // Rule at src/isa/x64/lower.isle line 2323. + // Rule at src/isa/x64/lower.isle line 2310. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtps(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -7604,7 +7604,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2557. + // Rule at src/isa/x64/lower.isle line 2544. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7626,7 +7626,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2282. + // Rule at src/isa/x64/lower.isle line 2269. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addpd(ctx, expr0_0, &expr1_0)?; @@ -7635,7 +7635,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2293. + // Rule at src/isa/x64/lower.isle line 2280. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subpd(ctx, expr0_0, &expr1_0)?; @@ -7644,7 +7644,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2304. + // Rule at src/isa/x64/lower.isle line 2291. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulpd(ctx, expr0_0, &expr1_0)?; @@ -7653,7 +7653,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2315. + // Rule at src/isa/x64/lower.isle line 2302. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divpd(ctx, expr0_0, &expr1_0)?; @@ -7662,7 +7662,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2390. + // Rule at src/isa/x64/lower.isle line 2377. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -7687,7 +7687,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2505. + // Rule at src/isa/x64/lower.isle line 2492. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -7696,7 +7696,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2453. + // Rule at src/isa/x64/lower.isle line 2440. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -7724,7 +7724,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2516. + // Rule at src/isa/x64/lower.isle line 2503. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -7740,7 +7740,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Sqrt => { - // Rule at src/isa/x64/lower.isle line 2325. + // Rule at src/isa/x64/lower.isle line 2312. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtpd(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -7770,7 +7770,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2559. + // Rule at src/isa/x64/lower.isle line 2546. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7829,7 +7829,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1742. + // Rule at src/isa/x64/lower.isle line 1740. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7847,7 +7847,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1694. + // Rule at src/isa/x64/lower.isle line 1692. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7863,7 +7863,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1697. + // Rule at src/isa/x64/lower.isle line 1695. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7879,7 +7879,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1717. + // Rule at src/isa/x64/lower.isle line 1715. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7895,7 +7895,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1720. + // Rule at src/isa/x64/lower.isle line 1718. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7911,7 +7911,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1745. + // Rule at src/isa/x64/lower.isle line 1743. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7929,7 +7929,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1688. + // Rule at src/isa/x64/lower.isle line 1686. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7945,7 +7945,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1691. + // Rule at src/isa/x64/lower.isle line 1689. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7961,7 +7961,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1723. + // Rule at src/isa/x64/lower.isle line 1721. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7977,7 +7977,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1726. + // Rule at src/isa/x64/lower.isle line 1724. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7993,7 +7993,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1700. + // Rule at src/isa/x64/lower.isle line 1698. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -8009,7 +8009,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1703. + // Rule at src/isa/x64/lower.isle line 1701. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -8038,21 +8038,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2239. + // Rule at src/isa/x64/lower.isle line 2226. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Bextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2220. + // Rule at src/isa/x64/lower.isle line 2207. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -8116,7 +8116,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2228. + // Rule at src/isa/x64/lower.isle line 2215. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } @@ -8124,14 +8124,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2121. + // Rule at src/isa/x64/lower.isle line 2108. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Sextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2214. + // Rule at src/isa/x64/lower.isle line 2201. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -9327,7 +9327,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1382. + // Rule at src/isa/x64/lower.isle line 1380. let expr0_0 = CC::L; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9414,7 +9414,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1376. + // Rule at src/isa/x64/lower.isle line 1374. let expr0_0 = CC::B; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9424,7 +9424,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1385. + // Rule at src/isa/x64/lower.isle line 1383. let expr0_0 = CC::NL; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9434,7 +9434,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1379. + // Rule at src/isa/x64/lower.isle line 1377. let expr0_0 = CC::NB; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9798,7 +9798,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2242. + // Rule at src/isa/x64/lower.isle line 2229. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -9806,7 +9806,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2250. + // Rule at src/isa/x64/lower.isle line 2237. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0: u32 = 1; let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; @@ -9816,7 +9816,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2234. + // Rule at src/isa/x64/lower.isle line 2221. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -9930,21 +9930,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1798. + // Rule at src/isa/x64/lower.isle line 1785. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_clz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Ctz => { - // Rule at src/isa/x64/lower.isle line 1840. + // Rule at src/isa/x64/lower.isle line 1827. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_ctz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Popcnt => { - // Rule at src/isa/x64/lower.isle line 1892. + // Rule at src/isa/x64/lower.isle line 1879. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_popcnt(ctx, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; @@ -10047,7 +10047,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1803. + // Rule at src/isa/x64/lower.isle line 1790. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -10058,7 +10058,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1845. + // Rule at src/isa/x64/lower.isle line 1832. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -10069,7 +10069,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1897. + // Rule at src/isa/x64/lower.isle line 1884. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -10095,7 +10095,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2536. + // Rule at src/isa/x64/lower.isle line 2523. let expr0_0 = ExtMode::BQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10143,7 +10143,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2538. + // Rule at src/isa/x64/lower.isle line 2525. let expr0_0 = ExtMode::BQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10153,7 +10153,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2540. + // Rule at src/isa/x64/lower.isle line 2527. let expr0_0 = ExtMode::WQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10163,7 +10163,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2542. + // Rule at src/isa/x64/lower.isle line 2529. let expr0_0 = ExtMode::WQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10173,7 +10173,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2544. + // Rule at src/isa/x64/lower.isle line 2531. let expr0_0 = ExtMode::LQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10183,7 +10183,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2546. + // Rule at src/isa/x64/lower.isle line 2533. let expr0_0 = ExtMode::LQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10787,14 +10787,14 @@ pub fn constructor_cmp_and_choose( let pattern3_0 = arg2; let pattern4_0 = arg3; // Rule at src/isa/x64/lower.isle line 1369. - let expr0_0 = constructor_put_in_gpr(ctx, pattern3_0)?; - let expr1_0 = constructor_put_in_gpr(ctx, pattern4_0)?; - let expr2_0 = C::raw_operand_size_of_type(ctx, pattern1_0); - let expr3_0 = C::gpr_to_gpr_mem_imm(ctx, expr0_0); - let expr4_0 = constructor_x64_cmp(ctx, &expr2_0, &expr3_0, expr1_0)?; - let expr5_0 = C::gpr_to_gpr_mem(ctx, expr1_0); - let expr6_0 = constructor_cmove(ctx, pattern1_0, pattern2_0, &expr5_0, expr0_0)?; - let expr7_0 = constructor_with_flags_reg(ctx, &expr4_0, &expr6_0)?; + let expr0_0 = C::raw_operand_size_of_type(ctx, pattern1_0); + let expr1_0 = constructor_put_in_gpr_mem_imm(ctx, pattern3_0)?; + let expr2_0 = constructor_put_in_gpr(ctx, pattern4_0)?; + let expr3_0 = constructor_x64_cmp(ctx, &expr0_0, &expr1_0, expr2_0)?; + let expr4_0 = constructor_put_in_gpr_mem(ctx, pattern4_0)?; + let expr5_0 = constructor_put_in_gpr(ctx, pattern3_0)?; + let expr6_0 = constructor_cmove(ctx, pattern1_0, pattern2_0, &expr4_0, expr5_0)?; + let expr7_0 = constructor_with_flags_reg(ctx, &expr3_0, &expr6_0)?; let expr8_0 = C::value_reg(ctx, expr7_0); return Some(expr8_0); } @@ -10811,7 +10811,7 @@ pub fn constructor_do_clz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1823. + // Rule at src/isa/x64/lower.isle line 1810. let expr0_0: Type = I64; let expr1_0: i64 = -1; let expr2_0 = constructor_imm_i64(ctx, expr0_0, expr1_0)?; @@ -10838,7 +10838,7 @@ pub fn constructor_do_ctz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1864. + // Rule at src/isa/x64/lower.isle line 1851. let expr0_0: Type = I64; let expr1_0 = C::ty_bits_u64(ctx, pattern1_0); let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -10852,7 +10852,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> let pattern0_0 = arg0; if pattern0_0 == I32 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1955. + // Rule at src/isa/x64/lower.isle line 1942. let expr0_0: Type = I32; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -10917,7 +10917,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> } if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1912. + // Rule at src/isa/x64/lower.isle line 1899. let expr0_0: Type = I64; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -10991,7 +10991,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2044. + // Rule at src/isa/x64/lower.isle line 2031. let expr0_0 = C::ty_mask(ctx, pattern0_0); let expr1_0: u64 = 6148914691236517205; let expr2_0 = C::u64_and(ctx, expr0_0, expr1_0); @@ -11054,7 +11054,7 @@ pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2067. + // Rule at src/isa/x64/lower.isle line 2054. let expr0_0 = constructor_do_bitrev8(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 71777214294589695; @@ -11082,7 +11082,7 @@ pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) - pub fn constructor_do_bitrev32(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2079. + // Rule at src/isa/x64/lower.isle line 2066. let expr0_0 = constructor_do_bitrev16(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 281470681808895; @@ -11111,7 +11111,7 @@ pub fn constructor_do_bitrev64(ctx: &mut C, arg0: Type, arg1: Gpr) - let pattern0_0 = arg0; if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2091. + // Rule at src/isa/x64/lower.isle line 2078. let expr0_0 = constructor_do_bitrev32(ctx, pattern0_0, pattern2_0)?; let expr1_0: u64 = 4294967295; let expr2_0 = constructor_imm(ctx, pattern0_0, expr1_0)?; @@ -11145,7 +11145,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::fits_in_32(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2211. + // Rule at src/isa/x64/lower.isle line 2198. let expr0_0: Type = I32; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -11153,7 +11153,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_64(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2207. + // Rule at src/isa/x64/lower.isle line 2194. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -11161,7 +11161,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2201. + // Rule at src/isa/x64/lower.isle line 2188. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -11176,7 +11176,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::ty_int_bool_64(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2197. + // Rule at src/isa/x64/lower.isle line 2184. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_put_in_gpr(ctx, pattern0_0)?; let expr2_0 = constructor_spread_sign_bit(ctx, expr1_0)?; @@ -11188,7 +11188,7 @@ pub fn constructor_generic_sextend( } let pattern2_0 = arg2; if pattern2_0 == pattern1_0 { - // Rule at src/isa/x64/lower.isle line 2187. + // Rule at src/isa/x64/lower.isle line 2174. let expr0_0 = constructor_output_value(ctx, pattern0_0)?; return Some(expr0_0); } @@ -11198,7 +11198,7 @@ pub fn constructor_generic_sextend( // Generated as internal constructor for term spread_sign_bit. pub fn constructor_spread_sign_bit(ctx: &mut C, arg0: Gpr) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 2193. + // Rule at src/isa/x64/lower.isle line 2180. let expr0_0: Type = I64; let expr1_0: u8 = 63; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 44c30872cc..aec7072d60 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -197,13 +197,49 @@ pub struct NonRegInput { /// computation (and side-effect if applicable) could occur at the /// current instruction's location instead. /// - /// If this instruction's operation is merged into the current instruction, - /// the backend must call [LowerCtx::sink_inst()]. - pub inst: Option<(Inst, usize)>, + /// If this instruction's operation is merged into the current + /// instruction, the backend must call [LowerCtx::sink_inst()]. + /// + /// This enum indicates whether this use of the source instruction + /// is unique or not. + pub inst: InputSourceInst, /// The value is a known constant. pub constant: Option, } +/// When examining an input to an instruction, this enum provides one +/// of several options: there is or isn't a single instruction (that +/// we can see and merge with) that produces that input's value, and +/// we are or aren't the single user of that instruction. +#[derive(Clone, Copy, Debug)] +pub enum InputSourceInst { + /// The input in question is the single, unique use of the given + /// instruction and output index, and it can be sunk to the + /// location of this input. + UniqueUse(Inst, usize), + /// The input in question is one of multiple uses of the given + /// instruction. It can still be sunk to the location of this + /// input. + Use(Inst, usize), + /// We cannot determine which instruction produced the input, or + /// it is one of several instructions (e.g., due to a control-flow + /// merge and blockparam), or the source instruction cannot be + /// allowed to sink to the current location due to side-effects. + None, +} + +impl InputSourceInst { + /// Get the instruction and output index for this source, whether + /// we are its single or one of many users. + pub fn as_inst(&self) -> Option<(Inst, usize)> { + match self { + &InputSourceInst::UniqueUse(inst, output_idx) + | &InputSourceInst::Use(inst, output_idx) => Some((inst, output_idx)), + &InputSourceInst::None => None, + } + } +} + /// A machine backend. pub trait LowerBackend { /// The machine instruction type. @@ -271,8 +307,13 @@ pub struct Lower<'func, I: VCodeInst> { /// Instruction constant values, if known. inst_constants: FxHashMap, - /// Use-counts per SSA value, as counted in the input IR. - value_uses: SecondaryMap, + /// Use-counts per SSA value, as counted in the input IR. These + /// are "coarsened", in the abstract-interpretation sense: we only + /// care about "0, 1, many" states, as this is all we need and + /// this lets us do an efficient fixpoint analysis. + /// + /// See doc comment on `ValueUseState` for more details. + value_ir_uses: SecondaryMap, /// Actual uses of each SSA value so far, incremented while lowering. value_lowered_uses: SecondaryMap, @@ -295,6 +336,108 @@ pub struct Lower<'func, I: VCodeInst> { vm_context: Option, } +/// How is a value used in the IR? +/// +/// This can be seen as a coarsening of an integer count. We only need +/// distinct states for zero, one, or many. +/// +/// This analysis deserves further explanation. The basic idea is that +/// we want to allow instruction lowering to know whether a value that +/// an instruction references is *only* referenced by that one use, or +/// by others as well. This is necessary to know when we might want to +/// move a side-effect: we cannot, for example, duplicate a load, so +/// we cannot let instruction lowering match a load as part of a +/// subpattern and potentially incorporate it. +/// +/// Note that a lot of subtlety comes into play once we have +/// *indirect* uses. The classical example of this in our development +/// history was the x86 compare instruction, which is incorporated +/// into flags users (e.g. `selectif`, `trueif`, branches) and can +/// subsequently incorporate loads, or at least we would like it +/// to. However, danger awaits: the compare might be the only user of +/// a load, so we might think we can just move the load (and nothing +/// is duplicated -- success!), except that the compare itself is +/// codegen'd in multiple places, where it is incorporated as a +/// subpattern itself. +/// +/// So we really want a notion of "unique all the way along the +/// matching path". Rust's `&T` and `&mut T` offer a partial analogy +/// to the semantics that we want here: we want to know when we've +/// matched a unique use of an instruction, and that instruction's +/// unique use of another instruction, etc, just as `&mut T` can only +/// be obtained by going through a chain of `&mut T`. If one has a +/// `&T` to a struct containing `&mut T` (one of several uses of an +/// instruction that itself has a unique use of an instruction), one +/// can only get a `&T` (one can only get a "I am one of several users +/// of this instruction" result). +/// +/// We could track these paths, either dynamically as one "looks up +/// the operand tree" or precomputed. But the former requires state +/// and means that the `LowerCtx` API carries that state implicitly, +/// which we'd like to avoid if we can. And the latter implies O(n^2) +/// storage: it is an all-pairs property (is inst `i` unique from the +/// point of view of `j`). +/// +/// To make matters even a little more complex still, a value that is +/// not uniquely used when initially viewing the IR can *become* +/// uniquely used, at least as a root allowing further unique uses of +/// e.g. loads to merge, if no other instruction actually merges +/// it. To be more concrete, if we have `v1 := load; v2 := op v1; v3 +/// := op v2; v4 := op v2` then `v2` is non-uniquely used, so from the +/// point of view of lowering `v4` or `v3`, we cannot merge the load +/// at `v1`. But if we decide just to use the assigned register for +/// `v2` at both `v3` and `v4`, then we only actually codegen `v2` +/// once, so it *is* a unique root at that point and we *can* merge +/// the load. +/// +/// Note also that the color scheme is not sufficient to give us this +/// information, for various reasons: reasoning about side-effects +/// does not tell us about potential duplication of uses through pure +/// ops. +/// +/// To keep things simple and avoid error-prone lowering APIs that +/// would extract more information about whether instruction merging +/// happens or not (we don't have that info now, and it would be +/// difficult to refactor to get it and make that refactor 100% +/// correct), we give up on the above "can become unique if not +/// actually merged" point. Instead, we compute a +/// transitive-uniqueness. That is what this enum represents. +/// +/// To define it plainly: a value is `Unused` if no references exist +/// to it; `Once` if only one other op refers to it, *and* that other +/// op is `Unused` or `Once`; and `Multiple` otherwise. In other +/// words, `Multiple` is contagious: even if an op's result value is +/// directly used only once in the CLIF, that value is `Multiple` if +/// the op that uses it is itself used multiple times (hence could be +/// codegen'd multiple times). In brief, this analysis tells us +/// whether, if every op merged all of its operand tree, a given op +/// could be codegen'd in more than one place. +/// +/// To compute this, we first consider direct uses. At this point +/// `Unused` answers are correct, `Multiple` answers are correct, but +/// some `Once`s may change to `Multiple`s. Then we propagate +/// `Multiple` transitively using a workqueue/fixpoint algorithm. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ValueUseState { + /// Not used at all. + Unused, + /// Used exactly once. + Once, + /// Used multiple times. + Multiple, +} + +impl ValueUseState { + /// Add one use. + fn inc(&mut self) { + let new = match self { + Self::Unused => Self::Once, + Self::Once | Self::Multiple => Self::Multiple, + }; + *self = new; + } +} + /// Notion of "relocation distance". This gives an estimate of how far away a symbol will be from a /// reference. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -408,7 +551,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> { let mut block_end_colors = SecondaryMap::with_default(InstColor::new(0)); let mut side_effect_inst_entry_colors = FxHashMap::default(); let mut inst_constants = FxHashMap::default(); - let mut value_uses = SecondaryMap::with_default(0); for bb in f.layout.blocks() { cur_color += 1; for inst in f.layout.block_insts(bb) { @@ -426,17 +568,13 @@ impl<'func, I: VCodeInst> Lower<'func, I> { log::trace!(" -> constant: {}", c); inst_constants.insert(inst, c); } - - // Count uses of all arguments. - for arg in f.dfg.inst_args(inst) { - let arg = f.dfg.resolve_aliases(*arg); - value_uses[arg] += 1; - } } block_end_colors[bb] = InstColor::new(cur_color); } + let value_ir_uses = Self::compute_use_states(f); + Ok(Lower { f, vcode, @@ -446,7 +584,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { side_effect_inst_entry_colors, inst_constants, next_vreg, - value_uses, + value_ir_uses, value_lowered_uses: SecondaryMap::default(), inst_sunk: FxHashSet::default(), cur_scan_entry_color: None, @@ -457,6 +595,114 @@ impl<'func, I: VCodeInst> Lower<'func, I> { }) } + /// Pre-analysis: compute `value_ir_uses`. See comment on + /// `ValueUseState` for a description of what this analysis + /// computes. + fn compute_use_states<'a>(f: &'a Function) -> SecondaryMap { + // We perform the analysis without recursion, so we don't + // overflow the stack on long chains of ops in the input. + // + // This is sort of a hybrid of a "shallow use-count" pass and + // a DFS. We iterate over all instructions and mark their args + // as used. However when we increment a use-count to + // "Multiple" we push its args onto the stack and do a DFS, + // immediately marking the whole dependency tree as + // Multiple. Doing both (shallow use-counting over all insts, + // and deep Multiple propagation) lets us trim both + // traversals, stopping recursion when a node is already at + // the appropriate state. + // + // In particular, note that the *coarsening* into {Unused, + // Once, Multiple} is part of what makes this pass more + // efficient than a full indirect-use-counting pass. + + let mut value_ir_uses: SecondaryMap = + SecondaryMap::with_default(ValueUseState::Unused); + + // Stack of iterators over Values as we do DFS to mark + // Multiple-state subtrees. + type StackVec<'a> = SmallVec<[std::slice::Iter<'a, Value>; 16]>; + let mut stack: StackVec = smallvec![]; + + // Push args for a given inst onto the DFS stack. + let push_args_on_stack = |stack: &mut StackVec<'a>, value| { + log::trace!(" -> pushing args for {} onto stack", value); + if let ValueDef::Result(src_inst, _) = f.dfg.value_def(value) { + stack.push(f.dfg.inst_args(src_inst).iter()); + } + }; + + // Do a DFS through `value_ir_uses` to mark a subtree as + // Multiple. + let mark_all_uses_as_multiple = + |value_ir_uses: &mut SecondaryMap, stack: &mut StackVec<'a>| { + while let Some(iter) = stack.last_mut() { + if let Some(&value) = iter.next() { + let value = f.dfg.resolve_aliases(value); + log::trace!(" -> DFS reaches {}", value); + if value_ir_uses[value] == ValueUseState::Multiple { + // Truncate DFS here: no need to go further, + // as whole subtree must already be Multiple. + #[cfg(debug_assertions)] + { + // With debug asserts, check one level + // of that invariant at least. + if let ValueDef::Result(src_inst, _) = f.dfg.value_def(value) { + debug_assert!(f.dfg.inst_args(src_inst).iter().all(|&arg| { + let arg = f.dfg.resolve_aliases(arg); + value_ir_uses[arg] == ValueUseState::Multiple + })); + } + } + continue; + } + value_ir_uses[value] = ValueUseState::Multiple; + log::trace!(" -> became Multiple"); + push_args_on_stack(stack, value); + } else { + // Empty iterator, discard. + stack.pop(); + } + } + }; + + for inst in f + .layout + .blocks() + .flat_map(|block| f.layout.block_insts(block)) + { + // If this inst produces multiple values, we must mark all + // of its args as Multiple, because otherwise two uses + // could come in as Once on our two different results. + let force_multiple = f.dfg.inst_results(inst).len() > 1; + + // Iterate over all args of all instructions, noting an + // additional use on each operand. If an operand becomes Multiple, + for &arg in f.dfg.inst_args(inst) { + let arg = f.dfg.resolve_aliases(arg); + let old = value_ir_uses[arg]; + if force_multiple { + log::trace!( + "forcing arg {} to Multiple because of multiple results of user inst", + arg + ); + value_ir_uses[arg] = ValueUseState::Multiple; + } else { + value_ir_uses[arg].inc(); + } + let new = value_ir_uses[arg]; + log::trace!("arg {} used, old state {:?}, new {:?}", arg, old, new,); + // On transition to Multiple, do DFS. + if old != ValueUseState::Multiple && new == ValueUseState::Multiple { + push_args_on_stack(&mut stack, arg); + mark_all_uses_as_multiple(&mut value_ir_uses, &mut stack); + } + } + } + + value_ir_uses + } + fn gen_arg_setup(&mut self) { if let Some(entry_bb) = self.f.layout.entry_block() { log::trace!( @@ -1050,9 +1296,11 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { // OK to merge source instruction if (i) we have a source // instruction, and: // - It has no side-effects, OR - // - It has a side-effect, has one output value, that one output has - // only one use (this one), and the instruction's color is *one less - // than* the current scan color. + // - It has a side-effect, has one output value, that one + // output has only one use, directly or indirectly (so + // cannot be duplicated -- see comment on + // `ValueUseState`), and the instruction's color is *one + // less than* the current scan color. // // This latter set of conditions is testing whether a // side-effecting instruction can sink to the current scan @@ -1071,15 +1319,26 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { log::trace!(" -> src inst {}", src_inst); log::trace!(" -> has lowering side effect: {}", src_side_effect); if !src_side_effect { - // Pure instruction: always possible to sink. - Some((src_inst, result_idx)) + // Pure instruction: always possible to + // sink. Let's determine whether we are the only + // user or not. + if self.value_ir_uses[val] == ValueUseState::Once { + InputSourceInst::UniqueUse(src_inst, result_idx) + } else { + InputSourceInst::Use(src_inst, result_idx) + } } else { // Side-effect: test whether this is the only use of the // only result of the instruction, and whether colors allow // the code-motion. + log::trace!( + " -> side-effecting op {} for val {}: use state {:?}", + src_inst, + val, + self.value_ir_uses[val] + ); if self.cur_scan_entry_color.is_some() - && self.value_uses[val] == 1 - && self.value_lowered_uses[val] == 0 + && self.value_ir_uses[val] == ValueUseState::Once && self.num_outputs(src_inst) == 1 && self .side_effect_inst_entry_colors @@ -1089,15 +1348,15 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { + 1 == self.cur_scan_entry_color.unwrap().get() { - Some((src_inst, 0)) + InputSourceInst::UniqueUse(src_inst, 0) } else { - None + InputSourceInst::None } } } - _ => None, + _ => InputSourceInst::None, }; - let constant = inst.and_then(|(inst, _)| self.get_constant(inst)); + let constant = inst.as_inst().and_then(|(inst, _)| self.get_constant(inst)); NonRegInput { inst, constant } } diff --git a/cranelift/filetests/filetests/isa/s390x/arithmetic.clif b/cranelift/filetests/filetests/isa/s390x/arithmetic.clif index 945e251371..765d824427 100644 --- a/cranelift/filetests/filetests/isa/s390x/arithmetic.clif +++ b/cranelift/filetests/filetests/isa/s390x/arithmetic.clif @@ -262,7 +262,8 @@ block0(v0: i64, v1: i64): } ; block0: -; alg %r2, 0(%r3) +; lg %r4, 0(%r3) +; algr %r2, %r4 ; br %r14 function %iadd_i64_mem_ext32(i64, i64) -> i64 { @@ -273,7 +274,8 @@ block0(v0: i64, v1: i64): } ; block0: -; algf %r2, 0(%r3) +; llgf %r4, 0(%r3) +; algr %r2, %r4 ; br %r14 function %iadd_i32(i32, i32) -> i32 { @@ -305,7 +307,8 @@ block0(v0: i32, v1: i64): } ; block0: -; al %r2, 0(%r3) +; l %r4, 0(%r3) +; alr %r2, %r4 ; br %r14 function %iadd_i32_memoff(i32, i64) -> i32 { @@ -316,7 +319,8 @@ block0(v0: i32, v1: i64): } ; block0: -; aly %r2, 4096(%r3) +; ly %r4, 4096(%r3) +; alr %r2, %r4 ; br %r14 function %isub_i64(i64, i64) -> i64 { diff --git a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif index 7793f4df75..178c364907 100644 --- a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif +++ b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif @@ -13,15 +13,15 @@ block0(v0: i64, v1: i32): ; block0: ; llgfr %r4, %r3 -; lghi %r3, 0 -; ag %r3, 0(%r2) -; clgr %r4, %r3 +; lg %r5, 0(%r2) +; aghi %r5, 0 +; clgr %r4, %r5 ; jgnh label1 ; jg label2 ; block1: ; agr %r2, %r4 -; lghi %r5, 0 -; clgr %r4, %r3 -; locgrh %r2, %r5 +; lghi %r3, 0 +; clgr %r4, %r5 +; locgrh %r2, %r3 ; br %r14 ; block2: ; trap diff --git a/cranelift/filetests/filetests/isa/x64/load-op.clif b/cranelift/filetests/filetests/isa/x64/load-op.clif index d2fa2b9f82..dcbe3f8570 100644 --- a/cranelift/filetests/filetests/isa/x64/load-op.clif +++ b/cranelift/filetests/filetests/isa/x64/load-op.clif @@ -68,3 +68,14 @@ block0(v0: i64): block1: return v2 } + +function %cmp_mem(i64) -> i64 { +block0(v0: i64): + v1 = load.i64 v0 + v2 = icmp eq v0, v1 + v3 = bint.i64 v2 + return v3 + + ; check: cmpq 0(%rdi), %rdi + ; nextln: setz %al +} diff --git a/cranelift/filetests/filetests/isa/x64/select-i128.clif b/cranelift/filetests/filetests/isa/x64/select-i128.clif index 5317a22872..c88e3c3c2a 100644 --- a/cranelift/filetests/filetests/isa/x64/select-i128.clif +++ b/cranelift/filetests/filetests/isa/x64/select-i128.clif @@ -13,8 +13,7 @@ block0(v0: i32, v1: i128, v2: i128): ; pushq %rbp ; movq %rsp, %rbp ; block0: -; movl $42, %r9d -; cmpl %r9d, %edi +; cmpl $42, %edi ; cmovzq %rsi, %rcx, %rcx ; cmovzq %rdx, %r8, %r8 ; movq %rcx, %rax