diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index c0b0822343..a69d7ee24b 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -306,7 +306,8 @@ fn put_input_in_rs>( narrow_mode: NarrowValueMode, ) -> ResultRS { let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); - if let Some((insn, 0)) = inputs.inst { + // Unique or non-unique use is fine for merging here. + if let Some((insn, 0)) = inputs.inst.as_inst() { let op = ctx.data(insn).opcode(); if op == Opcode::Ishl { @@ -353,7 +354,7 @@ fn get_as_extended_value>( narrow_mode: NarrowValueMode, ) -> Option<(Value, ExtendOp)> { let inputs = ctx.get_value_as_source_or_const(val); - let (insn, n) = inputs.inst?; + let (insn, n) = inputs.inst.as_inst()?; if n != 0 { return None; } @@ -1125,7 +1126,7 @@ pub(crate) fn maybe_input_insn>( inputs, op ); - if let Some((src_inst, _)) = inputs.inst { + if let Some((src_inst, _)) = inputs.inst.as_inst() { let data = c.data(src_inst); log::trace!(" -> input inst {:?}", data); if data.opcode() == op { @@ -1161,14 +1162,14 @@ pub(crate) fn maybe_input_insn_via_conv>( conv: Opcode, ) -> Option { let inputs = c.get_input_as_source_or_const(input.insn, input.input); - if let Some((src_inst, _)) = inputs.inst { + if let Some((src_inst, _)) = inputs.inst.as_inst() { let data = c.data(src_inst); if data.opcode() == op { return Some(src_inst); } if data.opcode() == conv { let inputs = c.get_input_as_source_or_const(src_inst, 0); - if let Some((src_inst, _)) = inputs.inst { + if let Some((src_inst, _)) = inputs.inst.as_inst() { let data = c.data(src_inst); if data.opcode() == op { return Some(src_inst); diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 7dda0dac37..73497dda54 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -12,7 +12,7 @@ use super::{ NZCV, }; use crate::isa::aarch64::settings::Flags as IsaFlags; -use crate::machinst::isle::*; +use crate::machinst::{isle::*, InputSourceInst}; use crate::settings::Flags; use crate::{ binemit::CodeOffset, @@ -245,7 +245,7 @@ where fn sinkable_atomic_load(&mut self, val: Value) -> Option { let input = self.lower_ctx.get_value_as_source_or_const(val); - if let Some((atomic_load, 0)) = input.inst { + if let InputSourceInst::UniqueUse(atomic_load, 0) = input.inst { if self.lower_ctx.data(atomic_load).opcode() == Opcode::AtomicLoad { let atomic_addr = self.lower_ctx.input_as_value(atomic_load, 0); return Some(SinkableAtomicLoad { diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index a9424d88c9..8ad35571fc 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -497,7 +497,7 @@ where #[inline] fn sinkable_inst(&mut self, val: Value) -> Option { let input = self.lower_ctx.get_value_as_source_or_const(val); - if let Some((inst, 0)) = input.inst { + if let Some((inst, 0)) = input.inst.as_inst() { return Some(inst); } None diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 544266fda3..d064e7a5f7 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1367,11 +1367,9 @@ (decl cmp_and_choose (Type CC Value Value) ValueRegs) (rule (cmp_and_choose (fits_in_64 ty) cc x y) - (let ((x_reg Gpr x) - (y_reg Gpr y) - (size OperandSize (raw_operand_size_of_type ty))) - (with_flags_reg (x64_cmp size x_reg y_reg) - (cmove ty cc y_reg x_reg)))) + (let ((size OperandSize (raw_operand_size_of_type ty))) + (with_flags_reg (x64_cmp size x y) + (cmove ty cc y x)))) (rule (lower (has_type (fits_in_64 ty) (umin x y))) (cmp_and_choose ty (CC.B) x y)) @@ -1751,19 +1749,8 @@ ;; than one instruction for certain types (e.g., XMM-held, I128). (rule (lower (has_type ty (select (icmp cc a @ (value_type (fits_in_64 a_ty)) b) x y))) - ;; N.B.: we force the comparison operators into registers, and disallow - ;; load-op fusion, because we do not have a transitive guarantee that this - ;; cmp-site will be the sole user of the value. Consider: the `icmp` might - ;; be the only user of a load, but there may be multiple users of the - ;; `icmp` (e.g., `select` or `bint` instructions) that each invoke emit a - ;; comparison. If we were to allow a load to sink to the *latest* one, but - ;; other sites did not permit sinking, then we would be missing the load - ;; for other cmp-sites. TODO: - ;; https://github.com/bytecodealliance/wasmtime/issues/3953. - (let ((gpr_a Gpr (put_in_gpr a)) - (gpr_b Gpr (put_in_gpr b)) - (size OperandSize (raw_operand_size_of_type a_ty))) - (with_flags (x64_cmp size gpr_b gpr_a) (cmove_from_values ty cc x y)))) + (let ((size OperandSize (raw_operand_size_of_type a_ty))) + (with_flags (x64_cmp size b a) (cmove_from_values ty cc x y)))) ;; Finally, we lower `select` from a condition value `c`. These rules are meant ;; to be the final, default lowerings if no other patterns matched above. diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d1aedd7b17..ac852160a7 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -61,7 +61,7 @@ fn matches_input>( op: Opcode, ) -> Option { let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); - inputs.inst.and_then(|(src_inst, _)| { + inputs.inst.as_inst().and_then(|(src_inst, _)| { let data = ctx.data(src_inst); if data.opcode() == op { return Some(src_inst); @@ -172,7 +172,7 @@ fn input_to_reg_mem>(ctx: &mut C, spec: InsnInput) -> RegM return RegMem::reg(generate_constant(ctx, ty, c).only_reg().unwrap()); } - if let Some((src_insn, 0)) = inputs.inst { + if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let Some((addr_input, offset)) = is_mergeable_load(ctx, src_insn) { ctx.sink_inst(src_insn); let amode = lower_to_amode(ctx, addr_input, offset); @@ -479,22 +479,11 @@ fn emit_cmp>(ctx: &mut C, insn: IRInst, cc: IntCC) -> IntC } else { // TODO Try to commute the operands (and invert the condition) if one is an immediate. let lhs = put_input_in_reg(ctx, inputs[0]); - // We force the RHS into a register, and disallow load-op fusion, because we - // do not have a transitive guarantee that this cmp-site will be the sole - // user of the value. Consider: the icmp might be the only user of a load, - // but there may be multiple users of the icmp (e.g. select or bint - // instructions) that each invoke `emit_cmp()`. If we were to allow a load - // to sink to the *latest* one, but other sites did not permit sinking, then - // we would be missing the load for other cmp-sites. - let rhs = put_input_in_reg(ctx, inputs[1]); + let rhs = input_to_reg_mem_imm(ctx, inputs[1]); // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives // us dst - src at the machine instruction level, so invert operands. - ctx.emit(Inst::cmp_rmi_r( - OperandSize::from_ty(ty), - RegMemImm::reg(rhs), - lhs, - )); + ctx.emit(Inst::cmp_rmi_r(OperandSize::from_ty(ty), rhs, lhs)); cc } } @@ -578,10 +567,8 @@ fn emit_fcmp>( (inputs[0], inputs[1]) }; let lhs = put_input_in_reg(ctx, lhs_input); - // See above in `emit_cmp()`. We must only use the reg/reg form of the - // comparison in order to avoid issues with merged loads. - let rhs = put_input_in_reg(ctx, rhs_input); - ctx.emit(Inst::xmm_cmp_rm_r(op, RegMem::reg(rhs), lhs)); + let rhs = input_to_reg_mem(ctx, rhs_input); + ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs)); let cond_result = match cond_code { FloatCC::Equal => FcmpCondResult::AndConditions(CC::NP, CC::Z), @@ -2406,6 +2393,7 @@ fn lower_insn_to_regs>( let cmp_insn = ctx .get_input_as_source_or_const(inputs[0].insn, inputs[0].input) .inst + .as_inst() .unwrap() .0; debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp); diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index c8032ecde9..733ef1f7a1 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -2,7 +2,7 @@ // Pull in the ISLE generated code. pub(crate) mod generated_code; -use crate::machinst::{Reg, Writable}; +use crate::machinst::{InputSourceInst, Reg, Writable}; use generated_code::MInst; // Types that the generated ISLE code uses via `use super::*`. @@ -84,7 +84,7 @@ where return RegMemImm::reg(generated_code::constructor_imm(self, ty, c).unwrap()); } - if let Some((src_insn, 0)) = inputs.inst { + if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) { self.lower_ctx.sink_inst(src_insn); let amode = lower_to_amode(self.lower_ctx, addr_input, offset); @@ -105,7 +105,7 @@ where return RegMem::reg(generated_code::constructor_imm(self, ty, c).unwrap()); } - if let Some((src_insn, 0)) = inputs.inst { + if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) { self.lower_ctx.sink_inst(src_insn); let amode = lower_to_amode(self.lower_ctx, addr_input, offset); @@ -237,7 +237,7 @@ where fn sinkable_load(&mut self, val: Value) -> Option { let input = self.lower_ctx.get_value_as_source_or_const(val); - if let Some((inst, 0)) = input.inst { + if let InputSourceInst::UniqueUse(inst, 0) = input.inst { if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, inst) { return Some(SinkableLoad { inst, diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 687ad644d3..4636672ed1 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 443b34b797fc8ace src/prelude.isle afd037c4d91c875c src/isa/x64/inst.isle cad03431447aca1b -src/isa/x64/lower.isle 42bd3982a6132a2f +src/isa/x64/lower.isle 803aac716f6f4c39 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 4d723587a8..d42160c20e 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -5115,7 +5115,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Debugtrap = pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2261. + // Rule at src/isa/x64/lower.isle line 2248. let expr0_0 = constructor_x64_hlt(ctx)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); @@ -5278,13 +5278,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern2_0 { &Opcode::Trap => { - // Rule at src/isa/x64/lower.isle line 1434. + // Rule at src/isa/x64/lower.isle line 1432. let expr0_0 = constructor_x64_ud2(ctx, pattern2_1)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); } &Opcode::ResumableTrap => { - // Rule at src/isa/x64/lower.isle line 1439. + // Rule at src/isa/x64/lower.isle line 1437. let expr0_0 = constructor_x64_ud2(ctx, pattern2_1)?; let expr1_0 = constructor_side_effect(ctx, &expr0_0)?; return Some(expr1_0); @@ -5321,7 +5321,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1571. + // Rule at src/isa/x64/lower.isle line 1569. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NP; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5341,7 +5341,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1599. + // Rule at src/isa/x64/lower.isle line 1597. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NBE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5350,7 +5350,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1601. + // Rule at src/isa/x64/lower.isle line 1599. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NB; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5359,7 +5359,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1611. + // Rule at src/isa/x64/lower.isle line 1609. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::NBE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5368,7 +5368,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1614. + // Rule at src/isa/x64/lower.isle line 1612. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::NB; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5377,7 +5377,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1580. + // Rule at src/isa/x64/lower.isle line 1578. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::P; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5397,7 +5397,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1591. + // Rule at src/isa/x64/lower.isle line 1589. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NP; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5406,7 +5406,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1595. + // Rule at src/isa/x64/lower.isle line 1593. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::NZ; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5415,7 +5415,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1593. + // Rule at src/isa/x64/lower.isle line 1591. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::P; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5424,7 +5424,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1597. + // Rule at src/isa/x64/lower.isle line 1595. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::Z; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5433,7 +5433,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1617. + // Rule at src/isa/x64/lower.isle line 1615. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::B; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5442,7 +5442,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1620. + // Rule at src/isa/x64/lower.isle line 1618. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_0, pattern4_1)?; let expr1_0 = CC::BE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5451,7 +5451,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1603. + // Rule at src/isa/x64/lower.isle line 1601. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::B; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5460,7 +5460,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1605. + // Rule at src/isa/x64/lower.isle line 1603. let expr0_0 = constructor_x64_ucomis(ctx, pattern4_1, pattern4_0)?; let expr1_0 = CC::BE; let expr2_0 = constructor_x64_setcc(ctx, &expr1_0)?; @@ -5474,7 +5474,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1628. + // Rule at src/isa/x64/lower.isle line 1626. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::Equal; @@ -5484,7 +5484,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1648. + // Rule at src/isa/x64/lower.isle line 1646. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::LessThan; @@ -5494,7 +5494,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1650. + // Rule at src/isa/x64/lower.isle line 1648. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::LessThanOrEqual; @@ -5504,7 +5504,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1632. + // Rule at src/isa/x64/lower.isle line 1630. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::LessThan; @@ -5514,7 +5514,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1634. + // Rule at src/isa/x64/lower.isle line 1632. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::LessThanOrEqual; @@ -5524,7 +5524,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1630. + // Rule at src/isa/x64/lower.isle line 1628. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::NotEqual; @@ -5534,7 +5534,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1636. + // Rule at src/isa/x64/lower.isle line 1634. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::Ordered; @@ -5544,7 +5544,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1638. + // Rule at src/isa/x64/lower.isle line 1636. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::Unordered; @@ -5554,7 +5554,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1640. + // Rule at src/isa/x64/lower.isle line 1638. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThan; @@ -5564,7 +5564,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1642. + // Rule at src/isa/x64/lower.isle line 1640. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThanOrEqual; @@ -5574,7 +5574,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1652. + // Rule at src/isa/x64/lower.isle line 1650. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThan; @@ -5584,7 +5584,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1654. + // Rule at src/isa/x64/lower.isle line 1652. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = FcmpImm::UnorderedOrGreaterThanOrEqual; @@ -5609,7 +5609,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1522. + // Rule at src/isa/x64/lower.isle line 1520. let expr0_0 = C::put_in_regs(ctx, pattern4_0); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -5653,7 +5653,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1541. + // Rule at src/isa/x64/lower.isle line 1539. let expr0_0 = C::put_in_regs(ctx, pattern4_0); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -5702,7 +5702,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1507. + // Rule at src/isa/x64/lower.isle line 1505. let expr0_0: Type = I64X2; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr2_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; @@ -5715,7 +5715,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1511. + // Rule at src/isa/x64/lower.isle line 1509. let expr0_0: Type = I64X2; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr2_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; @@ -5731,7 +5731,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1456. + // Rule at src/isa/x64/lower.isle line 1454. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5754,7 +5754,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1460. + // Rule at src/isa/x64/lower.isle line 1458. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5766,7 +5766,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1467. + // Rule at src/isa/x64/lower.isle line 1465. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5775,7 +5775,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1492. + // Rule at src/isa/x64/lower.isle line 1490. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5788,7 +5788,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1469. + // Rule at src/isa/x64/lower.isle line 1467. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr2_0 = @@ -5797,7 +5797,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1495. + // Rule at src/isa/x64/lower.isle line 1493. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5810,7 +5810,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1471. + // Rule at src/isa/x64/lower.isle line 1469. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); @@ -5826,7 +5826,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1498. + // Rule at src/isa/x64/lower.isle line 1496. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5839,7 +5839,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1481. + // Rule at src/isa/x64/lower.isle line 1479. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern4_1)?; let expr2_0 = C::xmm_to_xmm_mem(ctx, expr1_0); @@ -5855,7 +5855,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1501. + // Rule at src/isa/x64/lower.isle line 1499. let expr0_0 = constructor_put_in_xmm(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern4_1)?; let expr2_0 = @@ -5880,7 +5880,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 2104. + // Rule at src/isa/x64/lower.isle line 2091. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 0; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -5895,7 +5895,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern4_0 = C::value_type(ctx, pattern2_1); if pattern4_0 == R64 { - // Rule at src/isa/x64/lower.isle line 2112. + // Rule at src/isa/x64/lower.isle line 2099. let expr0_0 = OperandSize::Size64; let expr1_0: u32 = 4294967295; let expr2_0 = constructor_put_in_gpr(ctx, pattern2_1)?; @@ -6026,7 +6026,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2035. + // Rule at src/isa/x64/lower.isle line 2022. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0 = constructor_do_bitrev64(ctx, expr0_0, expr1_0)?; @@ -6100,7 +6100,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2151. + // Rule at src/isa/x64/lower.isle line 2138. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6108,7 +6108,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2157. + // Rule at src/isa/x64/lower.isle line 2144. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6116,7 +6116,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2160. + // Rule at src/isa/x64/lower.isle line 2147. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6124,7 +6124,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2154. + // Rule at src/isa/x64/lower.isle line 2141. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6132,7 +6132,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2163. + // Rule at src/isa/x64/lower.isle line 2150. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6140,7 +6140,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2166. + // Rule at src/isa/x64/lower.isle line 2153. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6148,7 +6148,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2169. + // Rule at src/isa/x64/lower.isle line 2156. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6156,7 +6156,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2172. + // Rule at src/isa/x64/lower.isle line 2159. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6164,7 +6164,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern14_0, pattern14_1) = C::unpack_value_array_2(ctx, pattern12_1); - // Rule at src/isa/x64/lower.isle line 2175. + // Rule at src/isa/x64/lower.isle line 2162. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6179,7 +6179,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Uload32 = pattern12_0 { - // Rule at src/isa/x64/lower.isle line 2178. + // Rule at src/isa/x64/lower.isle line 2165. let expr0_0 = constructor_output_value(ctx, pattern5_1)?; return Some(expr0_0); @@ -6192,7 +6192,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2038. + // Rule at src/isa/x64/lower.isle line 2025. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 1; @@ -6469,7 +6469,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1808. + // Rule at src/isa/x64/lower.isle line 1795. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -6507,7 +6507,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1850. + // Rule at src/isa/x64/lower.isle line 1837. let expr0_0: Type = I64; let expr1_0: Type = I64; let expr2_0 = C::put_in_regs(ctx, pattern5_1); @@ -6545,7 +6545,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1902. + // Rule at src/isa/x64/lower.isle line 1889. let expr0_0: Type = I64; let expr1_0 = C::put_in_regs(ctx, pattern5_1); let expr2_0: usize = 0; @@ -6568,7 +6568,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2253. + // Rule at src/isa/x64/lower.isle line 2240. let expr0_0: Type = I64; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr2_0: u32 = 1; @@ -6586,7 +6586,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern7_0 = C::value_type(ctx, pattern5_1); if pattern7_0 == I64 { - // Rule at src/isa/x64/lower.isle line 2125. + // Rule at src/isa/x64/lower.isle line 2112. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0: Type = I64; let expr2_0: u64 = 0; @@ -6596,7 +6596,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2276. + // Rule at src/isa/x64/lower.isle line 2263. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addss(ctx, expr0_0, &expr1_0)?; @@ -6666,7 +6666,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2287. + // Rule at src/isa/x64/lower.isle line 2274. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subss(ctx, expr0_0, &expr1_0)?; @@ -6675,7 +6675,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2298. + // Rule at src/isa/x64/lower.isle line 2285. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulss(ctx, expr0_0, &expr1_0)?; @@ -6684,7 +6684,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2309. + // Rule at src/isa/x64/lower.isle line 2296. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divss(ctx, expr0_0, &expr1_0)?; @@ -6693,7 +6693,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2330. + // Rule at src/isa/x64/lower.isle line 2317. let expr0_0: Type = F32; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6706,7 +6706,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2499. + // Rule at src/isa/x64/lower.isle line 2486. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minss(ctx, expr0_0, expr1_0)?; @@ -6715,7 +6715,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2402. + // Rule at src/isa/x64/lower.isle line 2389. let expr0_0: Type = F32; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6728,7 +6728,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2510. + // Rule at src/isa/x64/lower.isle line 2497. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxss(ctx, expr0_0, expr1_0)?; @@ -6743,7 +6743,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Sqrt = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2319. + // Rule at src/isa/x64/lower.isle line 2306. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtss(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -6757,7 +6757,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2553. + // Rule at src/isa/x64/lower.isle line 2540. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -6779,7 +6779,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2278. + // Rule at src/isa/x64/lower.isle line 2265. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addsd(ctx, expr0_0, &expr1_0)?; @@ -6788,7 +6788,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2289. + // Rule at src/isa/x64/lower.isle line 2276. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subsd(ctx, expr0_0, &expr1_0)?; @@ -6797,7 +6797,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2300. + // Rule at src/isa/x64/lower.isle line 2287. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulsd(ctx, expr0_0, &expr1_0)?; @@ -6806,7 +6806,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2311. + // Rule at src/isa/x64/lower.isle line 2298. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divsd(ctx, expr0_0, &expr1_0)?; @@ -6815,7 +6815,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2332. + // Rule at src/isa/x64/lower.isle line 2319. let expr0_0: Type = F64; let expr1_0: bool = true; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6828,7 +6828,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2501. + // Rule at src/isa/x64/lower.isle line 2488. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minsd(ctx, expr0_0, expr1_0)?; @@ -6837,7 +6837,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2404. + // Rule at src/isa/x64/lower.isle line 2391. let expr0_0: Type = F64; let expr1_0: bool = false; let expr2_0 = constructor_put_in_xmm(ctx, pattern7_0)?; @@ -6850,7 +6850,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2512. + // Rule at src/isa/x64/lower.isle line 2499. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxsd(ctx, expr0_0, expr1_0)?; @@ -6865,7 +6865,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Sqrt = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2321. + // Rule at src/isa/x64/lower.isle line 2308. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtsd(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -6879,7 +6879,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2555. + // Rule at src/isa/x64/lower.isle line 2542. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -6901,7 +6901,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1401. + // Rule at src/isa/x64/lower.isle line 1399. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminsb(ctx, expr0_0, &expr1_0)?; @@ -6910,7 +6910,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1423. + // Rule at src/isa/x64/lower.isle line 1421. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminub(ctx, expr0_0, &expr1_0)?; @@ -6919,7 +6919,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1390. + // Rule at src/isa/x64/lower.isle line 1388. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -6928,7 +6928,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1412. + // Rule at src/isa/x64/lower.isle line 1410. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxub(ctx, expr0_0, &expr1_0)?; @@ -7019,7 +7019,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2009. + // Rule at src/isa/x64/lower.isle line 1996. let expr0_0 = C::popcount_4bit_table(ctx); let expr1_0: Type = I8X16; let expr2_0 = C::popcount_low_mask(ctx); @@ -7064,7 +7064,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1404. + // Rule at src/isa/x64/lower.isle line 1402. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminsw(ctx, expr0_0, &expr1_0)?; @@ -7073,7 +7073,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1426. + // Rule at src/isa/x64/lower.isle line 1424. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminuw(ctx, expr0_0, &expr1_0)?; @@ -7082,7 +7082,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1393. + // Rule at src/isa/x64/lower.isle line 1391. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -7091,7 +7091,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1415. + // Rule at src/isa/x64/lower.isle line 1413. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxuw(ctx, expr0_0, &expr1_0)?; @@ -7165,7 +7165,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload8x8 => { - // Rule at src/isa/x64/lower.isle line 2568. + // Rule at src/isa/x64/lower.isle line 2555. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7174,7 +7174,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2566. + // Rule at src/isa/x64/lower.isle line 2553. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7198,7 +7198,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1407. + // Rule at src/isa/x64/lower.isle line 1405. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminsd(ctx, expr0_0, &expr1_0)?; @@ -7207,7 +7207,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1429. + // Rule at src/isa/x64/lower.isle line 1427. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pminud(ctx, expr0_0, &expr1_0)?; @@ -7216,7 +7216,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1396. + // Rule at src/isa/x64/lower.isle line 1394. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -7225,7 +7225,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1418. + // Rule at src/isa/x64/lower.isle line 1416. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaxud(ctx, expr0_0, &expr1_0)?; @@ -7264,7 +7264,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2266. + // Rule at src/isa/x64/lower.isle line 2253. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_pmaddwd(ctx, expr0_0, &expr1_0)?; @@ -7308,7 +7308,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload16x4 => { - // Rule at src/isa/x64/lower.isle line 2572. + // Rule at src/isa/x64/lower.isle line 2559. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7317,7 +7317,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2570. + // Rule at src/isa/x64/lower.isle line 2557. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7427,7 +7427,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Uload32x2 => { - // Rule at src/isa/x64/lower.isle line 2576. + // Rule at src/isa/x64/lower.isle line 2563. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7436,7 +7436,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2574. + // Rule at src/isa/x64/lower.isle line 2561. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7460,7 +7460,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2280. + // Rule at src/isa/x64/lower.isle line 2267. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addps(ctx, expr0_0, &expr1_0)?; @@ -7469,7 +7469,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2291. + // Rule at src/isa/x64/lower.isle line 2278. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subps(ctx, expr0_0, &expr1_0)?; @@ -7478,7 +7478,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2302. + // Rule at src/isa/x64/lower.isle line 2289. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulps(ctx, expr0_0, &expr1_0)?; @@ -7487,7 +7487,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2313. + // Rule at src/isa/x64/lower.isle line 2300. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divps(ctx, expr0_0, &expr1_0)?; @@ -7496,7 +7496,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2347. + // Rule at src/isa/x64/lower.isle line 2334. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -7521,7 +7521,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2503. + // Rule at src/isa/x64/lower.isle line 2490. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minps(ctx, expr0_0, expr1_0)?; @@ -7530,7 +7530,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2410. + // Rule at src/isa/x64/lower.isle line 2397. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -7558,7 +7558,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2514. + // Rule at src/isa/x64/lower.isle line 2501. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxps(ctx, expr0_0, expr1_0)?; @@ -7574,7 +7574,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Sqrt => { - // Rule at src/isa/x64/lower.isle line 2323. + // Rule at src/isa/x64/lower.isle line 2310. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtps(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -7604,7 +7604,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2557. + // Rule at src/isa/x64/lower.isle line 2544. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7626,7 +7626,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2282. + // Rule at src/isa/x64/lower.isle line 2269. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_addpd(ctx, expr0_0, &expr1_0)?; @@ -7635,7 +7635,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2293. + // Rule at src/isa/x64/lower.isle line 2280. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_subpd(ctx, expr0_0, &expr1_0)?; @@ -7644,7 +7644,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2304. + // Rule at src/isa/x64/lower.isle line 2291. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_mulpd(ctx, expr0_0, &expr1_0)?; @@ -7653,7 +7653,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2315. + // Rule at src/isa/x64/lower.isle line 2302. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; let expr2_0 = constructor_x64_divpd(ctx, expr0_0, &expr1_0)?; @@ -7662,7 +7662,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2390. + // Rule at src/isa/x64/lower.isle line 2377. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -7687,7 +7687,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2505. + // Rule at src/isa/x64/lower.isle line 2492. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_minpd(ctx, expr0_0, expr1_0)?; @@ -7696,7 +7696,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2453. + // Rule at src/isa/x64/lower.isle line 2440. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -7724,7 +7724,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 2516. + // Rule at src/isa/x64/lower.isle line 2503. let expr0_0 = constructor_put_in_xmm(ctx, pattern7_1)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern7_0)?; let expr2_0 = constructor_x64_maxpd(ctx, expr0_0, expr1_0)?; @@ -7740,7 +7740,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Sqrt => { - // Rule at src/isa/x64/lower.isle line 2325. + // Rule at src/isa/x64/lower.isle line 2312. let expr0_0 = constructor_put_in_xmm(ctx, pattern5_1)?; let expr1_0 = constructor_x64_sqrtpd(ctx, expr0_0)?; let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; @@ -7770,7 +7770,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Load = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 2559. + // Rule at src/isa/x64/lower.isle line 2546. let expr0_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; let expr1_0 = constructor_amode_to_xmm_mem(ctx, &expr0_0)?; @@ -7829,7 +7829,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1742. + // Rule at src/isa/x64/lower.isle line 1740. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7847,7 +7847,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1694. + // Rule at src/isa/x64/lower.isle line 1692. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7863,7 +7863,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1697. + // Rule at src/isa/x64/lower.isle line 1695. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7879,7 +7879,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1717. + // Rule at src/isa/x64/lower.isle line 1715. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7895,7 +7895,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1720. + // Rule at src/isa/x64/lower.isle line 1718. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7911,7 +7911,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1745. + // Rule at src/isa/x64/lower.isle line 1743. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7929,7 +7929,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1688. + // Rule at src/isa/x64/lower.isle line 1686. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7945,7 +7945,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1691. + // Rule at src/isa/x64/lower.isle line 1689. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -7961,7 +7961,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1723. + // Rule at src/isa/x64/lower.isle line 1721. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7977,7 +7977,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1726. + // Rule at src/isa/x64/lower.isle line 1724. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_0, @@ -7993,7 +7993,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1700. + // Rule at src/isa/x64/lower.isle line 1698. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -8009,7 +8009,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1703. + // Rule at src/isa/x64/lower.isle line 1701. let expr0_0 = constructor_x64_ucomis( ctx, pattern11_1, @@ -8038,21 +8038,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2239. + // Rule at src/isa/x64/lower.isle line 2226. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Bextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2220. + // Rule at src/isa/x64/lower.isle line 2207. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -8116,7 +8116,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2228. + // Rule at src/isa/x64/lower.isle line 2215. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } @@ -8124,14 +8124,14 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let pattern6_0 = C::value_type(ctx, pattern4_1); if pattern6_0 == pattern2_0 { - // Rule at src/isa/x64/lower.isle line 2121. + // Rule at src/isa/x64/lower.isle line 2108. let expr0_0 = constructor_output_value(ctx, pattern4_1)?; return Some(expr0_0); } } &Opcode::Sextend => { let pattern6_0 = C::value_type(ctx, pattern4_1); - // Rule at src/isa/x64/lower.isle line 2214. + // Rule at src/isa/x64/lower.isle line 2201. let expr0_0 = constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; return Some(expr0_0); @@ -9327,7 +9327,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1382. + // Rule at src/isa/x64/lower.isle line 1380. let expr0_0 = CC::L; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9414,7 +9414,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1376. + // Rule at src/isa/x64/lower.isle line 1374. let expr0_0 = CC::B; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9424,7 +9424,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1385. + // Rule at src/isa/x64/lower.isle line 1383. let expr0_0 = CC::NL; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9434,7 +9434,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - // Rule at src/isa/x64/lower.isle line 1379. + // Rule at src/isa/x64/lower.isle line 1377. let expr0_0 = CC::NB; let expr1_0 = constructor_cmp_and_choose( ctx, pattern3_0, &expr0_0, pattern7_0, pattern7_1, @@ -9798,7 +9798,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2242. + // Rule at src/isa/x64/lower.isle line 2229. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -9806,7 +9806,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2250. + // Rule at src/isa/x64/lower.isle line 2237. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0: u32 = 1; let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; @@ -9816,7 +9816,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2234. + // Rule at src/isa/x64/lower.isle line 2221. let expr0_0 = C::put_in_regs(ctx, pattern5_1); let expr1_0: usize = 0; let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; @@ -9930,21 +9930,21 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1798. + // Rule at src/isa/x64/lower.isle line 1785. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_clz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Ctz => { - // Rule at src/isa/x64/lower.isle line 1840. + // Rule at src/isa/x64/lower.isle line 1827. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_ctz(ctx, pattern3_0, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; return Some(expr2_0); } &Opcode::Popcnt => { - // Rule at src/isa/x64/lower.isle line 1892. + // Rule at src/isa/x64/lower.isle line 1879. let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; let expr1_0 = constructor_do_popcnt(ctx, pattern3_0, expr0_0)?; let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; @@ -10047,7 +10047,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { match pattern5_0 { &Opcode::Clz => { - // Rule at src/isa/x64/lower.isle line 1803. + // Rule at src/isa/x64/lower.isle line 1790. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -10058,7 +10058,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1845. + // Rule at src/isa/x64/lower.isle line 1832. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -10069,7 +10069,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 1897. + // Rule at src/isa/x64/lower.isle line 1884. let expr0_0: Type = I32; let expr1_0: Type = I32; let expr2_0 = ExtendKind::Zero; @@ -10095,7 +10095,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2536. + // Rule at src/isa/x64/lower.isle line 2523. let expr0_0 = ExtMode::BQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10143,7 +10143,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2538. + // Rule at src/isa/x64/lower.isle line 2525. let expr0_0 = ExtMode::BQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10153,7 +10153,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2540. + // Rule at src/isa/x64/lower.isle line 2527. let expr0_0 = ExtMode::WQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10163,7 +10163,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2542. + // Rule at src/isa/x64/lower.isle line 2529. let expr0_0 = ExtMode::WQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10173,7 +10173,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2544. + // Rule at src/isa/x64/lower.isle line 2531. let expr0_0 = ExtMode::LQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10183,7 +10183,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - // Rule at src/isa/x64/lower.isle line 2546. + // Rule at src/isa/x64/lower.isle line 2533. let expr0_0 = ExtMode::LQ; let expr1_0 = constructor_to_amode(ctx, pattern5_2, pattern5_1, pattern5_3)?; @@ -10787,14 +10787,14 @@ pub fn constructor_cmp_and_choose( let pattern3_0 = arg2; let pattern4_0 = arg3; // Rule at src/isa/x64/lower.isle line 1369. - let expr0_0 = constructor_put_in_gpr(ctx, pattern3_0)?; - let expr1_0 = constructor_put_in_gpr(ctx, pattern4_0)?; - let expr2_0 = C::raw_operand_size_of_type(ctx, pattern1_0); - let expr3_0 = C::gpr_to_gpr_mem_imm(ctx, expr0_0); - let expr4_0 = constructor_x64_cmp(ctx, &expr2_0, &expr3_0, expr1_0)?; - let expr5_0 = C::gpr_to_gpr_mem(ctx, expr1_0); - let expr6_0 = constructor_cmove(ctx, pattern1_0, pattern2_0, &expr5_0, expr0_0)?; - let expr7_0 = constructor_with_flags_reg(ctx, &expr4_0, &expr6_0)?; + let expr0_0 = C::raw_operand_size_of_type(ctx, pattern1_0); + let expr1_0 = constructor_put_in_gpr_mem_imm(ctx, pattern3_0)?; + let expr2_0 = constructor_put_in_gpr(ctx, pattern4_0)?; + let expr3_0 = constructor_x64_cmp(ctx, &expr0_0, &expr1_0, expr2_0)?; + let expr4_0 = constructor_put_in_gpr_mem(ctx, pattern4_0)?; + let expr5_0 = constructor_put_in_gpr(ctx, pattern3_0)?; + let expr6_0 = constructor_cmove(ctx, pattern1_0, pattern2_0, &expr4_0, expr5_0)?; + let expr7_0 = constructor_with_flags_reg(ctx, &expr3_0, &expr6_0)?; let expr8_0 = C::value_reg(ctx, expr7_0); return Some(expr8_0); } @@ -10811,7 +10811,7 @@ pub fn constructor_do_clz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1823. + // Rule at src/isa/x64/lower.isle line 1810. let expr0_0: Type = I64; let expr1_0: i64 = -1; let expr2_0 = constructor_imm_i64(ctx, expr0_0, expr1_0)?; @@ -10838,7 +10838,7 @@ pub fn constructor_do_ctz( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/lower.isle line 1864. + // Rule at src/isa/x64/lower.isle line 1851. let expr0_0: Type = I64; let expr1_0 = C::ty_bits_u64(ctx, pattern1_0); let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; @@ -10852,7 +10852,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> let pattern0_0 = arg0; if pattern0_0 == I32 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1955. + // Rule at src/isa/x64/lower.isle line 1942. let expr0_0: Type = I32; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -10917,7 +10917,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> } if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 1912. + // Rule at src/isa/x64/lower.isle line 1899. let expr0_0: Type = I64; let expr1_0: u8 = 1; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; @@ -10991,7 +10991,7 @@ pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2044. + // Rule at src/isa/x64/lower.isle line 2031. let expr0_0 = C::ty_mask(ctx, pattern0_0); let expr1_0: u64 = 6148914691236517205; let expr2_0 = C::u64_and(ctx, expr0_0, expr1_0); @@ -11054,7 +11054,7 @@ pub fn constructor_do_bitrev8(ctx: &mut C, arg0: Type, arg1: Gpr) -> pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2067. + // Rule at src/isa/x64/lower.isle line 2054. let expr0_0 = constructor_do_bitrev8(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 71777214294589695; @@ -11082,7 +11082,7 @@ pub fn constructor_do_bitrev16(ctx: &mut C, arg0: Type, arg1: Gpr) - pub fn constructor_do_bitrev32(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2079. + // Rule at src/isa/x64/lower.isle line 2066. let expr0_0 = constructor_do_bitrev16(ctx, pattern0_0, pattern1_0)?; let expr1_0 = C::ty_mask(ctx, pattern0_0); let expr2_0: u64 = 281470681808895; @@ -11111,7 +11111,7 @@ pub fn constructor_do_bitrev64(ctx: &mut C, arg0: Type, arg1: Gpr) - let pattern0_0 = arg0; if pattern0_0 == I64 { let pattern2_0 = arg1; - // Rule at src/isa/x64/lower.isle line 2091. + // Rule at src/isa/x64/lower.isle line 2078. let expr0_0 = constructor_do_bitrev32(ctx, pattern0_0, pattern2_0)?; let expr1_0: u64 = 4294967295; let expr2_0 = constructor_imm(ctx, pattern0_0, expr1_0)?; @@ -11145,7 +11145,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::fits_in_32(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2211. + // Rule at src/isa/x64/lower.isle line 2198. let expr0_0: Type = I32; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -11153,7 +11153,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_64(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2207. + // Rule at src/isa/x64/lower.isle line 2194. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -11161,7 +11161,7 @@ pub fn constructor_generic_sextend( return Some(expr3_0); } if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2201. + // Rule at src/isa/x64/lower.isle line 2188. let expr0_0: Type = I64; let expr1_0 = ExtendKind::Sign; let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; @@ -11176,7 +11176,7 @@ pub fn constructor_generic_sextend( if let Some(pattern2_0) = C::ty_int_bool_64(ctx, pattern1_0) { let pattern3_0 = arg2; if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { - // Rule at src/isa/x64/lower.isle line 2197. + // Rule at src/isa/x64/lower.isle line 2184. let expr0_0 = C::put_in_reg(ctx, pattern0_0); let expr1_0 = constructor_put_in_gpr(ctx, pattern0_0)?; let expr2_0 = constructor_spread_sign_bit(ctx, expr1_0)?; @@ -11188,7 +11188,7 @@ pub fn constructor_generic_sextend( } let pattern2_0 = arg2; if pattern2_0 == pattern1_0 { - // Rule at src/isa/x64/lower.isle line 2187. + // Rule at src/isa/x64/lower.isle line 2174. let expr0_0 = constructor_output_value(ctx, pattern0_0)?; return Some(expr0_0); } @@ -11198,7 +11198,7 @@ pub fn constructor_generic_sextend( // Generated as internal constructor for term spread_sign_bit. pub fn constructor_spread_sign_bit(ctx: &mut C, arg0: Gpr) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 2193. + // Rule at src/isa/x64/lower.isle line 2180. let expr0_0: Type = I64; let expr1_0: u8 = 63; let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 44c30872cc..aec7072d60 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -197,13 +197,49 @@ pub struct NonRegInput { /// computation (and side-effect if applicable) could occur at the /// current instruction's location instead. /// - /// If this instruction's operation is merged into the current instruction, - /// the backend must call [LowerCtx::sink_inst()]. - pub inst: Option<(Inst, usize)>, + /// If this instruction's operation is merged into the current + /// instruction, the backend must call [LowerCtx::sink_inst()]. + /// + /// This enum indicates whether this use of the source instruction + /// is unique or not. + pub inst: InputSourceInst, /// The value is a known constant. pub constant: Option, } +/// When examining an input to an instruction, this enum provides one +/// of several options: there is or isn't a single instruction (that +/// we can see and merge with) that produces that input's value, and +/// we are or aren't the single user of that instruction. +#[derive(Clone, Copy, Debug)] +pub enum InputSourceInst { + /// The input in question is the single, unique use of the given + /// instruction and output index, and it can be sunk to the + /// location of this input. + UniqueUse(Inst, usize), + /// The input in question is one of multiple uses of the given + /// instruction. It can still be sunk to the location of this + /// input. + Use(Inst, usize), + /// We cannot determine which instruction produced the input, or + /// it is one of several instructions (e.g., due to a control-flow + /// merge and blockparam), or the source instruction cannot be + /// allowed to sink to the current location due to side-effects. + None, +} + +impl InputSourceInst { + /// Get the instruction and output index for this source, whether + /// we are its single or one of many users. + pub fn as_inst(&self) -> Option<(Inst, usize)> { + match self { + &InputSourceInst::UniqueUse(inst, output_idx) + | &InputSourceInst::Use(inst, output_idx) => Some((inst, output_idx)), + &InputSourceInst::None => None, + } + } +} + /// A machine backend. pub trait LowerBackend { /// The machine instruction type. @@ -271,8 +307,13 @@ pub struct Lower<'func, I: VCodeInst> { /// Instruction constant values, if known. inst_constants: FxHashMap, - /// Use-counts per SSA value, as counted in the input IR. - value_uses: SecondaryMap, + /// Use-counts per SSA value, as counted in the input IR. These + /// are "coarsened", in the abstract-interpretation sense: we only + /// care about "0, 1, many" states, as this is all we need and + /// this lets us do an efficient fixpoint analysis. + /// + /// See doc comment on `ValueUseState` for more details. + value_ir_uses: SecondaryMap, /// Actual uses of each SSA value so far, incremented while lowering. value_lowered_uses: SecondaryMap, @@ -295,6 +336,108 @@ pub struct Lower<'func, I: VCodeInst> { vm_context: Option, } +/// How is a value used in the IR? +/// +/// This can be seen as a coarsening of an integer count. We only need +/// distinct states for zero, one, or many. +/// +/// This analysis deserves further explanation. The basic idea is that +/// we want to allow instruction lowering to know whether a value that +/// an instruction references is *only* referenced by that one use, or +/// by others as well. This is necessary to know when we might want to +/// move a side-effect: we cannot, for example, duplicate a load, so +/// we cannot let instruction lowering match a load as part of a +/// subpattern and potentially incorporate it. +/// +/// Note that a lot of subtlety comes into play once we have +/// *indirect* uses. The classical example of this in our development +/// history was the x86 compare instruction, which is incorporated +/// into flags users (e.g. `selectif`, `trueif`, branches) and can +/// subsequently incorporate loads, or at least we would like it +/// to. However, danger awaits: the compare might be the only user of +/// a load, so we might think we can just move the load (and nothing +/// is duplicated -- success!), except that the compare itself is +/// codegen'd in multiple places, where it is incorporated as a +/// subpattern itself. +/// +/// So we really want a notion of "unique all the way along the +/// matching path". Rust's `&T` and `&mut T` offer a partial analogy +/// to the semantics that we want here: we want to know when we've +/// matched a unique use of an instruction, and that instruction's +/// unique use of another instruction, etc, just as `&mut T` can only +/// be obtained by going through a chain of `&mut T`. If one has a +/// `&T` to a struct containing `&mut T` (one of several uses of an +/// instruction that itself has a unique use of an instruction), one +/// can only get a `&T` (one can only get a "I am one of several users +/// of this instruction" result). +/// +/// We could track these paths, either dynamically as one "looks up +/// the operand tree" or precomputed. But the former requires state +/// and means that the `LowerCtx` API carries that state implicitly, +/// which we'd like to avoid if we can. And the latter implies O(n^2) +/// storage: it is an all-pairs property (is inst `i` unique from the +/// point of view of `j`). +/// +/// To make matters even a little more complex still, a value that is +/// not uniquely used when initially viewing the IR can *become* +/// uniquely used, at least as a root allowing further unique uses of +/// e.g. loads to merge, if no other instruction actually merges +/// it. To be more concrete, if we have `v1 := load; v2 := op v1; v3 +/// := op v2; v4 := op v2` then `v2` is non-uniquely used, so from the +/// point of view of lowering `v4` or `v3`, we cannot merge the load +/// at `v1`. But if we decide just to use the assigned register for +/// `v2` at both `v3` and `v4`, then we only actually codegen `v2` +/// once, so it *is* a unique root at that point and we *can* merge +/// the load. +/// +/// Note also that the color scheme is not sufficient to give us this +/// information, for various reasons: reasoning about side-effects +/// does not tell us about potential duplication of uses through pure +/// ops. +/// +/// To keep things simple and avoid error-prone lowering APIs that +/// would extract more information about whether instruction merging +/// happens or not (we don't have that info now, and it would be +/// difficult to refactor to get it and make that refactor 100% +/// correct), we give up on the above "can become unique if not +/// actually merged" point. Instead, we compute a +/// transitive-uniqueness. That is what this enum represents. +/// +/// To define it plainly: a value is `Unused` if no references exist +/// to it; `Once` if only one other op refers to it, *and* that other +/// op is `Unused` or `Once`; and `Multiple` otherwise. In other +/// words, `Multiple` is contagious: even if an op's result value is +/// directly used only once in the CLIF, that value is `Multiple` if +/// the op that uses it is itself used multiple times (hence could be +/// codegen'd multiple times). In brief, this analysis tells us +/// whether, if every op merged all of its operand tree, a given op +/// could be codegen'd in more than one place. +/// +/// To compute this, we first consider direct uses. At this point +/// `Unused` answers are correct, `Multiple` answers are correct, but +/// some `Once`s may change to `Multiple`s. Then we propagate +/// `Multiple` transitively using a workqueue/fixpoint algorithm. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ValueUseState { + /// Not used at all. + Unused, + /// Used exactly once. + Once, + /// Used multiple times. + Multiple, +} + +impl ValueUseState { + /// Add one use. + fn inc(&mut self) { + let new = match self { + Self::Unused => Self::Once, + Self::Once | Self::Multiple => Self::Multiple, + }; + *self = new; + } +} + /// Notion of "relocation distance". This gives an estimate of how far away a symbol will be from a /// reference. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -408,7 +551,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> { let mut block_end_colors = SecondaryMap::with_default(InstColor::new(0)); let mut side_effect_inst_entry_colors = FxHashMap::default(); let mut inst_constants = FxHashMap::default(); - let mut value_uses = SecondaryMap::with_default(0); for bb in f.layout.blocks() { cur_color += 1; for inst in f.layout.block_insts(bb) { @@ -426,17 +568,13 @@ impl<'func, I: VCodeInst> Lower<'func, I> { log::trace!(" -> constant: {}", c); inst_constants.insert(inst, c); } - - // Count uses of all arguments. - for arg in f.dfg.inst_args(inst) { - let arg = f.dfg.resolve_aliases(*arg); - value_uses[arg] += 1; - } } block_end_colors[bb] = InstColor::new(cur_color); } + let value_ir_uses = Self::compute_use_states(f); + Ok(Lower { f, vcode, @@ -446,7 +584,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { side_effect_inst_entry_colors, inst_constants, next_vreg, - value_uses, + value_ir_uses, value_lowered_uses: SecondaryMap::default(), inst_sunk: FxHashSet::default(), cur_scan_entry_color: None, @@ -457,6 +595,114 @@ impl<'func, I: VCodeInst> Lower<'func, I> { }) } + /// Pre-analysis: compute `value_ir_uses`. See comment on + /// `ValueUseState` for a description of what this analysis + /// computes. + fn compute_use_states<'a>(f: &'a Function) -> SecondaryMap { + // We perform the analysis without recursion, so we don't + // overflow the stack on long chains of ops in the input. + // + // This is sort of a hybrid of a "shallow use-count" pass and + // a DFS. We iterate over all instructions and mark their args + // as used. However when we increment a use-count to + // "Multiple" we push its args onto the stack and do a DFS, + // immediately marking the whole dependency tree as + // Multiple. Doing both (shallow use-counting over all insts, + // and deep Multiple propagation) lets us trim both + // traversals, stopping recursion when a node is already at + // the appropriate state. + // + // In particular, note that the *coarsening* into {Unused, + // Once, Multiple} is part of what makes this pass more + // efficient than a full indirect-use-counting pass. + + let mut value_ir_uses: SecondaryMap = + SecondaryMap::with_default(ValueUseState::Unused); + + // Stack of iterators over Values as we do DFS to mark + // Multiple-state subtrees. + type StackVec<'a> = SmallVec<[std::slice::Iter<'a, Value>; 16]>; + let mut stack: StackVec = smallvec![]; + + // Push args for a given inst onto the DFS stack. + let push_args_on_stack = |stack: &mut StackVec<'a>, value| { + log::trace!(" -> pushing args for {} onto stack", value); + if let ValueDef::Result(src_inst, _) = f.dfg.value_def(value) { + stack.push(f.dfg.inst_args(src_inst).iter()); + } + }; + + // Do a DFS through `value_ir_uses` to mark a subtree as + // Multiple. + let mark_all_uses_as_multiple = + |value_ir_uses: &mut SecondaryMap, stack: &mut StackVec<'a>| { + while let Some(iter) = stack.last_mut() { + if let Some(&value) = iter.next() { + let value = f.dfg.resolve_aliases(value); + log::trace!(" -> DFS reaches {}", value); + if value_ir_uses[value] == ValueUseState::Multiple { + // Truncate DFS here: no need to go further, + // as whole subtree must already be Multiple. + #[cfg(debug_assertions)] + { + // With debug asserts, check one level + // of that invariant at least. + if let ValueDef::Result(src_inst, _) = f.dfg.value_def(value) { + debug_assert!(f.dfg.inst_args(src_inst).iter().all(|&arg| { + let arg = f.dfg.resolve_aliases(arg); + value_ir_uses[arg] == ValueUseState::Multiple + })); + } + } + continue; + } + value_ir_uses[value] = ValueUseState::Multiple; + log::trace!(" -> became Multiple"); + push_args_on_stack(stack, value); + } else { + // Empty iterator, discard. + stack.pop(); + } + } + }; + + for inst in f + .layout + .blocks() + .flat_map(|block| f.layout.block_insts(block)) + { + // If this inst produces multiple values, we must mark all + // of its args as Multiple, because otherwise two uses + // could come in as Once on our two different results. + let force_multiple = f.dfg.inst_results(inst).len() > 1; + + // Iterate over all args of all instructions, noting an + // additional use on each operand. If an operand becomes Multiple, + for &arg in f.dfg.inst_args(inst) { + let arg = f.dfg.resolve_aliases(arg); + let old = value_ir_uses[arg]; + if force_multiple { + log::trace!( + "forcing arg {} to Multiple because of multiple results of user inst", + arg + ); + value_ir_uses[arg] = ValueUseState::Multiple; + } else { + value_ir_uses[arg].inc(); + } + let new = value_ir_uses[arg]; + log::trace!("arg {} used, old state {:?}, new {:?}", arg, old, new,); + // On transition to Multiple, do DFS. + if old != ValueUseState::Multiple && new == ValueUseState::Multiple { + push_args_on_stack(&mut stack, arg); + mark_all_uses_as_multiple(&mut value_ir_uses, &mut stack); + } + } + } + + value_ir_uses + } + fn gen_arg_setup(&mut self) { if let Some(entry_bb) = self.f.layout.entry_block() { log::trace!( @@ -1050,9 +1296,11 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { // OK to merge source instruction if (i) we have a source // instruction, and: // - It has no side-effects, OR - // - It has a side-effect, has one output value, that one output has - // only one use (this one), and the instruction's color is *one less - // than* the current scan color. + // - It has a side-effect, has one output value, that one + // output has only one use, directly or indirectly (so + // cannot be duplicated -- see comment on + // `ValueUseState`), and the instruction's color is *one + // less than* the current scan color. // // This latter set of conditions is testing whether a // side-effecting instruction can sink to the current scan @@ -1071,15 +1319,26 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { log::trace!(" -> src inst {}", src_inst); log::trace!(" -> has lowering side effect: {}", src_side_effect); if !src_side_effect { - // Pure instruction: always possible to sink. - Some((src_inst, result_idx)) + // Pure instruction: always possible to + // sink. Let's determine whether we are the only + // user or not. + if self.value_ir_uses[val] == ValueUseState::Once { + InputSourceInst::UniqueUse(src_inst, result_idx) + } else { + InputSourceInst::Use(src_inst, result_idx) + } } else { // Side-effect: test whether this is the only use of the // only result of the instruction, and whether colors allow // the code-motion. + log::trace!( + " -> side-effecting op {} for val {}: use state {:?}", + src_inst, + val, + self.value_ir_uses[val] + ); if self.cur_scan_entry_color.is_some() - && self.value_uses[val] == 1 - && self.value_lowered_uses[val] == 0 + && self.value_ir_uses[val] == ValueUseState::Once && self.num_outputs(src_inst) == 1 && self .side_effect_inst_entry_colors @@ -1089,15 +1348,15 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { + 1 == self.cur_scan_entry_color.unwrap().get() { - Some((src_inst, 0)) + InputSourceInst::UniqueUse(src_inst, 0) } else { - None + InputSourceInst::None } } } - _ => None, + _ => InputSourceInst::None, }; - let constant = inst.and_then(|(inst, _)| self.get_constant(inst)); + let constant = inst.as_inst().and_then(|(inst, _)| self.get_constant(inst)); NonRegInput { inst, constant } } diff --git a/cranelift/filetests/filetests/isa/s390x/arithmetic.clif b/cranelift/filetests/filetests/isa/s390x/arithmetic.clif index 945e251371..765d824427 100644 --- a/cranelift/filetests/filetests/isa/s390x/arithmetic.clif +++ b/cranelift/filetests/filetests/isa/s390x/arithmetic.clif @@ -262,7 +262,8 @@ block0(v0: i64, v1: i64): } ; block0: -; alg %r2, 0(%r3) +; lg %r4, 0(%r3) +; algr %r2, %r4 ; br %r14 function %iadd_i64_mem_ext32(i64, i64) -> i64 { @@ -273,7 +274,8 @@ block0(v0: i64, v1: i64): } ; block0: -; algf %r2, 0(%r3) +; llgf %r4, 0(%r3) +; algr %r2, %r4 ; br %r14 function %iadd_i32(i32, i32) -> i32 { @@ -305,7 +307,8 @@ block0(v0: i32, v1: i64): } ; block0: -; al %r2, 0(%r3) +; l %r4, 0(%r3) +; alr %r2, %r4 ; br %r14 function %iadd_i32_memoff(i32, i64) -> i32 { @@ -316,7 +319,8 @@ block0(v0: i32, v1: i64): } ; block0: -; aly %r2, 4096(%r3) +; ly %r4, 4096(%r3) +; alr %r2, %r4 ; br %r14 function %isub_i64(i64, i64) -> i64 { diff --git a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif index 7793f4df75..178c364907 100644 --- a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif +++ b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif @@ -13,15 +13,15 @@ block0(v0: i64, v1: i32): ; block0: ; llgfr %r4, %r3 -; lghi %r3, 0 -; ag %r3, 0(%r2) -; clgr %r4, %r3 +; lg %r5, 0(%r2) +; aghi %r5, 0 +; clgr %r4, %r5 ; jgnh label1 ; jg label2 ; block1: ; agr %r2, %r4 -; lghi %r5, 0 -; clgr %r4, %r3 -; locgrh %r2, %r5 +; lghi %r3, 0 +; clgr %r4, %r5 +; locgrh %r2, %r3 ; br %r14 ; block2: ; trap diff --git a/cranelift/filetests/filetests/isa/x64/load-op.clif b/cranelift/filetests/filetests/isa/x64/load-op.clif index d2fa2b9f82..dcbe3f8570 100644 --- a/cranelift/filetests/filetests/isa/x64/load-op.clif +++ b/cranelift/filetests/filetests/isa/x64/load-op.clif @@ -68,3 +68,14 @@ block0(v0: i64): block1: return v2 } + +function %cmp_mem(i64) -> i64 { +block0(v0: i64): + v1 = load.i64 v0 + v2 = icmp eq v0, v1 + v3 = bint.i64 v2 + return v3 + + ; check: cmpq 0(%rdi), %rdi + ; nextln: setz %al +} diff --git a/cranelift/filetests/filetests/isa/x64/select-i128.clif b/cranelift/filetests/filetests/isa/x64/select-i128.clif index 5317a22872..c88e3c3c2a 100644 --- a/cranelift/filetests/filetests/isa/x64/select-i128.clif +++ b/cranelift/filetests/filetests/isa/x64/select-i128.clif @@ -13,8 +13,7 @@ block0(v0: i32, v1: i128, v2: i128): ; pushq %rbp ; movq %rsp, %rbp ; block0: -; movl $42, %r9d -; cmpl %r9d, %edi +; cmpl $42, %edi ; cmovzq %rsi, %rcx, %rcx ; cmovzq %rdx, %r8, %r8 ; movq %rcx, %rax