egraph-based midend: draw the rest of the owl (productionized). (#4953)

* egraph-based midend: draw the rest of the owl.

* Rename `egg` submodule of cranelift-codegen to `egraph`.

* Apply some feedback from @jsharp during code walkthrough.

* Remove recursion from find_best_node by doing a single pass.

Rather than recursively computing the lowest-cost node for a given
eclass and memoizing the answer at each eclass node, we can do a single
forward pass; because every eclass node refers only to earlier nodes,
this is sufficient. The behavior may slightly differ from the earlier
behavior because we cannot short-circuit costs to zero once a node is
elaborated; but in practice this should not matter.

* Make elaboration non-recursive.

Use an explicit stack instead (with `ElabStackEntry` entries,
alongside a result stack).

* Make elaboration traversal of the domtree non-recursive/stack-safe.

* Work analysis logic in Cranelift-side egraph glue into a general analysis framework in cranelift-egraph.

* Apply static recursion limit to rule application.

* Fix aarch64 wrt dynamic-vector support -- broken rebase.

* Topo-sort cranelift-egraph before cranelift-codegen in publish script, like the comment instructs me to!

* Fix multi-result call testcase.

* Include `cranelift-egraph` in `PUBLISHED_CRATES`.

* Fix atomic_rmw: not really a load.

* Remove now-unnecessary PartialOrd/Ord derivations.

* Address some code-review comments.

* Review feedback.

* Review feedback.

* No overlap in mid-end rules, because we are defining a multi-constructor.

* rustfmt

* Review feedback.

* Review feedback.

* Review feedback.

* Review feedback.

* Remove redundant `mut`.

* Add comment noting what rules can do.

* Review feedback.

* Clarify comment wording.

* Update `has_memory_fence_semantics`.

* Apply @jameysharp's improved loop-level computation.

Co-authored-by: Jamey Sharp <jamey@minilop.net>

* Fix suggestion commit.

* Fix off-by-one in new loop-nest analysis.

* Review feedback.

* Review feedback.

* Review feedback.

* Use `Default`, not `std::default::Default`, as per @fitzgen

Co-authored-by: Nick Fitzgerald <fitzgen@gmail.com>

* Apply @fitzgen's comment elaboration to a doc-comment.

Co-authored-by: Nick Fitzgerald <fitzgen@gmail.com>

* Add stat for hitting the rewrite-depth limit.

* Some code motion in split prelude to make the diff a little clearer wrt `main`.

* Take @jameysharp's suggested `try_into()` usage for blockparam indices.

Co-authored-by: Jamey Sharp <jamey@minilop.net>

* Take @jameysharp's suggestion to avoid double-match on load op.

Co-authored-by: Jamey Sharp <jamey@minilop.net>

* Fix suggestion (add import).

* Review feedback.

* Fix stack_load handling.

* Remove redundant can_store case.

* Take @jameysharp's suggested improvement to FuncEGraph::build() logic

Co-authored-by: Jamey Sharp <jamey@minilop.net>

* Tweaks to FuncEGraph::build() on top of suggestion.

* Take @jameysharp's suggested clarified condition

Co-authored-by: Jamey Sharp <jamey@minilop.net>

* Clean up after suggestion (unused variable).

* Fix loop analysis.

* loop level asserts

* Revert constant-space loop analysis -- edge cases were incorrect, so let's go with the simple thing for now.

* Take @jameysharp's suggestion re: result_tys

Co-authored-by: Jamey Sharp <jamey@minilop.net>

* Fix up after suggestion

* Take @jameysharp's suggestion to use fold rather than reduce

Co-authored-by: Jamey Sharp <jamey@minilop.net>

* Fixup after suggestion

* Take @jameysharp's suggestion to remove elaborate_eclass_use's return value.

* Clarifying comment in terminator insts.

Co-authored-by: Jamey Sharp <jamey@minilop.net>
Co-authored-by: Nick Fitzgerald <fitzgen@gmail.com>

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -34,6 +34,7 @@ use crate::{
         abi::ArgPair, ty_bits, InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData,
     },
 };
+use crate::{isle_common_prelude_methods, isle_lower_prelude_methods};
 use regalloc2::PReg;
 use std::boxed::Box;
 use std::convert::TryFrom;
@@ -96,7 +97,7 @@ impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
 }
 
 impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
-    isle_prelude_methods!();
+    isle_lower_prelude_methods!();
     isle_prelude_caller_methods!(crate::isa::aarch64::abi::AArch64MachineDeps, AArch64Caller);
 
     fn sign_return_address_disabled(&mut self) -> Option<()> {

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -41,10 +41,25 @@ pub(crate) fn lower_insn_to_regs(
     match op {
         Opcode::Iconst | Opcode::Bconst | Opcode::Null => implemented_in_isle(ctx),
 
-        Opcode::F32const | Opcode::F64const => unreachable!(
-            "Should never see constant ops at top level lowering entry
-            point, as constants are rematerialized at use-sites"
-        ),
+        Opcode::F32const => {
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+            let val = ctx.get_constant(insn).unwrap();
+            for inst in
+                Inst::load_fp_constant32(rd, val as u32, |ty| ctx.alloc_tmp(ty).only_reg().unwrap())
+            {
+                ctx.emit(inst);
+            }
+        }
+
+        Opcode::F64const => {
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+            let val = ctx.get_constant(insn).unwrap();
+            for inst in
+                Inst::load_fp_constant64(rd, val, |ty| ctx.alloc_tmp(ty).only_reg().unwrap())
+            {
+                ctx.emit(inst);
+            }
+        }
 
         Opcode::GetFramePointer | Opcode::GetStackPointer | Opcode::GetReturnAddress => {
             implemented_in_isle(ctx)

cranelift/codegen/src/isa/aarch64/mod.rs

@@ -62,7 +62,15 @@ impl AArch64Backend {
         let emit_info = EmitInfo::new(flags.clone());
         let sigs = SigSet::new::<abi::AArch64MachineDeps>(func, &self.flags)?;
         let abi = abi::AArch64Callee::new(func, self, &self.isa_flags, &sigs)?;
-        compile::compile::<AArch64Backend>(func, self, abi, &self.machine_env, emit_info, sigs)
+        compile::compile::<AArch64Backend>(
+            func,
+            flags,
+            self,
+            abi,
+            &self.machine_env,
+            emit_info,
+            sigs,
+        )
     }
 }
 

cranelift/codegen/src/isa/riscv64/lower/isle.rs

@@ -5,17 +5,14 @@
 pub mod generated_code;
 use generated_code::{Context, MInst};
 
-use target_lexicon::Triple;
-
 // Types that the generated ISLE code uses via `use super::*`.
 use super::{writable_zero_reg, zero_reg};
-use std::vec::Vec;
-
+use crate::isa::riscv64::abi::Riscv64ABICaller;
 use crate::isa::riscv64::settings::Flags as IsaFlags;
+use crate::machinst::Reg;
 use crate::machinst::{isle::*, MachInst, SmallInstVec};
-use crate::settings::Flags;
-
 use crate::machinst::{VCodeConstant, VCodeConstantData};
+use crate::settings::Flags;
 use crate::{
     ir::{
         immediates::*, types::*, AtomicRmwOp, ExternalName, Inst, InstructionData, MemFlags,
@@ -24,13 +21,12 @@ use crate::{
     isa::riscv64::inst::*,
     machinst::{ArgPair, InsnOutput, Lower},
 };
+use crate::{isle_common_prelude_methods, isle_lower_prelude_methods};
 use regalloc2::PReg;
-
-use crate::isa::riscv64::abi::Riscv64ABICaller;
 use std::boxed::Box;
 use std::convert::TryFrom;
-
-use crate::machinst::Reg;
+use std::vec::Vec;
+use target_lexicon::Triple;
 
 type BoxCallInfo = Box<CallInfo>;
 type BoxCallIndInfo = Box<CallIndInfo>;
@@ -64,7 +60,7 @@ impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
 }
 
 impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
-    isle_prelude_methods!();
+    isle_lower_prelude_methods!();
     isle_prelude_caller_methods!(Riscv64MachineDeps, Riscv64ABICaller);
 
     fn vec_writable_to_regs(&mut self, val: &VecWritableReg) -> ValueRegs {

cranelift/codegen/src/isa/riscv64/mod.rs

@@ -62,7 +62,7 @@ impl Riscv64Backend {
         let emit_info = EmitInfo::new(flags.clone(), self.isa_flags.clone());
         let sigs = SigSet::new::<abi::Riscv64MachineDeps>(func, &self.flags)?;
         let abi = abi::Riscv64Callee::new(func, self, &self.isa_flags, &sigs)?;
-        compile::compile::<Riscv64Backend>(func, self, abi, &self.mach_env, emit_info, sigs)
+        compile::compile::<Riscv64Backend>(func, flags, self, abi, &self.mach_env, emit_info, sigs)
     }
 }
 

cranelift/codegen/src/isa/s390x/lower/isle.rs

@@ -24,6 +24,7 @@ use crate::{
     machinst::abi::ABIMachineSpec,
     machinst::{ArgPair, InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData},
 };
+use crate::{isle_common_prelude_methods, isle_lower_prelude_methods};
 use regalloc2::PReg;
 use smallvec::{smallvec, SmallVec};
 use std::boxed::Box;
@@ -88,7 +89,7 @@ pub(crate) fn lower_branch(
 }
 
 impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
-    isle_prelude_methods!();
+    isle_lower_prelude_methods!();
 
     fn abi_sig(&mut self, sig_ref: SigRef) -> Sig {
         self.lower_ctx.sigs().abi_sig_for_sig_ref(sig_ref)

cranelift/codegen/src/isa/s390x/mod.rs

@@ -60,7 +60,15 @@ impl S390xBackend {
         let emit_info = EmitInfo::new(self.isa_flags.clone());
         let sigs = SigSet::new::<abi::S390xMachineDeps>(func, &self.flags)?;
         let abi = abi::S390xCallee::new(func, self, &self.isa_flags, &sigs)?;
-        compile::compile::<S390xBackend>(func, self, abi, &self.machine_env, emit_info, sigs)
+        compile::compile::<S390xBackend>(
+            func,
+            self.flags.clone(),
+            self,
+            abi,
+            &self.machine_env,
+            emit_info,
+            sigs,
+        )
     }
 }
 

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -7,6 +7,7 @@ use crate::{
     ir::AtomicRmwOp,
     machinst::{InputSourceInst, Reg, Writable},
 };
+use crate::{isle_common_prelude_methods, isle_lower_prelude_methods};
 use generated_code::{Context, MInst, RegisterClass};
 
 // Types that the generated ISLE code uses via `use super::*`.
@@ -92,7 +93,7 @@ pub(crate) fn lower_branch(
 }
 
 impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
-    isle_prelude_methods!();
+    isle_lower_prelude_methods!();
     isle_prelude_caller_methods!(X64ABIMachineSpec, X64Caller);
 
     #[inline]

cranelift/codegen/src/isa/x64/mod.rs

@@ -55,7 +55,7 @@ impl X64Backend {
         let emit_info = EmitInfo::new(flags.clone(), self.x64_flags.clone());
         let sigs = SigSet::new::<abi::X64ABIMachineSpec>(func, &self.flags)?;
         let abi = abi::X64Callee::new(&func, self, &self.x64_flags, &sigs)?;
-        compile::compile::<Self>(&func, self, abi, &self.reg_env, emit_info, sigs)
+        compile::compile::<Self>(&func, flags, self, abi, &self.reg_env, emit_info, sigs)
     }
 }