From d377b665c6ec2d6ad190c1e501e98123180a0a4f Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 12 Oct 2021 17:11:58 -0700 Subject: [PATCH] Initial ISLE integration with the x64 backend On the build side, this commit introduces two things: 1. The automatic generation of various ISLE definitions for working with CLIF. Specifically, it generates extern type definitions for clif opcodes and the clif instruction data `enum`, as well as extractors for matching each clif instructions. This happens inside the `cranelift-codegen-meta` crate. 2. The compilation of ISLE DSL sources to Rust code, that can be included in the main `cranelift-codegen` compilation. Next, this commit introduces the integration glue code required to get ISLE-generated Rust code hooked up in clif-to-x64 lowering. When lowering a clif instruction, we first try to use the ISLE code path. If it succeeds, then we are done lowering this instruction. If it fails, then we proceed along the existing hand-written code path for lowering. Finally, this commit ports many lowering rules over from hand-written, open-coded Rust to ISLE. In the process of supporting ISLE, this commit also makes the x64 `Inst` capable of expressing SSA by supporting 3-operand forms for all of the existing instructions that only have a 2-operand form encoding: dst = src1 op src2 Rather than only the typical x86-64 2-operand form: dst = dst op src This allows `MachInst` to be in SSA form, since `dst` and `src1` are disentangled. ("3-operand" and "2-operand" are a little bit of a misnomer since not all operations are binary operations, but we do the same thing for, e.g., unary operations by disentangling the sole operand from the result.) There are two motivations for this change: 1. To allow ISLE lowering code to have value-equivalence semantics. We want ISLE lowering to translate a CLIF expression that evaluates to some value into a `MachInst` expression that evaluates to the same value. We want both the lowering itself and the resulting `MachInst` to be pure and referentially transparent. This is both a nice paradigm for compiler writers that are authoring and maintaining lowering rules and is a prerequisite to any sort of formal verification of our lowering rules in the future. 2. Better align `MachInst` with `regalloc2`'s API, which requires that the input be in SSA form. --- .gitattributes | 7 + .github/workflows/main.yml | 15 + Cargo.lock | 2 + cranelift/codegen/Cargo.toml | 5 + cranelift/codegen/build.rs | 116 +- cranelift/codegen/meta/Cargo.toml | 3 + cranelift/codegen/meta/src/gen_inst.rs | 254 ++ cranelift/codegen/meta/src/lib.rs | 6 +- cranelift/codegen/meta/src/srcgen.rs | 1 + cranelift/codegen/src/clif.isle | 1635 ++++++++ cranelift/codegen/src/isa/x64/inst.isle | 933 +++++ cranelift/codegen/src/isa/x64/inst/args.rs | 26 +- cranelift/codegen/src/isa/x64/inst/emit.rs | 166 +- .../codegen/src/isa/x64/inst/emit_tests.rs | 111 +- cranelift/codegen/src/isa/x64/inst/mod.rs | 741 +++- cranelift/codegen/src/isa/x64/lower.isle | 890 +++++ cranelift/codegen/src/isa/x64/lower.rs | 675 +--- cranelift/codegen/src/isa/x64/lower/isle.rs | 428 ++ .../src/isa/x64/lower/isle/generated_code.rs | 3496 +++++++++++++++++ cranelift/codegen/src/isle.rs | 22 + cranelift/codegen/src/machinst/lower.rs | 195 +- cranelift/codegen/src/prelude.isle | 202 + cranelift/entity/src/list.rs | 4 +- .../filetests/filetests/isa/x64/i128.clif | 174 +- cranelift/isle/fuzz/README.md | 4 + cranelift/isle/isle/Cargo.toml | 2 +- cranelift/isle/islec/Cargo.toml | 3 +- deny.toml | 1 + scripts/publish.rs | 1 + 29 files changed, 9086 insertions(+), 1032 deletions(-) create mode 100644 cranelift/codegen/src/clif.isle create mode 100644 cranelift/codegen/src/isa/x64/inst.isle create mode 100644 cranelift/codegen/src/isa/x64/lower.isle create mode 100644 cranelift/codegen/src/isa/x64/lower/isle.rs create mode 100644 cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs create mode 100644 cranelift/codegen/src/isle.rs create mode 100644 cranelift/codegen/src/prelude.isle create mode 100644 cranelift/isle/fuzz/README.md diff --git a/.gitattributes b/.gitattributes index df94c94722..928feac126 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,10 @@ *.png binary *.ico binary *.wasm binary + +# ISLE should use lisp syntax highlighting. +*.isle linguist-language=lisp + +# Tell GitHub this is generated code, and doesn't need to be shown in diffs by +# default. +cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs linguist-generated diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f7b2f01c9..b880bfafb3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -189,6 +189,21 @@ jobs: working-directory: ./fuzz - run: cargo fuzz build --dev + rebuild_isle: + name: Rebuild ISLE + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: ./.github/actions/install-rust + - name: Rebuild ISLE DSL files + run: cargo build -p cranelift-codegen --features "rebuild-isle" + - name: Reformat + run: cargo fmt -p cranelift-codegen + - name: Check that the ISLE DSL files are up-to-date + run: git diff --exit-code + rebuild_peephole_optimizers: name: Rebuild Peephole Optimizers runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 2f8b1a4333..1e6102c16a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -552,7 +552,9 @@ dependencies = [ "criterion", "gimli", "hashbrown", + "isle", "log", + "miette", "peepmatic", "peepmatic-runtime", "peepmatic-traits", diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 5bc01a92ce..85a7dc3789 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -39,6 +39,8 @@ criterion = "0.3" [build-dependencies] cranelift-codegen-meta = { path = "meta", version = "0.78.0" } +isle = { path = "../isle/isle", version = "0.1.0", optional = true } +miette = { version = "3", features = ["fancy"] } [features] default = ["std", "unwind"] @@ -98,6 +100,9 @@ enable-peepmatic = ["peepmatic-runtime", "peepmatic-traits", "serde"] # Enable support for the Souper harvester. souper-harvest = ["souper-ir", "souper-ir/stringify"] +# Recompile ISLE DSL source files into their generated Rust code. +rebuild-isle = ["isle", "cranelift-codegen-meta/rebuild-isle"] + [badges] maintenance = { status = "experimental" } diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index 51fcf4cbaa..5932030fc3 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -46,9 +46,12 @@ fn main() { isa_targets }; + let cur_dir = env::current_dir().expect("Can't access current working directory"); + let crate_dir = cur_dir.as_path(); + println!("cargo:rerun-if-changed=build.rs"); - if let Err(err) = meta::generate(&isas, &out_dir) { + if let Err(err) = meta::generate(&isas, &out_dir, crate_dir) { eprintln!("Error: {}", err); process::exit(1); } @@ -74,6 +77,19 @@ fn main() { .unwrap() } + #[cfg(feature = "rebuild-isle")] + { + if let Err(e) = rebuild_isle(crate_dir) { + eprintln!("Error building ISLE files: {:?}", e); + let mut source = e.source(); + while let Some(e) = source { + eprintln!("{:?}", e); + source = e.source(); + } + std::process::abort(); + } + } + let pkg_version = env::var("CARGO_PKG_VERSION").unwrap(); let mut cmd = std::process::Command::new("git"); cmd.arg("rev-parse") @@ -110,3 +126,101 @@ fn main() { ) .unwrap(); } + +/// Rebuild ISLE DSL source text into generated Rust code. +/// +/// NB: This must happen *after* the `cranelift-codegen-meta` functions, since +/// it consumes files generated by them. +#[cfg(feature = "rebuild-isle")] +fn rebuild_isle(crate_dir: &std::path::Path) -> Result<(), Box> { + use std::sync::Once; + static SET_MIETTE_HOOK: Once = Once::new(); + SET_MIETTE_HOOK.call_once(|| { + let _ = miette::set_hook(Box::new(|_| { + Box::new( + miette::MietteHandlerOpts::new() + // `miette` mistakenly uses braille-optimized output for emacs's + // `M-x shell`. + .force_graphical(true) + .build(), + ) + })); + }); + + let clif_isle = crate_dir.join("src").join("clif.isle"); + let prelude_isle = crate_dir.join("src").join("prelude.isle"); + let src_isa_x64 = crate_dir.join("src").join("isa").join("x64"); + + // This is a set of ISLE compilation units. + // + // The format of each entry is: + // + // (output Rust code file, input ISLE source files) + // + // There should be one entry for each backend that uses ISLE for lowering, + // and if/when we replace our peephole optimization passes with ISLE, there + // should be an entry for each of those as well. + let isle_compilations = vec![ + // The x86-64 instruction selector. + ( + src_isa_x64 + .join("lower") + .join("isle") + .join("generated_code.rs"), + vec![ + clif_isle, + prelude_isle, + src_isa_x64.join("inst.isle"), + src_isa_x64.join("lower.isle"), + ], + ), + ]; + + let cur_dir = std::env::current_dir()?; + for (out_file, mut files) in isle_compilations { + for file in files.iter_mut() { + println!("cargo:rerun-if-changed={}", file.display()); + + // Strip the current directory from the file paths, because `islec` + // includes them in the generated source, and this helps us maintain + // deterministic builds that don't include those local file paths. + if let Ok(suffix) = file.strip_prefix(&cur_dir) { + *file = suffix.to_path_buf(); + } + } + + let code = (|| { + let lexer = isle::lexer::Lexer::from_files(files)?; + let defs = isle::parser::parse(lexer)?; + isle::compile::compile(&defs) + })() + .map_err(|e| { + // Make sure to include the source snippets location info along with + // the error messages. + + let report = miette::Report::new(e); + return DebugReport(report); + + struct DebugReport(miette::Report); + + impl std::fmt::Display for DebugReport { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.0.handler().debug(&*self.0, f) + } + } + + impl std::fmt::Debug for DebugReport { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(self, f) + } + } + + impl std::error::Error for DebugReport {} + })?; + + println!("Writing ISLE-generated Rust code to {}", out_file.display()); + std::fs::write(out_file, code)?; + } + + Ok(()) +} diff --git a/cranelift/codegen/meta/Cargo.toml b/cranelift/codegen/meta/Cargo.toml index 374f4f63db..041d59befd 100644 --- a/cranelift/codegen/meta/Cargo.toml +++ b/cranelift/codegen/meta/Cargo.toml @@ -17,3 +17,6 @@ cranelift-codegen-shared = { path = "../shared", version = "0.78.0" } [badges] maintenance = { status = "experimental" } + +[features] +rebuild-isle = [] diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs index 5e28e0b24b..1817bb7937 100644 --- a/cranelift/codegen/meta/src/gen_inst.rs +++ b/cranelift/codegen/meta/src/gen_inst.rs @@ -1,5 +1,6 @@ //! Generate instruction data (including opcodes, formats, builders, etc.). use std::fmt; +use std::path::Path; use cranelift_codegen_shared::constant_hash; @@ -1084,6 +1085,243 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo fmtln!(fmt, "}") } +#[cfg(feature = "rebuild-isle")] +fn gen_isle(formats: &[&InstructionFormat], instructions: &AllInstructions, fmt: &mut Formatter) { + use std::collections::BTreeSet; + use std::fmt::Write; + + fmt.multi_line( + r#" +;; GENERATED BY `gen_isle`. DO NOT EDIT!!! +;; +;; This ISLE file defines all the external type declarations for Cranelift's +;; data structures that ISLE will process, such as `InstructionData` and +;; `Opcode`. + "#, + ); + fmt.empty_line(); + + // Generate all the extern type declarations we need for various immediates. + fmt.line(";;;; Extern type declarations for immediates ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"); + fmt.empty_line(); + let imm_tys: BTreeSet<_> = formats + .iter() + .flat_map(|f| { + f.imm_fields + .iter() + .map(|i| i.kind.rust_type.rsplit("::").next().unwrap()) + .collect::>() + }) + .collect(); + for ty in imm_tys { + fmtln!(fmt, "(type {} (primitive {}))", ty, ty); + } + fmt.empty_line(); + + // Generate all of the value arrays we need for `InstructionData` as well as + // the constructors and extractors for them. + fmt.line(";;;; Value Arrays ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"); + fmt.empty_line(); + let value_array_arities: BTreeSet<_> = formats + .iter() + .filter(|f| f.typevar_operand.is_some() && !f.has_value_list && f.num_value_operands != 1) + .map(|f| f.num_value_operands) + .collect(); + for n in value_array_arities { + fmtln!(fmt, ";; ISLE representation of `[Value; {}]`.", n); + fmtln!(fmt, "(type ValueArray{} extern (enum))", n); + fmt.empty_line(); + + fmtln!( + fmt, + "(decl value_array_{} ({}) ValueArray{})", + n, + (0..n).map(|_| "Value").collect::>().join(" "), + n + ); + fmtln!( + fmt, + "(extern constructor value_array_{} pack_value_array_{})", + n, + n + ); + fmtln!( + fmt, + "(extern extractor infallible value_array_{} unpack_value_array_{})", + n, + n + ); + fmt.empty_line(); + } + + // Generate the extern type declaration for `Opcode`. + fmt.line(";;;; `Opcode` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"); + fmt.empty_line(); + fmt.line("(type Opcode extern"); + fmt.indent(|fmt| { + fmt.line("(enum"); + fmt.indent(|fmt| { + for inst in instructions { + fmtln!(fmt, "{}", inst.camel_name); + } + }); + fmt.line(")"); + }); + fmt.line(")"); + fmt.empty_line(); + + // Generate the extern type declaration for `InstructionData`. + fmt.line(";;;; `InstructionData` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"); + fmt.empty_line(); + fmt.line("(type InstructionData extern"); + fmt.indent(|fmt| { + fmt.line("(enum"); + fmt.indent(|fmt| { + for format in formats { + let mut s = format!("({} (opcode Opcode)", format.name); + if format.typevar_operand.is_some() { + if format.has_value_list { + s.push_str(" (args ValueList)"); + } else if format.num_value_operands == 1 { + s.push_str(" (arg Value)"); + } else { + write!(&mut s, " (args ValueArray{})", format.num_value_operands).unwrap(); + } + } + for field in &format.imm_fields { + write!( + &mut s, + " ({} {})", + field.member, + field.kind.rust_type.rsplit("::").next().unwrap() + ) + .unwrap(); + } + s.push(')'); + fmt.line(&s); + } + }); + fmt.line(")"); + }); + fmt.line(")"); + fmt.empty_line(); + + // Generate the helper extractors for each opcode's full instruction. + // + // TODO: if/when we port our peephole optimization passes to ISLE we will + // want helper constructors as well. + fmt.line(";;;; Extracting Opcode, Operands, and Immediates from `InstructionData` ;;;;;;;;"); + fmt.empty_line(); + for inst in instructions { + fmtln!( + fmt, + "(decl {} ({}) Inst)", + inst.name, + inst.operands_in + .iter() + .map(|o| { + let ty = o.kind.rust_type; + if ty == "&[Value]" { + "ValueSlice" + } else { + ty.rsplit("::").next().unwrap() + } + }) + .collect::>() + .join(" ") + ); + fmtln!(fmt, "(extractor"); + fmt.indent(|fmt| { + fmtln!( + fmt, + "({} {})", + inst.name, + inst.operands_in + .iter() + .map(|o| { o.name }) + .collect::>() + .join(" ") + ); + let mut s = format!( + "(inst_data (InstructionData.{} (Opcode.{})", + inst.format.name, inst.camel_name + ); + + // Immediates. + let imm_operands: Vec<_> = inst + .operands_in + .iter() + .filter(|o| !o.is_value() && !o.is_varargs()) + .collect(); + assert_eq!(imm_operands.len(), inst.format.imm_fields.len()); + for op in imm_operands { + write!(&mut s, " {}", op.name).unwrap(); + } + + // Value and varargs operands. + if inst.format.typevar_operand.is_some() { + if inst.format.has_value_list { + // The instruction format uses a value list, but the + // instruction itself might have not only a `&[Value]` + // varargs operand, but also one or more `Value` operands as + // well. If this is the case, then we need to read them off + // the front of the `ValueList`. + let values: Vec<_> = inst + .operands_in + .iter() + .filter(|o| o.is_value()) + .map(|o| o.name) + .collect(); + let varargs = inst + .operands_in + .iter() + .find(|o| o.is_varargs()) + .unwrap() + .name; + if values.is_empty() { + write!(&mut s, " (value_list_slice {})", varargs).unwrap(); + } else { + write!( + &mut s, + " (unwrap_head_value_list_{} {} {})", + values.len(), + values.join(" "), + varargs + ) + .unwrap(); + } + } else if inst.format.num_value_operands == 1 { + write!( + &mut s, + " {}", + inst.operands_in.iter().find(|o| o.is_value()).unwrap().name + ) + .unwrap(); + } else { + let values = inst + .operands_in + .iter() + .filter(|o| o.is_value()) + .map(|o| o.name) + .collect::>(); + assert_eq!(values.len(), inst.format.num_value_operands); + let values = values.join(" "); + write!( + &mut s, + " (value_array_{} {})", + inst.format.num_value_operands, values, + ) + .unwrap(); + } + } + s.push_str("))"); + fmt.line(&s); + }); + fmt.line(")"); + fmt.empty_line(); + } +} + /// Generate a Builder trait with methods for all instructions. fn gen_builder( instructions: &AllInstructions, @@ -1128,7 +1366,9 @@ pub(crate) fn generate( all_inst: &AllInstructions, opcode_filename: &str, inst_builder_filename: &str, + isle_filename: &str, out_dir: &str, + crate_dir: &Path, ) -> Result<(), error::Error> { // Opcodes. let mut fmt = Formatter::new(); @@ -1144,6 +1384,20 @@ pub(crate) fn generate( gen_try_from(all_inst, &mut fmt); fmt.update_file(opcode_filename, out_dir)?; + // ISLE DSL. + #[cfg(feature = "rebuild-isle")] + { + let mut fmt = Formatter::new(); + gen_isle(&formats, all_inst, &mut fmt); + let crate_src_dir = crate_dir.join("src"); + fmt.update_file(isle_filename, &crate_src_dir.display().to_string())?; + } + #[cfg(not(feature = "rebuild-isle"))] + { + // Silence unused variable warnings. + let _ = (isle_filename, crate_dir); + } + // Instruction builder. let mut fmt = Formatter::new(); gen_builder(all_inst, &formats, &mut fmt); diff --git a/cranelift/codegen/meta/src/lib.rs b/cranelift/codegen/meta/src/lib.rs index 30e5acc48c..e688f89b7d 100644 --- a/cranelift/codegen/meta/src/lib.rs +++ b/cranelift/codegen/meta/src/lib.rs @@ -1,5 +1,7 @@ //! This crate generates Rust sources for use by //! [`cranelift_codegen`](../cranelift_codegen/index.html). + +use std::path::Path; #[macro_use] mod cdsl; mod srcgen; @@ -21,7 +23,7 @@ pub fn isa_from_arch(arch: &str) -> Result { } /// Generates all the Rust source files used in Cranelift from the meta-language. -pub fn generate(isas: &[isa::Isa], out_dir: &str) -> Result<(), error::Error> { +pub fn generate(isas: &[isa::Isa], out_dir: &str, crate_dir: &Path) -> Result<(), error::Error> { // Create all the definitions: // - common definitions. let mut shared_defs = shared::define(); @@ -46,7 +48,9 @@ pub fn generate(isas: &[isa::Isa], out_dir: &str) -> Result<(), error::Error> { &shared_defs.all_instructions, "opcodes.rs", "inst_builder.rs", + "clif.isle", &out_dir, + crate_dir, )?; for isa in target_isas { diff --git a/cranelift/codegen/meta/src/srcgen.rs b/cranelift/codegen/meta/src/srcgen.rs index 21e3d5e904..c4fde06623 100644 --- a/cranelift/codegen/meta/src/srcgen.rs +++ b/cranelift/codegen/meta/src/srcgen.rs @@ -100,6 +100,7 @@ impl Formatter { let path_str = format!("{}/{}", directory, filename.as_ref()); let path = path::Path::new(&path_str); + println!("Writing generated file: {}", path.display()); let mut f = fs::File::create(path)?; for l in self.lines.iter().map(|l| l.as_bytes()) { diff --git a/cranelift/codegen/src/clif.isle b/cranelift/codegen/src/clif.isle new file mode 100644 index 0000000000..04601d5ea5 --- /dev/null +++ b/cranelift/codegen/src/clif.isle @@ -0,0 +1,1635 @@ +;; GENERATED BY `gen_isle`. DO NOT EDIT!!! +;; +;; This ISLE file defines all the external type declarations for Cranelift's +;; data structures that ISLE will process, such as `InstructionData` and +;; `Opcode`. + +;;;; Extern type declarations for immediates ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type AtomicRmwOp (primitive AtomicRmwOp)) +(type Block (primitive Block)) +(type Constant (primitive Constant)) +(type FloatCC (primitive FloatCC)) +(type FuncRef (primitive FuncRef)) +(type GlobalValue (primitive GlobalValue)) +(type Heap (primitive Heap)) +(type Ieee32 (primitive Ieee32)) +(type Ieee64 (primitive Ieee64)) +(type Imm64 (primitive Imm64)) +(type Immediate (primitive Immediate)) +(type IntCC (primitive IntCC)) +(type JumpTable (primitive JumpTable)) +(type MemFlags (primitive MemFlags)) +(type Offset32 (primitive Offset32)) +(type SigRef (primitive SigRef)) +(type StackSlot (primitive StackSlot)) +(type Table (primitive Table)) +(type TrapCode (primitive TrapCode)) +(type Uimm32 (primitive Uimm32)) +(type Uimm8 (primitive Uimm8)) +(type bool (primitive bool)) + +;;;; Value Arrays ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; ISLE representation of `[Value; 2]`. +(type ValueArray2 extern (enum)) + +(decl value_array_2 (Value Value) ValueArray2) +(extern constructor value_array_2 pack_value_array_2) +(extern extractor infallible value_array_2 unpack_value_array_2) + +;; ISLE representation of `[Value; 3]`. +(type ValueArray3 extern (enum)) + +(decl value_array_3 (Value Value Value) ValueArray3) +(extern constructor value_array_3 pack_value_array_3) +(extern extractor infallible value_array_3 unpack_value_array_3) + +;;;; `Opcode` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type Opcode extern + (enum + Jump + Brz + Brnz + BrIcmp + Brif + Brff + BrTable + Debugtrap + Trap + Trapz + ResumableTrap + Trapnz + ResumableTrapnz + Trapif + Trapff + Return + FallthroughReturn + Call + CallIndirect + FuncAddr + Splat + Swizzle + Insertlane + Extractlane + Imin + Umin + Imax + Umax + AvgRound + UaddSat + SaddSat + UsubSat + SsubSat + Load + LoadComplex + Store + StoreComplex + Uload8 + Uload8Complex + Sload8 + Sload8Complex + Istore8 + Istore8Complex + Uload16 + Uload16Complex + Sload16 + Sload16Complex + Istore16 + Istore16Complex + Uload32 + Uload32Complex + Sload32 + Sload32Complex + Istore32 + Istore32Complex + Uload8x8 + Uload8x8Complex + Sload8x8 + Sload8x8Complex + Uload16x4 + Uload16x4Complex + Sload16x4 + Sload16x4Complex + Uload32x2 + Uload32x2Complex + Sload32x2 + Sload32x2Complex + StackLoad + StackStore + StackAddr + GlobalValue + SymbolValue + TlsValue + HeapAddr + GetPinnedReg + SetPinnedReg + TableAddr + Iconst + F32const + F64const + Bconst + Vconst + ConstAddr + Shuffle + Null + Nop + Select + Selectif + SelectifSpectreGuard + Bitselect + Copy + IfcmpSp + Vsplit + Vconcat + Vselect + VanyTrue + VallTrue + VhighBits + Icmp + IcmpImm + Ifcmp + IfcmpImm + Iadd + Isub + Ineg + Iabs + Imul + Umulhi + Smulhi + SqmulRoundSat + Udiv + Sdiv + Urem + Srem + IaddImm + ImulImm + UdivImm + SdivImm + UremImm + SremImm + IrsubImm + IaddCin + IaddIfcin + IaddCout + IaddIfcout + IaddCarry + IaddIfcarry + IsubBin + IsubIfbin + IsubBout + IsubIfbout + IsubBorrow + IsubIfborrow + Band + Bor + Bxor + Bnot + BandNot + BorNot + BxorNot + BandImm + BorImm + BxorImm + Rotl + Rotr + RotlImm + RotrImm + Ishl + Ushr + Sshr + IshlImm + UshrImm + SshrImm + Bitrev + Clz + Cls + Ctz + Popcnt + Fcmp + Ffcmp + Fadd + Fsub + Fmul + Fdiv + Sqrt + Fma + Fneg + Fabs + Fcopysign + Fmin + FminPseudo + Fmax + FmaxPseudo + Ceil + Floor + Trunc + Nearest + IsNull + IsInvalid + Trueif + Trueff + Bitcast + RawBitcast + ScalarToVector + Breduce + Bextend + Bint + Bmask + Ireduce + Snarrow + Unarrow + Uunarrow + SwidenLow + SwidenHigh + UwidenLow + UwidenHigh + IaddPairwise + WideningPairwiseDotProductS + Uextend + Sextend + Fpromote + Fdemote + Fvdemote + FvpromoteLow + FcvtToUint + FcvtToUintSat + FcvtToSint + FcvtToSintSat + FcvtFromUint + FcvtFromSint + FcvtLowFromSint + Isplit + Iconcat + AtomicRmw + AtomicCas + AtomicLoad + AtomicStore + Fence + ) +) + +;;;; `InstructionData` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type InstructionData extern + (enum + (AtomicCas (opcode Opcode) (args ValueArray3) (flags MemFlags)) + (AtomicRmw (opcode Opcode) (args ValueArray2) (flags MemFlags) (op AtomicRmwOp)) + (Binary (opcode Opcode) (args ValueArray2)) + (BinaryImm64 (opcode Opcode) (arg Value) (imm Imm64)) + (BinaryImm8 (opcode Opcode) (arg Value) (imm Uimm8)) + (Branch (opcode Opcode) (args ValueList) (destination Block)) + (BranchFloat (opcode Opcode) (args ValueList) (cond FloatCC) (destination Block)) + (BranchIcmp (opcode Opcode) (args ValueList) (cond IntCC) (destination Block)) + (BranchInt (opcode Opcode) (args ValueList) (cond IntCC) (destination Block)) + (BranchTable (opcode Opcode) (arg Value) (destination Block) (table JumpTable)) + (Call (opcode Opcode) (func_ref FuncRef)) + (CallIndirect (opcode Opcode) (args ValueList) (sig_ref SigRef)) + (CondTrap (opcode Opcode) (arg Value) (code TrapCode)) + (FloatCompare (opcode Opcode) (args ValueArray2) (cond FloatCC)) + (FloatCond (opcode Opcode) (arg Value) (cond FloatCC)) + (FloatCondTrap (opcode Opcode) (arg Value) (cond FloatCC) (code TrapCode)) + (FuncAddr (opcode Opcode) (func_ref FuncRef)) + (HeapAddr (opcode Opcode) (arg Value) (heap Heap) (imm Uimm32)) + (IntCompare (opcode Opcode) (args ValueArray2) (cond IntCC)) + (IntCompareImm (opcode Opcode) (arg Value) (cond IntCC) (imm Imm64)) + (IntCond (opcode Opcode) (arg Value) (cond IntCC)) + (IntCondTrap (opcode Opcode) (arg Value) (cond IntCC) (code TrapCode)) + (IntSelect (opcode Opcode) (args ValueArray3) (cond IntCC)) + (Jump (opcode Opcode) (destination Block)) + (Load (opcode Opcode) (arg Value) (flags MemFlags) (offset Offset32)) + (LoadComplex (opcode Opcode) (flags MemFlags) (offset Offset32)) + (LoadNoOffset (opcode Opcode) (arg Value) (flags MemFlags)) + (MultiAry (opcode Opcode)) + (NullAry (opcode Opcode)) + (Shuffle (opcode Opcode) (args ValueArray2) (imm Immediate)) + (StackLoad (opcode Opcode) (stack_slot StackSlot) (offset Offset32)) + (StackStore (opcode Opcode) (arg Value) (stack_slot StackSlot) (offset Offset32)) + (Store (opcode Opcode) (args ValueArray2) (flags MemFlags) (offset Offset32)) + (StoreComplex (opcode Opcode) (args ValueList) (flags MemFlags) (offset Offset32)) + (StoreNoOffset (opcode Opcode) (args ValueArray2) (flags MemFlags)) + (TableAddr (opcode Opcode) (arg Value) (table Table) (offset Offset32)) + (Ternary (opcode Opcode) (args ValueArray3)) + (TernaryImm8 (opcode Opcode) (args ValueArray2) (imm Uimm8)) + (Trap (opcode Opcode) (code TrapCode)) + (Unary (opcode Opcode) (arg Value)) + (UnaryBool (opcode Opcode) (imm bool)) + (UnaryConst (opcode Opcode) (constant_handle Constant)) + (UnaryGlobalValue (opcode Opcode) (global_value GlobalValue)) + (UnaryIeee32 (opcode Opcode) (imm Ieee32)) + (UnaryIeee64 (opcode Opcode) (imm Ieee64)) + (UnaryImm (opcode Opcode) (imm Imm64)) + ) +) + +;;;; Extracting Opcode, Operands, and Immediates from `InstructionData` ;;;;;;;; + +(decl jump (Block ValueSlice) Inst) +(extractor + (jump block args) + (inst_data (InstructionData.Jump (Opcode.Jump) block)) +) + +(decl brz (Value Block ValueSlice) Inst) +(extractor + (brz c block args) + (inst_data (InstructionData.Branch (Opcode.Brz) block (unwrap_head_value_list_1 c args))) +) + +(decl brnz (Value Block ValueSlice) Inst) +(extractor + (brnz c block args) + (inst_data (InstructionData.Branch (Opcode.Brnz) block (unwrap_head_value_list_1 c args))) +) + +(decl br_icmp (IntCC Value Value Block ValueSlice) Inst) +(extractor + (br_icmp Cond x y block args) + (inst_data (InstructionData.BranchIcmp (Opcode.BrIcmp) Cond block (unwrap_head_value_list_2 x y args))) +) + +(decl brif (IntCC Value Block ValueSlice) Inst) +(extractor + (brif Cond f block args) + (inst_data (InstructionData.BranchInt (Opcode.Brif) Cond block (unwrap_head_value_list_1 f args))) +) + +(decl brff (FloatCC Value Block ValueSlice) Inst) +(extractor + (brff Cond f block args) + (inst_data (InstructionData.BranchFloat (Opcode.Brff) Cond block (unwrap_head_value_list_1 f args))) +) + +(decl br_table (Value Block JumpTable) Inst) +(extractor + (br_table x block JT) + (inst_data (InstructionData.BranchTable (Opcode.BrTable) block JT x)) +) + +(decl debugtrap () Inst) +(extractor + (debugtrap ) + (inst_data (InstructionData.NullAry (Opcode.Debugtrap))) +) + +(decl trap (TrapCode) Inst) +(extractor + (trap code) + (inst_data (InstructionData.Trap (Opcode.Trap) code)) +) + +(decl trapz (Value TrapCode) Inst) +(extractor + (trapz c code) + (inst_data (InstructionData.CondTrap (Opcode.Trapz) code c)) +) + +(decl resumable_trap (TrapCode) Inst) +(extractor + (resumable_trap code) + (inst_data (InstructionData.Trap (Opcode.ResumableTrap) code)) +) + +(decl trapnz (Value TrapCode) Inst) +(extractor + (trapnz c code) + (inst_data (InstructionData.CondTrap (Opcode.Trapnz) code c)) +) + +(decl resumable_trapnz (Value TrapCode) Inst) +(extractor + (resumable_trapnz c code) + (inst_data (InstructionData.CondTrap (Opcode.ResumableTrapnz) code c)) +) + +(decl trapif (IntCC Value TrapCode) Inst) +(extractor + (trapif Cond f code) + (inst_data (InstructionData.IntCondTrap (Opcode.Trapif) Cond code f)) +) + +(decl trapff (FloatCC Value TrapCode) Inst) +(extractor + (trapff Cond f code) + (inst_data (InstructionData.FloatCondTrap (Opcode.Trapff) Cond code f)) +) + +(decl return (ValueSlice) Inst) +(extractor + (return rvals) + (inst_data (InstructionData.MultiAry (Opcode.Return))) +) + +(decl fallthrough_return (ValueSlice) Inst) +(extractor + (fallthrough_return rvals) + (inst_data (InstructionData.MultiAry (Opcode.FallthroughReturn))) +) + +(decl call (FuncRef ValueSlice) Inst) +(extractor + (call FN args) + (inst_data (InstructionData.Call (Opcode.Call) FN)) +) + +(decl call_indirect (SigRef Value ValueSlice) Inst) +(extractor + (call_indirect SIG callee args) + (inst_data (InstructionData.CallIndirect (Opcode.CallIndirect) SIG (unwrap_head_value_list_1 callee args))) +) + +(decl func_addr (FuncRef) Inst) +(extractor + (func_addr FN) + (inst_data (InstructionData.FuncAddr (Opcode.FuncAddr) FN)) +) + +(decl splat (Value) Inst) +(extractor + (splat x) + (inst_data (InstructionData.Unary (Opcode.Splat) x)) +) + +(decl swizzle (Value Value) Inst) +(extractor + (swizzle x y) + (inst_data (InstructionData.Binary (Opcode.Swizzle) (value_array_2 x y))) +) + +(decl insertlane (Value Value Uimm8) Inst) +(extractor + (insertlane x y Idx) + (inst_data (InstructionData.TernaryImm8 (Opcode.Insertlane) Idx (value_array_2 x y))) +) + +(decl extractlane (Value Uimm8) Inst) +(extractor + (extractlane x Idx) + (inst_data (InstructionData.BinaryImm8 (Opcode.Extractlane) Idx x)) +) + +(decl imin (Value Value) Inst) +(extractor + (imin x y) + (inst_data (InstructionData.Binary (Opcode.Imin) (value_array_2 x y))) +) + +(decl umin (Value Value) Inst) +(extractor + (umin x y) + (inst_data (InstructionData.Binary (Opcode.Umin) (value_array_2 x y))) +) + +(decl imax (Value Value) Inst) +(extractor + (imax x y) + (inst_data (InstructionData.Binary (Opcode.Imax) (value_array_2 x y))) +) + +(decl umax (Value Value) Inst) +(extractor + (umax x y) + (inst_data (InstructionData.Binary (Opcode.Umax) (value_array_2 x y))) +) + +(decl avg_round (Value Value) Inst) +(extractor + (avg_round x y) + (inst_data (InstructionData.Binary (Opcode.AvgRound) (value_array_2 x y))) +) + +(decl uadd_sat (Value Value) Inst) +(extractor + (uadd_sat x y) + (inst_data (InstructionData.Binary (Opcode.UaddSat) (value_array_2 x y))) +) + +(decl sadd_sat (Value Value) Inst) +(extractor + (sadd_sat x y) + (inst_data (InstructionData.Binary (Opcode.SaddSat) (value_array_2 x y))) +) + +(decl usub_sat (Value Value) Inst) +(extractor + (usub_sat x y) + (inst_data (InstructionData.Binary (Opcode.UsubSat) (value_array_2 x y))) +) + +(decl ssub_sat (Value Value) Inst) +(extractor + (ssub_sat x y) + (inst_data (InstructionData.Binary (Opcode.SsubSat) (value_array_2 x y))) +) + +(decl load (MemFlags Value Offset32) Inst) +(extractor + (load MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Load) MemFlags Offset p)) +) + +(decl load_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (load_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.LoadComplex) MemFlags Offset)) +) + +(decl store (MemFlags Value Value Offset32) Inst) +(extractor + (store MemFlags x p Offset) + (inst_data (InstructionData.Store (Opcode.Store) MemFlags Offset (value_array_2 x p))) +) + +(decl store_complex (MemFlags Value ValueSlice Offset32) Inst) +(extractor + (store_complex MemFlags x args Offset) + (inst_data (InstructionData.StoreComplex (Opcode.StoreComplex) MemFlags Offset (unwrap_head_value_list_1 x args))) +) + +(decl uload8 (MemFlags Value Offset32) Inst) +(extractor + (uload8 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Uload8) MemFlags Offset p)) +) + +(decl uload8_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (uload8_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Uload8Complex) MemFlags Offset)) +) + +(decl sload8 (MemFlags Value Offset32) Inst) +(extractor + (sload8 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Sload8) MemFlags Offset p)) +) + +(decl sload8_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (sload8_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Sload8Complex) MemFlags Offset)) +) + +(decl istore8 (MemFlags Value Value Offset32) Inst) +(extractor + (istore8 MemFlags x p Offset) + (inst_data (InstructionData.Store (Opcode.Istore8) MemFlags Offset (value_array_2 x p))) +) + +(decl istore8_complex (MemFlags Value ValueSlice Offset32) Inst) +(extractor + (istore8_complex MemFlags x args Offset) + (inst_data (InstructionData.StoreComplex (Opcode.Istore8Complex) MemFlags Offset (unwrap_head_value_list_1 x args))) +) + +(decl uload16 (MemFlags Value Offset32) Inst) +(extractor + (uload16 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Uload16) MemFlags Offset p)) +) + +(decl uload16_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (uload16_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Uload16Complex) MemFlags Offset)) +) + +(decl sload16 (MemFlags Value Offset32) Inst) +(extractor + (sload16 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Sload16) MemFlags Offset p)) +) + +(decl sload16_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (sload16_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Sload16Complex) MemFlags Offset)) +) + +(decl istore16 (MemFlags Value Value Offset32) Inst) +(extractor + (istore16 MemFlags x p Offset) + (inst_data (InstructionData.Store (Opcode.Istore16) MemFlags Offset (value_array_2 x p))) +) + +(decl istore16_complex (MemFlags Value ValueSlice Offset32) Inst) +(extractor + (istore16_complex MemFlags x args Offset) + (inst_data (InstructionData.StoreComplex (Opcode.Istore16Complex) MemFlags Offset (unwrap_head_value_list_1 x args))) +) + +(decl uload32 (MemFlags Value Offset32) Inst) +(extractor + (uload32 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Uload32) MemFlags Offset p)) +) + +(decl uload32_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (uload32_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Uload32Complex) MemFlags Offset)) +) + +(decl sload32 (MemFlags Value Offset32) Inst) +(extractor + (sload32 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Sload32) MemFlags Offset p)) +) + +(decl sload32_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (sload32_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Sload32Complex) MemFlags Offset)) +) + +(decl istore32 (MemFlags Value Value Offset32) Inst) +(extractor + (istore32 MemFlags x p Offset) + (inst_data (InstructionData.Store (Opcode.Istore32) MemFlags Offset (value_array_2 x p))) +) + +(decl istore32_complex (MemFlags Value ValueSlice Offset32) Inst) +(extractor + (istore32_complex MemFlags x args Offset) + (inst_data (InstructionData.StoreComplex (Opcode.Istore32Complex) MemFlags Offset (unwrap_head_value_list_1 x args))) +) + +(decl uload8x8 (MemFlags Value Offset32) Inst) +(extractor + (uload8x8 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Uload8x8) MemFlags Offset p)) +) + +(decl uload8x8_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (uload8x8_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Uload8x8Complex) MemFlags Offset)) +) + +(decl sload8x8 (MemFlags Value Offset32) Inst) +(extractor + (sload8x8 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Sload8x8) MemFlags Offset p)) +) + +(decl sload8x8_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (sload8x8_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Sload8x8Complex) MemFlags Offset)) +) + +(decl uload16x4 (MemFlags Value Offset32) Inst) +(extractor + (uload16x4 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Uload16x4) MemFlags Offset p)) +) + +(decl uload16x4_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (uload16x4_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Uload16x4Complex) MemFlags Offset)) +) + +(decl sload16x4 (MemFlags Value Offset32) Inst) +(extractor + (sload16x4 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Sload16x4) MemFlags Offset p)) +) + +(decl sload16x4_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (sload16x4_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Sload16x4Complex) MemFlags Offset)) +) + +(decl uload32x2 (MemFlags Value Offset32) Inst) +(extractor + (uload32x2 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Uload32x2) MemFlags Offset p)) +) + +(decl uload32x2_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (uload32x2_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Uload32x2Complex) MemFlags Offset)) +) + +(decl sload32x2 (MemFlags Value Offset32) Inst) +(extractor + (sload32x2 MemFlags p Offset) + (inst_data (InstructionData.Load (Opcode.Sload32x2) MemFlags Offset p)) +) + +(decl sload32x2_complex (MemFlags ValueSlice Offset32) Inst) +(extractor + (sload32x2_complex MemFlags args Offset) + (inst_data (InstructionData.LoadComplex (Opcode.Sload32x2Complex) MemFlags Offset)) +) + +(decl stack_load (StackSlot Offset32) Inst) +(extractor + (stack_load SS Offset) + (inst_data (InstructionData.StackLoad (Opcode.StackLoad) SS Offset)) +) + +(decl stack_store (Value StackSlot Offset32) Inst) +(extractor + (stack_store x SS Offset) + (inst_data (InstructionData.StackStore (Opcode.StackStore) SS Offset x)) +) + +(decl stack_addr (StackSlot Offset32) Inst) +(extractor + (stack_addr SS Offset) + (inst_data (InstructionData.StackLoad (Opcode.StackAddr) SS Offset)) +) + +(decl global_value (GlobalValue) Inst) +(extractor + (global_value GV) + (inst_data (InstructionData.UnaryGlobalValue (Opcode.GlobalValue) GV)) +) + +(decl symbol_value (GlobalValue) Inst) +(extractor + (symbol_value GV) + (inst_data (InstructionData.UnaryGlobalValue (Opcode.SymbolValue) GV)) +) + +(decl tls_value (GlobalValue) Inst) +(extractor + (tls_value GV) + (inst_data (InstructionData.UnaryGlobalValue (Opcode.TlsValue) GV)) +) + +(decl heap_addr (Heap Value Uimm32) Inst) +(extractor + (heap_addr H p Size) + (inst_data (InstructionData.HeapAddr (Opcode.HeapAddr) H Size p)) +) + +(decl get_pinned_reg () Inst) +(extractor + (get_pinned_reg ) + (inst_data (InstructionData.NullAry (Opcode.GetPinnedReg))) +) + +(decl set_pinned_reg (Value) Inst) +(extractor + (set_pinned_reg addr) + (inst_data (InstructionData.Unary (Opcode.SetPinnedReg) addr)) +) + +(decl table_addr (Table Value Offset32) Inst) +(extractor + (table_addr T p Offset) + (inst_data (InstructionData.TableAddr (Opcode.TableAddr) T Offset p)) +) + +(decl iconst (Imm64) Inst) +(extractor + (iconst N) + (inst_data (InstructionData.UnaryImm (Opcode.Iconst) N)) +) + +(decl f32const (Ieee32) Inst) +(extractor + (f32const N) + (inst_data (InstructionData.UnaryIeee32 (Opcode.F32const) N)) +) + +(decl f64const (Ieee64) Inst) +(extractor + (f64const N) + (inst_data (InstructionData.UnaryIeee64 (Opcode.F64const) N)) +) + +(decl bconst (bool) Inst) +(extractor + (bconst N) + (inst_data (InstructionData.UnaryBool (Opcode.Bconst) N)) +) + +(decl vconst (Constant) Inst) +(extractor + (vconst N) + (inst_data (InstructionData.UnaryConst (Opcode.Vconst) N)) +) + +(decl const_addr (Constant) Inst) +(extractor + (const_addr constant) + (inst_data (InstructionData.UnaryConst (Opcode.ConstAddr) constant)) +) + +(decl shuffle (Value Value Immediate) Inst) +(extractor + (shuffle a b mask) + (inst_data (InstructionData.Shuffle (Opcode.Shuffle) mask (value_array_2 a b))) +) + +(decl null () Inst) +(extractor + (null ) + (inst_data (InstructionData.NullAry (Opcode.Null))) +) + +(decl nop () Inst) +(extractor + (nop ) + (inst_data (InstructionData.NullAry (Opcode.Nop))) +) + +(decl select (Value Value Value) Inst) +(extractor + (select c x y) + (inst_data (InstructionData.Ternary (Opcode.Select) (value_array_3 c x y))) +) + +(decl selectif (IntCC Value Value Value) Inst) +(extractor + (selectif cc flags x y) + (inst_data (InstructionData.IntSelect (Opcode.Selectif) cc (value_array_3 flags x y))) +) + +(decl selectif_spectre_guard (IntCC Value Value Value) Inst) +(extractor + (selectif_spectre_guard cc flags x y) + (inst_data (InstructionData.IntSelect (Opcode.SelectifSpectreGuard) cc (value_array_3 flags x y))) +) + +(decl bitselect (Value Value Value) Inst) +(extractor + (bitselect c x y) + (inst_data (InstructionData.Ternary (Opcode.Bitselect) (value_array_3 c x y))) +) + +(decl copy (Value) Inst) +(extractor + (copy x) + (inst_data (InstructionData.Unary (Opcode.Copy) x)) +) + +(decl ifcmp_sp (Value) Inst) +(extractor + (ifcmp_sp addr) + (inst_data (InstructionData.Unary (Opcode.IfcmpSp) addr)) +) + +(decl vsplit (Value) Inst) +(extractor + (vsplit x) + (inst_data (InstructionData.Unary (Opcode.Vsplit) x)) +) + +(decl vconcat (Value Value) Inst) +(extractor + (vconcat x y) + (inst_data (InstructionData.Binary (Opcode.Vconcat) (value_array_2 x y))) +) + +(decl vselect (Value Value Value) Inst) +(extractor + (vselect c x y) + (inst_data (InstructionData.Ternary (Opcode.Vselect) (value_array_3 c x y))) +) + +(decl vany_true (Value) Inst) +(extractor + (vany_true a) + (inst_data (InstructionData.Unary (Opcode.VanyTrue) a)) +) + +(decl vall_true (Value) Inst) +(extractor + (vall_true a) + (inst_data (InstructionData.Unary (Opcode.VallTrue) a)) +) + +(decl vhigh_bits (Value) Inst) +(extractor + (vhigh_bits a) + (inst_data (InstructionData.Unary (Opcode.VhighBits) a)) +) + +(decl icmp (IntCC Value Value) Inst) +(extractor + (icmp Cond x y) + (inst_data (InstructionData.IntCompare (Opcode.Icmp) Cond (value_array_2 x y))) +) + +(decl icmp_imm (IntCC Value Imm64) Inst) +(extractor + (icmp_imm Cond x Y) + (inst_data (InstructionData.IntCompareImm (Opcode.IcmpImm) Cond Y x)) +) + +(decl ifcmp (Value Value) Inst) +(extractor + (ifcmp x y) + (inst_data (InstructionData.Binary (Opcode.Ifcmp) (value_array_2 x y))) +) + +(decl ifcmp_imm (Value Imm64) Inst) +(extractor + (ifcmp_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.IfcmpImm) Y x)) +) + +(decl iadd (Value Value) Inst) +(extractor + (iadd x y) + (inst_data (InstructionData.Binary (Opcode.Iadd) (value_array_2 x y))) +) + +(decl isub (Value Value) Inst) +(extractor + (isub x y) + (inst_data (InstructionData.Binary (Opcode.Isub) (value_array_2 x y))) +) + +(decl ineg (Value) Inst) +(extractor + (ineg x) + (inst_data (InstructionData.Unary (Opcode.Ineg) x)) +) + +(decl iabs (Value) Inst) +(extractor + (iabs x) + (inst_data (InstructionData.Unary (Opcode.Iabs) x)) +) + +(decl imul (Value Value) Inst) +(extractor + (imul x y) + (inst_data (InstructionData.Binary (Opcode.Imul) (value_array_2 x y))) +) + +(decl umulhi (Value Value) Inst) +(extractor + (umulhi x y) + (inst_data (InstructionData.Binary (Opcode.Umulhi) (value_array_2 x y))) +) + +(decl smulhi (Value Value) Inst) +(extractor + (smulhi x y) + (inst_data (InstructionData.Binary (Opcode.Smulhi) (value_array_2 x y))) +) + +(decl sqmul_round_sat (Value Value) Inst) +(extractor + (sqmul_round_sat x y) + (inst_data (InstructionData.Binary (Opcode.SqmulRoundSat) (value_array_2 x y))) +) + +(decl udiv (Value Value) Inst) +(extractor + (udiv x y) + (inst_data (InstructionData.Binary (Opcode.Udiv) (value_array_2 x y))) +) + +(decl sdiv (Value Value) Inst) +(extractor + (sdiv x y) + (inst_data (InstructionData.Binary (Opcode.Sdiv) (value_array_2 x y))) +) + +(decl urem (Value Value) Inst) +(extractor + (urem x y) + (inst_data (InstructionData.Binary (Opcode.Urem) (value_array_2 x y))) +) + +(decl srem (Value Value) Inst) +(extractor + (srem x y) + (inst_data (InstructionData.Binary (Opcode.Srem) (value_array_2 x y))) +) + +(decl iadd_imm (Value Imm64) Inst) +(extractor + (iadd_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.IaddImm) Y x)) +) + +(decl imul_imm (Value Imm64) Inst) +(extractor + (imul_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.ImulImm) Y x)) +) + +(decl udiv_imm (Value Imm64) Inst) +(extractor + (udiv_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.UdivImm) Y x)) +) + +(decl sdiv_imm (Value Imm64) Inst) +(extractor + (sdiv_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.SdivImm) Y x)) +) + +(decl urem_imm (Value Imm64) Inst) +(extractor + (urem_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.UremImm) Y x)) +) + +(decl srem_imm (Value Imm64) Inst) +(extractor + (srem_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.SremImm) Y x)) +) + +(decl irsub_imm (Value Imm64) Inst) +(extractor + (irsub_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.IrsubImm) Y x)) +) + +(decl iadd_cin (Value Value Value) Inst) +(extractor + (iadd_cin x y c_in) + (inst_data (InstructionData.Ternary (Opcode.IaddCin) (value_array_3 x y c_in))) +) + +(decl iadd_ifcin (Value Value Value) Inst) +(extractor + (iadd_ifcin x y c_in) + (inst_data (InstructionData.Ternary (Opcode.IaddIfcin) (value_array_3 x y c_in))) +) + +(decl iadd_cout (Value Value) Inst) +(extractor + (iadd_cout x y) + (inst_data (InstructionData.Binary (Opcode.IaddCout) (value_array_2 x y))) +) + +(decl iadd_ifcout (Value Value) Inst) +(extractor + (iadd_ifcout x y) + (inst_data (InstructionData.Binary (Opcode.IaddIfcout) (value_array_2 x y))) +) + +(decl iadd_carry (Value Value Value) Inst) +(extractor + (iadd_carry x y c_in) + (inst_data (InstructionData.Ternary (Opcode.IaddCarry) (value_array_3 x y c_in))) +) + +(decl iadd_ifcarry (Value Value Value) Inst) +(extractor + (iadd_ifcarry x y c_in) + (inst_data (InstructionData.Ternary (Opcode.IaddIfcarry) (value_array_3 x y c_in))) +) + +(decl isub_bin (Value Value Value) Inst) +(extractor + (isub_bin x y b_in) + (inst_data (InstructionData.Ternary (Opcode.IsubBin) (value_array_3 x y b_in))) +) + +(decl isub_ifbin (Value Value Value) Inst) +(extractor + (isub_ifbin x y b_in) + (inst_data (InstructionData.Ternary (Opcode.IsubIfbin) (value_array_3 x y b_in))) +) + +(decl isub_bout (Value Value) Inst) +(extractor + (isub_bout x y) + (inst_data (InstructionData.Binary (Opcode.IsubBout) (value_array_2 x y))) +) + +(decl isub_ifbout (Value Value) Inst) +(extractor + (isub_ifbout x y) + (inst_data (InstructionData.Binary (Opcode.IsubIfbout) (value_array_2 x y))) +) + +(decl isub_borrow (Value Value Value) Inst) +(extractor + (isub_borrow x y b_in) + (inst_data (InstructionData.Ternary (Opcode.IsubBorrow) (value_array_3 x y b_in))) +) + +(decl isub_ifborrow (Value Value Value) Inst) +(extractor + (isub_ifborrow x y b_in) + (inst_data (InstructionData.Ternary (Opcode.IsubIfborrow) (value_array_3 x y b_in))) +) + +(decl band (Value Value) Inst) +(extractor + (band x y) + (inst_data (InstructionData.Binary (Opcode.Band) (value_array_2 x y))) +) + +(decl bor (Value Value) Inst) +(extractor + (bor x y) + (inst_data (InstructionData.Binary (Opcode.Bor) (value_array_2 x y))) +) + +(decl bxor (Value Value) Inst) +(extractor + (bxor x y) + (inst_data (InstructionData.Binary (Opcode.Bxor) (value_array_2 x y))) +) + +(decl bnot (Value) Inst) +(extractor + (bnot x) + (inst_data (InstructionData.Unary (Opcode.Bnot) x)) +) + +(decl band_not (Value Value) Inst) +(extractor + (band_not x y) + (inst_data (InstructionData.Binary (Opcode.BandNot) (value_array_2 x y))) +) + +(decl bor_not (Value Value) Inst) +(extractor + (bor_not x y) + (inst_data (InstructionData.Binary (Opcode.BorNot) (value_array_2 x y))) +) + +(decl bxor_not (Value Value) Inst) +(extractor + (bxor_not x y) + (inst_data (InstructionData.Binary (Opcode.BxorNot) (value_array_2 x y))) +) + +(decl band_imm (Value Imm64) Inst) +(extractor + (band_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.BandImm) Y x)) +) + +(decl bor_imm (Value Imm64) Inst) +(extractor + (bor_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.BorImm) Y x)) +) + +(decl bxor_imm (Value Imm64) Inst) +(extractor + (bxor_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.BxorImm) Y x)) +) + +(decl rotl (Value Value) Inst) +(extractor + (rotl x y) + (inst_data (InstructionData.Binary (Opcode.Rotl) (value_array_2 x y))) +) + +(decl rotr (Value Value) Inst) +(extractor + (rotr x y) + (inst_data (InstructionData.Binary (Opcode.Rotr) (value_array_2 x y))) +) + +(decl rotl_imm (Value Imm64) Inst) +(extractor + (rotl_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.RotlImm) Y x)) +) + +(decl rotr_imm (Value Imm64) Inst) +(extractor + (rotr_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.RotrImm) Y x)) +) + +(decl ishl (Value Value) Inst) +(extractor + (ishl x y) + (inst_data (InstructionData.Binary (Opcode.Ishl) (value_array_2 x y))) +) + +(decl ushr (Value Value) Inst) +(extractor + (ushr x y) + (inst_data (InstructionData.Binary (Opcode.Ushr) (value_array_2 x y))) +) + +(decl sshr (Value Value) Inst) +(extractor + (sshr x y) + (inst_data (InstructionData.Binary (Opcode.Sshr) (value_array_2 x y))) +) + +(decl ishl_imm (Value Imm64) Inst) +(extractor + (ishl_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.IshlImm) Y x)) +) + +(decl ushr_imm (Value Imm64) Inst) +(extractor + (ushr_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.UshrImm) Y x)) +) + +(decl sshr_imm (Value Imm64) Inst) +(extractor + (sshr_imm x Y) + (inst_data (InstructionData.BinaryImm64 (Opcode.SshrImm) Y x)) +) + +(decl bitrev (Value) Inst) +(extractor + (bitrev x) + (inst_data (InstructionData.Unary (Opcode.Bitrev) x)) +) + +(decl clz (Value) Inst) +(extractor + (clz x) + (inst_data (InstructionData.Unary (Opcode.Clz) x)) +) + +(decl cls (Value) Inst) +(extractor + (cls x) + (inst_data (InstructionData.Unary (Opcode.Cls) x)) +) + +(decl ctz (Value) Inst) +(extractor + (ctz x) + (inst_data (InstructionData.Unary (Opcode.Ctz) x)) +) + +(decl popcnt (Value) Inst) +(extractor + (popcnt x) + (inst_data (InstructionData.Unary (Opcode.Popcnt) x)) +) + +(decl fcmp (FloatCC Value Value) Inst) +(extractor + (fcmp Cond x y) + (inst_data (InstructionData.FloatCompare (Opcode.Fcmp) Cond (value_array_2 x y))) +) + +(decl ffcmp (Value Value) Inst) +(extractor + (ffcmp x y) + (inst_data (InstructionData.Binary (Opcode.Ffcmp) (value_array_2 x y))) +) + +(decl fadd (Value Value) Inst) +(extractor + (fadd x y) + (inst_data (InstructionData.Binary (Opcode.Fadd) (value_array_2 x y))) +) + +(decl fsub (Value Value) Inst) +(extractor + (fsub x y) + (inst_data (InstructionData.Binary (Opcode.Fsub) (value_array_2 x y))) +) + +(decl fmul (Value Value) Inst) +(extractor + (fmul x y) + (inst_data (InstructionData.Binary (Opcode.Fmul) (value_array_2 x y))) +) + +(decl fdiv (Value Value) Inst) +(extractor + (fdiv x y) + (inst_data (InstructionData.Binary (Opcode.Fdiv) (value_array_2 x y))) +) + +(decl sqrt (Value) Inst) +(extractor + (sqrt x) + (inst_data (InstructionData.Unary (Opcode.Sqrt) x)) +) + +(decl fma (Value Value Value) Inst) +(extractor + (fma x y z) + (inst_data (InstructionData.Ternary (Opcode.Fma) (value_array_3 x y z))) +) + +(decl fneg (Value) Inst) +(extractor + (fneg x) + (inst_data (InstructionData.Unary (Opcode.Fneg) x)) +) + +(decl fabs (Value) Inst) +(extractor + (fabs x) + (inst_data (InstructionData.Unary (Opcode.Fabs) x)) +) + +(decl fcopysign (Value Value) Inst) +(extractor + (fcopysign x y) + (inst_data (InstructionData.Binary (Opcode.Fcopysign) (value_array_2 x y))) +) + +(decl fmin (Value Value) Inst) +(extractor + (fmin x y) + (inst_data (InstructionData.Binary (Opcode.Fmin) (value_array_2 x y))) +) + +(decl fmin_pseudo (Value Value) Inst) +(extractor + (fmin_pseudo x y) + (inst_data (InstructionData.Binary (Opcode.FminPseudo) (value_array_2 x y))) +) + +(decl fmax (Value Value) Inst) +(extractor + (fmax x y) + (inst_data (InstructionData.Binary (Opcode.Fmax) (value_array_2 x y))) +) + +(decl fmax_pseudo (Value Value) Inst) +(extractor + (fmax_pseudo x y) + (inst_data (InstructionData.Binary (Opcode.FmaxPseudo) (value_array_2 x y))) +) + +(decl ceil (Value) Inst) +(extractor + (ceil x) + (inst_data (InstructionData.Unary (Opcode.Ceil) x)) +) + +(decl floor (Value) Inst) +(extractor + (floor x) + (inst_data (InstructionData.Unary (Opcode.Floor) x)) +) + +(decl trunc (Value) Inst) +(extractor + (trunc x) + (inst_data (InstructionData.Unary (Opcode.Trunc) x)) +) + +(decl nearest (Value) Inst) +(extractor + (nearest x) + (inst_data (InstructionData.Unary (Opcode.Nearest) x)) +) + +(decl is_null (Value) Inst) +(extractor + (is_null x) + (inst_data (InstructionData.Unary (Opcode.IsNull) x)) +) + +(decl is_invalid (Value) Inst) +(extractor + (is_invalid x) + (inst_data (InstructionData.Unary (Opcode.IsInvalid) x)) +) + +(decl trueif (IntCC Value) Inst) +(extractor + (trueif Cond f) + (inst_data (InstructionData.IntCond (Opcode.Trueif) Cond f)) +) + +(decl trueff (FloatCC Value) Inst) +(extractor + (trueff Cond f) + (inst_data (InstructionData.FloatCond (Opcode.Trueff) Cond f)) +) + +(decl bitcast (Value) Inst) +(extractor + (bitcast x) + (inst_data (InstructionData.Unary (Opcode.Bitcast) x)) +) + +(decl raw_bitcast (Value) Inst) +(extractor + (raw_bitcast x) + (inst_data (InstructionData.Unary (Opcode.RawBitcast) x)) +) + +(decl scalar_to_vector (Value) Inst) +(extractor + (scalar_to_vector s) + (inst_data (InstructionData.Unary (Opcode.ScalarToVector) s)) +) + +(decl breduce (Value) Inst) +(extractor + (breduce x) + (inst_data (InstructionData.Unary (Opcode.Breduce) x)) +) + +(decl bextend (Value) Inst) +(extractor + (bextend x) + (inst_data (InstructionData.Unary (Opcode.Bextend) x)) +) + +(decl bint (Value) Inst) +(extractor + (bint x) + (inst_data (InstructionData.Unary (Opcode.Bint) x)) +) + +(decl bmask (Value) Inst) +(extractor + (bmask x) + (inst_data (InstructionData.Unary (Opcode.Bmask) x)) +) + +(decl ireduce (Value) Inst) +(extractor + (ireduce x) + (inst_data (InstructionData.Unary (Opcode.Ireduce) x)) +) + +(decl snarrow (Value Value) Inst) +(extractor + (snarrow x y) + (inst_data (InstructionData.Binary (Opcode.Snarrow) (value_array_2 x y))) +) + +(decl unarrow (Value Value) Inst) +(extractor + (unarrow x y) + (inst_data (InstructionData.Binary (Opcode.Unarrow) (value_array_2 x y))) +) + +(decl uunarrow (Value Value) Inst) +(extractor + (uunarrow x y) + (inst_data (InstructionData.Binary (Opcode.Uunarrow) (value_array_2 x y))) +) + +(decl swiden_low (Value) Inst) +(extractor + (swiden_low x) + (inst_data (InstructionData.Unary (Opcode.SwidenLow) x)) +) + +(decl swiden_high (Value) Inst) +(extractor + (swiden_high x) + (inst_data (InstructionData.Unary (Opcode.SwidenHigh) x)) +) + +(decl uwiden_low (Value) Inst) +(extractor + (uwiden_low x) + (inst_data (InstructionData.Unary (Opcode.UwidenLow) x)) +) + +(decl uwiden_high (Value) Inst) +(extractor + (uwiden_high x) + (inst_data (InstructionData.Unary (Opcode.UwidenHigh) x)) +) + +(decl iadd_pairwise (Value Value) Inst) +(extractor + (iadd_pairwise x y) + (inst_data (InstructionData.Binary (Opcode.IaddPairwise) (value_array_2 x y))) +) + +(decl widening_pairwise_dot_product_s (Value Value) Inst) +(extractor + (widening_pairwise_dot_product_s x y) + (inst_data (InstructionData.Binary (Opcode.WideningPairwiseDotProductS) (value_array_2 x y))) +) + +(decl uextend (Value) Inst) +(extractor + (uextend x) + (inst_data (InstructionData.Unary (Opcode.Uextend) x)) +) + +(decl sextend (Value) Inst) +(extractor + (sextend x) + (inst_data (InstructionData.Unary (Opcode.Sextend) x)) +) + +(decl fpromote (Value) Inst) +(extractor + (fpromote x) + (inst_data (InstructionData.Unary (Opcode.Fpromote) x)) +) + +(decl fdemote (Value) Inst) +(extractor + (fdemote x) + (inst_data (InstructionData.Unary (Opcode.Fdemote) x)) +) + +(decl fvdemote (Value) Inst) +(extractor + (fvdemote x) + (inst_data (InstructionData.Unary (Opcode.Fvdemote) x)) +) + +(decl fvpromote_low (Value) Inst) +(extractor + (fvpromote_low a) + (inst_data (InstructionData.Unary (Opcode.FvpromoteLow) a)) +) + +(decl fcvt_to_uint (Value) Inst) +(extractor + (fcvt_to_uint x) + (inst_data (InstructionData.Unary (Opcode.FcvtToUint) x)) +) + +(decl fcvt_to_uint_sat (Value) Inst) +(extractor + (fcvt_to_uint_sat x) + (inst_data (InstructionData.Unary (Opcode.FcvtToUintSat) x)) +) + +(decl fcvt_to_sint (Value) Inst) +(extractor + (fcvt_to_sint x) + (inst_data (InstructionData.Unary (Opcode.FcvtToSint) x)) +) + +(decl fcvt_to_sint_sat (Value) Inst) +(extractor + (fcvt_to_sint_sat x) + (inst_data (InstructionData.Unary (Opcode.FcvtToSintSat) x)) +) + +(decl fcvt_from_uint (Value) Inst) +(extractor + (fcvt_from_uint x) + (inst_data (InstructionData.Unary (Opcode.FcvtFromUint) x)) +) + +(decl fcvt_from_sint (Value) Inst) +(extractor + (fcvt_from_sint x) + (inst_data (InstructionData.Unary (Opcode.FcvtFromSint) x)) +) + +(decl fcvt_low_from_sint (Value) Inst) +(extractor + (fcvt_low_from_sint x) + (inst_data (InstructionData.Unary (Opcode.FcvtLowFromSint) x)) +) + +(decl isplit (Value) Inst) +(extractor + (isplit x) + (inst_data (InstructionData.Unary (Opcode.Isplit) x)) +) + +(decl iconcat (Value Value) Inst) +(extractor + (iconcat lo hi) + (inst_data (InstructionData.Binary (Opcode.Iconcat) (value_array_2 lo hi))) +) + +(decl atomic_rmw (MemFlags AtomicRmwOp Value Value) Inst) +(extractor + (atomic_rmw MemFlags AtomicRmwOp p x) + (inst_data (InstructionData.AtomicRmw (Opcode.AtomicRmw) MemFlags AtomicRmwOp (value_array_2 p x))) +) + +(decl atomic_cas (MemFlags Value Value Value) Inst) +(extractor + (atomic_cas MemFlags p e x) + (inst_data (InstructionData.AtomicCas (Opcode.AtomicCas) MemFlags (value_array_3 p e x))) +) + +(decl atomic_load (MemFlags Value) Inst) +(extractor + (atomic_load MemFlags p) + (inst_data (InstructionData.LoadNoOffset (Opcode.AtomicLoad) MemFlags p)) +) + +(decl atomic_store (MemFlags Value Value) Inst) +(extractor + (atomic_store MemFlags x p) + (inst_data (InstructionData.StoreNoOffset (Opcode.AtomicStore) MemFlags (value_array_2 x p))) +) + +(decl fence () Inst) +(extractor + (fence ) + (inst_data (InstructionData.NullAry (Opcode.Fence))) +) + diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle new file mode 100644 index 0000000000..e89ccc3bcc --- /dev/null +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -0,0 +1,933 @@ +;; Extern type definitions and constructors for the x64 `MachInst` type. + +;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type MInst extern + (enum (Nop (len u8)) + (AluRmiR (size OperandSize) + (op AluRmiROpcode) + (src1 Reg) + (src2 RegMemImm) + (dst WritableReg)) + (XmmRmR (op SseOpcode) + (src1 Reg) + (src2 RegMem) + (dst WritableReg)) + (XmmUnaryRmR (op SseOpcode) + (src RegMem) + (dst WritableReg)) + (XmmRmiReg (opcode SseOpcode) + (src1 Reg) + (src2 RegMemImm) + (dst WritableReg)) + (XmmRmRImm (op SseOpcode) + (src1 Reg) + (src2 RegMem) + (dst WritableReg) + (imm u8) + (size OperandSize)) + (CmpRmiR (size OperandSize) + (opcode CmpOpcode) + (src RegMemImm) + (dst Reg)) + (Imm (dst_size OperandSize) + (simm64 u64) + (dst WritableReg)) + (ShiftR (size OperandSize) + (kind ShiftKind) + (src Reg) + (num_bits Imm8Reg) + (dst WritableReg)) + (MovzxRmR (ext_mode ExtMode) + (src RegMem) + (dst WritableReg)) + (MovsxRmR (ext_mode ExtMode) + (src RegMem) + (dst WritableReg)) + (Cmove (size OperandSize) + (cc CC) + (consequent RegMem) + (alternative Reg) + (dst WritableReg)) + (XmmRmREvex (op Avx512Opcode) + (src1 RegMem) + (src2 Reg) + (dst WritableReg)))) + +(type OperandSize extern + (enum Size8 + Size16 + Size32 + Size64)) + +;; Get the `OperandSize` for a given `Type`. +(decl operand_size_of_type (Type) OperandSize) +(extern constructor operand_size_of_type operand_size_of_type) + +;; Get the bit width of an `OperandSize`. +(decl operand_size_bits (OperandSize) u16) +(rule (operand_size_bits (OperandSize.Size8)) 8) +(rule (operand_size_bits (OperandSize.Size16)) 16) +(rule (operand_size_bits (OperandSize.Size32)) 32) +(rule (operand_size_bits (OperandSize.Size64)) 64) + +(type AluRmiROpcode extern + (enum Add + Adc + Sub + Sbb + And + Or + Xor + Mul + And8 + Or8)) + +(type SseOpcode extern + (enum Addps + Addpd + Addss + Addsd + Andps + Andpd + Andnps + Andnpd + Blendvpd + Blendvps + Comiss + Comisd + Cmpps + Cmppd + Cmpss + Cmpsd + Cvtdq2ps + Cvtdq2pd + Cvtpd2ps + Cvtps2pd + Cvtsd2ss + Cvtsd2si + Cvtsi2ss + Cvtsi2sd + Cvtss2si + Cvtss2sd + Cvttpd2dq + Cvttps2dq + Cvttss2si + Cvttsd2si + Divps + Divpd + Divss + Divsd + Insertps + Maxps + Maxpd + Maxss + Maxsd + Minps + Minpd + Minss + Minsd + Movaps + Movapd + Movd + Movdqa + Movdqu + Movlhps + Movmskps + Movmskpd + Movq + Movss + Movsd + Movups + Movupd + Mulps + Mulpd + Mulss + Mulsd + Orps + Orpd + Pabsb + Pabsw + Pabsd + Packssdw + Packsswb + Packusdw + Packuswb + Paddb + Paddd + Paddq + Paddw + Paddsb + Paddsw + Paddusb + Paddusw + Palignr + Pand + Pandn + Pavgb + Pavgw + Pblendvb + Pcmpeqb + Pcmpeqw + Pcmpeqd + Pcmpeqq + Pcmpgtb + Pcmpgtw + Pcmpgtd + Pcmpgtq + Pextrb + Pextrw + Pextrd + Pinsrb + Pinsrw + Pinsrd + Pmaddubsw + Pmaddwd + Pmaxsb + Pmaxsw + Pmaxsd + Pmaxub + Pmaxuw + Pmaxud + Pminsb + Pminsw + Pminsd + Pminub + Pminuw + Pminud + Pmovmskb + Pmovsxbd + Pmovsxbw + Pmovsxbq + Pmovsxwd + Pmovsxwq + Pmovsxdq + Pmovzxbd + Pmovzxbw + Pmovzxbq + Pmovzxwd + Pmovzxwq + Pmovzxdq + Pmuldq + Pmulhw + Pmulhuw + Pmulhrsw + Pmulld + Pmullw + Pmuludq + Por + Pshufb + Pshufd + Psllw + Pslld + Psllq + Psraw + Psrad + Psrlw + Psrld + Psrlq + Psubb + Psubd + Psubq + Psubw + Psubsb + Psubsw + Psubusb + Psubusw + Ptest + Punpckhbw + Punpckhwd + Punpcklbw + Punpcklwd + Pxor + Rcpss + Roundps + Roundpd + Roundss + Roundsd + Rsqrtss + Shufps + Sqrtps + Sqrtpd + Sqrtss + Sqrtsd + Subps + Subpd + Subss + Subsd + Ucomiss + Ucomisd + Unpcklps + Xorps + Xorpd)) + +(type CmpOpcode extern + (enum Cmp + Test)) + +(type RegMemImm extern + (enum + (Reg (reg Reg)) + (Mem (addr SyntheticAmode)) + (Imm (simm32 u32)))) + +(type RegMem extern + (enum + (Reg (reg Reg)) + (Mem (addr SyntheticAmode)))) + +;; Put the given clif value into a `RegMem` operand. +;; +;; Asserts that the value fits into a single register, and doesn't require +;; multiple registers for its representation (like `i128` for example). +;; +;; As a side effect, this marks the value as used. +(decl put_in_reg_mem (Value) RegMem) +(extern constructor put_in_reg_mem put_in_reg_mem) + +(type SyntheticAmode extern (enum)) + +(type ShiftKind extern + (enum ShiftLeft + ShiftRightLogical + ShiftRightArithmetic + RotateLeft + RotateRight)) + +(type Imm8Reg extern + (enum (Imm8 (imm u8)) + (Reg (reg Reg)))) + +(type CC extern + (enum O + NO + B + NB + Z + NZ + BE + NBE + S + NS + L + NL + LE + NLE + P + NP)) + +(type Avx512Opcode extern + (enum Vcvtudq2ps + Vpabsq + Vpermi2b + Vpmullq + Vpopcntb)) + +;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl avx512vl_enabled () Type) +(extern extractor avx512vl_enabled avx512vl_enabled) + +(decl avx512dq_enabled () Type) +(extern extractor avx512dq_enabled avx512dq_enabled) + +;;;; Helpers for Merging and Sinking Immediates/Loads ;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Extract a constant `Imm8Reg.Imm8` from a value operand. +(decl imm8_from_value (Imm8Reg) Value) +(extern extractor imm8_from_value imm8_from_value) + +;; Extract a constant `RegMemImm.Imm` from a value operand. +(decl simm32_from_value (RegMemImm) Value) +(extern extractor simm32_from_value simm32_from_value) + +;; Extract a constant `RegMemImm.Imm` from an `Imm64` immediate. +(decl simm32_from_imm64 (RegMemImm) Imm64) +(extern extractor simm32_from_imm64 simm32_from_imm64) + +;; A load that can be sunk into another operation. +(type SinkableLoad extern (enum)) + +;; Extract a `SinkableLoad` that works with `RegMemImm.Mem` from a value +;; operand. +(decl sinkable_load (SinkableLoad) Value) +(extern extractor sinkable_load sinkable_load) + +;; Sink a `SinkableLoad` into a `RegMemImm.Mem`. +;; +;; This is a side-effectful operation that notifies the context that the +;; instruction that produced the `SinkableImm` has been sunk into another +;; instruction, and no longer needs to be lowered. +(decl sink_load (SinkableLoad) RegMemImm) +(extern constructor sink_load sink_load) + +;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Newtype wrapper around `MInst` for instructions that are used for their +;; effect on flags. +(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg)))) + +;; Newtype wrapper around `MInst` for instructions that consume flags. +(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg)))) + +;; Combine flags-producing and -consuming instructions together, ensuring that +;; they are emitted back-to-back and no other instructions can be emitted +;; between them and potentially clobber the flags. +;; +;; Returns a `ValueRegs` where the first register is the result of the +;; `ProducesFlags` instruction and the second is the result of the +;; `ConsumesFlags` instruction. +(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs) +(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst))) + (value_regs producer_result consumer_result))) + +;; Like `with_flags` but returns only the result of the consumer operation. +(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg) +(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst consumer_result)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst))) + consumer_result)) + +;; Like `with_flags` but allows two consumers of the same flags. The result is a +;; `ValueRegs` containing the first consumer's result and then the second +;; consumer's result. +(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs) +(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result) + (ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1) + (ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2)) + (let ((_x Unit (emit producer_inst)) + (_y Unit (emit consumer_inst_1)) + (_z Unit (emit consumer_inst_2))) + (value_regs consumer_result_1 consumer_result_2))) + +;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type ExtendKind (enum Sign Zero)) + +(type ExtMode extern (enum BL BQ WL WQ LQ)) + +;; `ExtMode::new` +(decl ext_mode (u16 u16) ExtMode) +(extern constructor ext_mode ext_mode) + +;; Put the given value into a register, but extended as the given type. +(decl extend_to_reg (Value Type ExtendKind) Reg) + +;; If the value is already of the requested type, no extending is necessary. +(rule (extend_to_reg (and val (value_type ty)) =ty _kind) + (put_in_reg val)) + +(rule (extend_to_reg (and val (value_type from_ty)) + to_ty + kind) + (let ((from_bits u16 (ty_bits from_ty)) + ;; Use `operand_size_of_type` so that the we clamp the output to 32- + ;; or 64-bit width types. + (to_bits u16 (operand_size_bits (operand_size_of_type to_ty)))) + (extend kind + to_ty + (ext_mode from_bits to_bits) + (put_in_reg_mem val)))) + +;; Do a sign or zero extension of the given `RegMem`. +(decl extend (ExtendKind Type ExtMode RegMem) Reg) + +;; Zero extending uses `movzx`. +(rule (extend (ExtendKind.Zero) ty mode src) + (movzx ty mode src)) + +;; Sign extending uses `movsx`. +(rule (extend (ExtendKind.Sign) ty mode src) + (movsx ty mode src)) + +;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; These constructors create SSA-style `MInst`s. It is their responsibility to +;; maintain the invariant that each temporary register they allocate and define +;; only gets defined the once. + +;; Emit an instruction. +;; +;; This is low-level and side-effectful; it should only be used as an +;; implementation detail by helpers that preserve the SSA facade themselves. +(decl emit (MInst) Unit) +(extern constructor emit emit) + +;; Helper for emitting `MInst.AluRmiR` instructions. +(decl alu_rmi_r (Type AluRmiROpcode Reg RegMemImm) Reg) +(rule (alu_rmi_r ty opcode src1 src2) + (let ((dst WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type ty)) + (_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst)))) + (writable_reg_to_reg dst))) + +;; Helper for emitting `add` instructions. +(decl add (Type Reg RegMemImm) Reg) +(rule (add ty src1 src2) + (alu_rmi_r ty + (AluRmiROpcode.Add) + src1 + src2)) + +;; Helper for creating `add` instructions whose flags are also used. +(decl add_with_flags (Type Reg RegMemImm) ProducesFlags) +(rule (add_with_flags ty src1 src2) + (let ((dst WritableReg (temp_writable_reg ty))) + (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type ty) + (AluRmiROpcode.Add) + src1 + src2 + dst) + (writable_reg_to_reg dst)))) + +;; Helper for creating `adc` instructions. +(decl adc (Type Reg RegMemImm) ConsumesFlags) +(rule (adc ty src1 src2) + (let ((dst WritableReg (temp_writable_reg ty))) + (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type ty) + (AluRmiROpcode.Adc) + src1 + src2 + dst) + (writable_reg_to_reg dst)))) + +;; Helper for emitting `sub` instructions. +(decl sub (Type Reg RegMemImm) Reg) +(rule (sub ty src1 src2) + (alu_rmi_r ty + (AluRmiROpcode.Sub) + src1 + src2)) + +;; Helper for creating `sub` instructions whose flags are also used. +(decl sub_with_flags (Type Reg RegMemImm) ProducesFlags) +(rule (sub_with_flags ty src1 src2) + (let ((dst WritableReg (temp_writable_reg ty))) + (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type ty) + (AluRmiROpcode.Sub) + src1 + src2 + dst) + (writable_reg_to_reg dst)))) + +;; Helper for creating `sbb` instructions. +(decl sbb (Type Reg RegMemImm) ConsumesFlags) +(rule (sbb ty src1 src2) + (let ((dst WritableReg (temp_writable_reg ty))) + (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type ty) + (AluRmiROpcode.Sbb) + src1 + src2 + dst) + (writable_reg_to_reg dst)))) + +;; Helper for creating `mul` instructions. +(decl mul (Type Reg RegMemImm) Reg) +(rule (mul ty src1 src2) + (alu_rmi_r ty + (AluRmiROpcode.Mul) + src1 + src2)) + +;; Helper for emitting `and` instructions. +;; +;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin +;; `and` operator. +(decl m_and (Type Reg RegMemImm) Reg) +(rule (m_and ty src1 src2) + (alu_rmi_r ty + (AluRmiROpcode.And) + src1 + src2)) + +;; Helper for emitting `or` instructions. +(decl or (Type Reg RegMemImm) Reg) +(rule (or ty src1 src2) + (alu_rmi_r ty + (AluRmiROpcode.Or) + src1 + src2)) + +;; Helper for emitting `xor` instructions. +(decl xor (Type Reg RegMemImm) Reg) +(rule (xor ty src1 src2) + (alu_rmi_r ty + (AluRmiROpcode.Xor) + src1 + src2)) + +;; Helper for emitting immediates. +(decl imm (Type u64) Reg) +(rule (imm ty simm64) + (let ((dst WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type ty)) + (_ Unit (emit (MInst.Imm size simm64 dst)))) + (writable_reg_to_reg dst))) + +(decl nonzero_u64_fits_in_u32 (u64) u64) +(extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32) + +;; Special case for when a 64-bit immediate fits into 32-bits. We can use a +;; 32-bit move that zero-extends the value, which has a smaller encoding. +(rule (imm $I64 (nonzero_u64_fits_in_u32 x)) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.Imm (OperandSize.Size32) x dst)))) + (writable_reg_to_reg dst))) + +;; Special case for zero immediates: turn them into an `xor r, r`. +(rule (imm ty 0) + (let ((wr WritableReg (temp_writable_reg ty)) + (r Reg (writable_reg_to_reg wr)) + (size OperandSize (operand_size_of_type ty)) + (_ Unit (emit (MInst.AluRmiR size + (AluRmiROpcode.Xor) + r + (RegMemImm.Reg r) + wr)))) + r)) + +;; Helper for creating `MInst.ShifR` instructions. +(decl shift_r (Type ShiftKind Reg Imm8Reg) Reg) +(rule (shift_r ty kind src1 src2) + (let ((dst WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type ty)) + (_ Unit (emit (MInst.ShiftR size kind src1 src2 dst)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach +;; inst", to disambiguate this from clif's `rotl`). +(decl m_rotl (Type Reg Imm8Reg) Reg) +(rule (m_rotl ty src1 src2) + (shift_r ty (ShiftKind.RotateLeft) src1 src2)) + +;; Helper for creating `shl` instructions. +(decl shl (Type Reg Imm8Reg) Reg) +(rule (shl ty src1 src2) + (shift_r ty (ShiftKind.ShiftLeft) src1 src2)) + +;; Helper for creating logical shift-right instructions. +(decl shr (Type Reg Imm8Reg) Reg) +(rule (shr ty src1 src2) + (shift_r ty (ShiftKind.ShiftRightLogical) src1 src2)) + +;; Helper for creating arithmetic shift-right instructions. +(decl sar (Type Reg Imm8Reg) Reg) +(rule (sar ty src1 src2) + (shift_r ty (ShiftKind.ShiftRightArithmetic) src1 src2)) + +;; Helper for creating `MInst.CmpRmiR` instructions. +(decl cmp_rmi_r (OperandSize CmpOpcode RegMemImm Reg) ProducesFlags) +(rule (cmp_rmi_r size opcode src1 src2) + (ProducesFlags.ProducesFlags (MInst.CmpRmiR size + opcode + src1 + src2) + (invalid_reg))) + +;; Helper for creating `cmp` instructions. +(decl cmp (OperandSize RegMemImm Reg) ProducesFlags) +(rule (cmp size src1 src2) + (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2)) + +;; Helper for creating `test` instructions. +(decl test (OperandSize RegMemImm Reg) ProducesFlags) +(rule (test size src1 src2) + (cmp_rmi_r size (CmpOpcode.Test) src1 src2)) + +;; Helper for creating `MInst.Cmove` instructions. +(decl cmove (Type CC RegMem Reg) ConsumesFlags) +(rule (cmove ty cc consequent alternative) + (let ((dst WritableReg (temp_writable_reg ty)) + (size OperandSize (operand_size_of_type ty))) + (ConsumesFlags.ConsumesFlags (MInst.Cmove size cc consequent alternative dst) + (writable_reg_to_reg dst)))) + +;; Helper for creating `MInst.MovzxRmR` instructions. +(decl movzx (Type ExtMode RegMem) Reg) +(rule (movzx ty mode src) + (let ((dst WritableReg (temp_writable_reg ty)) + (_ Unit (emit (MInst.MovzxRmR mode src dst)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `MInst.MovsxRmR` instructions. +(decl movsx (Type ExtMode RegMem) Reg) +(rule (movsx ty mode src) + (let ((dst WritableReg (temp_writable_reg ty)) + (_ Unit (emit (MInst.MovsxRmR mode src dst)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `MInst.XmmRmR` instructions. +(decl xmm_rm_r (Type SseOpcode Reg RegMem) Reg) +(rule (xmm_rm_r ty op src1 src2) + (let ((dst WritableReg (temp_writable_reg ty)) + (_ Unit (emit (MInst.XmmRmR op src1 src2 dst)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `paddb` instructions. +(decl paddb (Reg RegMem) Reg) +(rule (paddb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2)) + +;; Helper for creating `paddw` instructions. +(decl paddw (Reg RegMem) Reg) +(rule (paddw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2)) + +;; Helper for creating `paddd` instructions. +(decl paddd (Reg RegMem) Reg) +(rule (paddd src1 src2) + (xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2)) + +;; Helper for creating `paddq` instructions. +(decl paddq (Reg RegMem) Reg) +(rule (paddq src1 src2) + (xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2)) + +;; Helper for creating `paddsb` instructions. +(decl paddsb (Reg RegMem) Reg) +(rule (paddsb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2)) + +;; Helper for creating `paddsw` instructions. +(decl paddsw (Reg RegMem) Reg) +(rule (paddsw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2)) + +;; Helper for creating `paddusb` instructions. +(decl paddusb (Reg RegMem) Reg) +(rule (paddusb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2)) + +;; Helper for creating `paddusw` instructions. +(decl paddusw (Reg RegMem) Reg) +(rule (paddusw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2)) + +;; Helper for creating `psubb` instructions. +(decl psubb (Reg RegMem) Reg) +(rule (psubb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2)) + +;; Helper for creating `psubw` instructions. +(decl psubw (Reg RegMem) Reg) +(rule (psubw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2)) + +;; Helper for creating `psubd` instructions. +(decl psubd (Reg RegMem) Reg) +(rule (psubd src1 src2) + (xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2)) + +;; Helper for creating `psubq` instructions. +(decl psubq (Reg RegMem) Reg) +(rule (psubq src1 src2) + (xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2)) + +;; Helper for creating `psubsb` instructions. +(decl psubsb (Reg RegMem) Reg) +(rule (psubsb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2)) + +;; Helper for creating `psubsw` instructions. +(decl psubsw (Reg RegMem) Reg) +(rule (psubsw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2)) + +;; Helper for creating `psubusb` instructions. +(decl psubusb (Reg RegMem) Reg) +(rule (psubusb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2)) + +;; Helper for creating `psubusw` instructions. +(decl psubusw (Reg RegMem) Reg) +(rule (psubusw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2)) + +;; Helper for creating `pavgb` instructions. +(decl pavgb (Reg RegMem) Reg) +(rule (pavgb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2)) + +;; Helper for creating `pavgw` instructions. +(decl pavgw (Reg RegMem) Reg) +(rule (pavgw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2)) + +;; Helper for creating `pand` instructions. +(decl pand (Reg RegMem) Reg) +(rule (pand src1 src2) + (xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2)) + +;; Helper for creating `andps` instructions. +(decl andps (Reg RegMem) Reg) +(rule (andps src1 src2) + (xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2)) + +;; Helper for creating `andpd` instructions. +(decl andpd (Reg RegMem) Reg) +(rule (andpd src1 src2) + (xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2)) + +;; Helper for creating `por` instructions. +(decl por (Reg RegMem) Reg) +(rule (por src1 src2) + (xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2)) + +;; Helper for creating `orps` instructions. +(decl orps (Reg RegMem) Reg) +(rule (orps src1 src2) + (xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2)) + +;; Helper for creating `orpd` instructions. +(decl orpd (Reg RegMem) Reg) +(rule (orpd src1 src2) + (xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2)) + +;; Helper for creating `pxor` instructions. +(decl pxor (Reg RegMem) Reg) +(rule (pxor src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2)) + +;; Helper for creating `xorps` instructions. +(decl xorps (Reg RegMem) Reg) +(rule (xorps src1 src2) + (xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2)) + +;; Helper for creating `xorpd` instructions. +(decl xorpd (Reg RegMem) Reg) +(rule (xorpd src1 src2) + (xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2)) + +;; Helper for creating `pmullw` instructions. +(decl pmullw (Reg RegMem) Reg) +(rule (pmullw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2)) + +;; Helper for creating `pmulld` instructions. +(decl pmulld (Reg RegMem) Reg) +(rule (pmulld src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2)) + +;; Helper for creating `pmulhw` instructions. +(decl pmulhw (Reg RegMem) Reg) +(rule (pmulhw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2)) + +;; Helper for creating `pmulhuw` instructions. +(decl pmulhuw (Reg RegMem) Reg) +(rule (pmulhuw src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2)) + +;; Helper for creating `pmuldq` instructions. +(decl pmuldq (Reg RegMem) Reg) +(rule (pmuldq src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2)) + +;; Helper for creating `pmuludq` instructions. +(decl pmuludq (Reg RegMem) Reg) +(rule (pmuludq src1 src2) + (xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2)) + +;; Helper for creating `punpckhwd` instructions. +(decl punpckhwd (Reg RegMem) Reg) +(rule (punpckhwd src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2)) + +;; Helper for creating `punpcklwd` instructions. +(decl punpcklwd (Reg RegMem) Reg) +(rule (punpcklwd src1 src2) + (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2)) + +;; Helper for creating `MInst.XmmRmRImm` instructions. +(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg) +(rule (xmm_rm_r_imm op src1 src2 imm size) + (let ((dst WritableReg (temp_writable_reg $I8X16)) + (_ Unit (emit (MInst.XmmRmRImm op + src1 + src2 + dst + imm + size)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `palignr` instructions. +(decl palignr (Reg RegMem u8 OperandSize) Reg) +(rule (palignr src1 src2 imm size) + (xmm_rm_r_imm (SseOpcode.Palignr) + src1 + src2 + imm + size)) + +;; Helper for creating `pshufd` instructions. +(decl pshufd (RegMem u8 OperandSize) Reg) +(rule (pshufd src imm size) + (let ((w_dst WritableReg (temp_writable_reg $I8X16)) + (dst Reg (writable_reg_to_reg w_dst)) + (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd) + dst + src + w_dst + imm + size)))) + dst)) + +;; Helper for creating `MInst.XmmUnaryRmR` instructions. +(decl xmm_unary_rm_r (SseOpcode RegMem) Reg) +(rule (xmm_unary_rm_r op src) + (let ((dst WritableReg (temp_writable_reg $I8X16)) + (_ Unit (emit (MInst.XmmUnaryRmR op src dst)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `pmovsxbw` instructions. +(decl pmovsxbw (RegMem) Reg) +(rule (pmovsxbw src) + (xmm_unary_rm_r (SseOpcode.Pmovsxbw) src)) + +;; Helper for creating `pmovzxbw` instructions. +(decl pmovzxbw (RegMem) Reg) +(rule (pmovzxbw src) + (xmm_unary_rm_r (SseOpcode.Pmovzxbw) src)) + +;; Helper for creating `MInst.XmmRmREvex` instructions. +(decl xmm_rm_r_evex (Avx512Opcode RegMem Reg) Reg) +(rule (xmm_rm_r_evex op src1 src2) + (let ((dst WritableReg (temp_writable_reg $I8X16)) + (_ Unit (emit (MInst.XmmRmREvex op + src1 + src2 + dst)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `vpmullq` instructions. +;; +;; Requires AVX-512 vl and dq. +(decl vpmullq (RegMem Reg) Reg) +(rule (vpmullq src1 src2) + (xmm_rm_r_evex (Avx512Opcode.Vpmullq) + src1 + src2)) + +;; Helper for creating `MInst.XmmRmiReg` instructions. +(decl xmm_rmi_reg (SseOpcode Reg RegMemImm) Reg) +(rule (xmm_rmi_reg op src1 src2) + (let ((dst WritableReg (temp_writable_reg $I8X16)) + (_ Unit (emit (MInst.XmmRmiReg op + src1 + src2 + dst)))) + (writable_reg_to_reg dst))) + +;; Helper for creating `psllq` instructions. +(decl psllq (Reg RegMemImm) Reg) +(rule (psllq src1 src2) + (xmm_rmi_reg (SseOpcode.Psllq) src1 src2)) + +;; Helper for creating `psrlq` instructions. +(decl psrlq (Reg RegMemImm) Reg) +(rule (psrlq src1 src2) + (xmm_rmi_reg (SseOpcode.Psrlq) src1 src2)) diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index f279ee9096..e7aaf108fe 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -1,14 +1,13 @@ //! Instruction operand sub-components (aka "parts"): definitions and printing. use super::regs::{self, show_ireg_sized}; -use super::EmitState; +use super::{EmitState, RegMapper}; use crate::ir::condcodes::{FloatCC, IntCC}; use crate::ir::{MemFlags, Type}; use crate::isa::x64::inst::Inst; use crate::machinst::*; use regalloc::{ - PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, - RegUsageMapper, Writable, + PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, Writable, }; use smallvec::{smallvec, SmallVec}; use std::fmt; @@ -175,7 +174,7 @@ impl SyntheticAmode { } } - pub(crate) fn map_uses(&mut self, map: &RUM) { + pub(crate) fn map_uses(&mut self, map: &RM) { match self { SyntheticAmode::Real(addr) => addr.map_uses(map), SyntheticAmode::NominalSPOffset { .. } => { @@ -285,6 +284,25 @@ impl PrettyPrintSized for RegMemImm { } } +/// An operand which is either an 8-bit integer immediate or a register. +#[derive(Clone, Debug)] +pub enum Imm8Reg { + Imm8 { imm: u8 }, + Reg { reg: Reg }, +} + +impl From for Imm8Reg { + fn from(imm: u8) -> Self { + Self::Imm8 { imm } + } +} + +impl From for Imm8Reg { + fn from(reg: Reg) -> Self { + Self::Reg { reg } + } +} + /// An operand which is either an integer Register or a value in Memory. This can denote an 8, 16, /// 32, 64, or 128 bit value. #[derive(Clone, Debug)] diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 93132a2aab..53f67d7346 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -147,14 +147,16 @@ pub(crate) fn emit( Inst::AluRmiR { size, op, - src, + src1, + src2, dst: reg_g, } => { + debug_assert_eq!(*src1, reg_g.to_reg()); let mut rex = RexFlags::from(*size); if *op == AluRmiROpcode::Mul { // We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so // we have to special-case it. - match src { + match src2 { RegMemImm::Reg { reg: reg_e } => { emit_std_reg_reg( sink, @@ -213,7 +215,7 @@ pub(crate) fn emit( }; assert!(!(is_8bit && *size == OperandSize::Size64)); - match src { + match src2 { RegMemImm::Reg { reg: reg_e } => { if is_8bit { rex.always_emit_if_8bit_needed(*reg_e); @@ -323,8 +325,9 @@ pub(crate) fn emit( } } - Inst::Not { size, src } => { - let rex_flags = RexFlags::from((*size, src.to_reg())); + Inst::Not { size, src, dst } => { + debug_assert_eq!(*src, dst.to_reg()); + let rex_flags = RexFlags::from((*size, dst.to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -333,12 +336,13 @@ pub(crate) fn emit( }; let subopcode = 2; - let enc_src = int_reg_enc(src.to_reg()); + let enc_src = int_reg_enc(dst.to_reg()); emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags) } - Inst::Neg { size, src } => { - let rex_flags = RexFlags::from((*size, src.to_reg())); + Inst::Neg { size, src, dst } => { + debug_assert_eq!(*src, dst.to_reg()); + let rex_flags = RexFlags::from((*size, dst.to_reg())); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -347,15 +351,21 @@ pub(crate) fn emit( }; let subopcode = 3; - let enc_src = int_reg_enc(src.to_reg()); + let enc_src = int_reg_enc(dst.to_reg()); emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags) } Inst::Div { size, signed, + dividend, divisor, + dst_quotient, + dst_remainder, } => { + debug_assert_eq!(*dividend, regs::rax()); + debug_assert_eq!(dst_quotient.to_reg(), regs::rax()); + debug_assert_eq!(dst_remainder.to_reg(), regs::rdx()); let (opcode, prefix) = match size { OperandSize::Size8 => (0xF6, LegacyPrefixes::None), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66), @@ -397,7 +407,18 @@ pub(crate) fn emit( } } - Inst::MulHi { size, signed, rhs } => { + Inst::MulHi { + size, + signed, + src1, + src2, + dst_lo, + dst_hi, + } => { + debug_assert_eq!(*src1, regs::rax()); + debug_assert_eq!(dst_lo.to_reg(), regs::rax()); + debug_assert_eq!(dst_hi.to_reg(), regs::rdx()); + let rex_flags = RexFlags::from(*size); let prefix = match size { OperandSize::Size16 => LegacyPrefixes::_66, @@ -407,7 +428,7 @@ pub(crate) fn emit( }; let subopcode = if *signed { 5 } else { 4 }; - match rhs { + match src2 { RegMem::Reg { reg } => { let src = int_reg_enc(*reg); emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags) @@ -421,28 +442,39 @@ pub(crate) fn emit( } } - Inst::SignExtendData { size } => match size { - OperandSize::Size8 => { - sink.put1(0x66); - sink.put1(0x98); + Inst::SignExtendData { size, src, dst } => { + debug_assert_eq!(*src, regs::rax()); + debug_assert_eq!(dst.to_reg(), regs::rdx()); + match size { + OperandSize::Size8 => { + sink.put1(0x66); + sink.put1(0x98); + } + OperandSize::Size16 => { + sink.put1(0x66); + sink.put1(0x99); + } + OperandSize::Size32 => sink.put1(0x99), + OperandSize::Size64 => { + sink.put1(0x48); + sink.put1(0x99); + } } - OperandSize::Size16 => { - sink.put1(0x66); - sink.put1(0x99); - } - OperandSize::Size32 => sink.put1(0x99), - OperandSize::Size64 => { - sink.put1(0x48); - sink.put1(0x99); - } - }, + } Inst::CheckedDivOrRemSeq { kind, size, + dividend, divisor, tmp, + dst_quotient, + dst_remainder, } => { + debug_assert_eq!(*dividend, regs::rax()); + debug_assert_eq!(dst_quotient.to_reg(), regs::rax()); + debug_assert_eq!(dst_remainder.to_reg(), regs::rdx()); + // Generates the following code sequence: // // ;; check divide by zero: @@ -792,9 +824,11 @@ pub(crate) fn emit( Inst::ShiftR { size, kind, + src, num_bits, dst, } => { + debug_assert_eq!(*src, dst.to_reg()); let subopcode = match kind { ShiftKind::RotateLeft => 0, ShiftKind::RotateRight => 1, @@ -805,7 +839,8 @@ pub(crate) fn emit( let enc_dst = int_reg_enc(dst.to_reg()); let rex_flags = RexFlags::from((*size, dst.to_reg())); match num_bits { - None => { + Imm8Reg::Reg { reg } => { + debug_assert_eq!(*reg, regs::rcx()); let (opcode, prefix) = match size { OperandSize::Size8 => (0xD2, LegacyPrefixes::None), OperandSize::Size16 => (0xD3, LegacyPrefixes::_66), @@ -820,7 +855,7 @@ pub(crate) fn emit( emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags); } - Some(num_bits) => { + Imm8Reg::Imm8 { imm: num_bits } => { let (opcode, prefix) = match size { OperandSize::Size8 => (0xC0, LegacyPrefixes::None), OperandSize::Size16 => (0xC1, LegacyPrefixes::_66), @@ -840,10 +875,16 @@ pub(crate) fn emit( } } - Inst::XmmRmiReg { opcode, src, dst } => { + Inst::XmmRmiReg { + opcode, + src1, + src2, + dst, + } => { + debug_assert_eq!(*src1, dst.to_reg()); let rex = RexFlags::clear_w(); let prefix = LegacyPrefixes::_66; - if let RegMemImm::Imm { simm32 } = src { + if let RegMemImm::Imm { simm32 } = src2 { let (opcode_bytes, reg_digit) = match opcode { SseOpcode::Psllw => (0x0F71, 6), SseOpcode::Pslld => (0x0F72, 6), @@ -874,7 +915,7 @@ pub(crate) fn emit( _ => panic!("invalid opcode: {}", opcode), }; - match src { + match src2 { RegMemImm::Reg { reg } => { emit_std_reg_reg(sink, prefix, opcode_bytes, 2, dst.to_reg(), *reg, rex); } @@ -993,9 +1034,11 @@ pub(crate) fn emit( Inst::Cmove { size, cc, - src, + consequent, + alternative, dst: reg_g, } => { + debug_assert_eq!(*alternative, reg_g.to_reg()); let rex_flags = RexFlags::from(*size); let prefix = match size { OperandSize::Size16 => LegacyPrefixes::_66, @@ -1004,7 +1047,7 @@ pub(crate) fn emit( _ => unreachable!("invalid size spec for cmove"), }; let opcode = 0x0F40 + cc.get_enc() as u32; - match src { + match consequent { RegMem::Reg { reg: reg_e } => { emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex_flags); } @@ -1433,9 +1476,11 @@ pub(crate) fn emit( Inst::XmmRmR { op, - src: src_e, + src1, + src2: src_e, dst: reg_g, } => { + debug_assert_eq!(*src1, reg_g.to_reg()); let rex = RexFlags::clear_w(); let (prefix, opcode, length) = match op { SseOpcode::Addps => (LegacyPrefixes::None, 0x0F58, 2), @@ -1678,11 +1723,13 @@ pub(crate) fn emit( Inst::XmmRmRImm { op, - src, + src1, + src2, dst, imm, size, } => { + debug_assert_eq!(*src1, dst.to_reg()); let (prefix, opcode, len) = match op { SseOpcode::Cmpps => (LegacyPrefixes::None, 0x0FC2, 2), SseOpcode::Cmppd => (LegacyPrefixes::_66, 0x0FC2, 2), @@ -1713,7 +1760,7 @@ pub(crate) fn emit( // `src` in ModRM's r/m field. _ => false, }; - match src { + match src2 { RegMem::Reg { reg } => { if regs_swapped { emit_std_reg_reg(sink, prefix, opcode, len, *reg, dst.to_reg(), rex); @@ -2403,8 +2450,17 @@ pub(crate) fn emit( } } - Inst::LockCmpxchg { ty, src, dst } => { - // lock cmpxchg{b,w,l,q} %src, (dst) + Inst::LockCmpxchg { + ty, + replacement, + expected, + mem, + dst_old, + } => { + debug_assert_eq!(*expected, regs::rax()); + debug_assert_eq!(dst_old.to_reg(), regs::rax()); + + // lock cmpxchg{b,w,l,q} %replacement, (mem) // Note that 0xF0 is the Lock prefix. let (prefix, opcodes) = match *ty { types::I8 => (LegacyPrefixes::_F0, 0x0FB0), @@ -2413,12 +2469,34 @@ pub(crate) fn emit( types::I64 => (LegacyPrefixes::_F0, 0x0FB1), _ => unreachable!(), }; - let rex = RexFlags::from((OperandSize::from_ty(*ty), *src)); - let amode = dst.finalize(state, sink); - emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex); + let rex = RexFlags::from((OperandSize::from_ty(*ty), *replacement)); + let amode = mem.finalize(state, sink); + emit_std_reg_mem( + sink, + state, + info, + prefix, + opcodes, + 2, + *replacement, + &amode, + rex, + ); } - Inst::AtomicRmwSeq { ty, op } => { + Inst::AtomicRmwSeq { + ty, + op, + address, + operand, + temp, + dst_old, + } => { + debug_assert_eq!(*address, regs::r9()); + debug_assert_eq!(*operand, regs::r10()); + debug_assert_eq!(temp.to_reg(), regs::r11()); + debug_assert_eq!(dst_old.to_reg(), regs::rax()); + // Emit this: // // mov{zbq,zwq,zlq,q} (%r9), %rax // rax = old value @@ -2516,8 +2594,10 @@ pub(crate) fn emit( // No need to call `add_trap` here, since the `i4` emit will do that. let i4 = Inst::LockCmpxchg { ty: *ty, - src: r11, - dst: amode.into(), + replacement: r11, + expected: regs::rax(), + mem: amode.into(), + dst_old: Writable::from_reg(regs::rax()), }; i4.emit(sink, info, state); diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 1a81191141..2f753ace1a 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -4199,8 +4199,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I8, - src: rbx, - dst: am1, + mem: am1, + replacement: rbx, + expected: rax, + dst_old: w_rax, }, "F0410FB09C9241010000", "lock cmpxchgb %bl, 321(%r10,%rdx,4)", @@ -4209,8 +4211,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I8, - src: rdx, - dst: am2.clone(), + mem: am2.clone(), + replacement: rdx, + expected: rax, + dst_old: w_rax, }, "F00FB094F1C7CFFFFF", "lock cmpxchgb %dl, -12345(%rcx,%rsi,8)", @@ -4218,8 +4222,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I8, - src: rsi, - dst: am2.clone(), + mem: am2.clone(), + replacement: rsi, + expected: rax, + dst_old: w_rax, }, "F0400FB0B4F1C7CFFFFF", "lock cmpxchgb %sil, -12345(%rcx,%rsi,8)", @@ -4227,8 +4233,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I8, - src: r10, - dst: am2.clone(), + mem: am2.clone(), + replacement: r10, + expected: rax, + dst_old: w_rax, }, "F0440FB094F1C7CFFFFF", "lock cmpxchgb %r10b, -12345(%rcx,%rsi,8)", @@ -4236,8 +4244,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I8, - src: r15, - dst: am2.clone(), + mem: am2.clone(), + replacement: r15, + expected: rax, + dst_old: w_rax, }, "F0440FB0BCF1C7CFFFFF", "lock cmpxchgb %r15b, -12345(%rcx,%rsi,8)", @@ -4246,8 +4256,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I16, - src: rsi, - dst: am2.clone(), + mem: am2.clone(), + replacement: rsi, + expected: rax, + dst_old: w_rax, }, "66F00FB1B4F1C7CFFFFF", "lock cmpxchgw %si, -12345(%rcx,%rsi,8)", @@ -4255,8 +4267,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I16, - src: r10, - dst: am2.clone(), + mem: am2.clone(), + replacement: r10, + expected: rax, + dst_old: w_rax, }, "66F0440FB194F1C7CFFFFF", "lock cmpxchgw %r10w, -12345(%rcx,%rsi,8)", @@ -4265,8 +4279,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I32, - src: rsi, - dst: am2.clone(), + mem: am2.clone(), + replacement: rsi, + expected: rax, + dst_old: w_rax, }, "F00FB1B4F1C7CFFFFF", "lock cmpxchgl %esi, -12345(%rcx,%rsi,8)", @@ -4274,8 +4290,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I32, - src: r10, - dst: am2.clone(), + mem: am2.clone(), + replacement: r10, + expected: rax, + dst_old: w_rax, }, "F0440FB194F1C7CFFFFF", "lock cmpxchgl %r10d, -12345(%rcx,%rsi,8)", @@ -4284,8 +4302,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I64, - src: rsi, - dst: am2.clone(), + mem: am2.clone(), + replacement: rsi, + expected: rax, + dst_old: w_rax, }, "F0480FB1B4F1C7CFFFFF", "lock cmpxchgq %rsi, -12345(%rcx,%rsi,8)", @@ -4293,8 +4313,10 @@ fn test_x64_emit() { insns.push(( Inst::LockCmpxchg { ty: types::I64, - src: r10, - dst: am2.clone(), + mem: am2.clone(), + replacement: r10, + expected: rax, + dst_old: w_rax, }, "F04C0FB194F1C7CFFFFF", "lock cmpxchgq %r10, -12345(%rcx,%rsi,8)", @@ -4302,27 +4324,62 @@ fn test_x64_emit() { // AtomicRmwSeq insns.push(( - Inst::AtomicRmwSeq { ty: types::I8, op: inst_common::AtomicRmwOp::Or, }, + Inst::AtomicRmwSeq { + ty: types::I8, + op: inst_common::AtomicRmwOp::Or, + address: r9, + operand: r10, + temp: w_r11, + dst_old: w_rax + }, "490FB6014989C34D09D3F0450FB0190F85EFFFFFFF", "atomically { 8_bits_at_[%r9]) Or= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }" )); insns.push(( - Inst::AtomicRmwSeq { ty: types::I16, op: inst_common::AtomicRmwOp::And, }, + Inst::AtomicRmwSeq { + ty: types::I16, + op: inst_common::AtomicRmwOp::And, + address: r9, + operand: r10, + temp: w_r11, + dst_old: w_rax + }, "490FB7014989C34D21D366F0450FB1190F85EEFFFFFF", "atomically { 16_bits_at_[%r9]) And= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }" )); insns.push(( - Inst::AtomicRmwSeq { ty: types::I32, op: inst_common::AtomicRmwOp::Xchg, }, + Inst::AtomicRmwSeq { + ty: types::I32, + op: inst_common::AtomicRmwOp::Xchg, + address: r9, + operand: r10, + temp: w_r11, + dst_old: w_rax + }, "418B014989C34D89D3F0450FB1190F85EFFFFFFF", "atomically { 32_bits_at_[%r9]) Xchg= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }" )); insns.push(( - Inst::AtomicRmwSeq { ty: types::I32, op: inst_common::AtomicRmwOp::Umin, }, + Inst::AtomicRmwSeq { + ty: types::I32, + op: inst_common::AtomicRmwOp::Umin, + address: r9, + operand: r10, + temp: w_r11, + dst_old: w_rax + }, "418B014989C34539DA4D0F46DAF0450FB1190F85EBFFFFFF", "atomically { 32_bits_at_[%r9]) Umin= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }" )); insns.push(( - Inst::AtomicRmwSeq { ty: types::I64, op: inst_common::AtomicRmwOp::Add, }, + Inst::AtomicRmwSeq { + ty: types::I64, + op: inst_common::AtomicRmwOp::Add, + address: r9, + operand: r10, + temp: w_r11, + dst_old: w_rax + }, "498B014989C34D01D3F04D0FB1190F85EFFFFFFF", "atomically { 64_bits_at_[%r9]) Add= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }" )); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index e682c6f51c..5d67835941 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -11,8 +11,8 @@ use crate::{settings, settings::Flags, CodegenError, CodegenResult}; use alloc::boxed::Box; use alloc::vec::Vec; use regalloc::{ - PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, - RegUsageMapper, SpillSlot, VirtualReg, Writable, + PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, SpillSlot, + VirtualReg, Writable, }; use smallvec::{smallvec, SmallVec}; use std::fmt; @@ -33,7 +33,7 @@ use regs::{create_reg_universe_systemv, show_ireg_sized}; // Don't build these directly. Instead use the Inst:: functions to create them. -/// Instructions. Destinations are on the RIGHT (a la AT&T syntax). +/// Instructions. #[derive(Clone)] pub enum Inst { /// Nops of various sizes, including zero. @@ -45,7 +45,8 @@ pub enum Inst { AluRmiR { size: OperandSize, // 4 or 8 op: AluRmiROpcode, - src: RegMemImm, + src1: Reg, + src2: RegMemImm, dst: Writable, }, @@ -60,13 +61,15 @@ pub enum Inst { /// Bitwise not Not { size: OperandSize, // 1, 2, 4 or 8 - src: Writable, + src: Reg, + dst: Writable, }, /// Integer negation Neg { size: OperandSize, // 1, 2, 4 or 8 - src: Writable, + src: Reg, + dst: Writable, }, /// Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) @@ -74,19 +77,27 @@ pub enum Inst { size: OperandSize, // 1, 2, 4 or 8 signed: bool, divisor: RegMem, + dividend: Reg, + dst_quotient: Writable, + dst_remainder: Writable, }, /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. MulHi { size: OperandSize, // 2, 4, or 8 signed: bool, - rhs: RegMem, + src1: Reg, + src2: RegMem, + dst_lo: Writable, + dst_hi: Writable, }, /// A synthetic sequence to implement the right inline checks for remainder and division, /// assuming the dividend is in %rax. + /// /// Puts the result back into %rax if is_div, %rdx if !is_div, to mimic what the div /// instruction does. + /// /// The generated code sequence is described in the emit's function match arm for this /// instruction. /// @@ -97,9 +108,12 @@ pub enum Inst { CheckedDivOrRemSeq { kind: DivOrRemKind, size: OperandSize, + dividend: Reg, /// The divisor operand. Note it's marked as modified so that it gets assigned a register /// different from the temporary. divisor: Writable, + dst_quotient: Writable, + dst_remainder: Writable, tmp: Option>, }, @@ -107,9 +121,12 @@ pub enum Inst { /// or al into ah: (cbw) SignExtendData { size: OperandSize, // 1, 2, 4 or 8 + src: Reg, + dst: Writable, }, /// Constant materialization: (imm32 imm64) reg. + /// /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. Imm { dst_size: OperandSize, // 4 or 8 @@ -163,15 +180,17 @@ pub enum Inst { ShiftR { size: OperandSize, // 1, 2, 4 or 8 kind: ShiftKind, + src: Reg, /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl". - num_bits: Option, + num_bits: Imm8Reg, dst: Writable, }, /// Arithmetic SIMD shifts. XmmRmiReg { opcode: SseOpcode, - src: RegMemImm, + src1: Reg, + src2: RegMemImm, dst: Writable, }, @@ -191,7 +210,8 @@ pub enum Inst { Cmove { size: OperandSize, // 2, 4, or 8 cc: CC, - src: RegMem, + consequent: RegMem, + alternative: Reg, dst: Writable, }, @@ -208,7 +228,8 @@ pub enum Inst { /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg XmmRmR { op: SseOpcode, - src: RegMem, + src1: Reg, + src2: RegMem, dst: Writable, }, @@ -337,7 +358,8 @@ pub enum Inst { /// A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg XmmRmRImm { op: SseOpcode, - src: RegMem, + src1: Reg, + src2: RegMem, dst: Writable, imm: u8, size: OperandSize, // 4 or 8 @@ -428,17 +450,19 @@ pub enum Inst { // Instructions pertaining to atomic memory accesses. /// A standard (native) `lock cmpxchg src, (amode)`, with register conventions: /// - /// `dst` (read) address - /// `src` (read) replacement value - /// %rax (modified) in: expected value, out: value that was actually at `dst` + /// `mem` (read) address + /// `replacement` (read) replacement value + /// %rax (modified) in: expected value, out: value that was actually at `dst` /// %rflags is written. Do not assume anything about it after the instruction. /// /// The instruction "succeeded" iff the lowest `ty` bits of %rax afterwards are the same as /// they were before. LockCmpxchg { ty: Type, // I8, I16, I32 or I64 - src: Reg, - dst: SyntheticAmode, + replacement: Reg, + expected: Reg, + mem: SyntheticAmode, + dst_old: Writable, }, /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction. @@ -467,6 +491,10 @@ pub enum Inst { AtomicRmwSeq { ty: Type, // I8, I16, I32 or I64 op: inst_common::AtomicRmwOp, + address: Reg, + operand: Reg, + temp: Writable, + dst_old: Writable, }, /// A memory fence (mfence, lfence or sfence). @@ -606,7 +634,13 @@ impl Inst { debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); src.assert_regclass_is(RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64); - Self::AluRmiR { size, op, src, dst } + Self::AluRmiR { + size, + op, + src1: dst.to_reg(), + src2: src, + dst, + } } pub(crate) fn unary_rm_r( @@ -627,12 +661,20 @@ impl Inst { pub(crate) fn not(size: OperandSize, src: Writable) -> Inst { debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); - Inst::Not { size, src } + Inst::Not { + size, + src: src.to_reg(), + dst: src, + } } pub(crate) fn neg(size: OperandSize, src: Writable) -> Inst { debug_assert_eq!(src.to_reg().get_class(), RegClass::I64); - Inst::Neg { size, src } + Inst::Neg { + size, + src: src.to_reg(), + dst: src, + } } pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst { @@ -641,6 +683,9 @@ impl Inst { size, signed, divisor, + dividend: regs::rax(), + dst_quotient: Writable::from_reg(regs::rax()), + dst_remainder: Writable::from_reg(regs::rdx()), } } @@ -651,7 +696,14 @@ impl Inst { OperandSize::Size64 ])); rhs.assert_regclass_is(RegClass::I64); - Inst::MulHi { size, signed, rhs } + Inst::MulHi { + size, + signed, + src1: regs::rax(), + src2: rhs, + dst_lo: Writable::from_reg(regs::rax()), + dst_hi: Writable::from_reg(regs::rdx()), + } } pub(crate) fn checked_div_or_rem_seq( @@ -668,12 +720,19 @@ impl Inst { kind, size, divisor, + dividend: regs::rax(), + dst_quotient: Writable::from_reg(regs::rax()), + dst_remainder: Writable::from_reg(regs::rdx()), tmp, } } pub(crate) fn sign_extend_data(size: OperandSize) -> Inst { - Inst::SignExtendData { size } + Inst::SignExtendData { + size, + src: regs::rax(), + dst: Writable::from_reg(regs::rdx()), + } } pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable) -> Inst { @@ -728,7 +787,12 @@ impl Inst { pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmRmR { op, src, dst } + Inst::XmmRmR { + op, + src1: dst.to_reg(), + src2: src, + dst, + } } pub(crate) fn xmm_rm_r_evex( @@ -902,7 +966,8 @@ impl Inst { debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); Inst::XmmRmRImm { op, - src, + src1: dst.to_reg(), + src2: src, dst, imm, size, @@ -918,7 +983,12 @@ impl Inst { pub(crate) fn xmm_rmi_reg(opcode: SseOpcode, src: RegMemImm, dst: Writable) -> Inst { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XmmRmiReg { opcode, src, dst } + Inst::XmmRmiReg { + opcode, + src1: dst.to_reg(), + src2: src, + dst, + } } pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable) -> Inst { @@ -976,7 +1046,11 @@ impl Inst { Inst::ShiftR { size, kind, - num_bits, + src: dst.to_reg(), + num_bits: match num_bits { + Some(imm) => Imm8Reg::Imm8 { imm }, + None => Imm8Reg::Reg { reg: regs::rcx() }, + }, dst, } } @@ -1024,7 +1098,13 @@ impl Inst { OperandSize::Size64 ])); debug_assert!(dst.to_reg().get_class() == RegClass::I64); - Inst::Cmove { size, cc, src, dst } + Inst::Cmove { + size, + cc, + consequent: src, + alternative: dst.to_reg(), + dst, + } } pub(crate) fn xmm_cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable) -> Inst { @@ -1193,13 +1273,13 @@ impl Inst { /// same as the first register (already handled). fn produces_const(&self) -> bool { match self { - Self::AluRmiR { op, src, dst, .. } => { - src.to_reg() == Some(dst.to_reg()) + Self::AluRmiR { op, src2, dst, .. } => { + src2.to_reg() == Some(dst.to_reg()) && (*op == AluRmiROpcode::Xor || *op == AluRmiROpcode::Sub) } - Self::XmmRmR { op, src, dst, .. } => { - src.to_reg() == Some(dst.to_reg()) + Self::XmmRmR { op, src2, dst, .. } => { + src2.to_reg() == Some(dst.to_reg()) && (*op == SseOpcode::Xorps || *op == SseOpcode::Xorpd || *op == SseOpcode::Pxor @@ -1210,9 +1290,9 @@ impl Inst { } Self::XmmRmRImm { - op, src, dst, imm, .. + op, src2, dst, imm, .. } => { - src.to_reg() == Some(dst.to_reg()) + src2.to_reg() == Some(dst.to_reg()) && (*op == SseOpcode::Cmppd || *op == SseOpcode::Cmpps) && *imm == FcmpImm::Equal.encode() } @@ -1285,6 +1365,265 @@ impl Inst { _ => unimplemented!("unimplemented type for Inst::xor: {}", ty), } } + + /// Translate three-operand instructions into a sequence of two-operand + /// instructions. + /// + /// For example: + /// + /// ```text + /// x = add a, b + /// ``` + /// + /// Becomes: + /// + /// ```text + /// mov x, a + /// add x, b + /// ``` + /// + /// The three-operand form for instructions allows our ISLE DSL code to have + /// a value-based, SSA view of the world. This method is responsible for + /// undoing that. + /// + /// Note that register allocation cleans up most of these inserted `mov`s + /// with its move coalescing. + pub(crate) fn mov_mitosis(mut self) -> impl Iterator { + log::trace!("mov_mitosis({:?})", self); + + let mut insts = SmallVec::<[Self; 4]>::new(); + + match &mut self { + Inst::AluRmiR { src1, dst, .. } => { + if *src1 != dst.to_reg() { + debug_assert!(src1.is_virtual()); + insts.push(Self::gen_move(*dst, *src1, types::I64)); + *src1 = dst.to_reg(); + } + insts.push(self); + } + Inst::XmmRmiReg { src1, dst, .. } => { + if *src1 != dst.to_reg() { + debug_assert!(src1.is_virtual()); + insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + *src1 = dst.to_reg(); + } + insts.push(self); + } + Inst::XmmRmR { src1, dst, .. } => { + if *src1 != dst.to_reg() { + debug_assert!(src1.is_virtual()); + insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + *src1 = dst.to_reg(); + } + insts.push(self); + } + Inst::XmmRmRImm { src1, dst, .. } => { + if *src1 != dst.to_reg() { + debug_assert!(src1.is_virtual()); + insts.push(Self::gen_move(*dst, *src1, types::I8X16)); + *src1 = dst.to_reg(); + } + insts.push(self); + } + Inst::Cmove { + size, + alternative, + dst, + .. + } => { + if *alternative != dst.to_reg() { + debug_assert!(alternative.is_virtual()); + insts.push(Self::mov_r_r(*size, *alternative, *dst)); + *alternative = dst.to_reg(); + } + insts.push(self); + } + Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { + if *src != dst.to_reg() { + debug_assert!(src.is_virtual()); + insts.push(Self::gen_move(*dst, *src, types::I64)); + *src = dst.to_reg(); + } + insts.push(self); + } + Inst::Div { + dividend, + dst_quotient, + dst_remainder, + .. + } + | Inst::CheckedDivOrRemSeq { + dividend, + dst_quotient, + dst_remainder, + .. + } => { + if *dividend != regs::rax() { + debug_assert!(dividend.is_virtual()); + insts.push(Self::gen_move( + Writable::from_reg(regs::rax()), + *dividend, + types::I64, + )); + *dividend = regs::rax(); + } + let mut quotient_mov = None; + if dst_quotient.to_reg() != regs::rax() { + debug_assert!(dst_quotient.to_reg().is_virtual()); + quotient_mov = Some(Self::gen_move(*dst_quotient, regs::rax(), types::I64)); + *dst_quotient = Writable::from_reg(regs::rax()); + } + let mut remainder_mov = None; + if dst_remainder.to_reg() != regs::rdx() { + debug_assert!(dst_remainder.to_reg().is_virtual()); + remainder_mov = Some(Self::gen_move(*dst_remainder, regs::rdx(), types::I64)); + *dst_remainder = Writable::from_reg(regs::rdx()); + } + insts.push(self); + insts.extend(quotient_mov); + insts.extend(remainder_mov); + } + Inst::MulHi { + src1, + dst_lo, + dst_hi, + .. + } => { + if *src1 != regs::rax() { + debug_assert!(src1.is_virtual()); + insts.push(Self::gen_move( + Writable::from_reg(regs::rax()), + *src1, + types::I64, + )); + *src1 = regs::rax(); + } + let mut dst_lo_mov = None; + if dst_lo.to_reg() != regs::rax() { + debug_assert!(dst_lo.to_reg().is_virtual()); + dst_lo_mov = Some(Self::gen_move(*dst_lo, regs::rax(), types::I64)); + *dst_lo = Writable::from_reg(regs::rax()); + } + let mut dst_hi_mov = None; + if dst_hi.to_reg() != regs::rdx() { + debug_assert!(dst_hi.to_reg().is_virtual()); + dst_hi_mov = Some(Self::gen_move(*dst_hi, regs::rdx(), types::I64)); + *dst_hi = Writable::from_reg(regs::rdx()); + } + insts.push(self); + insts.extend(dst_lo_mov); + insts.extend(dst_hi_mov); + } + Inst::SignExtendData { src, dst, .. } => { + if *src != regs::rax() { + debug_assert!(src.is_virtual()); + insts.push(Self::gen_move( + Writable::from_reg(regs::rax()), + *src, + types::I64, + )); + *src = regs::rax(); + } + let mut dst_mov = None; + if dst.to_reg() != regs::rax() { + debug_assert!(dst.to_reg().is_virtual()); + dst_mov = Some(Self::gen_move(*dst, dst.to_reg(), types::I64)); + *dst = Writable::from_reg(regs::rax()); + } + insts.push(self); + insts.extend(dst_mov); + } + Inst::ShiftR { + src, num_bits, dst, .. + } => { + if *src != dst.to_reg() { + debug_assert!(src.is_virtual()); + insts.push(Self::gen_move(*dst, *src, types::I64)); + *src = dst.to_reg(); + } + if let Imm8Reg::Reg { reg } = num_bits { + if *reg != regs::rcx() { + debug_assert!(reg.is_virtual()); + insts.push(Self::gen_move( + Writable::from_reg(regs::rcx()), + *reg, + types::I64, + )); + *reg = regs::rcx(); + } + } + insts.push(self); + } + Inst::LockCmpxchg { + ty, + expected, + dst_old, + .. + } => { + if *expected != regs::rax() { + debug_assert!(expected.is_virtual()); + insts.push(Self::gen_move( + Writable::from_reg(regs::rax()), + *expected, + *ty, + )); + } + let mut dst_old_mov = None; + if dst_old.to_reg() != regs::rax() { + debug_assert!(dst_old.to_reg().is_virtual()); + dst_old_mov = Some(Self::gen_move(*dst_old, regs::rax(), *ty)); + *dst_old = Writable::from_reg(regs::rax()); + } + insts.push(self); + insts.extend(dst_old_mov); + } + Inst::AtomicRmwSeq { + ty, + address, + operand, + dst_old, + .. + } => { + if *address != regs::r9() { + debug_assert!(address.is_virtual()); + insts.push(Self::gen_move( + Writable::from_reg(regs::r9()), + *address, + types::I64, + )); + *address = regs::r9(); + } + if *operand != regs::r10() { + debug_assert!(operand.is_virtual()); + insts.push(Self::gen_move( + Writable::from_reg(regs::r10()), + *operand, + *ty, + )); + *address = regs::r10(); + } + let mut dst_old_mov = None; + if dst_old.to_reg() != regs::rax() { + debug_assert!(dst_old.to_reg().is_virtual()); + dst_old_mov = Some(Self::gen_move(*dst_old, regs::rax(), *ty)); + *dst_old = Writable::from_reg(regs::rax()); + } + insts.push(self); + insts.extend(dst_old_mov); + } + // No other instruction needs 3-operand to 2-operand legalization. + _ => insts.push(self), + } + + if log::log_enabled!(log::Level::Trace) { + for inst in &insts { + log::trace!(" -> {:?}", inst); + } + } + + insts.into_iter() + } } //============================================================================= @@ -1344,10 +1683,16 @@ impl PrettyPrint for Inst { match self { Inst::Nop { len } => format!("{} len={}", ljustify("nop".to_string()), len), - Inst::AluRmiR { size, op, src, dst } => format!( + Inst::AluRmiR { + size, + op, + src1: _, + src2, + dst, + } => format!( "{} {}, {}", ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())), - src.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())), + src2.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())), show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*size, op.is_8bit())), ), @@ -1358,16 +1703,16 @@ impl PrettyPrint for Inst { show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()), ), - Inst::Not { size, src } => format!( + Inst::Not { size, src: _, dst } => format!( "{} {}", ljustify2("not".to_string(), suffix_bwlq(*size)), - show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) ), - Inst::Neg { size, src } => format!( + Inst::Neg { size, src: _, dst } => format!( "{} {}", ljustify2("neg".to_string(), suffix_bwlq(*size)), - show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()) + show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) ), Inst::Div { @@ -1386,7 +1731,7 @@ impl PrettyPrint for Inst { ), Inst::MulHi { - size, signed, rhs, .. + size, signed, src2, .. } => format!( "{} {}", ljustify(if *signed { @@ -1394,7 +1739,7 @@ impl PrettyPrint for Inst { } else { "mul".to_string() }), - rhs.show_rru_sized(mb_rru, size.to_bytes()) + src2.show_rru_sized(mb_rru, size.to_bytes()) ), Inst::CheckedDivOrRemSeq { @@ -1413,7 +1758,7 @@ impl PrettyPrint for Inst { show_ireg_sized(divisor.to_reg(), mb_rru, size.to_bytes()), ), - Inst::SignExtendData { size } => match size { + Inst::SignExtendData { size, .. } => match size { OperandSize::Size8 => "cbw", OperandSize::Size16 => "cwd", OperandSize::Size32 => "cdq", @@ -1442,10 +1787,10 @@ impl PrettyPrint for Inst { dst.show_rru(mb_rru), ), - Inst::XmmRmR { op, src, dst, .. } => format!( + Inst::XmmRmR { op, src2, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), - src.show_rru_sized(mb_rru, 8), + src2.show_rru_sized(mb_rru, 8), show_ireg_sized(dst.to_reg(), mb_rru, 8), ), @@ -1484,7 +1829,7 @@ impl PrettyPrint for Inst { Inst::XmmRmRImm { op, - src, + src2, dst, imm, size, @@ -1501,7 +1846,7 @@ impl PrettyPrint for Inst { } )), imm, - src.show_rru(mb_rru), + src2.show_rru(mb_rru), dst.show_rru(mb_rru), ), @@ -1681,14 +2026,16 @@ impl PrettyPrint for Inst { kind, num_bits, dst, + .. } => match num_bits { - None => format!( - "{} %cl, {}", + Imm8Reg::Reg { reg } => format!( + "{} {}, {}", ljustify2(kind.to_string(), suffix_bwlq(*size)), + show_ireg_sized(*reg, mb_rru, 1), show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()) ), - Some(num_bits) => format!( + Imm8Reg::Imm8 { imm: num_bits } => format!( "{} ${}, {}", ljustify2(kind.to_string(), suffix_bwlq(*size)), num_bits, @@ -1696,10 +2043,12 @@ impl PrettyPrint for Inst { ), }, - Inst::XmmRmiReg { opcode, src, dst } => format!( + Inst::XmmRmiReg { + opcode, src2, dst, .. + } => format!( "{} {}, {}", ljustify(opcode.to_string()), - src.show_rru(mb_rru), + src2.show_rru(mb_rru), dst.to_reg().show_rru(mb_rru) ), @@ -1727,7 +2076,13 @@ impl PrettyPrint for Inst { show_ireg_sized(dst.to_reg(), mb_rru, 1) ), - Inst::Cmove { size, cc, src, dst } => format!( + Inst::Cmove { + size, + cc, + consequent: src, + alternative: _, + dst, + } => format!( "{} {}, {}", ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))), src.show_rru_sized(mb_rru, size.to_bytes()), @@ -1813,13 +2168,18 @@ impl PrettyPrint for Inst { show_ireg_sized(dst.to_reg(), mb_rru, 8), ), - Inst::LockCmpxchg { ty, src, dst, .. } => { + Inst::LockCmpxchg { + ty, + replacement, + mem, + .. + } => { let size = ty.bytes() as u8; format!( "lock cmpxchg{} {}, {}", suffix_bwlq(OperandSize::from_bytes(size as u32)), - show_ireg_sized(*src, mb_rru, size), - dst.show_rru(mb_rru) + show_ireg_sized(*replacement, mb_rru, size), + mem.show_rru(mb_rru) ) } @@ -1874,36 +2234,74 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { // regalloc.rs will "fix" this for us by removing the modified set from the use and def // sets. match inst { - Inst::AluRmiR { src, dst, .. } => { + Inst::AluRmiR { + src1, src2, dst, .. + } => { + debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { - // No need to account for src, since src == dst. + // No need to account for src2, since src2 == dst. collector.add_def(*dst); } else { - src.get_regs_as_uses(collector); + src2.get_regs_as_uses(collector); collector.add_mod(*dst); } } - Inst::Not { src, .. } => { - collector.add_mod(*src); + Inst::Not { src, dst, .. } => { + debug_assert_eq!(*src, dst.to_reg()); + collector.add_mod(*dst); } - Inst::Neg { src, .. } => { - collector.add_mod(*src); + Inst::Neg { src, dst, .. } => { + debug_assert_eq!(*src, dst.to_reg()); + collector.add_mod(*dst); } - Inst::Div { size, divisor, .. } => { + Inst::Div { + size, + divisor, + dividend, + dst_quotient, + dst_remainder, + .. + } => { + debug_assert_eq!(*dividend, regs::rax()); + debug_assert_eq!(dst_quotient.to_reg(), regs::rax()); collector.add_mod(Writable::from_reg(regs::rax())); + + debug_assert_eq!(dst_remainder.to_reg(), regs::rdx()); if *size == OperandSize::Size8 { collector.add_def(Writable::from_reg(regs::rdx())); } else { collector.add_mod(Writable::from_reg(regs::rdx())); } + divisor.get_regs_as_uses(collector); } - Inst::MulHi { rhs, .. } => { + Inst::MulHi { + src1, + src2, + dst_lo, + dst_hi, + .. + } => { + debug_assert_eq!(*src1, regs::rax()); + debug_assert_eq!(dst_lo.to_reg(), regs::rax()); collector.add_mod(Writable::from_reg(regs::rax())); + + debug_assert_eq!(dst_hi.to_reg(), regs::rdx()); collector.add_def(Writable::from_reg(regs::rdx())); - rhs.get_regs_as_uses(collector); + + src2.get_regs_as_uses(collector); } - Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => { + Inst::CheckedDivOrRemSeq { + divisor, + dividend, + dst_quotient, + dst_remainder, + tmp, + .. + } => { + debug_assert_eq!(*dividend, regs::rax()); + debug_assert_eq!(dst_quotient.to_reg(), regs::rax()); + debug_assert_eq!(dst_remainder.to_reg(), regs::rdx()); // Mark both fixed registers as mods, to avoid an early clobber problem in codegen // (i.e. the temporary is allocated one of the fixed registers). This requires writing // the rdx register *before* the instruction, which is not too bad. @@ -1914,25 +2312,36 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(*tmp); } } - Inst::SignExtendData { size } => match size { - OperandSize::Size8 => collector.add_mod(Writable::from_reg(regs::rax())), - _ => { - collector.add_use(regs::rax()); - collector.add_def(Writable::from_reg(regs::rdx())); + Inst::SignExtendData { size, src, dst } => { + debug_assert_eq!(*src, regs::rax()); + debug_assert_eq!(dst.to_reg(), regs::rdx()); + match size { + OperandSize::Size8 => collector.add_mod(Writable::from_reg(regs::rax())), + _ => { + collector.add_use(regs::rax()); + collector.add_def(Writable::from_reg(regs::rdx())); + } } - }, + } Inst::UnaryRmR { src, dst, .. } | Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmREvex { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } - Inst::XmmRmR { src, dst, op, .. } => { + Inst::XmmRmR { + src1, + src2, + dst, + op, + .. + } => { + debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { // No need to account for src, since src == dst. collector.add_def(*dst); } else { - src.get_regs_as_uses(collector); + src2.get_regs_as_uses(collector); collector.add_mod(*dst); // Some instructions have an implicit use of XMM0. if *op == SseOpcode::Blendvpd @@ -1957,9 +2366,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { _ => collector.add_def(*dst), } } - Inst::XmmRmRImm { op, src, dst, .. } => { + Inst::XmmRmRImm { + op, + src1, + src2, + dst, + .. + } => { + debug_assert_eq!(*src1, dst.to_reg()); if inst.produces_const() { - // No need to account for src, since src == dst. + // No need to account for src2, since src2 == dst. + debug_assert_eq!(src2.to_reg(), Some(dst.to_reg())); collector.add_def(*dst); } else if *op == SseOpcode::Pextrb || *op == SseOpcode::Pextrw @@ -1970,10 +2387,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { || *op == SseOpcode::Roundps || *op == SseOpcode::Roundpd { - src.get_regs_as_uses(collector); + src2.get_regs_as_uses(collector); collector.add_def(*dst); } else { - src.get_regs_as_uses(collector); + src2.get_regs_as_uses(collector); collector.add_mod(*dst); } } @@ -1983,8 +2400,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_use(*lhs); collector.add_mod(*rhs_dst); } - Inst::XmmRmiReg { src, dst, .. } => { - src.get_regs_as_uses(collector); + Inst::XmmRmiReg { + src1, src2, dst, .. + } => { + debug_assert_eq!(*src1, dst.to_reg()); + src2.get_regs_as_uses(collector); collector.add_mod(*dst); } Inst::XmmMovRM { src, dst, .. } => { @@ -2054,7 +2474,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { dst.get_regs_as_uses(collector); } Inst::ShiftR { num_bits, dst, .. } => { - if num_bits.is_none() { + if let Imm8Reg::Reg { reg } = num_bits { + debug_assert_eq!(*reg, regs::rcx()); collector.add_use(regs::rcx()); } collector.add_mod(*dst); @@ -2066,7 +2487,12 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::Setcc { dst, .. } => { collector.add_def(*dst); } - Inst::Cmove { src, dst, .. } | Inst::XmmCmove { src, dst, .. } => { + Inst::Cmove { + consequent: src, + dst, + .. + } + | Inst::XmmCmove { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_mod(*dst); } @@ -2115,9 +2541,18 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(*dst); } - Inst::LockCmpxchg { src, dst, .. } => { - dst.get_regs_as_uses(collector); - collector.add_use(*src); + Inst::LockCmpxchg { + replacement, + expected, + mem, + dst_old, + .. + } => { + mem.get_regs_as_uses(collector); + collector.add_use(*replacement); + + debug_assert_eq!(*expected, regs::rax()); + debug_assert_eq!(dst_old.to_reg(), regs::rax()); collector.add_mod(Writable::from_reg(regs::rax())); } @@ -2165,29 +2600,55 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { //============================================================================= // Instructions and subcomponents: map_regs -fn map_use(m: &RUM, r: &mut Reg) { - if let Some(reg) = r.as_virtual_reg() { - let new = m.get_use(reg).unwrap().to_reg(); +// Define our own register-mapping trait so we can do arbitrary register +// renaming that are more free form than what `regalloc` constrains us to with +// its `RegUsageMapper` trait definition. +pub trait RegMapper { + fn get_use(&self, reg: Reg) -> Option; + fn get_def(&self, reg: Reg) -> Option; + fn get_mod(&self, reg: Reg) -> Option; +} + +impl RegMapper for T +where + T: regalloc::RegUsageMapper, +{ + fn get_use(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_use(v).map(|r| r.to_reg()) + } + + fn get_def(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_def(v).map(|r| r.to_reg()) + } + + fn get_mod(&self, reg: Reg) -> Option { + let v = reg.as_virtual_reg()?; + self.get_mod(v).map(|r| r.to_reg()) + } +} + +fn map_use(m: &RM, r: &mut Reg) { + if let Some(new) = m.get_use(*r) { *r = new; } } -fn map_def(m: &RUM, r: &mut Writable) { - if let Some(reg) = r.to_reg().as_virtual_reg() { - let new = m.get_def(reg).unwrap().to_reg(); +fn map_def(m: &RM, r: &mut Writable) { + if let Some(new) = m.get_def(r.to_reg()) { *r = Writable::from_reg(new); } } -fn map_mod(m: &RUM, r: &mut Writable) { - if let Some(reg) = r.to_reg().as_virtual_reg() { - let new = m.get_mod(reg).unwrap().to_reg(); +fn map_mod(m: &RM, r: &mut Writable) { + if let Some(new) = m.get_mod(r.to_reg()) { *r = Writable::from_reg(new); } } impl Amode { - fn map_uses(&mut self, map: &RUM) { + fn map_uses(&mut self, map: &RM) { match self { Amode::ImmReg { ref mut base, .. } => map_use(map, base), Amode::ImmRegRegShift { @@ -2217,7 +2678,7 @@ impl Amode { } impl RegMemImm { - fn map_uses(&mut self, map: &RUM) { + fn map_uses(&mut self, map: &RM) { match self { RegMemImm::Reg { ref mut reg } => map_use(map, reg), RegMemImm::Mem { ref mut addr } => addr.map_uses(map), @@ -2225,7 +2686,7 @@ impl RegMemImm { } } - fn map_as_def(&mut self, mapper: &RUM) { + fn map_as_def(&mut self, mapper: &RM) { match self { Self::Reg { reg } => { let mut writable_src = Writable::from_reg(*reg); @@ -2238,14 +2699,14 @@ impl RegMemImm { } impl RegMem { - fn map_uses(&mut self, map: &RUM) { + fn map_uses(&mut self, map: &RM) { match self { RegMem::Reg { ref mut reg } => map_use(map, reg), RegMem::Mem { ref mut addr, .. } => addr.map_uses(map), } } - fn map_as_def(&mut self, mapper: &RUM) { + fn map_as_def(&mut self, mapper: &RM) { match self { Self::Reg { reg } => { let mut writable_src = Writable::from_reg(*reg); @@ -2257,28 +2718,36 @@ impl RegMem { } } -fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { +pub(crate) fn x64_map_regs(inst: &mut Inst, mapper: &RM) { // Note this must be carefully synchronized with x64_get_regs. let produces_const = inst.produces_const(); match inst { // ** Nop Inst::AluRmiR { - ref mut src, + ref mut src1, + ref mut src2, ref mut dst, .. } => { + debug_assert_eq!(*src1, dst.to_reg()); if produces_const { - src.map_as_def(mapper); + src2.map_as_def(mapper); map_def(mapper, dst); + *src1 = dst.to_reg(); } else { - src.map_uses(mapper); + src2.map_uses(mapper); map_mod(mapper, dst); + *src1 = dst.to_reg(); } } - Inst::Not { src, .. } | Inst::Neg { src, .. } => map_mod(mapper, src), + Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => { + debug_assert_eq!(*src, dst.to_reg()); + map_mod(mapper, dst); + *src = dst.to_reg(); + } Inst::Div { divisor, .. } => divisor.map_uses(mapper), - Inst::MulHi { rhs, .. } => rhs.map_uses(mapper), + Inst::MulHi { src2, .. } => src2.map_uses(mapper), Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => { map_mod(mapper, divisor); if let Some(tmp) = tmp { @@ -2306,13 +2775,16 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { } Inst::XmmRmRImm { ref op, - ref mut src, + ref mut src1, + ref mut src2, ref mut dst, .. } => { + debug_assert_eq!(*src1, dst.to_reg()); if produces_const { - src.map_as_def(mapper); + src2.map_as_def(mapper); map_def(mapper, dst); + *src1 = dst.to_reg(); } else if *op == SseOpcode::Pextrb || *op == SseOpcode::Pextrw || *op == SseOpcode::Pextrd @@ -2322,24 +2794,30 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { || *op == SseOpcode::Roundps || *op == SseOpcode::Roundpd { - src.map_uses(mapper); + src2.map_uses(mapper); map_def(mapper, dst); + *src1 = dst.to_reg(); } else { - src.map_uses(mapper); + src2.map_uses(mapper); map_mod(mapper, dst); + *src1 = dst.to_reg(); } } Inst::XmmRmR { - ref mut src, + ref mut src1, + ref mut src2, ref mut dst, .. } => { + debug_assert_eq!(*src1, dst.to_reg()); if produces_const { - src.map_as_def(mapper); + src2.map_as_def(mapper); map_def(mapper, dst); + *src1 = dst.to_reg(); } else { - src.map_uses(mapper); + src2.map_uses(mapper); map_mod(mapper, dst); + *src1 = dst.to_reg(); } } Inst::XmmRmREvex { @@ -2357,12 +2835,15 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { } } Inst::XmmRmiReg { - ref mut src, + ref mut src1, + ref mut src2, ref mut dst, .. } => { - src.map_uses(mapper); + debug_assert_eq!(*src1, dst.to_reg()); + src2.map_uses(mapper); map_mod(mapper, dst); + *src1 = dst.to_reg(); } Inst::XmmUninitializedValue { ref mut dst, .. } => { map_def(mapper, dst); @@ -2475,8 +2956,14 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { map_use(mapper, src); dst.map_uses(mapper); } - Inst::ShiftR { ref mut dst, .. } => { + Inst::ShiftR { + ref mut src, + ref mut dst, + .. + } => { + debug_assert_eq!(*src, dst.to_reg()); map_mod(mapper, dst); + *src = dst.to_reg(); } Inst::CmpRmiR { ref mut src, @@ -2488,17 +2975,22 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { } Inst::Setcc { ref mut dst, .. } => map_def(mapper, dst), Inst::Cmove { - ref mut src, + consequent: ref mut src, ref mut dst, + ref mut alternative, .. + } => { + src.map_uses(mapper); + map_mod(mapper, dst); + *alternative = dst.to_reg(); } - | Inst::XmmCmove { + Inst::XmmCmove { ref mut src, ref mut dst, .. } => { src.map_uses(mapper); - map_mod(mapper, dst) + map_mod(mapper, dst); } Inst::Push64 { ref mut src } => src.map_uses(mapper), Inst::Pop64 { ref mut dst } => { @@ -2549,12 +3041,12 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { Inst::LoadExtName { ref mut dst, .. } => map_def(mapper, dst), Inst::LockCmpxchg { - ref mut src, - ref mut dst, + ref mut replacement, + ref mut mem, .. } => { - map_use(mapper, src); - dst.map_uses(mapper); + map_use(mapper, replacement); + mem.map_uses(mapper); } Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg), @@ -2588,7 +3080,10 @@ impl MachInst for Inst { x64_get_regs(&self, collector) } - fn map_regs(&mut self, mapper: &RUM) { + fn map_regs(&mut self, mapper: &RUM) + where + RUM: regalloc::RegUsageMapper, + { x64_map_regs(self, mapper); } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle new file mode 100644 index 0000000000..f89caccfe4 --- /dev/null +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -0,0 +1,890 @@ +;; x86-64 instruction selection and CLIF-to-MachInst lowering. + +;; The main lowering constructor term: takes a clif `Inst` and returns the +;; register(s) within which the lowered instruction's result values live. +(decl lower (Inst) ValueRegs) + +;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. +(rule (lower (has_type (fits_in_64 ty) + (iconst (u64_from_imm64 x)))) + (value_reg (imm ty x))) + +;; `i128` +(rule (lower (has_type $I128 + (iconst (u64_from_imm64 x)))) + (value_regs (imm $I64 x) + (imm $I64 0))) + +;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `b64` and smaller. + +(rule (lower (has_type (fits_in_64 ty) + (bconst $false))) + (value_reg (imm ty 0))) + +(rule (lower (has_type (fits_in_64 ty) + (bconst $true))) + (value_reg (imm ty 1))) + +;; `b128` + +(rule (lower (has_type $B128 + (bconst $false))) + (value_regs (imm $B64 0) + (imm $B64 0))) + +(rule (lower (has_type $B128 + (bconst $true))) + (value_regs (imm $B64 1) + (imm $B64 0))) + +;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type ty (null))) + (value_reg (imm ty 0))) + +;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +;; Add two registers. +(rule (lower (has_type (fits_in_64 ty) + (iadd x y))) + (value_reg (add ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) + +;; Add a register and an immediate. + +(rule (lower (has_type (fits_in_64 ty) + (iadd x (simm32_from_value y)))) + (value_reg (add ty (put_in_reg x) y))) + +(rule (lower (has_type (fits_in_64 ty) + (iadd (simm32_from_value x) y))) + (value_reg (add ty (put_in_reg y) x))) + +;; Add a register and memory. + +(rule (lower (has_type (fits_in_64 ty) + (iadd x (sinkable_load y)))) + (value_reg (add ty + (put_in_reg x) + (sink_load y)))) + +(rule (lower (has_type (fits_in_64 ty) + (iadd (sinkable_load x) y))) + (value_reg (add ty + (put_in_reg y) + (sink_load x)))) + +;; SSE. + +(rule (lower (has_type (multi_lane 8 16) + (iadd x y))) + (value_reg (paddb (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 16 8) + (iadd x y))) + (value_reg (paddw (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 32 4) + (iadd x y))) + (value_reg (paddd (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 64 2) + (iadd x y))) + (value_reg (paddq (put_in_reg x) + (put_in_reg_mem y)))) + +;; `i128` +(rule (lower (has_type $I128 (iadd x y))) + ;; Get the high/low registers for `x`. + (let ((x_regs ValueRegs (put_in_regs x)) + (x_lo Reg (value_regs_get x_regs 0)) + (x_hi Reg (value_regs_get x_regs 1))) + ;; Get the high/low registers for `y`. + (let ((y_regs ValueRegs (put_in_regs y)) + (y_lo Reg (value_regs_get y_regs 0)) + (y_hi Reg (value_regs_get y_regs 1))) + ;; Do an add followed by an add-with-carry. + (with_flags (add_with_flags $I64 x_lo (RegMemImm.Reg y_lo)) + (adc $I64 x_hi (RegMemImm.Reg y_hi)))))) + +;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (multi_lane 8 16) + (sadd_sat x y))) + (value_reg (paddsb (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 16 8) + (sadd_sat x y))) + (value_reg (paddsw (put_in_reg x) + (put_in_reg_mem y)))) + +;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (multi_lane 8 16) + (uadd_sat x y))) + (value_reg (paddusb (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 16 8) + (uadd_sat x y))) + (value_reg (paddusw (put_in_reg x) + (put_in_reg_mem y)))) + +;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Add two registers. +(rule (lower (has_type (fits_in_64 ty) + (iadd_ifcout x y))) + (value_reg (add ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) + +;; Add a register and an immediate. + +(rule (lower (has_type (fits_in_64 ty) + (iadd_ifcout x (simm32_from_value y)))) + (value_reg (add ty (put_in_reg x) y))) + +(rule (lower (has_type (fits_in_64 ty) + (iadd_ifcout (simm32_from_value x) y))) + (value_reg (add ty (put_in_reg y) x))) + +;; Add a register and memory. + +(rule (lower (has_type (fits_in_64 ty) + (iadd_ifcout x (sinkable_load y)))) + (value_reg (add ty + (put_in_reg x) + (sink_load y)))) + +(rule (lower (has_type (fits_in_64 ty) + (iadd_ifcout (sinkable_load x) y))) + (value_reg (add ty + (put_in_reg y) + (sink_load x)))) + +;; (No `iadd_ifcout` for `i128`.) + +;;;; Rules for `iadd_imm` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +;; When the immediate fits in a `RegMemImm.Imm`, use that. +(rule (lower (has_type (fits_in_64 ty) (iadd_imm (simm32_from_imm64 x) y))) + (value_reg (add ty (put_in_reg y) x))) + +;; Otherwise, put the immediate into a register. +(rule (lower (has_type (fits_in_64 ty) (iadd_imm (u64_from_imm64 x) y))) + (value_reg (add ty (put_in_reg y) (RegMemImm.Reg (imm ty x))))) + +;; `i128` + +;; When the immediate fits in a `RegMemImm.Imm`, use that. +(rule (lower (has_type $I128 (iadd_imm (simm32_from_imm64 x) y))) + (let ((y_regs ValueRegs (put_in_regs y)) + (y_lo Reg (value_regs_get y_regs 0)) + (y_hi Reg (value_regs_get y_regs 1))) + (with_flags (add_with_flags $I64 y_lo x) + (adc $I64 y_hi (RegMemImm.Imm 0))))) + +;; Otherwise, put the immediate into a register. +(rule (lower (has_type $I128 (iadd_imm (u64_from_imm64 x) y))) + (let ((y_regs ValueRegs (put_in_regs y)) + (y_lo Reg (value_regs_get y_regs 0)) + (y_hi Reg (value_regs_get y_regs 1)) + (x_lo Reg (imm $I64 x))) + (with_flags (add_with_flags $I64 y_lo (RegMemImm.Reg x_lo)) + (adc $I64 y_hi (RegMemImm.Imm 0))))) + +;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +;; Sub two registers. +(rule (lower (has_type (fits_in_64 ty) + (isub x y))) + (value_reg (sub ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) + +;; Sub a register and an immediate. +(rule (lower (has_type (fits_in_64 ty) + (isub x (simm32_from_value y)))) + (value_reg (sub ty (put_in_reg x) y))) + +;; Sub a register and memory. +(rule (lower (has_type (fits_in_64 ty) + (isub x (sinkable_load y)))) + (value_reg (sub ty + (put_in_reg x) + (sink_load y)))) + +;; SSE. + +(rule (lower (has_type (multi_lane 8 16) + (isub x y))) + (value_reg (psubb (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 16 8) + (isub x y))) + (value_reg (psubw (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 32 4) + (isub x y))) + (value_reg (psubd (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 64 2) + (isub x y))) + (value_reg (psubq (put_in_reg x) + (put_in_reg_mem y)))) + +;; `i128` +(rule (lower (has_type $I128 (isub x y))) + ;; Get the high/low registers for `x`. + (let ((x_regs ValueRegs (put_in_regs x)) + (x_lo Reg (value_regs_get x_regs 0)) + (x_hi Reg (value_regs_get x_regs 1))) + ;; Get the high/low registers for `y`. + (let ((y_regs ValueRegs (put_in_regs y)) + (y_lo Reg (value_regs_get y_regs 0)) + (y_hi Reg (value_regs_get y_regs 1))) + ;; Do a sub followed by an sub-with-borrow. + (with_flags (sub_with_flags $I64 x_lo (RegMemImm.Reg y_lo)) + (sbb $I64 x_hi (RegMemImm.Reg y_hi)))))) + +;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (multi_lane 8 16) + (ssub_sat x y))) + (value_reg (psubsb (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 16 8) + (ssub_sat x y))) + (value_reg (psubsw (put_in_reg x) + (put_in_reg_mem y)))) + +;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (multi_lane 8 16) + (usub_sat x y))) + (value_reg (psubusb (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 16 8) + (usub_sat x y))) + (value_reg (psubusw (put_in_reg x) + (put_in_reg_mem y)))) + +;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `{i,b}64` and smaller. + +;; And two registers. +(rule (lower (has_type (fits_in_64 ty) (band x y))) + (value_reg (m_and ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) + +;; And with a memory operand. + +(rule (lower (has_type (fits_in_64 ty) + (band x (sinkable_load y)))) + (value_reg (m_and ty + (put_in_reg x) + (sink_load y)))) + +(rule (lower (has_type (fits_in_64 ty) + (band (sinkable_load x) y))) + (value_reg (m_and ty + (put_in_reg y) + (sink_load x)))) + +;; And with an immediate. + +(rule (lower (has_type (fits_in_64 ty) + (band x (simm32_from_value y)))) + (value_reg (m_and ty + (put_in_reg x) + y))) + +(rule (lower (has_type (fits_in_64 ty) + (band (simm32_from_value x) y))) + (value_reg (m_and ty + (put_in_reg y) + x))) + +;; SSE. + +(rule (lower (has_type $F32X4 (band x y))) + (value_reg (andps (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type $F64X2 (band x y))) + (value_reg (andpd (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane _bits _lanes) + (band x y))) + (value_reg (pand (put_in_reg x) + (put_in_reg_mem y)))) + +;; `{i,b}128`. + +(rule (lower (has_type $I128 (band x y))) + (let ((x_regs ValueRegs (put_in_regs x)) + (x_lo Reg (value_regs_get x_regs 0)) + (x_hi Reg (value_regs_get x_regs 1)) + (y_regs ValueRegs (put_in_regs y)) + (y_lo Reg (value_regs_get y_regs 0)) + (y_hi Reg (value_regs_get y_regs 1))) + (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo)) + (m_and $I64 x_hi (RegMemImm.Reg y_hi))))) + +(rule (lower (has_type $B128 (band x y))) + ;; Booleans are always `0` or `1`, so we only need to do the `and` on the + ;; low half. The high half is always zero but, rather than generate a new + ;; zero, we just reuse `x`'s high half which is already zero. + (let ((x_regs ValueRegs (put_in_regs x)) + (x_lo Reg (value_regs_get x_regs 0)) + (x_hi Reg (value_regs_get x_regs 1)) + (y_lo Reg (lo_reg y))) + (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo)) + x_hi))) + +;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `{i,b}64` and smaller. + +;; Or two registers. +(rule (lower (has_type (fits_in_64 ty) (bor x y))) + (value_reg (or ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) + +;; Or with a memory operand. + +(rule (lower (has_type (fits_in_64 ty) + (bor x (sinkable_load y)))) + (value_reg (or ty + (put_in_reg x) + (sink_load y)))) + +(rule (lower (has_type (fits_in_64 ty) + (bor (sinkable_load x) y))) + (value_reg (or ty + (put_in_reg y) + (sink_load x)))) + +;; Or with an immediate. + +(rule (lower (has_type (fits_in_64 ty) + (bor x (simm32_from_value y)))) + (value_reg (or ty + (put_in_reg x) + y))) + +(rule (lower (has_type (fits_in_64 ty) + (bor (simm32_from_value x) y))) + (value_reg (or ty + (put_in_reg y) + x))) + +;; SSE. + +(rule (lower (has_type $F32X4 (bor x y))) + (value_reg (orps (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type $F64X2 (bor x y))) + (value_reg (orpd (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane _bits _lanes) + (bor x y))) + (value_reg (por (put_in_reg x) + (put_in_reg_mem y)))) + +;; `{i,b}128`. + +(decl or_i128 (ValueRegs ValueRegs) ValueRegs) +(rule (or_i128 x y) + (let ((x_lo Reg (value_regs_get x 0)) + (x_hi Reg (value_regs_get x 1)) + (y_lo Reg (value_regs_get y 0)) + (y_hi Reg (value_regs_get y 1))) + (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo)) + (or $I64 x_hi (RegMemImm.Reg y_hi))))) + +(rule (lower (has_type $I128 (bor x y))) + (or_i128 (put_in_regs x) (put_in_regs y))) + +(rule (lower (has_type $B128 (bor x y))) + ;; Booleans are always `0` or `1`, so we only need to do the `or` on the + ;; low half. The high half is always zero but, rather than generate a new + ;; zero, we just reuse `x`'s high half which is already zero. + (let ((x_regs ValueRegs (put_in_regs x)) + (x_lo Reg (value_regs_get x_regs 0)) + (x_hi Reg (value_regs_get x_regs 1)) + (y_lo Reg (lo_reg y))) + (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo)) + x_hi))) + +;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `{i,b}64` and smaller. + +;; Xor two registers. +(rule (lower (has_type (fits_in_64 ty) (bxor x y))) + (value_reg (xor ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) + +;; Xor with a memory operand. + +(rule (lower (has_type (fits_in_64 ty) + (bxor x (sinkable_load y)))) + (value_reg (xor ty + (put_in_reg x) + (sink_load y)))) + +(rule (lower (has_type (fits_in_64 ty) + (bxor (sinkable_load x) y))) + (value_reg (xor ty + (put_in_reg y) + (sink_load x)))) + +;; Xor with an immediate. + +(rule (lower (has_type (fits_in_64 ty) + (bxor x (simm32_from_value y)))) + (value_reg (xor ty + (put_in_reg x) + y))) + +(rule (lower (has_type (fits_in_64 ty) + (bxor (simm32_from_value x) y))) + (value_reg (xor ty + (put_in_reg y) + x))) + +;; SSE. + +(rule (lower (has_type $F32X4 (bxor x y))) + (value_reg (xorps (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type $F64X2 (bxor x y))) + (value_reg (xorpd (put_in_reg x) + (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane _bits _lanes) + (bxor x y))) + (value_reg (pxor (put_in_reg x) + (put_in_reg_mem y)))) + +;; `{i,b}128`. + +(rule (lower (has_type $I128 (bxor x y))) + (let ((x_regs ValueRegs (put_in_regs x)) + (x_lo Reg (value_regs_get x_regs 0)) + (x_hi Reg (value_regs_get x_regs 1)) + (y_regs ValueRegs (put_in_regs y)) + (y_lo Reg (value_regs_get y_regs 0)) + (y_hi Reg (value_regs_get y_regs 1))) + (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo)) + (xor $I64 x_hi (RegMemImm.Reg y_hi))))) + +(rule (lower (has_type $B128 (bxor x y))) + ;; Booleans are always `0` or `1`, so we only need to do the `xor` on the + ;; low half. The high half is always zero but, rather than generate a new + ;; zero, we just reuse `x`'s high half which is already zero. + (let ((x_regs ValueRegs (put_in_regs x)) + (x_lo Reg (value_regs_get x_regs 0)) + (x_hi Reg (value_regs_get x_regs 1)) + (y_lo Reg (lo_reg y))) + (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo)) + x_hi))) + +;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +(rule (lower (has_type (fits_in_64 ty) (ishl src amt))) + ;; NB: Only the low bits of `amt` matter since we logically mask the shift + ;; amount to the value's bit width. + (let ((amt_ Reg (lo_reg amt))) + (value_reg (shl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (fits_in_64 ty) (ishl src (imm8_from_value amt)))) + (value_reg (shl ty (put_in_reg src) amt))) + +;; `i128`. + +(decl shl_i128 (ValueRegs Reg) ValueRegs) +(rule (shl_i128 src amt) + ;; Unpack the registers that make up the 128-bit value being shifted. + (let ((src_lo Reg (value_regs_get src 0)) + (src_hi Reg (value_regs_get src 1)) + ;; Do two 64-bit shifts. + (lo_shifted Reg (shl $I64 src_lo (Imm8Reg.Reg amt))) + (hi_shifted Reg (shl $I64 src_hi (Imm8Reg.Reg amt))) + ;; `src_lo >> (64 - amt)` are the bits to carry over from the lo + ;; into the hi. + (carry Reg (shr $I64 src_lo (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt))))) + (zero Reg (imm $I64 0)) + ;; Nullify the carry if we are shifting in by a multiple of 128. + (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt) + (cmove $I64 (CC.Z) (RegMem.Reg zero) carry))) + ;; Add the carry into the high half. + (hi_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg hi_shifted)))) + ;; Combine the two shifted halves. However, if we are shifting by >= 64 + ;; (modulo 128), then the low bits are zero and the high bits are our + ;; low bits. + (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt) + (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted) zero) + (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted_) lo_shifted)))) + +(rule (lower (has_type $I128 (ishl src amt))) + ;; NB: Only the low bits of `amt` matter since we logically mask the shift + ;; amount to the value's bit width. + (let ((amt_ Reg (lo_reg amt))) + (shl_i128 (put_in_regs src) amt_))) + +;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +(rule (lower (has_type (fits_in_64 ty) (ushr src amt))) + (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Zero))) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; shift amount to the value's bit width. + (amt_ Reg (lo_reg amt))) + (value_reg (shr ty src_ (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (fits_in_64 ty) (ushr src (imm8_from_value amt)))) + (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Zero)))) + (value_reg (shr ty src_ amt)))) + +;; `i128`. + +(decl shr_i128 (ValueRegs Reg) ValueRegs) +(rule (shr_i128 src amt) + ;; Unpack the lo/hi halves of `src`. + (let ((src_lo Reg (value_regs_get src 0)) + (src_hi Reg (value_regs_get src 1)) + ;; Do a shift on each half. + (lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt))) + (hi_shifted Reg (shr $I64 src_hi (Imm8Reg.Reg amt))) + ;; `src_hi << (64 - amt)` are the bits to carry over from the hi + ;; into the lo. + (carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt))))) + ;; Nullify the carry if we are shifting by a multiple of 128. + (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt) + (cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry))) + ;; Add the carry bits into the lo. + (lo_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg lo_shifted)))) + ;; Combine the two shifted halves. However, if we are shifting by >= 64 + ;; (modulo 128), then the hi bits are zero and the lo bits are what + ;; would otherwise be our hi bits. + (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt) + (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted) + (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) (imm $I64 0))))) + +(rule (lower (has_type $I128 (ushr src amt))) + ;; NB: Only the low bits of `amt` matter since we logically mask the shift + ;; amount to the value's bit width. + (let ((amt_ Reg (lo_reg amt))) + (shr_i128 (put_in_regs src) amt_))) + +;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +(rule (lower (has_type (fits_in_64 ty) (rotl src amt))) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; shift amount to the value's bit width. + (let ((amt_ Reg (lo_reg amt))) + (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (fits_in_64 ty) (rotl src (imm8_from_value amt)))) + (value_reg (m_rotl ty (put_in_reg src) amt))) + +;; `i128`. + +(rule (lower (has_type $I128 (rotl src amt))) + (let ((src_ ValueRegs (put_in_regs src)) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; rotation amount to the value's bit width. + (amt_ Reg (lo_reg amt))) + (or_i128 (shl_i128 src_ amt_) + (shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_)))))) + +;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (multi_lane 8 16) + (avg_round x y))) + (value_reg (pavgb (put_in_reg x) (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 16 8) + (avg_round x y))) + (value_reg (pavgw (put_in_reg x) (put_in_reg_mem y)))) + +;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +;; Multiply two registers. +(rule (lower (has_type (fits_in_64 ty) (imul x y))) + (value_reg (mul ty + (put_in_reg x) + (RegMemImm.Reg (put_in_reg y))))) + +;; Multiply a register and an immediate. + +(rule (lower (has_type (fits_in_64 ty) + (imul x (simm32_from_value y)))) + (value_reg (mul ty (put_in_reg x) y))) + +(rule (lower (has_type (fits_in_64 ty) + (imul (simm32_from_value x) y))) + (value_reg (mul ty (put_in_reg y) x))) + +;; Multiply a register and a memory load. + +(rule (lower (has_type (fits_in_64 ty) + (imul x (sinkable_load y)))) + (value_reg (mul ty + (put_in_reg x) + (sink_load y)))) + +(rule (lower (has_type (fits_in_64 ty) + (imul (sinkable_load x) y))) + (value_reg (mul ty + (put_in_reg y) + (sink_load x)))) + +;; SSE. + +;; (No i8x16 multiply.) + +(rule (lower (has_type (multi_lane 16 8) (imul x y))) + (value_reg (pmullw (put_in_reg x) (put_in_reg_mem y)))) + +(rule (lower (has_type (multi_lane 32 4) (imul x y))) + (value_reg (pmulld (put_in_reg x) (put_in_reg_mem y)))) + +;; With AVX-512 we can implement `i64x2` multiplication with a single +;; instruction. +(rule (lower (has_type (and (avx512vl_enabled) + (avx512dq_enabled) + (multi_lane 64 2)) + (imul x y))) + (value_reg (vpmullq (put_in_reg_mem x) (put_in_reg y)))) + +;; Otherwise, for i64x2 multiplication we describe a lane A as being composed of +;; a 32-bit upper half "Ah" and a 32-bit lower half "Al". The 32-bit long hand +;; multiplication can then be written as: +;; +;; Ah Al +;; * Bh Bl +;; ----- +;; Al * Bl +;; + (Ah * Bl) << 32 +;; + (Al * Bh) << 32 +;; +;; So for each lane we will compute: +;; +;; A * B = (Al * Bl) + ((Ah * Bl) + (Al * Bh)) << 32 +;; +;; Note, the algorithm will use `pmuldq` which operates directly on the lower +;; 32-bit (`Al` or `Bl`) of a lane and writes the result to the full 64-bits of +;; the lane of the destination. For this reason we don't need shifts to isolate +;; the lower 32-bits, however, we will need to use shifts to isolate the high +;; 32-bits when doing calculations, i.e., `Ah == A >> 32`. +(rule (lower (has_type (multi_lane 64 2) + (imul a b))) + (let ((a0 Reg (put_in_reg a)) + (b0 Reg (put_in_reg b)) + ;; a_hi = A >> 32 + (a_hi Reg (psrlq a0 (RegMemImm.Imm 32))) + ;; ah_bl = Ah * Bl + (ah_bl Reg (pmuludq a_hi (RegMem.Reg b0))) + ;; b_hi = B >> 32 + (b_hi Reg (psrlq b0 (RegMemImm.Imm 32))) + ;; al_bh = Al * Bh + (al_bh Reg (pmuludq a0 (RegMem.Reg b_hi))) + ;; aa_bb = ah_bl + al_bh + (aa_bb Reg (paddq ah_bl (RegMem.Reg al_bh))) + ;; aa_bb_shifted = aa_bb << 32 + (aa_bb_shifted Reg (psllq aa_bb (RegMemImm.Imm 32))) + ;; al_bl = Al * Bl + (al_bl Reg (pmuludq a0 (RegMem.Reg b0)))) + ;; al_bl + aa_bb_shifted + (value_reg (paddq al_bl (RegMem.Reg aa_bb_shifted))))) + +;; Special case for `i16x8.extmul_high_i8x16_s`. +(rule (lower (has_type (multi_lane 16 8) + (imul (def_inst (swiden_high (and (value_type (multi_lane 8 16)) + x))) + (def_inst (swiden_high (and (value_type (multi_lane 8 16)) + y)))))) + (let ((x1 Reg (put_in_reg x)) + (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) + (x3 Reg (pmovsxbw (RegMem.Reg x2))) + (y1 Reg (put_in_reg y)) + (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) + (y3 Reg (pmovsxbw (RegMem.Reg y2)))) + (value_reg (pmullw x3 (RegMem.Reg y3))))) + +;; Special case for `i32x4.extmul_high_i16x8_s`. +(rule (lower (has_type (multi_lane 32 4) + (imul (def_inst (swiden_high (and (value_type (multi_lane 16 8)) + x))) + (def_inst (swiden_high (and (value_type (multi_lane 16 8)) + y)))))) + (let ((x2 Reg (put_in_reg x)) + (y2 Reg (put_in_reg y)) + (lo Reg (pmullw x2 (RegMem.Reg y2))) + (hi Reg (pmulhw x2 (RegMem.Reg y2)))) + (value_reg (punpckhwd lo (RegMem.Reg hi))))) + +;; Special case for `i64x2.extmul_high_i32x4_s`. +(rule (lower (has_type (multi_lane 64 2) + (imul (def_inst (swiden_high (and (value_type (multi_lane 32 4)) + x))) + (def_inst (swiden_high (and (value_type (multi_lane 32 4)) + y)))))) + (let ((x2 Reg (pshufd (put_in_reg_mem x) + 0xFA + (OperandSize.Size32))) + (y2 Reg (pshufd (put_in_reg_mem y) + 0xFA + (OperandSize.Size32)))) + (value_reg (pmuldq x2 (RegMem.Reg y2))))) + +;; Special case for `i16x8.extmul_low_i8x16_s`. +(rule (lower (has_type (multi_lane 16 8) + (imul (def_inst (swiden_low (and (value_type (multi_lane 8 16)) + x))) + (def_inst (swiden_low (and (value_type (multi_lane 8 16)) + y)))))) + (let ((x2 Reg (pmovsxbw (put_in_reg_mem x))) + (y2 Reg (pmovsxbw (put_in_reg_mem y)))) + (value_reg (pmullw x2 (RegMem.Reg y2))))) + +;; Special case for `i32x4.extmul_low_i16x8_s`. +(rule (lower (has_type (multi_lane 32 4) + (imul (def_inst (swiden_low (and (value_type (multi_lane 16 8)) + x))) + (def_inst (swiden_low (and (value_type (multi_lane 16 8)) + y)))))) + (let ((x2 Reg (put_in_reg x)) + (y2 Reg (put_in_reg y)) + (lo Reg (pmullw x2 (RegMem.Reg y2))) + (hi Reg (pmulhw x2 (RegMem.Reg y2)))) + (value_reg (punpcklwd lo (RegMem.Reg hi))))) + +;; Special case for `i64x2.extmul_low_i32x4_s`. +(rule (lower (has_type (multi_lane 64 2) + (imul (def_inst (swiden_low (and (value_type (multi_lane 32 4)) + x))) + (def_inst (swiden_low (and (value_type (multi_lane 32 4)) + y)))))) + (let ((x2 Reg (pshufd (put_in_reg_mem x) + 0x50 + (OperandSize.Size32))) + (y2 Reg (pshufd (put_in_reg_mem y) + 0x50 + (OperandSize.Size32)))) + (value_reg (pmuldq x2 (RegMem.Reg y2))))) + +;; Special case for `i16x8.extmul_high_i8x16_u`. +(rule (lower (has_type (multi_lane 16 8) + (imul (def_inst (uwiden_high (and (value_type (multi_lane 8 16)) + x))) + (def_inst (uwiden_high (and (value_type (multi_lane 8 16)) + y)))))) + (let ((x1 Reg (put_in_reg x)) + (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32))) + (x3 Reg (pmovzxbw (RegMem.Reg x2))) + (y1 Reg (put_in_reg y)) + (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32))) + (y3 Reg (pmovzxbw (RegMem.Reg y2)))) + (value_reg (pmullw x3 (RegMem.Reg y3))))) + +;; Special case for `i32x4.extmul_high_i16x8_u`. +(rule (lower (has_type (multi_lane 32 4) + (imul (def_inst (uwiden_high (and (value_type (multi_lane 16 8)) + x))) + (def_inst (uwiden_high (and (value_type (multi_lane 16 8)) + y)))))) + (let ((x2 Reg (put_in_reg x)) + (y2 Reg (put_in_reg y)) + (lo Reg (pmullw x2 (RegMem.Reg y2))) + (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) + (value_reg (punpckhwd lo (RegMem.Reg hi))))) + +;; Special case for `i64x2.extmul_high_i32x4_u`. +(rule (lower (has_type (multi_lane 64 2) + (imul (def_inst (uwiden_high (and (value_type (multi_lane 32 4)) + x))) + (def_inst (uwiden_high (and (value_type (multi_lane 32 4)) + y)))))) + (let ((x2 Reg (pshufd (put_in_reg_mem x) + 0xFA + (OperandSize.Size32))) + (y2 Reg (pshufd (put_in_reg_mem y) + 0xFA + (OperandSize.Size32)))) + (value_reg (pmuludq x2 (RegMem.Reg y2))))) + +;; Special case for `i16x8.extmul_low_i8x16_u`. +(rule (lower (has_type (multi_lane 16 8) + (imul (def_inst (uwiden_low (and (value_type (multi_lane 8 16)) + x))) + (def_inst (uwiden_low (and (value_type (multi_lane 8 16)) + y)))))) + (let ((x2 Reg (pmovzxbw (put_in_reg_mem x))) + (y2 Reg (pmovzxbw (put_in_reg_mem y)))) + (value_reg (pmullw x2 (RegMem.Reg y2))))) + +;; Special case for `i32x4.extmul_low_i16x8_u`. +(rule (lower (has_type (multi_lane 32 4) + (imul (def_inst (uwiden_low (and (value_type (multi_lane 16 8)) + x))) + (def_inst (uwiden_low (and (value_type (multi_lane 16 8)) + y)))))) + (let ((x2 Reg (put_in_reg x)) + (y2 Reg (put_in_reg y)) + (lo Reg (pmullw x2 (RegMem.Reg y2))) + (hi Reg (pmulhuw x2 (RegMem.Reg y2)))) + (value_reg (punpcklwd lo (RegMem.Reg hi))))) + +;; Special case for `i64x2.extmul_low_i32x4_u`. +(rule (lower (has_type (multi_lane 64 2) + (imul (def_inst (uwiden_low (and (value_type (multi_lane 32 4)) + x))) + (def_inst (uwiden_low (and (value_type (multi_lane 32 4)) + y)))))) + (let ((x2 Reg (pshufd (put_in_reg_mem x) + 0x50 + (OperandSize.Size32))) + (y2 Reg (pshufd (put_in_reg_mem y) + 0x50 + (OperandSize.Size32)))) + (value_reg (pmuludq x2 (RegMem.Reg y2))))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index c67dab3dde..65e94e396a 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1,5 +1,8 @@ //! Lowering rules for X64. +// ISLE integration glue. +mod isle; + use crate::data_value::DataValue; use crate::ir::{ condcodes::{CondCode, FloatCC, IntCC}, @@ -1497,20 +1500,15 @@ fn lower_insn_to_regs>( None }; - match op { - Opcode::Iconst | Opcode::Bconst | Opcode::Null => { - let value = ctx - .get_constant(insn) - .expect("constant value for iconst et al"); - let dst = get_output_reg(ctx, outputs[0]); - for inst in Inst::gen_constant(dst, value as u128, ty.unwrap(), |ty| { - ctx.alloc_tmp(ty).only_reg().unwrap() - }) { - ctx.emit(inst); - } - } + if let Ok(()) = isle::lower(ctx, isa_flags, &outputs, insn) { + return Ok(()); + } - Opcode::Iadd + match op { + Opcode::Iconst + | Opcode::Bconst + | Opcode::Null + | Opcode::Iadd | Opcode::IaddIfcout | Opcode::SaddSat | Opcode::UaddSat @@ -1521,149 +1519,11 @@ fn lower_insn_to_regs>( | Opcode::Band | Opcode::Bor | Opcode::Bxor => { - let ty = ty.unwrap(); - if ty.lane_count() > 1 { - let sse_op = match op { - Opcode::Iadd => match ty { - types::I8X16 => SseOpcode::Paddb, - types::I16X8 => SseOpcode::Paddw, - types::I32X4 => SseOpcode::Paddd, - types::I64X2 => SseOpcode::Paddq, - _ => panic!("Unsupported type for packed iadd instruction: {}", ty), - }, - Opcode::SaddSat => match ty { - types::I8X16 => SseOpcode::Paddsb, - types::I16X8 => SseOpcode::Paddsw, - _ => panic!("Unsupported type for packed sadd_sat instruction: {}", ty), - }, - Opcode::UaddSat => match ty { - types::I8X16 => SseOpcode::Paddusb, - types::I16X8 => SseOpcode::Paddusw, - _ => panic!("Unsupported type for packed uadd_sat instruction: {}", ty), - }, - Opcode::Isub => match ty { - types::I8X16 => SseOpcode::Psubb, - types::I16X8 => SseOpcode::Psubw, - types::I32X4 => SseOpcode::Psubd, - types::I64X2 => SseOpcode::Psubq, - _ => panic!("Unsupported type for packed isub instruction: {}", ty), - }, - Opcode::SsubSat => match ty { - types::I8X16 => SseOpcode::Psubsb, - types::I16X8 => SseOpcode::Psubsw, - _ => panic!("Unsupported type for packed ssub_sat instruction: {}", ty), - }, - Opcode::UsubSat => match ty { - types::I8X16 => SseOpcode::Psubusb, - types::I16X8 => SseOpcode::Psubusw, - _ => panic!("Unsupported type for packed usub_sat instruction: {}", ty), - }, - Opcode::AvgRound => match ty { - types::I8X16 => SseOpcode::Pavgb, - types::I16X8 => SseOpcode::Pavgw, - _ => panic!("Unsupported type for packed avg_round instruction: {}", ty), - }, - Opcode::Band => match ty { - types::F32X4 => SseOpcode::Andps, - types::F64X2 => SseOpcode::Andpd, - _ => SseOpcode::Pand, - }, - Opcode::Bor => match ty { - types::F32X4 => SseOpcode::Orps, - types::F64X2 => SseOpcode::Orpd, - _ => SseOpcode::Por, - }, - Opcode::Bxor => match ty { - types::F32X4 => SseOpcode::Xorps, - types::F64X2 => SseOpcode::Xorpd, - _ => SseOpcode::Pxor, - }, - _ => panic!("Unsupported packed instruction: {}", op), - }; - let lhs = put_input_in_reg(ctx, inputs[0]); - let rhs = input_to_reg_mem(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - // Move the `lhs` to the same register as `dst`. - ctx.emit(Inst::gen_move(dst, lhs, ty)); - ctx.emit(Inst::xmm_rm_r(sse_op, rhs, dst)); - } else if ty == types::I128 || ty == types::B128 { - let alu_ops = match op { - Opcode::Iadd => (AluRmiROpcode::Add, AluRmiROpcode::Adc), - Opcode::Isub => (AluRmiROpcode::Sub, AluRmiROpcode::Sbb), - Opcode::Band => (AluRmiROpcode::And, AluRmiROpcode::And), - Opcode::Bor => (AluRmiROpcode::Or, AluRmiROpcode::Or), - Opcode::Bxor => (AluRmiROpcode::Xor, AluRmiROpcode::Xor), - _ => panic!("Unsupported opcode with 128-bit integers: {:?}", op), - }; - let lhs = put_input_in_regs(ctx, inputs[0]); - let rhs = put_input_in_regs(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]); - assert_eq!(lhs.len(), 2); - assert_eq!(rhs.len(), 2); - assert_eq!(dst.len(), 2); - - // For add, sub, and, or, xor: just do ops on lower then upper - // half. Carry-flag propagation is implicit (add/adc, sub/sbb). - ctx.emit(Inst::gen_move(dst.regs()[0], lhs.regs()[0], types::I64)); - ctx.emit(Inst::gen_move(dst.regs()[1], lhs.regs()[1], types::I64)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - alu_ops.0, - RegMemImm::reg(rhs.regs()[0]), - dst.regs()[0], - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - alu_ops.1, - RegMemImm::reg(rhs.regs()[1]), - dst.regs()[1], - )); - } else { - let size = if ty == types::I64 { - OperandSize::Size64 - } else { - OperandSize::Size32 - }; - let alu_op = match op { - Opcode::Iadd | Opcode::IaddIfcout => AluRmiROpcode::Add, - Opcode::Isub => AluRmiROpcode::Sub, - Opcode::Band => AluRmiROpcode::And, - Opcode::Bor => AluRmiROpcode::Or, - Opcode::Bxor => AluRmiROpcode::Xor, - _ => unreachable!(), - }; - - let (lhs, rhs) = match op { - Opcode::Iadd - | Opcode::IaddIfcout - | Opcode::Band - | Opcode::Bor - | Opcode::Bxor => { - // For commutative operations, try to commute operands if one is an - // immediate or direct memory reference. Do so by converting LHS to RMI; if - // reg, then always convert RHS to RMI; else, use LHS as RMI and convert - // RHS to reg. - let lhs = input_to_reg_mem_imm(ctx, inputs[0]); - if let RegMemImm::Reg { reg: lhs_reg } = lhs { - let rhs = input_to_reg_mem_imm(ctx, inputs[1]); - (lhs_reg, rhs) - } else { - let rhs_reg = put_input_in_reg(ctx, inputs[1]); - (rhs_reg, lhs) - } - } - Opcode::Isub => ( - put_input_in_reg(ctx, inputs[0]), - input_to_reg_mem_imm(ctx, inputs[1]), - ), - _ => unreachable!(), - }; - - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::mov_r_r(OperandSize::Size64, lhs, dst)); - ctx.emit(Inst::alu_rmi_r(size, alu_op, rhs, dst)); - } + unreachable!( + "implemented in ISLE: inst = `{}`, type = `{:?}`", + ctx.dfg().display_inst(insn), + ty + ); } Opcode::Imul => { @@ -1681,469 +1541,9 @@ fn lower_insn_to_regs>( Opcode::UwidenLow, ], ) { - // Optimized ext_mul_* lowerings are based on optimized lowerings - // here: https://github.com/WebAssembly/simd/pull/376 - if let Some(swiden0_high) = matches_input(ctx, inputs[0], Opcode::SwidenHigh) { - if let Some(swiden1_high) = matches_input(ctx, inputs[1], Opcode::SwidenHigh) { - let swiden_input = &[ - InsnInput { - insn: swiden0_high, - input: 0, - }, - InsnInput { - insn: swiden1_high, - input: 0, - }, - ]; - let input0_ty = ctx.input_ty(swiden0_high, 0); - let input1_ty = ctx.input_ty(swiden1_high, 0); - let output_ty = ctx.output_ty(insn, 0); - let lhs = put_input_in_reg(ctx, swiden_input[0]); - let rhs = put_input_in_reg(ctx, swiden_input[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - match (input0_ty, input1_ty, output_ty) { - (types::I8X16, types::I8X16, types::I16X8) => { - // i16x8.extmul_high_i8x16_s - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Palignr, - RegMem::reg(lhs), - Writable::from_reg(lhs), - 8, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_mov( - SseOpcode::Pmovsxbw, - RegMem::reg(lhs), - Writable::from_reg(lhs), - )); - - ctx.emit(Inst::gen_move(dst, rhs, output_ty)); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Palignr, - RegMem::reg(rhs), - dst, - 8, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_mov( - SseOpcode::Pmovsxbw, - RegMem::reg(dst.to_reg()), - dst, - )); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(lhs), dst)); - } - (types::I16X8, types::I16X8, types::I32X4) => { - // i32x4.extmul_high_i16x8_s - ctx.emit(Inst::gen_move(dst, lhs, input0_ty)); - let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap(); - ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty)); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmulhw, - RegMem::reg(rhs), - tmp_reg, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Punpckhwd, - RegMem::from(tmp_reg), - dst, - )); - } - (types::I32X4, types::I32X4, types::I64X2) => { - // i64x2.extmul_high_i32x4_s - let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap(); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(lhs), - tmp_reg, - 0xFA, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(rhs), - dst, - 0xFA, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmuldq, - RegMem::reg(tmp_reg.to_reg()), - dst, - )); - } - // Note swiden_high only allows types: I8X16, I16X8, and I32X4 - _ => panic!("Unsupported extmul_low_signed type"), - } - } - } else if let Some(swiden0_low) = matches_input(ctx, inputs[0], Opcode::SwidenLow) { - if let Some(swiden1_low) = matches_input(ctx, inputs[1], Opcode::SwidenLow) { - let swiden_input = &[ - InsnInput { - insn: swiden0_low, - input: 0, - }, - InsnInput { - insn: swiden1_low, - input: 0, - }, - ]; - let input0_ty = ctx.input_ty(swiden0_low, 0); - let input1_ty = ctx.input_ty(swiden1_low, 0); - let output_ty = ctx.output_ty(insn, 0); - let lhs = put_input_in_reg(ctx, swiden_input[0]); - let rhs = put_input_in_reg(ctx, swiden_input[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - match (input0_ty, input1_ty, output_ty) { - (types::I8X16, types::I8X16, types::I16X8) => { - // i32x4.extmul_low_i8x16_s - let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap(); - ctx.emit(Inst::xmm_mov( - SseOpcode::Pmovsxbw, - RegMem::reg(lhs), - tmp_reg, - )); - ctx.emit(Inst::xmm_mov(SseOpcode::Pmovsxbw, RegMem::reg(rhs), dst)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmullw, - RegMem::reg(tmp_reg.to_reg()), - dst, - )); - } - (types::I16X8, types::I16X8, types::I32X4) => { - // i32x4.extmul_low_i16x8_s - ctx.emit(Inst::gen_move(dst, lhs, input0_ty)); - let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap(); - ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty)); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmulhw, - RegMem::reg(rhs), - tmp_reg, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Punpcklwd, - RegMem::from(tmp_reg), - dst, - )); - } - (types::I32X4, types::I32X4, types::I64X2) => { - // i64x2.extmul_low_i32x4_s - let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap(); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(lhs), - tmp_reg, - 0x50, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(rhs), - dst, - 0x50, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmuldq, - RegMem::reg(tmp_reg.to_reg()), - dst, - )); - } - // Note swiden_low only allows types: I8X16, I16X8, and I32X4 - _ => panic!("Unsupported extmul_low_signed type"), - } - } - } else if let Some(uwiden0_high) = matches_input(ctx, inputs[0], Opcode::UwidenHigh) - { - if let Some(uwiden1_high) = matches_input(ctx, inputs[1], Opcode::UwidenHigh) { - let uwiden_input = &[ - InsnInput { - insn: uwiden0_high, - input: 0, - }, - InsnInput { - insn: uwiden1_high, - input: 0, - }, - ]; - let input0_ty = ctx.input_ty(uwiden0_high, 0); - let input1_ty = ctx.input_ty(uwiden1_high, 0); - let output_ty = ctx.output_ty(insn, 0); - let lhs = put_input_in_reg(ctx, uwiden_input[0]); - let rhs = put_input_in_reg(ctx, uwiden_input[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - match (input0_ty, input1_ty, output_ty) { - (types::I8X16, types::I8X16, types::I16X8) => { - // i16x8.extmul_high_i8x16_u - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Palignr, - RegMem::reg(lhs), - Writable::from_reg(lhs), - 8, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_mov( - SseOpcode::Pmovzxbw, - RegMem::reg(lhs), - Writable::from_reg(lhs), - )); - ctx.emit(Inst::gen_move(dst, rhs, output_ty)); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Palignr, - RegMem::reg(rhs), - dst, - 8, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_mov( - SseOpcode::Pmovzxbw, - RegMem::reg(dst.to_reg()), - dst, - )); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(lhs), dst)); - } - (types::I16X8, types::I16X8, types::I32X4) => { - // i32x4.extmul_high_i16x8_u - ctx.emit(Inst::gen_move(dst, lhs, input0_ty)); - let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap(); - ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty)); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmulhuw, - RegMem::reg(rhs), - tmp_reg, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Punpckhwd, - RegMem::from(tmp_reg), - dst, - )); - } - (types::I32X4, types::I32X4, types::I64X2) => { - // i64x2.extmul_high_i32x4_u - let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap(); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(lhs), - tmp_reg, - 0xFA, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(rhs), - dst, - 0xFA, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmuludq, - RegMem::reg(tmp_reg.to_reg()), - dst, - )); - } - // Note uwiden_high only allows types: I8X16, I16X8, and I32X4 - _ => panic!("Unsupported extmul_high_unsigned type"), - } - } - } else if let Some(uwiden0_low) = matches_input(ctx, inputs[0], Opcode::UwidenLow) { - if let Some(uwiden1_low) = matches_input(ctx, inputs[1], Opcode::UwidenLow) { - let uwiden_input = &[ - InsnInput { - insn: uwiden0_low, - input: 0, - }, - InsnInput { - insn: uwiden1_low, - input: 0, - }, - ]; - - let input0_ty = ctx.input_ty(uwiden0_low, 0); - let input1_ty = ctx.input_ty(uwiden1_low, 0); - let output_ty = ctx.output_ty(insn, 0); - let lhs = put_input_in_reg(ctx, uwiden_input[0]); - let rhs = put_input_in_reg(ctx, uwiden_input[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - match (input0_ty, input1_ty, output_ty) { - (types::I8X16, types::I8X16, types::I16X8) => { - // i16x8.extmul_low_i8x16_u - let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap(); - ctx.emit(Inst::xmm_mov( - SseOpcode::Pmovzxbw, - RegMem::reg(lhs), - tmp_reg, - )); - ctx.emit(Inst::xmm_mov(SseOpcode::Pmovzxbw, RegMem::reg(rhs), dst)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmullw, - RegMem::reg(tmp_reg.to_reg()), - dst, - )); - } - (types::I16X8, types::I16X8, types::I32X4) => { - // i32x4.extmul_low_i16x8_u - ctx.emit(Inst::gen_move(dst, lhs, input0_ty)); - let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap(); - ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty)); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmulhuw, - RegMem::reg(rhs), - tmp_reg, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Punpcklwd, - RegMem::from(tmp_reg), - dst, - )); - } - (types::I32X4, types::I32X4, types::I64X2) => { - // i64x2.extmul_low_i32x4_u - let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap(); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(lhs), - tmp_reg, - 0x50, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r_imm( - SseOpcode::Pshufd, - RegMem::reg(rhs), - dst, - 0x50, - OperandSize::Size32, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmuludq, - RegMem::reg(tmp_reg.to_reg()), - dst, - )); - } - // Note uwiden_low only allows types: I8X16, I16X8, and I32X4 - _ => panic!("Unsupported extmul_low_unsigned type"), - } - } - } else { - panic!("Unsupported imul operation for type: {}", ty); - } + unreachable!("implemented in ISLE: {}", ctx.dfg().display_inst(insn)); } else if ty == types::I64X2 { - // Eventually one of these should be `input_to_reg_mem` (TODO). - let lhs = put_input_in_reg(ctx, inputs[0]); - let rhs = put_input_in_reg(ctx, inputs[1]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - if isa_flags.use_avx512vl_simd() && isa_flags.use_avx512dq_simd() { - // With the right AVX512 features (VL + DQ) this operation - // can lower to a single operation. - ctx.emit(Inst::xmm_rm_r_evex( - Avx512Opcode::Vpmullq, - RegMem::reg(rhs), - lhs, - dst, - )); - } else { - // Otherwise, for I64X2 multiplication we describe a lane A as being - // composed of a 32-bit upper half "Ah" and a 32-bit lower half - // "Al". The 32-bit long hand multiplication can then be written - // as: - // Ah Al - // * Bh Bl - // ----- - // Al * Bl - // + (Ah * Bl) << 32 - // + (Al * Bh) << 32 - // - // So for each lane we will compute: - // A * B = (Al * Bl) + ((Ah * Bl) + (Al * Bh)) << 32 - // - // Note, the algorithm will use pmuldq which operates directly - // on the lower 32-bit (Al or Bl) of a lane and writes the - // result to the full 64-bits of the lane of the destination. - // For this reason we don't need shifts to isolate the lower - // 32-bits, however, we will need to use shifts to isolate the - // high 32-bits when doing calculations, i.e., Ah == A >> 32. - // - // The full sequence then is as follows: - // A' = A - // A' = A' >> 32 - // A' = Ah' * Bl - // B' = B - // B' = B' >> 32 - // B' = Bh' * Al - // B' = B' + A' - // B' = B' << 32 - // A' = A - // A' = Al' * Bl - // A' = A' + B' - // dst = A' - - // A' = A - let rhs_1 = ctx.alloc_tmp(types::I64X2).only_reg().unwrap(); - ctx.emit(Inst::gen_move(rhs_1, rhs, ty)); - - // A' = A' >> 32 - // A' = Ah' * Bl - ctx.emit(Inst::xmm_rmi_reg( - SseOpcode::Psrlq, - RegMemImm::imm(32), - rhs_1, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmuludq, - RegMem::reg(lhs.clone()), - rhs_1, - )); - - // B' = B - let lhs_1 = ctx.alloc_tmp(types::I64X2).only_reg().unwrap(); - ctx.emit(Inst::gen_move(lhs_1, lhs, ty)); - - // B' = B' >> 32 - // B' = Bh' * Al - ctx.emit(Inst::xmm_rmi_reg( - SseOpcode::Psrlq, - RegMemImm::imm(32), - lhs_1, - )); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmuludq, RegMem::reg(rhs), lhs_1)); - - // B' = B' + A' - // B' = B' << 32 - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Paddq, - RegMem::reg(rhs_1.to_reg()), - lhs_1, - )); - ctx.emit(Inst::xmm_rmi_reg( - SseOpcode::Psllq, - RegMemImm::imm(32), - lhs_1, - )); - - // A' = A - // A' = Al' * Bl - // A' = A' + B' - // dst = A' - ctx.emit(Inst::gen_move(rhs_1, rhs, ty)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pmuludq, - RegMem::reg(lhs.clone()), - rhs_1, - )); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Paddq, - RegMem::reg(lhs_1.to_reg()), - rhs_1, - )); - ctx.emit(Inst::gen_move(dst, rhs_1.to_reg(), ty)); - } + unreachable!("implemented in ISLE: {}", ctx.dfg().display_inst(insn)); } else if ty.lane_count() > 1 { // Emit single instruction lowerings for the remaining vector // multiplications. @@ -2228,29 +1628,7 @@ fn lower_insn_to_regs>( dst.regs()[1], )); } else { - let size = if ty == types::I64 { - OperandSize::Size64 - } else { - OperandSize::Size32 - }; - let alu_op = AluRmiROpcode::Mul; - - // For commutative operations, try to commute operands if one is - // an immediate or direct memory reference. Do so by converting - // LHS to RMI; if reg, then always convert RHS to RMI; else, use - // LHS as RMI and convert RHS to reg. - let lhs = input_to_reg_mem_imm(ctx, inputs[0]); - let (lhs, rhs) = if let RegMemImm::Reg { reg: lhs_reg } = lhs { - let rhs = input_to_reg_mem_imm(ctx, inputs[1]); - (lhs_reg, rhs) - } else { - let rhs_reg = put_input_in_reg(ctx, inputs[1]); - (rhs_reg, lhs) - }; - - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::mov_r_r(OperandSize::Size64, lhs, dst)); - ctx.emit(Inst::alu_rmi_r(size, alu_op, rhs, dst)); + unreachable!("implemented in ISLE") } } @@ -5801,7 +5179,14 @@ fn lower_insn_to_regs>( // Now the AtomicRmwSeq (pseudo-) instruction itself let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap()); - ctx.emit(Inst::AtomicRmwSeq { ty: ty_access, op }); + ctx.emit(Inst::AtomicRmwSeq { + ty: ty_access, + op, + address: regs::r9(), + operand: regs::r10(), + temp: Writable::from_reg(regs::r11()), + dst_old: Writable::from_reg(regs::rax()), + }); // And finally, copy the preordained AtomicRmwSeq output reg to its destination. ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64)); @@ -5827,8 +5212,10 @@ fn lower_insn_to_regs>( )); ctx.emit(Inst::LockCmpxchg { ty: ty_access, - src: replacement, - dst: addr.into(), + mem: addr.into(), + replacement, + expected: regs::rax(), + dst_old: Writable::from_reg(regs::rax()), }); // And finally, copy the old value at the location to its destination reg. ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64)); diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs new file mode 100644 index 0000000000..2a4c960301 --- /dev/null +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -0,0 +1,428 @@ +//! ISLE integration glue code for x64 lowering. + +// Pull in the ISLE generated code. +mod generated_code; + +// Types that the generated ISLE code uses via `use super::*`. +use super::{ + is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm, + Writable, +}; +use crate::isa::x64::settings as x64_settings; +use crate::{ + ir::{immediates::*, types::*, Inst, InstructionData, Opcode, Value, ValueList}, + isa::x64::inst::{ + args::{ + Amode, Avx512Opcode, CmpOpcode, ExtMode, Imm8Reg, RegMem, ShiftKind, SseOpcode, CC, + }, + x64_map_regs, RegMapper, + }, + machinst::{get_output_reg, InsnInput, InsnOutput, LowerCtx}, +}; +use smallvec::SmallVec; +use std::convert::TryFrom; + +type Unit = (); +type ValueSlice<'a> = &'a [Value]; +type ValueArray2 = [Value; 2]; +type ValueArray3 = [Value; 3]; +type WritableReg = Writable; +type ValueRegs = crate::machinst::ValueRegs; + +pub struct SinkableLoad { + inst: Inst, + addr_input: InsnInput, + offset: i32, +} + +#[derive(Default)] +struct RegRenamer { + // Map of `(old, new)` register names. Use a `SmallVec` because we typically + // only have one or two renamings. + renames: SmallVec<[(Reg, Reg); 2]>, +} + +impl RegRenamer { + fn add_rename(&mut self, old: Reg, new: Reg) { + self.renames.push((old, new)); + } + + fn get_rename(&self, reg: Reg) -> Option { + self.renames + .iter() + .find(|(old, _)| reg == *old) + .map(|(_, new)| *new) + } +} + +impl RegMapper for RegRenamer { + fn get_use(&self, reg: Reg) -> Option { + self.get_rename(reg) + } + + fn get_def(&self, reg: Reg) -> Option { + self.get_rename(reg) + } + + fn get_mod(&self, reg: Reg) -> Option { + self.get_rename(reg) + } +} + +/// The main entry point for lowering with ISLE. +pub(crate) fn lower( + lower_ctx: &mut C, + isa_flags: &x64_settings::Flags, + outputs: &[InsnOutput], + inst: Inst, +) -> Result<(), ()> +where + C: LowerCtx, +{ + // TODO: reuse the ISLE context across lowerings so we can reuse its + // internal heap allocations. + let mut isle_ctx = IsleContext::new(lower_ctx, isa_flags); + + let temp_regs = generated_code::constructor_lower(&mut isle_ctx, inst).ok_or(())?; + let mut temp_regs = temp_regs.regs().iter(); + + // The ISLE generated code emits its own registers to define the + // instruction's lowered values in. We rename those registers to the + // registers they were assigned when their value was used as an operand in + // earlier lowerings. + let mut renamer = RegRenamer::default(); + for output in outputs { + let dsts = get_output_reg(isle_ctx.lower_ctx, *output); + for (temp, dst) in temp_regs.by_ref().zip(dsts.regs()) { + renamer.add_rename(*temp, dst.to_reg()); + } + } + + for mut inst in isle_ctx.into_emitted_insts() { + x64_map_regs(&mut inst, &renamer); + lower_ctx.emit(inst); + } + + Ok(()) +} + +pub struct IsleContext<'a, C> { + lower_ctx: &'a mut C, + isa_flags: &'a x64_settings::Flags, + emitted_insts: SmallVec<[MInst; 6]>, +} + +impl<'a, C> IsleContext<'a, C> { + pub fn new(lower_ctx: &'a mut C, isa_flags: &'a x64_settings::Flags) -> Self { + IsleContext { + lower_ctx, + isa_flags, + emitted_insts: SmallVec::new(), + } + } + + pub fn into_emitted_insts(self) -> SmallVec<[MInst; 6]> { + self.emitted_insts + } +} + +impl<'a, C> generated_code::Context for IsleContext<'a, C> +where + C: LowerCtx, +{ + #[inline] + fn unpack_value_array_2(&mut self, arr: &ValueArray2) -> (Value, Value) { + let [a, b] = *arr; + (a, b) + } + + #[inline] + fn pack_value_array_2(&mut self, a: Value, b: Value) -> ValueArray2 { + [a, b] + } + + #[inline] + fn unpack_value_array_3(&mut self, arr: &ValueArray3) -> (Value, Value, Value) { + let [a, b, c] = *arr; + (a, b, c) + } + + #[inline] + fn pack_value_array_3(&mut self, a: Value, b: Value, c: Value) -> ValueArray3 { + [a, b, c] + } + + #[inline] + fn value_reg(&mut self, reg: Reg) -> ValueRegs { + ValueRegs::one(reg) + } + + #[inline] + fn value_regs(&mut self, r1: Reg, r2: Reg) -> ValueRegs { + ValueRegs::two(r1, r2) + } + + #[inline] + fn temp_writable_reg(&mut self, ty: Type) -> WritableReg { + let value_regs = self.lower_ctx.alloc_tmp(ty); + value_regs.only_reg().unwrap() + } + + #[inline] + fn invalid_reg(&mut self) -> Reg { + Reg::invalid() + } + + #[inline] + fn put_in_reg(&mut self, val: Value) -> Reg { + self.lower_ctx.put_value_in_regs(val).only_reg().unwrap() + } + + #[inline] + fn put_in_regs(&mut self, val: Value) -> ValueRegs { + self.lower_ctx.put_value_in_regs(val) + } + + #[inline] + fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg { + regs.regs()[i] + } + + #[inline] + fn u8_as_u64(&mut self, x: u8) -> u64 { + x.into() + } + + #[inline] + fn u16_as_u64(&mut self, x: u16) -> u64 { + x.into() + } + + #[inline] + fn u32_as_u64(&mut self, x: u32) -> u64 { + x.into() + } + + #[inline] + fn ty_bits(&mut self, ty: Type) -> u16 { + ty.bits() + } + + #[inline] + fn fits_in_64(&mut self, ty: Type) -> Option { + if ty.bits() <= 64 { + Some(ty) + } else { + None + } + } + + #[inline] + fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { + list.as_slice(&self.lower_ctx.dfg().value_lists) + } + + #[inline] + fn unwrap_head_value_list_1(&mut self, list: ValueList) -> (Value, ValueSlice) { + match self.value_list_slice(list) { + [head, tail @ ..] => (*head, tail), + _ => out_of_line_panic("`unwrap_head_value_list_1` on empty `ValueList`"), + } + } + + #[inline] + fn unwrap_head_value_list_2(&mut self, list: ValueList) -> (Value, Value, ValueSlice) { + match self.value_list_slice(list) { + [head1, head2, tail @ ..] => (*head1, *head2, tail), + _ => out_of_line_panic( + "`unwrap_head_value_list_2` on list without at least two elements", + ), + } + } + + #[inline] + fn writable_reg_to_reg(&mut self, r: WritableReg) -> Reg { + r.to_reg() + } + + #[inline] + fn u64_from_imm64(&mut self, imm: Imm64) -> u64 { + imm.bits() as u64 + } + + #[inline] + fn inst_results(&mut self, inst: Inst) -> ValueSlice { + self.lower_ctx.dfg().inst_results(inst) + } + + #[inline] + fn first_result(&mut self, inst: Inst) -> Option { + self.lower_ctx.dfg().inst_results(inst).first().copied() + } + + #[inline] + fn inst_data(&mut self, inst: Inst) -> InstructionData { + self.lower_ctx.dfg()[inst].clone() + } + + #[inline] + fn value_type(&mut self, val: Value) -> Type { + self.lower_ctx.dfg().value_type(val) + } + + #[inline] + fn multi_lane(&mut self, ty: Type) -> Option<(u8, u16)> { + if ty.lane_count() > 1 { + Some((ty.lane_bits(), ty.lane_count())) + } else { + None + } + } + + #[inline] + fn def_inst(&mut self, val: Value) -> Option { + self.lower_ctx.dfg().value_def(val).inst() + } + + #[inline] + fn operand_size_of_type(&mut self, ty: Type) -> OperandSize { + if ty.bits() == 64 { + OperandSize::Size64 + } else { + OperandSize::Size32 + } + } + + fn put_in_reg_mem(&mut self, val: Value) -> RegMem { + let inputs = self.lower_ctx.get_value_as_source_or_const(val); + + if let Some(c) = inputs.constant { + // Generate constants fresh at each use to minimize long-range + // register pressure. + let ty = self.value_type(val); + return RegMem::reg(generated_code::constructor_imm(self, ty, c).unwrap()); + } + + if let Some((src_insn, 0)) = inputs.inst { + if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) { + self.lower_ctx.sink_inst(src_insn); + let amode = lower_to_amode(self.lower_ctx, addr_input, offset); + return RegMem::mem(amode); + } + } + + RegMem::reg(self.put_in_reg(val)) + } + + #[inline] + fn avx512vl_enabled(&mut self, _: Type) -> Option<()> { + if self.isa_flags.use_avx512vl_simd() { + Some(()) + } else { + None + } + } + + #[inline] + fn avx512dq_enabled(&mut self, _: Type) -> Option<()> { + if self.isa_flags.use_avx512dq_simd() { + Some(()) + } else { + None + } + } + + #[inline] + fn imm8_from_value(&mut self, val: Value) -> Option { + let inst = self.lower_ctx.dfg().value_def(val).inst()?; + let constant = self.lower_ctx.get_constant(inst)?; + let imm = u8::try_from(constant).ok()?; + Some(Imm8Reg::Imm8 { imm }) + } + + #[inline] + fn simm32_from_value(&mut self, val: Value) -> Option { + let inst = self.lower_ctx.dfg().value_def(val).inst()?; + let constant: u64 = self.lower_ctx.get_constant(inst)?; + let constant = constant as i64; + to_simm32(constant) + } + + #[inline] + fn simm32_from_imm64(&mut self, imm: Imm64) -> Option { + to_simm32(imm.bits()) + } + + fn sinkable_load(&mut self, val: Value) -> Option { + let input = self.lower_ctx.get_value_as_source_or_const(val); + if let Some((inst, 0)) = input.inst { + if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, inst) { + return Some(SinkableLoad { + inst, + addr_input, + offset, + }); + } + } + None + } + + fn sink_load(&mut self, load: &SinkableLoad) -> RegMemImm { + self.lower_ctx.sink_inst(load.inst); + + let flags = self + .lower_ctx + .memflags(load.inst) + .expect("sinkable loads should have memflags"); + + let base = self + .lower_ctx + .put_input_in_regs(load.addr_input.insn, load.addr_input.input) + .only_reg() + .unwrap(); + + RegMemImm::Mem { + addr: Amode::imm_reg(load.offset as u32, base) + .with_flags(flags) + .into(), + } + } + + #[inline] + fn ext_mode(&mut self, from_bits: u16, to_bits: u16) -> ExtMode { + ExtMode::new(from_bits, to_bits).unwrap() + } + + fn emit(&mut self, inst: &MInst) -> Unit { + for inst in inst.clone().mov_mitosis() { + self.emitted_insts.push(inst); + } + } + + #[inline] + fn nonzero_u64_fits_in_u32(&mut self, x: u64) -> Option { + if x != 0 && x < u64::from(u32::MAX) { + Some(x) + } else { + None + } + } +} + +#[inline] +fn to_simm32(constant: i64) -> Option { + if constant == ((constant << 32) >> 32) { + Some(RegMemImm::Imm { + simm32: constant as u32, + }) + } else { + None + } +} + +#[inline(never)] +#[cold] +#[track_caller] +fn out_of_line_panic(msg: &str) -> ! { + panic!("{}", msg); +} diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs new file mode 100644 index 0000000000..8050f87947 --- /dev/null +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -0,0 +1,3496 @@ +// GENERATED BY ISLE. DO NOT EDIT! +// +// Generated automatically from the instruction-selection DSL code in: +// - src/clif.isle +// - src/prelude.isle +// - src/isa/x64/inst.isle +// - src/isa/x64/lower.isle + +#![allow(dead_code, unreachable_code, unreachable_patterns)] +#![allow(unused_imports, unused_variables, non_snake_case)] +#![allow(irrefutable_let_patterns)] + +use super::*; // Pulls in all external types. + +/// Context during lowering: an implementation of this trait +/// must be provided with all external constructors and extractors. +/// A mutable borrow is passed along through all lowering logic. +pub trait Context { + fn unpack_value_array_2(&mut self, arg0: &ValueArray2) -> (Value, Value); + fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> ValueArray2; + fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value); + fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3; + fn value_reg(&mut self, arg0: Reg) -> ValueRegs; + fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; + fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; + fn invalid_reg(&mut self) -> Reg; + fn put_in_reg(&mut self, arg0: Value) -> Reg; + fn put_in_regs(&mut self, arg0: Value) -> ValueRegs; + fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg; + fn u8_as_u64(&mut self, arg0: u8) -> u64; + fn u16_as_u64(&mut self, arg0: u16) -> u64; + fn u32_as_u64(&mut self, arg0: u32) -> u64; + fn ty_bits(&mut self, arg0: Type) -> u16; + fn fits_in_64(&mut self, arg0: Type) -> Option; + fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; + fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice); + fn unwrap_head_value_list_2(&mut self, arg0: ValueList) -> (Value, Value, ValueSlice); + fn writable_reg_to_reg(&mut self, arg0: WritableReg) -> Reg; + fn u64_from_imm64(&mut self, arg0: Imm64) -> u64; + fn inst_results(&mut self, arg0: Inst) -> ValueSlice; + fn first_result(&mut self, arg0: Inst) -> Option; + fn inst_data(&mut self, arg0: Inst) -> InstructionData; + fn value_type(&mut self, arg0: Value) -> Type; + fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>; + fn def_inst(&mut self, arg0: Value) -> Option; + fn operand_size_of_type(&mut self, arg0: Type) -> OperandSize; + fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; + fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; + fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; + fn imm8_from_value(&mut self, arg0: Value) -> Option; + fn simm32_from_value(&mut self, arg0: Value) -> Option; + fn simm32_from_imm64(&mut self, arg0: Imm64) -> Option; + fn sinkable_load(&mut self, arg0: Value) -> Option; + fn sink_load(&mut self, arg0: &SinkableLoad) -> RegMemImm; + fn ext_mode(&mut self, arg0: u16, arg1: u16) -> ExtMode; + fn emit(&mut self, arg0: &MInst) -> Unit; + fn nonzero_u64_fits_in_u32(&mut self, arg0: u64) -> Option; +} + +/// Internal type ProducesFlags: defined at src/isa/x64/inst.isle line 368. +#[derive(Clone, Debug)] +pub enum ProducesFlags { + ProducesFlags { inst: MInst, result: Reg }, +} + +/// Internal type ConsumesFlags: defined at src/isa/x64/inst.isle line 371. +#[derive(Clone, Debug)] +pub enum ConsumesFlags { + ConsumesFlags { inst: MInst, result: Reg }, +} + +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 409. +#[derive(Clone, Debug)] +pub enum ExtendKind { + Sign, + Zero, +} + +// Generated as internal constructor for term temp_reg. +pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { + let pattern0_0 = arg0; + // Rule at src/prelude.isle line 57. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr1_0); +} + +// Generated as internal constructor for term lo_reg. +pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { + let pattern0_0 = arg0; + // Rule at src/prelude.isle line 92. + let expr0_0 = C::put_in_regs(ctx, pattern0_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term operand_size_bits. +pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &OperandSize::Size8 => { + // Rule at src/isa/x64/inst.isle line 69. + let expr0_0: u16 = 8; + return Some(expr0_0); + } + &OperandSize::Size16 => { + // Rule at src/isa/x64/inst.isle line 70. + let expr0_0: u16 = 16; + return Some(expr0_0); + } + &OperandSize::Size32 => { + // Rule at src/isa/x64/inst.isle line 71. + let expr0_0: u16 = 32; + return Some(expr0_0); + } + &OperandSize::Size64 => { + // Rule at src/isa/x64/inst.isle line 72. + let expr0_0: u16 = 64; + return Some(expr0_0); + } + _ => {} + } + return None; +} + +// Generated as internal constructor for term with_flags. +pub fn constructor_with_flags( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + // Rule at src/isa/x64/inst.isle line 381. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); + return Some(expr2_0); + } + } + return None; +} + +// Generated as internal constructor for term with_flags_1. +pub fn constructor_with_flags_1( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + // Rule at src/isa/x64/inst.isle line 389. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + return Some(pattern3_1); + } + } + return None; +} + +// Generated as internal constructor for term with_flags_2. +pub fn constructor_with_flags_2( + ctx: &mut C, + arg0: &ProducesFlags, + arg1: &ConsumesFlags, + arg2: &ConsumesFlags, +) -> Option { + let pattern0_0 = arg0; + if let &ProducesFlags::ProducesFlags { + inst: ref pattern1_0, + result: pattern1_1, + } = pattern0_0 + { + let pattern2_0 = arg1; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern3_0, + result: pattern3_1, + } = pattern2_0 + { + let pattern4_0 = arg2; + if let &ConsumesFlags::ConsumesFlags { + inst: ref pattern5_0, + result: pattern5_1, + } = pattern4_0 + { + // Rule at src/isa/x64/inst.isle line 399. + let expr0_0 = C::emit(ctx, &pattern1_0); + let expr1_0 = C::emit(ctx, &pattern3_0); + let expr2_0 = C::emit(ctx, &pattern5_0); + let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1); + return Some(expr3_0); + } + } + } + return None; +} + +// Generated as internal constructor for term extend_to_reg. +pub fn constructor_extend_to_reg( + ctx: &mut C, + arg0: Value, + arg1: Type, + arg2: &ExtendKind, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = C::value_type(ctx, pattern0_0); + let pattern2_0 = arg1; + if pattern2_0 == pattern1_0 { + let pattern4_0 = arg2; + // Rule at src/isa/x64/inst.isle line 421. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + return Some(expr0_0); + } + let pattern3_0 = arg2; + // Rule at src/isa/x64/inst.isle line 424. + let expr0_0 = C::ty_bits(ctx, pattern1_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern2_0); + let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; + let expr3_0 = C::ext_mode(ctx, expr0_0, expr2_0); + let expr4_0 = C::put_in_reg_mem(ctx, pattern0_0); + let expr5_0 = constructor_extend(ctx, pattern3_0, pattern2_0, &expr3_0, &expr4_0)?; + return Some(expr5_0); +} + +// Generated as internal constructor for term extend. +pub fn constructor_extend( + ctx: &mut C, + arg0: &ExtendKind, + arg1: Type, + arg2: &ExtMode, + arg3: &RegMem, +) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &ExtendKind::Sign => { + let pattern2_0 = arg1; + let pattern3_0 = arg2; + let pattern4_0 = arg3; + // Rule at src/isa/x64/inst.isle line 444. + let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; + return Some(expr0_0); + } + &ExtendKind::Zero => { + let pattern2_0 = arg1; + let pattern3_0 = arg2; + let pattern4_0 = arg3; + // Rule at src/isa/x64/inst.isle line 440. + let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; + return Some(expr0_0); + } + _ => {} + } + return None; +} + +// Generated as internal constructor for term alu_rmi_r. +pub fn constructor_alu_rmi_r( + ctx: &mut C, + arg0: Type, + arg1: &AluRmiROpcode, + arg2: Reg, + arg3: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 462. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = MInst::AluRmiR { + size: expr1_0, + op: pattern1_0.clone(), + src1: pattern2_0, + src2: pattern3_0.clone(), + dst: expr0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term add. +pub fn constructor_add( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 470. + let expr0_0 = AluRmiROpcode::Add; + let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term add_with_flags. +pub fn constructor_add_with_flags( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 478. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = AluRmiROpcode::Add; + let expr3_0 = MInst::AluRmiR { + size: expr1_0, + op: expr2_0, + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr5_0 = ProducesFlags::ProducesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term adc. +pub fn constructor_adc( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 489. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = AluRmiROpcode::Adc; + let expr3_0 = MInst::AluRmiR { + size: expr1_0, + op: expr2_0, + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr5_0 = ConsumesFlags::ConsumesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term sub. +pub fn constructor_sub( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 500. + let expr0_0 = AluRmiROpcode::Sub; + let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term sub_with_flags. +pub fn constructor_sub_with_flags( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 508. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = AluRmiROpcode::Sub; + let expr3_0 = MInst::AluRmiR { + size: expr1_0, + op: expr2_0, + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr5_0 = ProducesFlags::ProducesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term sbb. +pub fn constructor_sbb( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 519. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = AluRmiROpcode::Sbb; + let expr3_0 = MInst::AluRmiR { + size: expr1_0, + op: expr2_0, + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr0_0, + }; + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr5_0 = ConsumesFlags::ConsumesFlags { + inst: expr3_0, + result: expr4_0, + }; + return Some(expr5_0); +} + +// Generated as internal constructor for term mul. +pub fn constructor_mul( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 530. + let expr0_0 = AluRmiROpcode::Mul; + let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term m_and. +pub fn constructor_m_and( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 541. + let expr0_0 = AluRmiROpcode::And; + let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term or. +pub fn constructor_or( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 549. + let expr0_0 = AluRmiROpcode::Or; + let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term xor. +pub fn constructor_xor( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 557. + let expr0_0 = AluRmiROpcode::Xor; + let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term imm. +pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option { + let pattern0_0 = arg0; + if pattern0_0 == I64 { + let pattern2_0 = arg1; + if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { + // Rule at src/isa/x64/inst.isle line 576. + let expr0_0: Type = I64; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = OperandSize::Size32; + let expr3_0 = MInst::Imm { + dst_size: expr2_0, + simm64: pattern3_0, + dst: expr1_0, + }; + let expr4_0 = C::emit(ctx, &expr3_0); + let expr5_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr5_0); + } + } + let pattern1_0 = arg1; + if pattern1_0 == 0 { + // Rule at src/isa/x64/inst.isle line 582. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr2_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr3_0 = AluRmiROpcode::Xor; + let expr4_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr5_0 = MInst::AluRmiR { + size: expr2_0, + op: expr3_0, + src1: expr1_0, + src2: expr4_0, + dst: expr0_0, + }; + let expr6_0 = C::emit(ctx, &expr5_0); + return Some(expr1_0); + } + // Rule at src/isa/x64/inst.isle line 565. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = MInst::Imm { + dst_size: expr1_0, + simm64: pattern1_0, + dst: expr0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term shift_r. +pub fn constructor_shift_r( + ctx: &mut C, + arg0: Type, + arg1: &ShiftKind, + arg2: Reg, + arg3: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 595. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = MInst::ShiftR { + size: expr1_0, + kind: pattern1_0.clone(), + src: pattern2_0, + num_bits: pattern3_0.clone(), + dst: expr0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term m_rotl. +pub fn constructor_m_rotl( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 604. + let expr0_0 = ShiftKind::RotateLeft; + let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term shl. +pub fn constructor_shl( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 609. + let expr0_0 = ShiftKind::ShiftLeft; + let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term shr. +pub fn constructor_shr( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 614. + let expr0_0 = ShiftKind::ShiftRightLogical; + let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term sar. +pub fn constructor_sar( + ctx: &mut C, + arg0: Type, + arg1: Reg, + arg2: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 619. + let expr0_0 = ShiftKind::ShiftRightArithmetic; + let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term cmp_rmi_r. +pub fn constructor_cmp_rmi_r( + ctx: &mut C, + arg0: &OperandSize, + arg1: &CmpOpcode, + arg2: &RegMemImm, + arg3: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 624. + let expr0_0 = MInst::CmpRmiR { + size: pattern0_0.clone(), + opcode: pattern1_0.clone(), + src: pattern2_0.clone(), + dst: pattern3_0, + }; + let expr1_0 = C::invalid_reg(ctx); + let expr2_0 = ProducesFlags::ProducesFlags { + inst: expr0_0, + result: expr1_0, + }; + return Some(expr2_0); +} + +// Generated as internal constructor for term cmp. +pub fn constructor_cmp( + ctx: &mut C, + arg0: &OperandSize, + arg1: &RegMemImm, + arg2: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 633. + let expr0_0 = CmpOpcode::Cmp; + let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term test. +pub fn constructor_test( + ctx: &mut C, + arg0: &OperandSize, + arg1: &RegMemImm, + arg2: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 638. + let expr0_0 = CmpOpcode::Test; + let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term cmove. +pub fn constructor_cmove( + ctx: &mut C, + arg0: Type, + arg1: &CC, + arg2: &RegMem, + arg3: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 643. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::operand_size_of_type(ctx, pattern0_0); + let expr2_0 = MInst::Cmove { + size: expr1_0, + cc: pattern1_0.clone(), + consequent: pattern2_0.clone(), + alternative: pattern3_0, + dst: expr0_0, + }; + let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr4_0 = ConsumesFlags::ConsumesFlags { + inst: expr2_0, + result: expr3_0, + }; + return Some(expr4_0); +} + +// Generated as internal constructor for term movzx. +pub fn constructor_movzx( + ctx: &mut C, + arg0: Type, + arg1: &ExtMode, + arg2: &RegMem, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 651. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = MInst::MovzxRmR { + ext_mode: pattern1_0.clone(), + src: pattern2_0.clone(), + dst: expr0_0, + }; + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr3_0); +} + +// Generated as internal constructor for term movsx. +pub fn constructor_movsx( + ctx: &mut C, + arg0: Type, + arg1: &ExtMode, + arg2: &RegMem, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 658. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = MInst::MovsxRmR { + ext_mode: pattern1_0.clone(), + src: pattern2_0.clone(), + dst: expr0_0, + }; + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr3_0); +} + +// Generated as internal constructor for term xmm_rm_r. +pub fn constructor_xmm_rm_r( + ctx: &mut C, + arg0: Type, + arg1: &SseOpcode, + arg2: Reg, + arg3: &RegMem, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 665. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = MInst::XmmRmR { + op: pattern1_0.clone(), + src1: pattern2_0, + src2: pattern3_0.clone(), + dst: expr0_0, + }; + let expr2_0 = C::emit(ctx, &expr1_0); + let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0); + return Some(expr3_0); +} + +// Generated as internal constructor for term paddb. +pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 672. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Paddb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term paddw. +pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 677. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Paddw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term paddd. +pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 682. + let expr0_0: Type = I32X4; + let expr1_0 = SseOpcode::Paddd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term paddq. +pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 687. + let expr0_0: Type = I64X2; + let expr1_0 = SseOpcode::Paddq; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term paddsb. +pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 692. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Paddsb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term paddsw. +pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 697. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Paddsw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term paddusb. +pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 702. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Paddusb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term paddusw. +pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 707. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Paddusw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubb. +pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 712. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Psubb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubw. +pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 717. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Psubw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubd. +pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 722. + let expr0_0: Type = I32X4; + let expr1_0 = SseOpcode::Psubd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubq. +pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 727. + let expr0_0: Type = I64X2; + let expr1_0 = SseOpcode::Psubq; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubsb. +pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 732. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Psubsb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubsw. +pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 737. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Psubsw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubusb. +pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 742. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Psubusb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term psubusw. +pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 747. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Psubusw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pavgb. +pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 752. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Pavgb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pavgw. +pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 757. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Pavgw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pand. +pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 762. + let expr0_0: Type = F32X4; + let expr1_0 = SseOpcode::Pand; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term andps. +pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 767. + let expr0_0: Type = F32X4; + let expr1_0 = SseOpcode::Andps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term andpd. +pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 772. + let expr0_0: Type = F64X2; + let expr1_0 = SseOpcode::Andpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term por. +pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 777. + let expr0_0: Type = F32X4; + let expr1_0 = SseOpcode::Por; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term orps. +pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 782. + let expr0_0: Type = F32X4; + let expr1_0 = SseOpcode::Orps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term orpd. +pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 787. + let expr0_0: Type = F64X2; + let expr1_0 = SseOpcode::Orpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pxor. +pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 792. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Pxor; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term xorps. +pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 797. + let expr0_0: Type = F32X4; + let expr1_0 = SseOpcode::Xorps; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term xorpd. +pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 802. + let expr0_0: Type = F64X2; + let expr1_0 = SseOpcode::Xorpd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pmullw. +pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 807. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Pmullw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pmulld. +pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 812. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Pmulld; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pmulhw. +pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 817. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Pmulhw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pmulhuw. +pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 822. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Pmulhuw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pmuldq. +pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 827. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Pmuldq; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term pmuludq. +pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 832. + let expr0_0: Type = I64X2; + let expr1_0 = SseOpcode::Pmuludq; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term punpckhwd. +pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 837. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Punpckhwd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term punpcklwd. +pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 842. + let expr0_0: Type = I16X8; + let expr1_0 = SseOpcode::Punpcklwd; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term xmm_rm_r_imm. +pub fn constructor_xmm_rm_r_imm( + ctx: &mut C, + arg0: &SseOpcode, + arg1: Reg, + arg2: &RegMem, + arg3: u8, + arg4: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + let pattern4_0 = arg4; + // Rule at src/isa/x64/inst.isle line 847. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::XmmRmRImm { + op: pattern0_0.clone(), + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr1_0, + imm: pattern3_0, + size: pattern4_0.clone(), + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term palignr. +pub fn constructor_palignr( + ctx: &mut C, + arg0: Reg, + arg1: &RegMem, + arg2: u8, + arg3: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + let pattern3_0 = arg3; + // Rule at src/isa/x64/inst.isle line 859. + let expr0_0 = SseOpcode::Palignr; + let expr1_0 = constructor_xmm_rm_r_imm( + ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, + )?; + return Some(expr1_0); +} + +// Generated as internal constructor for term pshufd. +pub fn constructor_pshufd( + ctx: &mut C, + arg0: &RegMem, + arg1: u8, + arg2: &OperandSize, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 868. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr3_0 = SseOpcode::Pshufd; + let expr4_0 = MInst::XmmRmRImm { + op: expr3_0, + src1: expr2_0, + src2: pattern0_0.clone(), + dst: expr1_0, + imm: pattern1_0, + size: pattern2_0.clone(), + }; + let expr5_0 = C::emit(ctx, &expr4_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term xmm_unary_rm_r. +pub fn constructor_xmm_unary_rm_r( + ctx: &mut C, + arg0: &SseOpcode, + arg1: &RegMem, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 881. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::XmmUnaryRmR { + op: pattern0_0.clone(), + src: pattern1_0.clone(), + dst: expr1_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term pmovsxbw. +pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 888. + let expr0_0 = SseOpcode::Pmovsxbw; + let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term pmovzxbw. +pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 893. + let expr0_0 = SseOpcode::Pmovzxbw; + let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term xmm_rm_r_evex. +pub fn constructor_xmm_rm_r_evex( + ctx: &mut C, + arg0: &Avx512Opcode, + arg1: &RegMem, + arg2: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 898. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::XmmRmREvex { + op: pattern0_0.clone(), + src1: pattern1_0.clone(), + src2: pattern2_0, + dst: expr1_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term vpmullq. +pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 910. + let expr0_0 = Avx512Opcode::Vpmullq; + let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term xmm_rmi_reg. +pub fn constructor_xmm_rmi_reg( + ctx: &mut C, + arg0: &SseOpcode, + arg1: Reg, + arg2: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 917. + let expr0_0: Type = I8X16; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = MInst::XmmRmiReg { + opcode: pattern0_0.clone(), + src1: pattern1_0, + src2: pattern2_0.clone(), + dst: expr1_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0); + return Some(expr4_0); +} + +// Generated as internal constructor for term psllq. +pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 927. + let expr0_0 = SseOpcode::Psllq; + let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psrlq. +pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 932. + let expr0_0 = SseOpcode::Psrlq; + let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term lower. +pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) { + let pattern2_0 = C::value_type(ctx, pattern1_0); + if pattern2_0 == B128 { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + match &pattern4_0 { + &InstructionData::UnaryBool { + opcode: ref pattern5_0, + imm: pattern5_1, + } => { + if let &Opcode::Bconst = &pattern5_0 { + if pattern5_1 == true { + // Rule at src/isa/x64/lower.isle line 39. + let expr0_0: Type = B64; + let expr1_0: u64 = 1; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0: Type = B64; + let expr4_0: u64 = 0; + let expr5_0 = constructor_imm(ctx, expr3_0, expr4_0)?; + let expr6_0 = C::value_regs(ctx, expr2_0, expr5_0); + return Some(expr6_0); + } + if pattern5_1 == false { + // Rule at src/isa/x64/lower.isle line 34. + let expr0_0: Type = B64; + let expr1_0: u64 = 0; + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0: Type = B64; + let expr4_0: u64 = 0; + let expr5_0 = constructor_imm(ctx, expr3_0, expr4_0)?; + let expr6_0 = C::value_regs(ctx, expr2_0, expr5_0); + return Some(expr6_0); + } + } + } + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match &pattern5_0 { + &Opcode::Band => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 358. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr6_0: Type = I64; + let expr7_0 = RegMemImm::Reg { reg: expr5_0 }; + let expr8_0 = constructor_m_and(ctx, expr6_0, expr2_0, &expr7_0)?; + let expr9_0 = C::value_regs(ctx, expr8_0, expr4_0); + return Some(expr9_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 436. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr6_0: Type = I64; + let expr7_0 = RegMemImm::Reg { reg: expr5_0 }; + let expr8_0 = constructor_or(ctx, expr6_0, expr2_0, &expr7_0)?; + let expr9_0 = C::value_regs(ctx, expr8_0, expr4_0); + return Some(expr9_0); + } + &Opcode::Bxor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 512. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr6_0: Type = I64; + let expr7_0 = RegMemImm::Reg { reg: expr5_0 }; + let expr8_0 = constructor_xor(ctx, expr6_0, expr2_0, &expr7_0)?; + let expr9_0 = C::value_regs(ctx, expr8_0, expr4_0); + return Some(expr9_0); + } + _ => {} + } + } + _ => {} + } + } + if pattern2_0 == I128 { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + match &pattern4_0 { + &InstructionData::UnaryImm { + opcode: ref pattern5_0, + imm: pattern5_1, + } => { + if let &Opcode::Iconst = &pattern5_0 { + let pattern7_0 = C::u64_from_imm64(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 15. + let expr0_0: Type = I64; + let expr1_0 = constructor_imm(ctx, expr0_0, pattern7_0)?; + let expr2_0: Type = I64; + let expr3_0: u64 = 0; + let expr4_0 = constructor_imm(ctx, expr2_0, expr3_0)?; + let expr5_0 = C::value_regs(ctx, expr1_0, expr4_0); + return Some(expr5_0); + } + } + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match &pattern5_0 { + &Opcode::Iadd => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 107. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0: Type = I64; + let expr11_0 = RegMemImm::Reg { reg: expr7_0 }; + let expr12_0 = + constructor_add_with_flags(ctx, expr10_0, expr2_0, &expr11_0)?; + let expr13_0: Type = I64; + let expr14_0 = RegMemImm::Reg { reg: expr9_0 }; + let expr15_0 = constructor_adc(ctx, expr13_0, expr4_0, &expr14_0)?; + let expr16_0 = constructor_with_flags(ctx, &expr12_0, &expr15_0)?; + return Some(expr16_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 256. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0: Type = I64; + let expr11_0 = RegMemImm::Reg { reg: expr7_0 }; + let expr12_0 = + constructor_sub_with_flags(ctx, expr10_0, expr2_0, &expr11_0)?; + let expr13_0: Type = I64; + let expr14_0 = RegMemImm::Reg { reg: expr9_0 }; + let expr15_0 = constructor_sbb(ctx, expr13_0, expr4_0, &expr14_0)?; + let expr16_0 = constructor_with_flags(ctx, &expr12_0, &expr15_0)?; + return Some(expr16_0); + } + &Opcode::Band => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 348. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0: Type = I64; + let expr11_0 = RegMemImm::Reg { reg: expr7_0 }; + let expr12_0 = constructor_m_and(ctx, expr10_0, expr2_0, &expr11_0)?; + let expr13_0: Type = I64; + let expr14_0 = RegMemImm::Reg { reg: expr9_0 }; + let expr15_0 = constructor_m_and(ctx, expr13_0, expr4_0, &expr14_0)?; + let expr16_0 = C::value_regs(ctx, expr12_0, expr15_0); + return Some(expr16_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 433. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = C::put_in_regs(ctx, pattern7_1); + let expr2_0 = constructor_or_i128(ctx, expr0_0, expr1_0)?; + return Some(expr2_0); + } + &Opcode::Bxor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 502. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0 = C::put_in_regs(ctx, pattern7_1); + let expr6_0: usize = 0; + let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0); + let expr8_0: usize = 1; + let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0); + let expr10_0: Type = I64; + let expr11_0 = RegMemImm::Reg { reg: expr7_0 }; + let expr12_0 = constructor_xor(ctx, expr10_0, expr2_0, &expr11_0)?; + let expr13_0: Type = I64; + let expr14_0 = RegMemImm::Reg { reg: expr9_0 }; + let expr15_0 = constructor_xor(ctx, expr13_0, expr4_0, &expr14_0)?; + let expr16_0 = C::value_regs(ctx, expr12_0, expr15_0); + return Some(expr16_0); + } + &Opcode::Rotl => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 629. + let expr0_0 = C::put_in_regs(ctx, pattern7_0); + let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; + let expr3_0: Type = I64; + let expr4_0: Type = I64; + let expr5_0: u64 = 128; + let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?; + let expr7_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr8_0 = constructor_sub(ctx, expr3_0, expr6_0, &expr7_0)?; + let expr9_0 = constructor_shr_i128(ctx, expr0_0, expr8_0)?; + let expr10_0 = constructor_or_i128(ctx, expr2_0, expr9_0)?; + return Some(expr10_0); + } + &Opcode::Ishl => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 562. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_regs(ctx, pattern7_0); + let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?; + return Some(expr2_0); + } + &Opcode::Ushr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 608. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_regs(ctx, pattern7_0); + let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?; + return Some(expr2_0); + } + _ => {} + } + } + &InstructionData::BinaryImm64 { + opcode: ref pattern5_0, + arg: pattern5_1, + imm: pattern5_2, + } => { + if let &Opcode::IaddImm = &pattern5_0 { + let pattern7_0 = C::u64_from_imm64(ctx, pattern5_2); + // Rule at src/isa/x64/lower.isle line 202. + let expr0_0 = C::put_in_regs(ctx, pattern5_1); + let expr1_0: usize = 0; + let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); + let expr3_0: usize = 1; + let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0); + let expr5_0: Type = I64; + let expr6_0 = constructor_imm(ctx, expr5_0, pattern7_0)?; + let expr7_0: Type = I64; + let expr8_0 = RegMemImm::Reg { reg: expr6_0 }; + let expr9_0 = constructor_add_with_flags(ctx, expr7_0, expr2_0, &expr8_0)?; + let expr10_0: Type = I64; + let expr11_0: u32 = 0; + let expr12_0 = RegMemImm::Imm { simm32: expr11_0 }; + let expr13_0 = constructor_adc(ctx, expr10_0, expr4_0, &expr12_0)?; + let expr14_0 = constructor_with_flags(ctx, &expr9_0, &expr13_0)?; + return Some(expr14_0); + } + } + _ => {} + } + } + if pattern2_0 == F32X4 { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Band => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 333. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_andps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 409. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_orps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bxor => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 487. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_xorps(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} + } + } + } + if pattern2_0 == F64X2 { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Band => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 337. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_andpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 413. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_orpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bxor => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 491. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_xorpd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} + } + } + } + let pattern3_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::NullAry { + opcode: ref pattern4_0, + } = &pattern3_0 + { + if let &Opcode::Null = &pattern4_0 { + // Rule at src/isa/x64/lower.isle line 46. + let expr0_0: u64 = 0; + let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + } + if let Some(()) = C::avx512vl_enabled(ctx, pattern2_0) { + if let Some(()) = C::avx512dq_enabled(ctx, pattern2_0) { + if let Some((pattern5_0, pattern5_1)) = C::multi_lane(ctx, pattern2_0) { + if pattern5_0 == 64 { + if pattern5_1 == 2 { + let pattern8_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern9_0, + args: ref pattern9_1, + } = &pattern8_0 + { + if let &Opcode::Imul = &pattern9_0 { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, &pattern9_1); + // Rule at src/isa/x64/lower.isle line 693. + let expr0_0 = C::put_in_reg_mem(ctx, pattern11_0); + let expr1_0 = C::put_in_reg(ctx, pattern11_1); + let expr2_0 = constructor_vpmullq(ctx, &expr0_0, expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + } + } + } + } + } + } + if let Some((pattern3_0, pattern3_1)) = C::multi_lane(ctx, pattern2_0) { + if pattern3_0 == 8 { + if pattern3_1 == 16 { + let pattern6_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern7_0, + args: ref pattern7_1, + } = &pattern6_0 + { + match &pattern7_0 { + &Opcode::AvgRound => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 639. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::UaddSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 134. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddusb(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::SaddSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 122. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddsb(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::UsubSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 283. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubusb(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::SsubSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 271. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubsb(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Iadd => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 86. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddb(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Isub => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 235. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubb(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} + } + } + } + } + if pattern3_0 == 16 { + if pattern3_1 == 8 { + let pattern6_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern7_0, + args: ref pattern7_1, + } = &pattern6_0 + { + match &pattern7_0 { + &Opcode::AvgRound => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 643. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::UaddSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 139. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddusw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::SaddSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 127. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddsw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::UsubSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 288. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubusw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::SsubSat => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 276. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubsw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Iadd => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 91. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Isub => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 240. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Imul => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + if let Some(pattern10_0) = C::def_inst(ctx, pattern9_0) { + let pattern11_0 = C::inst_data(ctx, pattern10_0); + if let &InstructionData::Unary { + opcode: ref pattern12_0, + arg: pattern12_1, + } = &pattern11_0 + { + match &pattern12_0 { + &Opcode::SwidenLow => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 8 { + if pattern15_1 == 16 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::SwidenLow = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 8 { + if pattern23_1 == 16 + { + // Rule at src/isa/x64/lower.isle line 781. + let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1); + let expr1_0 = constructor_pmovsxbw(ctx, &expr0_0)?; + let expr2_0 = C::put_in_reg_mem(ctx, pattern20_1); + let expr3_0 = constructor_pmovsxbw(ctx, &expr2_0)?; + let expr4_0 = RegMem::Reg { + reg: expr3_0, + }; + let expr5_0 = constructor_pmullw(ctx, expr1_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some( + expr6_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::SwidenHigh => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 8 { + if pattern15_1 == 16 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::SwidenHigh = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 8 { + if pattern23_1 == 16 + { + // Rule at src/isa/x64/lower.isle line 741. + let expr0_0 = C::put_in_reg(ctx, pattern12_1); + let expr1_0 = RegMem::Reg { + reg: expr0_0, + }; + let expr2_0: u8 = 8; + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_palignr(ctx, expr0_0, &expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = RegMem::Reg { + reg: expr4_0, + }; + let expr6_0 = constructor_pmovsxbw(ctx, &expr5_0)?; + let expr7_0 = C::put_in_reg(ctx, pattern20_1); + let expr8_0 = RegMem::Reg { + reg: expr7_0, + }; + let expr9_0: u8 = 8; + let expr10_0 = OperandSize::Size32; + let expr11_0 = constructor_palignr(ctx, expr7_0, &expr8_0, expr9_0, &expr10_0)?; + let expr12_0 = RegMem::Reg { + reg: expr11_0, + }; + let expr13_0 = constructor_pmovsxbw(ctx, &expr12_0)?; + let expr14_0 = RegMem::Reg { + reg: expr13_0, + }; + let expr15_0 = constructor_pmullw(ctx, expr6_0, &expr14_0)?; + let expr16_0 = C::value_reg(ctx, expr15_0); + return Some( + expr16_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::UwidenLow => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 8 { + if pattern15_1 == 16 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::UwidenLow = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 8 { + if pattern23_1 == 16 + { + // Rule at src/isa/x64/lower.isle line 857. + let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1); + let expr1_0 = constructor_pmovzxbw(ctx, &expr0_0)?; + let expr2_0 = C::put_in_reg_mem(ctx, pattern20_1); + let expr3_0 = constructor_pmovzxbw(ctx, &expr2_0)?; + let expr4_0 = RegMem::Reg { + reg: expr3_0, + }; + let expr5_0 = constructor_pmullw(ctx, expr1_0, &expr4_0)?; + let expr6_0 = C::value_reg(ctx, expr5_0); + return Some( + expr6_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::UwidenHigh => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 8 { + if pattern15_1 == 16 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::UwidenHigh = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 8 { + if pattern23_1 == 16 + { + // Rule at src/isa/x64/lower.isle line 817. + let expr0_0 = C::put_in_reg(ctx, pattern12_1); + let expr1_0 = RegMem::Reg { + reg: expr0_0, + }; + let expr2_0: u8 = 8; + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_palignr(ctx, expr0_0, &expr1_0, expr2_0, &expr3_0)?; + let expr5_0 = RegMem::Reg { + reg: expr4_0, + }; + let expr6_0 = constructor_pmovzxbw(ctx, &expr5_0)?; + let expr7_0 = C::put_in_reg(ctx, pattern20_1); + let expr8_0 = RegMem::Reg { + reg: expr7_0, + }; + let expr9_0: u8 = 8; + let expr10_0 = OperandSize::Size32; + let expr11_0 = constructor_palignr(ctx, expr7_0, &expr8_0, expr9_0, &expr10_0)?; + let expr12_0 = RegMem::Reg { + reg: expr11_0, + }; + let expr13_0 = constructor_pmovzxbw(ctx, &expr12_0)?; + let expr14_0 = RegMem::Reg { + reg: expr13_0, + }; + let expr15_0 = constructor_pmullw(ctx, expr6_0, &expr14_0)?; + let expr16_0 = C::value_reg(ctx, expr15_0); + return Some( + expr16_0, + ); + } + } + } + } + } + } + } + } + } + } + _ => {} + } + } + } + // Rule at src/isa/x64/lower.isle line 685. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_pmullw(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} + } + } + } + } + if pattern3_0 == 32 { + if pattern3_1 == 4 { + let pattern6_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern7_0, + args: ref pattern7_1, + } = &pattern6_0 + { + match &pattern7_0 { + &Opcode::Iadd => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 96. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Isub => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 245. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubd(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Imul => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + if let Some(pattern10_0) = C::def_inst(ctx, pattern9_0) { + let pattern11_0 = C::inst_data(ctx, pattern10_0); + if let &InstructionData::Unary { + opcode: ref pattern12_0, + arg: pattern12_1, + } = &pattern11_0 + { + match &pattern12_0 { + &Opcode::SwidenLow => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 16 { + if pattern15_1 == 8 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::SwidenLow = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 16 { + if pattern23_1 == 8 + { + // Rule at src/isa/x64/lower.isle line 791. + let expr0_0 = C::put_in_reg(ctx, pattern12_1); + let expr1_0 = C::put_in_reg(ctx, pattern20_1); + let expr2_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?; + let expr4_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr5_0 = constructor_pmulhw(ctx, expr0_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { + reg: expr5_0, + }; + let expr7_0 = constructor_punpcklwd(ctx, expr3_0, &expr6_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some( + expr8_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::SwidenHigh => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 16 { + if pattern15_1 == 8 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::SwidenHigh = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 16 { + if pattern23_1 == 8 + { + // Rule at src/isa/x64/lower.isle line 755. + let expr0_0 = C::put_in_reg(ctx, pattern12_1); + let expr1_0 = C::put_in_reg(ctx, pattern20_1); + let expr2_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?; + let expr4_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr5_0 = constructor_pmulhw(ctx, expr0_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { + reg: expr5_0, + }; + let expr7_0 = constructor_punpckhwd(ctx, expr3_0, &expr6_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some( + expr8_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::UwidenLow => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 16 { + if pattern15_1 == 8 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::UwidenLow = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 16 { + if pattern23_1 == 8 + { + // Rule at src/isa/x64/lower.isle line 867. + let expr0_0 = C::put_in_reg(ctx, pattern12_1); + let expr1_0 = C::put_in_reg(ctx, pattern20_1); + let expr2_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?; + let expr4_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr5_0 = constructor_pmulhuw(ctx, expr0_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { + reg: expr5_0, + }; + let expr7_0 = constructor_punpcklwd(ctx, expr3_0, &expr6_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some( + expr8_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::UwidenHigh => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 16 { + if pattern15_1 == 8 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::UwidenHigh = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 16 { + if pattern23_1 == 8 + { + // Rule at src/isa/x64/lower.isle line 831. + let expr0_0 = C::put_in_reg(ctx, pattern12_1); + let expr1_0 = C::put_in_reg(ctx, pattern20_1); + let expr2_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?; + let expr4_0 = RegMem::Reg { + reg: expr1_0, + }; + let expr5_0 = constructor_pmulhuw(ctx, expr0_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { + reg: expr5_0, + }; + let expr7_0 = constructor_punpckhwd(ctx, expr3_0, &expr6_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some( + expr8_0, + ); + } + } + } + } + } + } + } + } + } + } + _ => {} + } + } + } + // Rule at src/isa/x64/lower.isle line 688. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_pmulld(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} + } + } + } + } + if pattern3_0 == 64 { + if pattern3_1 == 2 { + let pattern6_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern7_0, + args: ref pattern7_1, + } = &pattern6_0 + { + match &pattern7_0 { + &Opcode::Iadd => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 101. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_paddq(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Isub => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + // Rule at src/isa/x64/lower.isle line 250. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); + let expr2_0 = constructor_psubq(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Imul => { + let (pattern9_0, pattern9_1) = + C::unpack_value_array_2(ctx, &pattern7_1); + if let Some(pattern10_0) = C::def_inst(ctx, pattern9_0) { + let pattern11_0 = C::inst_data(ctx, pattern10_0); + if let &InstructionData::Unary { + opcode: ref pattern12_0, + arg: pattern12_1, + } = &pattern11_0 + { + match &pattern12_0 { + &Opcode::SwidenLow => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 32 { + if pattern15_1 == 4 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::SwidenLow = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 32 { + if pattern23_1 == 4 + { + // Rule at src/isa/x64/lower.isle line 803. + let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1); + let expr1_0: u8 = 80; + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?; + let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1); + let expr5_0: u8 = 80; + let expr6_0 = OperandSize::Size32; + let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?; + let expr8_0 = RegMem::Reg { + reg: expr7_0, + }; + let expr9_0 = constructor_pmuldq(ctx, expr3_0, &expr8_0)?; + let expr10_0 = C::value_reg(ctx, expr9_0); + return Some( + expr10_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::SwidenHigh => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 32 { + if pattern15_1 == 4 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::SwidenHigh = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 32 { + if pattern23_1 == 4 + { + // Rule at src/isa/x64/lower.isle line 767. + let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1); + let expr1_0: u8 = 250; + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?; + let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1); + let expr5_0: u8 = 250; + let expr6_0 = OperandSize::Size32; + let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?; + let expr8_0 = RegMem::Reg { + reg: expr7_0, + }; + let expr9_0 = constructor_pmuldq(ctx, expr3_0, &expr8_0)?; + let expr10_0 = C::value_reg(ctx, expr9_0); + return Some( + expr10_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::UwidenLow => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 32 { + if pattern15_1 == 4 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::UwidenLow = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 32 { + if pattern23_1 == 4 + { + // Rule at src/isa/x64/lower.isle line 879. + let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1); + let expr1_0: u8 = 80; + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?; + let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1); + let expr5_0: u8 = 80; + let expr6_0 = OperandSize::Size32; + let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?; + let expr8_0 = RegMem::Reg { + reg: expr7_0, + }; + let expr9_0 = constructor_pmuludq(ctx, expr3_0, &expr8_0)?; + let expr10_0 = C::value_reg(ctx, expr9_0); + return Some( + expr10_0, + ); + } + } + } + } + } + } + } + } + } + } + &Opcode::UwidenHigh => { + let pattern14_0 = C::value_type(ctx, pattern12_1); + if let Some((pattern15_0, pattern15_1)) = + C::multi_lane(ctx, pattern14_0) + { + if pattern15_0 == 32 { + if pattern15_1 == 4 { + if let Some(pattern18_0) = + C::def_inst(ctx, pattern9_1) + { + let pattern19_0 = + C::inst_data(ctx, pattern18_0); + if let &InstructionData::Unary { + opcode: ref pattern20_0, + arg: pattern20_1, + } = &pattern19_0 + { + if let &Opcode::UwidenHigh = + &pattern20_0 + { + let pattern22_0 = + C::value_type( + ctx, + pattern20_1, + ); + if let Some(( + pattern23_0, + pattern23_1, + )) = C::multi_lane( + ctx, + pattern22_0, + ) { + if pattern23_0 == 32 { + if pattern23_1 == 4 + { + // Rule at src/isa/x64/lower.isle line 843. + let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1); + let expr1_0: u8 = 250; + let expr2_0 = OperandSize::Size32; + let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?; + let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1); + let expr5_0: u8 = 250; + let expr6_0 = OperandSize::Size32; + let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?; + let expr8_0 = RegMem::Reg { + reg: expr7_0, + }; + let expr9_0 = constructor_pmuludq(ctx, expr3_0, &expr8_0)?; + let expr10_0 = C::value_reg(ctx, expr9_0); + return Some( + expr10_0, + ); + } + } + } + } + } + } + } + } + } + } + _ => {} + } + } + } + // Rule at src/isa/x64/lower.isle line 719. + let expr0_0 = C::put_in_reg(ctx, pattern9_0); + let expr1_0 = C::put_in_reg(ctx, pattern9_1); + let expr2_0: u32 = 32; + let expr3_0 = RegMemImm::Imm { simm32: expr2_0 }; + let expr4_0 = constructor_psrlq(ctx, expr0_0, &expr3_0)?; + let expr5_0 = RegMem::Reg { reg: expr1_0 }; + let expr6_0 = constructor_pmuludq(ctx, expr4_0, &expr5_0)?; + let expr7_0: u32 = 32; + let expr8_0 = RegMemImm::Imm { simm32: expr7_0 }; + let expr9_0 = constructor_psrlq(ctx, expr1_0, &expr8_0)?; + let expr10_0 = RegMem::Reg { reg: expr9_0 }; + let expr11_0 = constructor_pmuludq(ctx, expr0_0, &expr10_0)?; + let expr12_0 = RegMem::Reg { reg: expr11_0 }; + let expr13_0 = constructor_paddq(ctx, expr6_0, &expr12_0)?; + let expr14_0: u32 = 32; + let expr15_0 = RegMemImm::Imm { simm32: expr14_0 }; + let expr16_0 = constructor_psllq(ctx, expr13_0, &expr15_0)?; + let expr17_0 = RegMem::Reg { reg: expr1_0 }; + let expr18_0 = constructor_pmuludq(ctx, expr0_0, &expr17_0)?; + let expr19_0 = RegMem::Reg { reg: expr16_0 }; + let expr20_0 = constructor_paddq(ctx, expr18_0, &expr19_0)?; + let expr21_0 = C::value_reg(ctx, expr20_0); + return Some(expr21_0); + } + _ => {} + } + } + } + } + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } = &pattern4_0 + { + match &pattern5_0 { + &Opcode::Band => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 341. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_pand(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 417. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_por(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + &Opcode::Bxor => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 495. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); + let expr2_0 = constructor_pxor(ctx, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + _ => {} + } + } + } + if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + match &pattern4_0 { + &InstructionData::UnaryImm { + opcode: ref pattern5_0, + imm: pattern5_1, + } => { + if let &Opcode::Iconst = &pattern5_0 { + let pattern7_0 = C::u64_from_imm64(ctx, pattern5_1); + // Rule at src/isa/x64/lower.isle line 10. + let expr0_0 = constructor_imm(ctx, pattern3_0, pattern7_0)?; + let expr1_0 = C::value_reg(ctx, expr0_0); + return Some(expr1_0); + } + } + &InstructionData::UnaryBool { + opcode: ref pattern5_0, + imm: pattern5_1, + } => { + if let &Opcode::Bconst = &pattern5_0 { + if pattern5_1 == true { + // Rule at src/isa/x64/lower.isle line 28. + let expr0_0: u64 = 1; + let expr1_0 = constructor_imm(ctx, pattern3_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if pattern5_1 == false { + // Rule at src/isa/x64/lower.isle line 24. + let expr0_0: u64 = 0; + let expr1_0 = constructor_imm(ctx, pattern3_0, expr0_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + } + } + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match &pattern5_0 { + &Opcode::Iadd => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 66. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = + constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 78. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 62. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 72. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 54. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Isub => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 222. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_sub(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 227. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_sub(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 215. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_sub(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Imul => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 663. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = + constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 675. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 659. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 669. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 652. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::IaddIfcout => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 159. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = + constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 171. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 155. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 165. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 147. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Band => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 325. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = + constructor_m_and(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 311. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = + constructor_m_and(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 319. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_m_and(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 305. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = + constructor_m_and(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 298. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_m_and(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Bor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 401. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = + constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 387. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 395. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 381. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 374. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Bxor => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 479. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = + constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 465. + let expr0_0 = C::put_in_reg(ctx, pattern7_1); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 473. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 459. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::sink_load(ctx, &pattern8_0); + let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 452. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg(ctx, pattern7_1); + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Rotl => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 624. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_m_rotl(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + // Rule at src/isa/x64/lower.isle line 618. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Ishl => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 533. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = + constructor_shl(ctx, pattern3_0, expr0_0, &pattern8_0)?; + let expr2_0 = C::value_reg(ctx, expr1_0); + return Some(expr2_0); + } + // Rule at src/isa/x64/lower.isle line 527. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_reg(ctx, pattern7_0); + let expr2_0 = Imm8Reg::Reg { reg: expr0_0 }; + let expr3_0 = constructor_shl(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + &Opcode::Ushr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 579. + let expr0_0 = ExtendKind::Zero; + let expr1_0 = constructor_extend_to_reg( + ctx, pattern7_0, pattern3_0, &expr0_0, + )?; + let expr2_0 = + constructor_shr(ctx, pattern3_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 572. + let expr0_0 = ExtendKind::Zero; + let expr1_0 = + constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; + let expr2_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr3_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr4_0 = constructor_shr(ctx, pattern3_0, expr1_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } + _ => {} + } + } + &InstructionData::BinaryImm64 { + opcode: ref pattern5_0, + arg: pattern5_1, + imm: pattern5_2, + } => { + if let &Opcode::IaddImm = &pattern5_0 { + let pattern7_0 = C::u64_from_imm64(ctx, pattern5_2); + // Rule at src/isa/x64/lower.isle line 188. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0 = constructor_imm(ctx, pattern3_0, pattern7_0)?; + let expr2_0 = RegMemImm::Reg { reg: expr1_0 }; + let expr3_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } + } + _ => {} + } + } + } + return None; +} + +// Generated as internal constructor for term or_i128. +pub fn constructor_or_i128( + ctx: &mut C, + arg0: ValueRegs, + arg1: ValueRegs, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/lower.isle line 425. + let expr0_0: usize = 0; + let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr2_0: usize = 1; + let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr4_0: usize = 0; + let expr5_0 = C::value_regs_get(ctx, pattern1_0, expr4_0); + let expr6_0: usize = 1; + let expr7_0 = C::value_regs_get(ctx, pattern1_0, expr6_0); + let expr8_0: Type = I64; + let expr9_0 = RegMemImm::Reg { reg: expr5_0 }; + let expr10_0 = constructor_or(ctx, expr8_0, expr1_0, &expr9_0)?; + let expr11_0: Type = I64; + let expr12_0 = RegMemImm::Reg { reg: expr7_0 }; + let expr13_0 = constructor_or(ctx, expr11_0, expr3_0, &expr12_0)?; + let expr14_0 = C::value_regs(ctx, expr10_0, expr13_0); + return Some(expr14_0); +} + +// Generated as internal constructor for term shl_i128. +pub fn constructor_shl_i128( + ctx: &mut C, + arg0: ValueRegs, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/lower.isle line 539. + let expr0_0: usize = 0; + let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr2_0: usize = 1; + let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr4_0: Type = I64; + let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr6_0 = constructor_shl(ctx, expr4_0, expr1_0, &expr5_0)?; + let expr7_0: Type = I64; + let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr9_0 = constructor_shl(ctx, expr7_0, expr3_0, &expr8_0)?; + let expr10_0: Type = I64; + let expr11_0: Type = I64; + let expr12_0: Type = I64; + let expr13_0: u64 = 64; + let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?; + let expr15_0 = RegMemImm::Reg { reg: pattern1_0 }; + let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?; + let expr17_0 = Imm8Reg::Reg { reg: expr16_0 }; + let expr18_0 = constructor_shr(ctx, expr10_0, expr1_0, &expr17_0)?; + let expr19_0: Type = I64; + let expr20_0: u64 = 0; + let expr21_0 = constructor_imm(ctx, expr19_0, expr20_0)?; + let expr22_0 = OperandSize::Size64; + let expr23_0: u32 = 127; + let expr24_0 = RegMemImm::Imm { simm32: expr23_0 }; + let expr25_0 = constructor_test(ctx, &expr22_0, &expr24_0, pattern1_0)?; + let expr26_0: Type = I64; + let expr27_0 = CC::Z; + let expr28_0 = RegMem::Reg { reg: expr21_0 }; + let expr29_0 = constructor_cmove(ctx, expr26_0, &expr27_0, &expr28_0, expr18_0)?; + let expr30_0 = constructor_with_flags_1(ctx, &expr25_0, &expr29_0)?; + let expr31_0: Type = I64; + let expr32_0 = RegMemImm::Reg { reg: expr9_0 }; + let expr33_0 = constructor_or(ctx, expr31_0, expr30_0, &expr32_0)?; + let expr34_0 = OperandSize::Size64; + let expr35_0: u32 = 64; + let expr36_0 = RegMemImm::Imm { simm32: expr35_0 }; + let expr37_0 = constructor_test(ctx, &expr34_0, &expr36_0, pattern1_0)?; + let expr38_0: Type = I64; + let expr39_0 = CC::Z; + let expr40_0 = RegMem::Reg { reg: expr6_0 }; + let expr41_0 = constructor_cmove(ctx, expr38_0, &expr39_0, &expr40_0, expr21_0)?; + let expr42_0: Type = I64; + let expr43_0 = CC::Z; + let expr44_0 = RegMem::Reg { reg: expr33_0 }; + let expr45_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr6_0)?; + let expr46_0 = constructor_with_flags_2(ctx, &expr37_0, &expr41_0, &expr45_0)?; + return Some(expr46_0); +} + +// Generated as internal constructor for term shr_i128. +pub fn constructor_shr_i128( + ctx: &mut C, + arg0: ValueRegs, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/lower.isle line 586. + let expr0_0: usize = 0; + let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr2_0: usize = 1; + let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr4_0: Type = I64; + let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr6_0 = constructor_shr(ctx, expr4_0, expr1_0, &expr5_0)?; + let expr7_0: Type = I64; + let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr9_0 = constructor_shr(ctx, expr7_0, expr3_0, &expr8_0)?; + let expr10_0: Type = I64; + let expr11_0: Type = I64; + let expr12_0: Type = I64; + let expr13_0: u64 = 64; + let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?; + let expr15_0 = RegMemImm::Reg { reg: pattern1_0 }; + let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?; + let expr17_0 = Imm8Reg::Reg { reg: expr16_0 }; + let expr18_0 = constructor_shl(ctx, expr10_0, expr3_0, &expr17_0)?; + let expr19_0 = OperandSize::Size64; + let expr20_0: u32 = 127; + let expr21_0 = RegMemImm::Imm { simm32: expr20_0 }; + let expr22_0 = constructor_test(ctx, &expr19_0, &expr21_0, pattern1_0)?; + let expr23_0: Type = I64; + let expr24_0 = CC::Z; + let expr25_0: Type = I64; + let expr26_0: u64 = 0; + let expr27_0 = constructor_imm(ctx, expr25_0, expr26_0)?; + let expr28_0 = RegMem::Reg { reg: expr27_0 }; + let expr29_0 = constructor_cmove(ctx, expr23_0, &expr24_0, &expr28_0, expr18_0)?; + let expr30_0 = constructor_with_flags_1(ctx, &expr22_0, &expr29_0)?; + let expr31_0: Type = I64; + let expr32_0 = RegMemImm::Reg { reg: expr6_0 }; + let expr33_0 = constructor_or(ctx, expr31_0, expr30_0, &expr32_0)?; + let expr34_0 = OperandSize::Size64; + let expr35_0: u32 = 64; + let expr36_0 = RegMemImm::Imm { simm32: expr35_0 }; + let expr37_0 = constructor_test(ctx, &expr34_0, &expr36_0, pattern1_0)?; + let expr38_0: Type = I64; + let expr39_0 = CC::Z; + let expr40_0 = RegMem::Reg { reg: expr33_0 }; + let expr41_0 = constructor_cmove(ctx, expr38_0, &expr39_0, &expr40_0, expr9_0)?; + let expr42_0: Type = I64; + let expr43_0 = CC::Z; + let expr44_0 = RegMem::Reg { reg: expr9_0 }; + let expr45_0: Type = I64; + let expr46_0: u64 = 0; + let expr47_0 = constructor_imm(ctx, expr45_0, expr46_0)?; + let expr48_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr47_0)?; + let expr49_0 = constructor_with_flags_2(ctx, &expr37_0, &expr41_0, &expr48_0)?; + return Some(expr49_0); +} diff --git a/cranelift/codegen/src/isle.rs b/cranelift/codegen/src/isle.rs new file mode 100644 index 0000000000..f8794716b9 --- /dev/null +++ b/cranelift/codegen/src/isle.rs @@ -0,0 +1,22 @@ +// GENERATED BY ISLE. DO NOT EDIT! +// +// Generated automatically from the instruction-selection DSL code in: +// - src/clif.isle + +#![allow(dead_code, unreachable_code, unreachable_patterns)] +#![allow(unused_imports, unused_variables, non_snake_case)] + +use super::*; // Pulls in all external types. + +/// Context during lowering: an implementation of this trait +/// must be provided with all external constructors and extractors. +/// A mutable borrow is passed along through all lowering logic. +pub trait Context { + fn value_list_slice(&mut self, arg0: &ValueList) -> (ValueSlice,); + fn unwrap_head_value_list_1(&mut self, arg0: &ValueList) -> (Value, ValueSlice,); + fn unwrap_head_value_list_2(&mut self, arg0: &ValueList) -> (Value, Value, ValueSlice,); + fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> (ValueArray2,); + fn unpack_value_array_2(&mut self, arg0: &ValueArray2) -> (Value, Value,); + fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> (ValueArray3,); + fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value,); +} diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 0a35ad3ae6..e6e608ec3d 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -11,9 +11,9 @@ use crate::fx::{FxHashMap, FxHashSet}; use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit}; use crate::ir::instructions::BranchInfo; use crate::ir::{ - ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst, - InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef, - ValueLabelAssignments, ValueLabelStart, + ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function, + GlobalValueData, Inst, InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, + ValueDef, ValueLabelAssignments, ValueLabelStart, }; use crate::machinst::{ writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode, @@ -61,6 +61,8 @@ pub trait LowerCtx { /// The instruction type for which this lowering framework is instantiated. type I: VCodeInst; + fn dfg(&self) -> &DataFlowGraph; + // Function-level queries: /// Get the `ABICallee`. @@ -124,8 +126,12 @@ pub trait LowerCtx { /// instruction's result(s) must have *no* uses remaining, because it will /// not be codegen'd (it has been integrated into the current instruction). fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput; + /// Like `get_input_as_source_or_const` but with a `Value`. + fn get_value_as_source_or_const(&self, value: Value) -> NonRegInput; /// Put the `idx`th input into register(s) and return the assigned register. fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs; + /// Put the given value into register(s) and return the assigned register. + fn put_value_in_regs(&mut self, value: Value) -> ValueRegs; /// Get the `idx`th output register(s) of the given IR instruction. When /// `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected that /// the backend will write results to these output register(s). This @@ -1002,101 +1008,15 @@ impl<'func, I: VCodeInst> Lower<'func, I> { Ok((vcode, stack_map_info)) } - - fn put_value_in_regs(&mut self, val: Value) -> ValueRegs { - log::trace!("put_value_in_reg: val {}", val); - let mut regs = self.value_regs[val]; - log::trace!(" -> regs {:?}", regs); - assert!(regs.is_valid()); - - self.value_lowered_uses[val] += 1; - - // Pinned-reg hack: if backend specifies a fixed pinned register, use it - // directly when we encounter a GetPinnedReg op, rather than lowering - // the actual op, and do not return the source inst to the caller; the - // value comes "out of the ether" and we will not force generation of - // the superfluous move. - if let ValueDef::Result(i, 0) = self.f.dfg.value_def(val) { - if self.f.dfg[i].opcode() == Opcode::GetPinnedReg { - if let Some(pr) = self.pinned_reg { - regs = ValueRegs::one(pr); - } - } - } - - regs - } - - /// Get the actual inputs for a value. This is the implementation for - /// `get_input()` but starting from the SSA value, which is not exposed to - /// the backend. - fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput { - log::trace!( - "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}", - val, - self.cur_inst, - self.cur_scan_entry_color, - ); - let inst = match self.f.dfg.value_def(val) { - // OK to merge source instruction if (i) we have a source - // instruction, and: - // - It has no side-effects, OR - // - It has a side-effect, has one output value, that one output has - // only one use (this one), and the instruction's color is *one less - // than* the current scan color. - // - // This latter set of conditions is testing whether a - // side-effecting instruction can sink to the current scan - // location; this is possible if the in-color of this inst is - // equal to the out-color of the producing inst, so no other - // side-effecting ops occur between them (which will only be true - // if they are in the same BB, because color increments at each BB - // start). - // - // If it is actually sunk, then in `merge_inst()`, we update the - // scan color so that as we scan over the range past which the - // instruction was sunk, we allow other instructions (that came - // prior to the sunk instruction) to sink. - ValueDef::Result(src_inst, result_idx) => { - let src_side_effect = has_lowering_side_effect(self.f, src_inst); - log::trace!(" -> src inst {}", src_inst); - log::trace!(" -> has lowering side effect: {}", src_side_effect); - if !src_side_effect { - // Pure instruction: always possible to sink. - Some((src_inst, result_idx)) - } else { - // Side-effect: test whether this is the only use of the - // only result of the instruction, and whether colors allow - // the code-motion. - if self.cur_scan_entry_color.is_some() - && self.value_uses[val] == 1 - && self.value_lowered_uses[val] == 0 - && self.num_outputs(src_inst) == 1 - && self - .side_effect_inst_entry_colors - .get(&src_inst) - .unwrap() - .get() - + 1 - == self.cur_scan_entry_color.unwrap().get() - { - Some((src_inst, 0)) - } else { - None - } - } - } - _ => None, - }; - let constant = inst.and_then(|(inst, _)| self.get_constant(inst)); - - NonRegInput { inst, constant } - } } impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { type I = I; + fn dfg(&self) -> &DataFlowGraph { + &self.f.dfg + } + fn abi(&mut self) -> &mut dyn ABICallee { self.vcode.abi() } @@ -1207,12 +1127,99 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { self.get_value_as_source_or_const(val) } + fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput { + log::trace!( + "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}", + val, + self.cur_inst, + self.cur_scan_entry_color, + ); + let inst = match self.f.dfg.value_def(val) { + // OK to merge source instruction if (i) we have a source + // instruction, and: + // - It has no side-effects, OR + // - It has a side-effect, has one output value, that one output has + // only one use (this one), and the instruction's color is *one less + // than* the current scan color. + // + // This latter set of conditions is testing whether a + // side-effecting instruction can sink to the current scan + // location; this is possible if the in-color of this inst is + // equal to the out-color of the producing inst, so no other + // side-effecting ops occur between them (which will only be true + // if they are in the same BB, because color increments at each BB + // start). + // + // If it is actually sunk, then in `merge_inst()`, we update the + // scan color so that as we scan over the range past which the + // instruction was sunk, we allow other instructions (that came + // prior to the sunk instruction) to sink. + ValueDef::Result(src_inst, result_idx) => { + let src_side_effect = has_lowering_side_effect(self.f, src_inst); + log::trace!(" -> src inst {}", src_inst); + log::trace!(" -> has lowering side effect: {}", src_side_effect); + if !src_side_effect { + // Pure instruction: always possible to sink. + Some((src_inst, result_idx)) + } else { + // Side-effect: test whether this is the only use of the + // only result of the instruction, and whether colors allow + // the code-motion. + if self.cur_scan_entry_color.is_some() + && self.value_uses[val] == 1 + && self.value_lowered_uses[val] == 0 + && self.num_outputs(src_inst) == 1 + && self + .side_effect_inst_entry_colors + .get(&src_inst) + .unwrap() + .get() + + 1 + == self.cur_scan_entry_color.unwrap().get() + { + Some((src_inst, 0)) + } else { + None + } + } + } + _ => None, + }; + let constant = inst.and_then(|(inst, _)| self.get_constant(inst)); + + NonRegInput { inst, constant } + } + fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs { let val = self.f.dfg.inst_args(ir_inst)[idx]; - let val = self.f.dfg.resolve_aliases(val); self.put_value_in_regs(val) } + fn put_value_in_regs(&mut self, val: Value) -> ValueRegs { + let val = self.f.dfg.resolve_aliases(val); + log::trace!("put_value_in_reg: val {}", val); + let mut regs = self.value_regs[val]; + log::trace!(" -> regs {:?}", regs); + assert!(regs.is_valid()); + + self.value_lowered_uses[val] += 1; + + // Pinned-reg hack: if backend specifies a fixed pinned register, use it + // directly when we encounter a GetPinnedReg op, rather than lowering + // the actual op, and do not return the source inst to the caller; the + // value comes "out of the ether" and we will not force generation of + // the superfluous move. + if let ValueDef::Result(i, 0) = self.f.dfg.value_def(val) { + if self.f.dfg[i].opcode() == Opcode::GetPinnedReg { + if let Some(pr) = self.pinned_reg { + regs = ValueRegs::one(pr); + } + } + } + + regs + } + fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs> { let val = self.f.dfg.inst_results(ir_inst)[idx]; writable_value_regs(self.value_regs[val]) diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle new file mode 100644 index 0000000000..07cc34e87e --- /dev/null +++ b/cranelift/codegen/src/prelude.isle @@ -0,0 +1,202 @@ +;; This is a prelude of standard definitions for ISLE, the instruction-selector +;; DSL, as we use it bound to our interfaces. + +;;;; Primitive and External Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `()` +(type Unit (primitive Unit)) + +;; `bool` is declared in `clif.isle`. +(extern const $true bool) +(extern const $false bool) + +(type u8 (primitive u8)) +(type u16 (primitive u16)) +(type u32 (primitive u32)) +(type u64 (primitive u64)) +(type u128 (primitive u128)) +(type usize (primitive usize)) + +(type i8 (primitive i8)) +(type i16 (primitive i16)) +(type i32 (primitive i32)) +(type i64 (primitive i64)) +(type i128 (primitive i128)) +(type isize (primitive isize)) + +;; `cranelift-entity`-based identifiers. +(type Inst (primitive Inst)) +(type Type (primitive Type)) +(type Value (primitive Value)) + +;; ISLE representation of `&[Value]`. +(type ValueSlice (primitive ValueSlice)) + +(type ValueList (primitive ValueList)) +(type ValueRegs (primitive ValueRegs)) + +;;;; Registers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type Reg (primitive Reg)) +(type WritableReg (primitive WritableReg)) + +;; Construct a `ValueRegs` of one register. +(decl value_reg (Reg) ValueRegs) +(extern constructor value_reg value_reg) + +;; Construct a `ValueRegs` of two registers. +(decl value_regs (Reg Reg) ValueRegs) +(extern constructor value_regs value_regs) + +;; Get a temporary register for writing. +(decl temp_writable_reg (Type) WritableReg) +(extern constructor temp_writable_reg temp_writable_reg) + +;; Get a temporary register for reading. +(decl temp_reg (Type) Reg) +(rule (temp_reg ty) + (writable_reg_to_reg (temp_writable_reg ty))) + +;; Get the invalid register. +(decl invalid_reg () Reg) +(extern constructor invalid_reg invalid_reg) + +;; Put the given value into a register. +;; +;; Asserts that the value fits into a single register, and doesn't require +;; multiple registers for its representation (like `i128` on x64 for example). +;; +;; As a side effect, this marks the value as used. +(decl put_in_reg (Value) Reg) +(extern constructor put_in_reg put_in_reg) + +;; Put the given value into one or more registers. +;; +;; As a side effect, this marks the value as used. +(decl put_in_regs (Value) ValueRegs) +(extern constructor put_in_regs put_in_regs) + +;; Get the `n`th register inside a `ValueRegs`. +(decl value_regs_get (ValueRegs usize) Reg) +(extern constructor value_regs_get value_regs_get) + +;; Put the value into one or more registers and return the first register. +;; +;; Unlike `put_in_reg`, this does not assert that the value fits in a single +;; register. This is useful for things like a `i128` shift amount, where we mask +;; the shift amount to the bit width of the value being shifted, and so the high +;; half of the `i128` won't ever be used. +;; +;; As a side efect, this marks that value as used. +(decl lo_reg (Value) Reg) +(rule (lo_reg val) + (let ((regs ValueRegs (put_in_regs val))) + (value_regs_get regs 0))) + +;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl u8_as_u64 (u8) u64) +(extern constructor u8_as_u64 u8_as_u64) + +(decl u16_as_u64 (u16) u64) +(extern constructor u16_as_u64 u16_as_u64) + +(decl u32_as_u64 (u32) u64) +(extern constructor u32_as_u64 u32_as_u64) + +;;;; `cranelift_codegen::ir::Type` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(extern const $B1 Type) +(extern const $B8 Type) +(extern const $B16 Type) +(extern const $B32 Type) +(extern const $B64 Type) +(extern const $B128 Type) + +(extern const $I8 Type) +(extern const $I16 Type) +(extern const $I32 Type) +(extern const $I64 Type) +(extern const $I128 Type) + +(extern const $B8X16 Type) +(extern const $B16X8 Type) +(extern const $B32X4 Type) +(extern const $B64X2 Type) + +(extern const $I8X16 Type) +(extern const $I16X8 Type) +(extern const $I32X4 Type) +(extern const $I64X2 Type) + +(extern const $F32X4 Type) +(extern const $F64X2 Type) + +;; Get the bit width of a given type. +(decl ty_bits (Type) u16) +(extern constructor ty_bits ty_bits) + +;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; An extractor that only matches types that can fit in 64 bits. +(decl fits_in_64 (Type) Type) +(extern extractor fits_in_64 fits_in_64) + +;; Extractor to get a `ValueSlice` out of a `ValueList`. +(decl value_list_slice (ValueSlice) ValueList) +(extern extractor infallible value_list_slice value_list_slice) + +;; Extractor to get the first element from a value list, along with its tail as +;; a `ValueSlice`. +(decl unwrap_head_value_list_1 (Value ValueSlice) ValueList) +(extern extractor infallible unwrap_head_value_list_1 unwrap_head_value_list_1) + +;; Extractor to get the first two elements from a value list, along with its +;; tail as a `ValueSlice`. +(decl unwrap_head_value_list_2 (Value Value ValueSlice) ValueList) +(extern extractor infallible unwrap_head_value_list_2 unwrap_head_value_list_2) + +;; Turn a `Writable` into a `Reg` via `Writable::to_reg`. +(decl writable_reg_to_reg (WritableReg) Reg) +(extern constructor writable_reg_to_reg writable_reg_to_reg) + +;; Extract a `u64` from an `Imm64`. +(decl u64_from_imm64 (u64) Imm64) +(extern extractor infallible u64_from_imm64 u64_from_imm64) + +;; Extract the result values for the given instruction. +(decl inst_results (ValueSlice) Inst) +(extern extractor infallible inst_results inst_results) + +;; Extract the first result value of the given instruction. +(decl first_result (Value) Inst) +(extern extractor first_result first_result) + +;; Extract the `InstructionData` for an `Inst`. +(decl inst_data (InstructionData) Inst) +(extern extractor infallible inst_data inst_data) + +;; Extract the type of a `Value`. +(decl value_type (Type) Value) +(extern extractor infallible value_type value_type) + +;; Extract the type of the instruction's first result. +(decl result_type (Type) Inst) +(extractor (result_type ty) + (first_result (value_type ty))) + +;; Extract the type of the instruction's first result and pass along the +;; instruction as well. +(decl has_type (Type Inst) Inst) +(extractor (has_type ty inst) + (and (result_type ty) + inst)) + +;; Match a multi-lane type, extracting (# bits per lane, # lanes) from the given +;; type. Will only match when there is more than one lane. +(decl multi_lane (u8 u16) Type) +(extern extractor multi_lane multi_lane) + +;; Match the instruction that defines the given value, if any. +(decl def_inst (Inst) Value) +(extern extractor def_inst def_inst) diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs index d37ea8ae09..d4a057bf4e 100644 --- a/cranelift/entity/src/list.rs +++ b/cranelift/entity/src/list.rs @@ -62,7 +62,7 @@ use serde::{Deserialize, Serialize}; /// /// The index stored in an `EntityList` points to part 2, the list elements. The value 0 is /// reserved for the empty list which isn't allocated in the vector. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct EntityList { index: u32, @@ -271,7 +271,7 @@ impl EntityList { } /// Get the list as a slice. - pub fn as_slice<'a>(&'a self, pool: &'a ListPool) -> &'a [T] { + pub fn as_slice<'a>(&self, pool: &'a ListPool) -> &'a [T] { let idx = self.index as usize; match pool.len_of(self) { None => &[], diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index 96520af906..4e6ce01a0d 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -700,31 +700,30 @@ block2(v6: i128): v8 = iadd.i128 v6, v7 return v8 -; check: pushq %rbp +; check: Block 0: +; check: pushq %rbp ; nextln: movq %rsp, %rbp +; nextln: xorq %rdi, %rdi +; nextln: xorq %rsi, %rsi ; nextln: testb $$1, %dl ; nextln: jnz label1; j label2 ; check: Block 1: -; check: movl $$0, %esi -; nextln: movl $$0, %edi -; nextln: movl $$1, %eax -; nextln: movl $$0, %ecx -; nextln: addq %rax, %rsi -; nextln: adcq %rcx, %rdi -; nextln: movq %rsi, %rax -; nextln: movq %rdi, %rdx +; check: movl $$1, %ecx +; nextln: xorq %rax, %rax +; nextln: addq %rcx, %rdi +; nextln: adcq %rax, %rsi +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret ; check: Block 2: -; check: movl $$0, %esi -; nextln: movl $$0, %edi -; nextln: movl $$2, %eax -; nextln: movl $$0, %ecx -; nextln: addq %rax, %rsi -; nextln: adcq %rcx, %rdi -; nextln: movq %rsi, %rax -; nextln: movq %rdi, %rdx +; check: movl $$2, %ecx +; nextln: xorq %rax, %rax +; nextln: addq %rcx, %rdi +; nextln: adcq %rax, %rsi +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret @@ -744,34 +743,32 @@ block0(v0: i128, v1: i128, v2: i64, v3: i128, v4: i128, v5: i128): ; check: pushq %rbp ; nextln: movq %rsp, %rbp -; nextln: subq $$32, %rsp +; nextln: subq $$16, %rsp ; nextln: movq %r12, 0(%rsp) ; nextln: movq %r13, 8(%rsp) -; nextln: movq %r14, 16(%rsp) -; nextln: movq %r8, %r14 -; nextln: movq 16(%rbp), %r10 +; nextln: movq %r9, %r11 +; nextln: movq 16(%rbp), %r13 ; nextln: movq 24(%rbp), %r12 -; nextln: movq 32(%rbp), %r11 -; nextln: movq 40(%rbp), %rax -; nextln: movq 48(%rbp), %r13 -; nextln: movq %rsi, %r8 +; nextln: movq 32(%rbp), %r10 +; nextln: movq 40(%rbp), %r9 +; nextln: movq 48(%rbp), %rax ; nextln: addq %rdx, %rdi -; nextln: adcq %rcx, %r8 +; nextln: movq %rsi, %rdx +; nextln: adcq %rcx, %rdx ; nextln: xorq %rsi, %rsi -; nextln: addq %r14, %r9 -; nextln: adcq %rsi, %r10 -; nextln: addq %rax, %r12 -; nextln: adcq %r13, %r11 -; nextln: addq %r9, %rdi -; nextln: adcq %r10, %r8 +; nextln: addq %r8, %r11 +; nextln: adcq %rsi, %r13 +; nextln: addq %r9, %r12 +; nextln: adcq %rax, %r10 +; nextln: addq %r11, %rdi +; nextln: adcq %r13, %rdx ; nextln: addq %rdi, %r12 -; nextln: adcq %r8, %r11 +; nextln: adcq %rdx, %r10 ; nextln: movq %r12, %rax -; nextln: movq %r11, %rdx +; nextln: movq %r10, %rdx ; nextln: movq 0(%rsp), %r12 ; nextln: movq 8(%rsp), %r13 -; nextln: movq 16(%rsp), %r14 -; nextln: addq $$32, %rsp +; nextln: addq $$16, %rsp ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret @@ -907,26 +904,25 @@ block0(v0: i128, v1: i128): ; check: pushq %rbp ; nextln: movq %rsp, %rbp -; nextln: movq %rsi, %rax -; nextln: movq %rdi, %rsi +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdi +; nextln: movq %rax, %rsi ; nextln: movq %rdx, %rcx ; nextln: shlq %cl, %rsi ; nextln: movq %rdx, %rcx -; nextln: shlq %cl, %rax +; nextln: shlq %cl, %rdi ; nextln: movl $$64, %ecx ; nextln: subq %rdx, %rcx -; nextln: shrq %cl, %rdi +; nextln: shrq %cl, %rax ; nextln: xorq %rcx, %rcx ; nextln: testq $$127, %rdx -; nextln: cmovzq %rcx, %rdi -; nextln: orq %rax, %rdi -; nextln: xorq %rax, %rax -; nextln: andq $$64, %rdx -; nextln: cmovzq %rdi, %rax +; nextln: cmovzq %rcx, %rax +; nextln: orq %rdi, %rax +; nextln: testq $$64, %rdx ; nextln: cmovzq %rsi, %rcx -; nextln: cmovnzq %rsi, %rax -; nextln: movq %rax, %rdx +; nextln: cmovzq %rax, %rsi ; nextln: movq %rcx, %rax +; nextln: movq %rsi, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret @@ -939,28 +935,26 @@ block0(v0: i128, v1: i128): ; check: pushq %rbp ; nextln: movq %rsp, %rbp -; nextln: movq %rdi, %rax -; nextln: movq %rsi, %rdi -; nextln: movq %rdi, %rsi +; nextln: movq %rsi, %rax +; nextln: movq %rdx, %rcx +; nextln: shrq %cl, %rdi +; nextln: movq %rax, %rsi ; nextln: movq %rdx, %rcx ; nextln: shrq %cl, %rsi -; nextln: movq %rdx, %rcx -; nextln: shrq %cl, %rax ; nextln: movl $$64, %ecx ; nextln: subq %rdx, %rcx -; nextln: shlq %cl, %rdi +; nextln: shlq %cl, %rax ; nextln: xorq %rcx, %rcx ; nextln: testq $$127, %rdx -; nextln: cmovzq %rcx, %rdi -; nextln: orq %rax, %rdi -; nextln: xorq %rax, %rax +; nextln: cmovzq %rcx, %rax +; nextln: orq %rdi, %rax ; nextln: xorq %rcx, %rcx -; nextln: andq $$64, %rdx -; nextln: cmovzq %rsi, %rax -; nextln: cmovzq %rdi, %rcx -; nextln: cmovnzq %rsi, %rcx -; nextln: movq %rax, %rdx -; nextln: movq %rcx, %rax +; nextln: testq $$64, %rdx +; nextln: movq %rsi, %rdi +; nextln: cmovzq %rax, %rdi +; nextln: cmovzq %rsi, %rcx +; nextln: movq %rdi, %rax +; nextln: movq %rcx, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret @@ -1006,53 +1000,51 @@ block0(v0: i128, v1: i128): return v2 } -; check: pushq %rbp +; check: pushq %rbp ; nextln: movq %rsp, %rbp -; nextln: movq %rdi, %r8 -; nextln: movq %r8, %r9 -; nextln: movq %rdx, %rcx -; nextln: shlq %cl, %r9 -; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rax ; nextln: movq %rdx, %rcx ; nextln: shlq %cl, %rax +; nextln: movq %rsi, %r8 +; nextln: movq %rdx, %rcx +; nextln: shlq %cl, %r8 ; nextln: movl $$64, %ecx ; nextln: subq %rdx, %rcx -; nextln: movq %r8, %r10 -; nextln: shrq %cl, %r10 -; nextln: xorq %rdi, %rdi +; nextln: movq %rdi, %r9 +; nextln: shrq %cl, %r9 +; nextln: xorq %rcx, %rcx ; nextln: testq $$127, %rdx -; nextln: cmovzq %rdi, %r10 -; nextln: orq %rax, %r10 -; nextln: xorq %rax, %rax -; nextln: movq %rdx, %rcx -; nextln: andq $$64, %rcx -; nextln: cmovzq %r10, %rax -; nextln: cmovzq %r9, %rdi -; nextln: cmovnzq %r9, %rax +; nextln: cmovzq %rcx, %r9 +; nextln: orq %r8, %r9 +; nextln: testq $$64, %rdx +; nextln: movq %rcx, %r8 +; nextln: cmovzq %rax, %r8 +; nextln: cmovzq %r9, %rax ; nextln: movl $$128, %r9d ; nextln: subq %rdx, %r9 -; nextln: movq %rsi, %rdx +; nextln: movq %rdi, %rdx ; nextln: movq %r9, %rcx ; nextln: shrq %cl, %rdx +; nextln: movq %rsi, %rdi ; nextln: movq %r9, %rcx -; nextln: shrq %cl, %r8 +; nextln: shrq %cl, %rdi ; nextln: movl $$64, %ecx ; nextln: subq %r9, %rcx ; nextln: shlq %cl, %rsi ; nextln: xorq %rcx, %rcx ; nextln: testq $$127, %r9 ; nextln: cmovzq %rcx, %rsi -; nextln: orq %r8, %rsi -; nextln: xorq %rcx, %rcx -; nextln: xorq %r8, %r8 -; nextln: andq $$64, %r9 -; nextln: cmovzq %rdx, %rcx -; nextln: cmovzq %rsi, %r8 -; nextln: cmovnzq %rdx, %r8 -; nextln: orq %rdi, %r8 -; nextln: orq %rax, %rcx +; nextln: orq %rdx, %rsi +; nextln: xorq %rdx, %rdx +; nextln: testq $$64, %r9 +; nextln: movq %rdi, %rcx +; nextln: cmovzq %rsi, %rcx +; nextln: movq %rdx, %rsi +; nextln: cmovzq %rdi, %rsi +; nextln: orq %rcx, %r8 +; nextln: orq %rsi, %rax +; nextln: movq %rax, %rdx ; nextln: movq %r8, %rax -; nextln: movq %rcx, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret diff --git a/cranelift/isle/fuzz/README.md b/cranelift/isle/fuzz/README.md new file mode 100644 index 0000000000..a6d695df10 --- /dev/null +++ b/cranelift/isle/fuzz/README.md @@ -0,0 +1,4 @@ +# ISLE Fuzz Targets + +These are separate from the top-level `wasmtime/fuzz` fuzz targets because we +don't intend to run them on OSS-Fuzz. They are just for local ISLE hacking. diff --git a/cranelift/isle/isle/Cargo.toml b/cranelift/isle/isle/Cargo.toml index 5c7acee3a4..14d709ef07 100644 --- a/cranelift/isle/isle/Cargo.toml +++ b/cranelift/isle/isle/Cargo.toml @@ -1,5 +1,5 @@ [package] -authors = ["Chris Fallin ", "Nick Fitzgerald "] +authors = ["The Cranelift Project Developers"] description = "ISLE: Instruction Selection and Lowering Expressions. A domain-specific language for instruction selection in Cranelift." edition = "2018" license = "Apache-2.0 WITH LLVM-exception" diff --git a/cranelift/isle/islec/Cargo.toml b/cranelift/isle/islec/Cargo.toml index cc80b44fc0..3a6ab3d115 100644 --- a/cranelift/isle/islec/Cargo.toml +++ b/cranelift/isle/islec/Cargo.toml @@ -1,9 +1,10 @@ [package] name = "islec" version = "0.1.0" -authors = ["Chris Fallin "] +authors = ["The Cranelift Project Developers"] edition = "2018" license = "Apache-2.0 WITH LLVM-exception" +publish = false [dependencies] log = "0.4" diff --git a/deny.toml b/deny.toml index 1bfbbf46c8..bcfd4b41a2 100644 --- a/deny.toml +++ b/deny.toml @@ -35,4 +35,5 @@ skip = [ { name = "wast" }, # old one pulled in by witx { name = "itertools" }, # 0.9 pulled in by criterion-plot { name = "quick-error" }, # transitive dependencies + { name = "textwrap" }, # `miette` and `clap` depend on different versions ] diff --git a/scripts/publish.rs b/scripts/publish.rs index e4aa342246..1b8083eb4a 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -26,6 +26,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "peepmatic", "peepmatic-souper", // cranelift + "isle", "cranelift-entity", "wasmtime-types", "cranelift-bforest",