T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix iadd_ifcout lowering in ISLE to return a register corresponding to the iflags.

Browse Source 
This register is not initialized, but we protect against its being used
by never allowing an iflags/fflags-typed value to be used with
`put_value_in_regs`. All `iflags`/`fflags` usages should be handled by
pattern-matching: e.g., `trapif` explicitly matches an `iadd_ifcout`
input.

Eventually (#3249) we need to simplify this by removing
iflags/fflags-tyepd values and using bool flags instead,
pattern-matching to get the same efficient lowerings as today. For now,
this allows the ISLE assertions to pass.
This commit is contained in:
Chris Fallin
2021-12-08 11:40:58 -08:00
committed by Andrew Brown
parent acaa84068d
commit 7bc17fda39
4 changed files with 183 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -153,36 +153,56 @@
;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; N.B.: the second output of `iadd_ifcout` is meant to be the
;; `iflags` value containing the carry result. However, we plan to
;; replace this with a bool carry flag, and all consumers of `iflags`
;; remain in the handwritten pattern-matching code and explicitly
;; match on the flags producer. So we can get away with just
;; allocating a second temp so that the reg-renaming code does the
;; right thing, for now. For safety, we assert elsewhere that no one
;; actually uses the register assigned to the SSA `iflags`-typed
;; `Value`.
;; Add two registers.
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout x y)))
(value_reg (add ty
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
(RegMemImm.Reg (put_in_reg y)))
unused_iflags)))
;; Add a register and an immediate.
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout x (simm32_from_value y))))
(value_reg (add ty (put_in_reg x) y)))
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty (put_in_reg x) y)
unused_iflags)))
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout (simm32_from_value x) y)))
(value_reg (add ty (put_in_reg y) x)))
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty (put_in_reg y) x)
unused_iflags)))
;; Add a register and memory.
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout x (sinkable_load y))))
(value_reg (add ty
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty
(put_in_reg x)
(sink_load y))))
(sink_load y))
unused_iflags)))
(rule (lower (has_type (fits_in_64 ty)
(iadd_ifcout (sinkable_load x) y)))
(value_reg (add ty
(let ((unused_iflags Reg (writable_reg_to_reg (temp_writable_reg $I64))))
(value_regs (add ty
(put_in_reg y)
(sink_load x))))
(sink_load x))
unused_iflags)))
;; (No `iadd_ifcout` for `i128`.)

2
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9c0563583e5500de00ec5e226edc0547ac3ea789c8d76f1da0401c80ec619320fdc9a6f17fd76bbcac74a5894f85385c1f51c900c2b83bc9906d03d0f29bf5cb
src/prelude.isle 9bd1fcb6a3604a24cf2e05e6b7eb04dcb3b9dc8fa9a2f1c8f29c25b6e3bf7f679b3b1b72dff87501497b72bc30fc92fd755b898db7e03f380235fae931b6a74b
src/isa/x64/inst.isle c93db8fe2819e952af629c4f4c31163545efcf5158f0747987eca30b5e42f2b6e75290bbccc72d7f44c2b43aab67fcbc339ef5a687918f796dd9317bad77a695
src/isa/x64/lower.isle 333e1be62f602bb835a3cebc3299290a3d386438e9190d2db219263d974e097bfc3f1afdaac9401853806d21d548cad70bab2ffbc3b1cf5c3bebdd971a961f70
src/isa/x64/lower.isle fe3dd6583a2415c7940b1ea414b14d4dd106f9cd46b605d52dcb0dfbe2d00bc4507925b3bb35e74ac3444138d256cd896b7c750957844d699397ffed21907d3a

273
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

@@ -1951,7 +1951,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, &pattern2_1);
let pattern5_0 = C::value_type(ctx, pattern4_0);
let pattern6_0 = C::u8_from_uimm8(ctx, pattern2_2);
// Rule at src/isa/x64/lower.isle line 997.
// Rule at src/isa/x64/lower.isle line 1017.
let expr0_0 = C::put_in_reg(ctx, pattern4_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern4_1);
let expr2_0 =
@@ -2004,7 +2004,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Band => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 368.
// Rule at src/isa/x64/lower.isle line 388.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2020,7 +2020,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 446.
// Rule at src/isa/x64/lower.isle line 466.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2036,7 +2036,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bxor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 512.
// Rule at src/isa/x64/lower.isle line 532.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2106,7 +2106,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Isub => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 266.
// Rule at src/isa/x64/lower.isle line 286.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2130,7 +2130,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Imul => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 696.
// Rule at src/isa/x64/lower.isle line 716.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2166,7 +2166,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Band => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 358.
// Rule at src/isa/x64/lower.isle line 378.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2189,7 +2189,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 443.
// Rule at src/isa/x64/lower.isle line 463.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0 = C::put_in_regs(ctx, pattern7_1);
let expr2_0 = constructor_or_i128(ctx, expr0_0, expr1_0)?;
@@ -2198,7 +2198,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bxor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 502.
// Rule at src/isa/x64/lower.isle line 522.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2221,7 +2221,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Rotl => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 629.
// Rule at src/isa/x64/lower.isle line 649.
let expr0_0 = C::put_in_regs(ctx, pattern7_0);
let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?;
let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?;
@@ -2238,7 +2238,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Ishl => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 562.
// Rule at src/isa/x64/lower.isle line 582.
let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
let expr1_0 = C::put_in_regs(ctx, pattern7_0);
let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?;
@@ -2247,7 +2247,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Ushr => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 608.
// Rule at src/isa/x64/lower.isle line 628.
let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
let expr1_0 = C::put_in_regs(ctx, pattern7_0);
let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?;
@@ -2263,7 +2263,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
} => {
if let &Opcode::IaddImm = &pattern5_0 {
let pattern7_0 = C::u64_from_imm64(ctx, pattern5_2);
// Rule at src/isa/x64/lower.isle line 212.
// Rule at src/isa/x64/lower.isle line 232.
let expr0_0 = C::put_in_regs(ctx, pattern5_1);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -2293,7 +2293,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
} = &pattern4_0
{
if let &Opcode::Iabs = &pattern5_0 {
// Rule at src/isa/x64/lower.isle line 951.
// Rule at src/isa/x64/lower.isle line 971.
let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1);
let expr1_0 = constructor_pabsb(ctx, &expr0_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
@@ -2309,7 +2309,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
} = &pattern4_0
{
if let &Opcode::Iabs = &pattern5_0 {
// Rule at src/isa/x64/lower.isle line 954.
// Rule at src/isa/x64/lower.isle line 974.
let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1);
let expr1_0 = constructor_pabsw(ctx, &expr0_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
@@ -2325,7 +2325,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
} = &pattern4_0
{
if let &Opcode::Iabs = &pattern5_0 {
// Rule at src/isa/x64/lower.isle line 957.
// Rule at src/isa/x64/lower.isle line 977.
let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1);
let expr1_0 = constructor_pabsd(ctx, &expr0_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
@@ -2341,7 +2341,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
} = &pattern4_0
{
if let &Opcode::Iabs = &pattern5_0 {
// Rule at src/isa/x64/lower.isle line 971.
// Rule at src/isa/x64/lower.isle line 991.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0: Type = I64X2;
let expr2_0: u64 = 0;
@@ -2366,7 +2366,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Band => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 343.
// Rule at src/isa/x64/lower.isle line 363.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr2_0 = constructor_andps(ctx, expr0_0, &expr1_0)?;
@@ -2376,7 +2376,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 419.
// Rule at src/isa/x64/lower.isle line 439.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr2_0 = constructor_orps(ctx, expr0_0, &expr1_0)?;
@@ -2386,7 +2386,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::BandNot => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 940.
// Rule at src/isa/x64/lower.isle line 960.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0);
let expr2_0 = constructor_andnps(ctx, expr0_0, &expr1_0)?;
@@ -2401,7 +2401,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
arg: pattern5_1,
} => {
if let &Opcode::Fabs = &pattern5_0 {
// Rule at src/isa/x64/lower.isle line 979.
// Rule at src/isa/x64/lower.isle line 999.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0: Type = F32X4;
let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?;
@@ -2428,7 +2428,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Band => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 347.
// Rule at src/isa/x64/lower.isle line 367.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr2_0 = constructor_andpd(ctx, expr0_0, &expr1_0)?;
@@ -2438,7 +2438,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 423.
// Rule at src/isa/x64/lower.isle line 443.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr2_0 = constructor_orpd(ctx, expr0_0, &expr1_0)?;
@@ -2448,7 +2448,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::BandNot => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 943.
// Rule at src/isa/x64/lower.isle line 963.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0);
let expr2_0 = constructor_andnpd(ctx, expr0_0, &expr1_0)?;
@@ -2463,7 +2463,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
arg: pattern5_1,
} => {
if let &Opcode::Fabs = &pattern5_0 {
// Rule at src/isa/x64/lower.isle line 984.
// Rule at src/isa/x64/lower.isle line 1004.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0: Type = F64X2;
let expr2_0 = constructor_vector_all_ones(ctx, expr1_0)?;
@@ -2506,7 +2506,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if let &Opcode::Imul = &pattern9_0 {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, &pattern9_1);
// Rule at src/isa/x64/lower.isle line 731.
// Rule at src/isa/x64/lower.isle line 751.
let expr0_0 = C::put_in_reg_mem(ctx, pattern11_0);
let expr1_0 = C::put_in_reg(ctx, pattern11_1);
let expr2_0 = constructor_vpmullq(ctx, &expr0_0, expr1_0)?;
@@ -2527,7 +2527,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
} = &pattern6_0
{
if let &Opcode::Iabs = &pattern7_0 {
// Rule at src/isa/x64/lower.isle line 961.
// Rule at src/isa/x64/lower.isle line 981.
let expr0_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr1_0 = constructor_vpabsq(ctx, &expr0_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
@@ -2550,7 +2550,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::AvgRound => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 639.
// Rule at src/isa/x64/lower.isle line 659.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?;
@@ -2580,7 +2580,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::UsubSat => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 293.
// Rule at src/isa/x64/lower.isle line 313.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubusb(ctx, expr0_0, &expr1_0)?;
@@ -2590,7 +2590,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::SsubSat => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 281.
// Rule at src/isa/x64/lower.isle line 301.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubsb(ctx, expr0_0, &expr1_0)?;
@@ -2610,7 +2610,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Isub => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 245.
// Rule at src/isa/x64/lower.isle line 265.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubb(ctx, expr0_0, &expr1_0)?;
@@ -2634,7 +2634,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::AvgRound => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 643.
// Rule at src/isa/x64/lower.isle line 663.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?;
@@ -2664,7 +2664,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::UsubSat => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 298.
// Rule at src/isa/x64/lower.isle line 318.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubusw(ctx, expr0_0, &expr1_0)?;
@@ -2674,7 +2674,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::SsubSat => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 286.
// Rule at src/isa/x64/lower.isle line 306.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubsw(ctx, expr0_0, &expr1_0)?;
@@ -2694,7 +2694,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Isub => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 250.
// Rule at src/isa/x64/lower.isle line 270.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubw(ctx, expr0_0, &expr1_0)?;
@@ -2747,7 +2747,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 8 {
if pattern23_1 == 16
{
// Rule at src/isa/x64/lower.isle line 819.
// Rule at src/isa/x64/lower.isle line 839.
let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
let expr1_0 = constructor_pmovsxbw(ctx, &expr0_0)?;
let expr2_0 = C::put_in_reg_mem(ctx, pattern20_1);
@@ -2805,7 +2805,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 8 {
if pattern23_1 == 16
{
// Rule at src/isa/x64/lower.isle line 779.
// Rule at src/isa/x64/lower.isle line 799.
let expr0_0 = C::put_in_reg(ctx, pattern12_1);
let expr1_0 = RegMem::Reg {
reg: expr0_0,
@@ -2881,7 +2881,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 8 {
if pattern23_1 == 16
{
// Rule at src/isa/x64/lower.isle line 895.
// Rule at src/isa/x64/lower.isle line 915.
let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
let expr1_0 = constructor_pmovzxbw(ctx, &expr0_0)?;
let expr2_0 = C::put_in_reg_mem(ctx, pattern20_1);
@@ -2939,7 +2939,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 8 {
if pattern23_1 == 16
{
// Rule at src/isa/x64/lower.isle line 855.
// Rule at src/isa/x64/lower.isle line 875.
let expr0_0 = C::put_in_reg(ctx, pattern12_1);
let expr1_0 = RegMem::Reg {
reg: expr0_0,
@@ -2984,7 +2984,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
}
}
}
// Rule at src/isa/x64/lower.isle line 723.
// Rule at src/isa/x64/lower.isle line 743.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_pmullw(ctx, expr0_0, &expr1_0)?;
@@ -3018,7 +3018,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Isub => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 255.
// Rule at src/isa/x64/lower.isle line 275.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubd(ctx, expr0_0, &expr1_0)?;
@@ -3071,7 +3071,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 16 {
if pattern23_1 == 8
{
// Rule at src/isa/x64/lower.isle line 829.
// Rule at src/isa/x64/lower.isle line 849.
let expr0_0 = C::put_in_reg(ctx, pattern12_1);
let expr1_0 = C::put_in_reg(ctx, pattern20_1);
let expr2_0 = RegMem::Reg {
@@ -3135,7 +3135,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 16 {
if pattern23_1 == 8
{
// Rule at src/isa/x64/lower.isle line 793.
// Rule at src/isa/x64/lower.isle line 813.
let expr0_0 = C::put_in_reg(ctx, pattern12_1);
let expr1_0 = C::put_in_reg(ctx, pattern20_1);
let expr2_0 = RegMem::Reg {
@@ -3199,7 +3199,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 16 {
if pattern23_1 == 8
{
// Rule at src/isa/x64/lower.isle line 905.
// Rule at src/isa/x64/lower.isle line 925.
let expr0_0 = C::put_in_reg(ctx, pattern12_1);
let expr1_0 = C::put_in_reg(ctx, pattern20_1);
let expr2_0 = RegMem::Reg {
@@ -3263,7 +3263,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 16 {
if pattern23_1 == 8
{
// Rule at src/isa/x64/lower.isle line 869.
// Rule at src/isa/x64/lower.isle line 889.
let expr0_0 = C::put_in_reg(ctx, pattern12_1);
let expr1_0 = C::put_in_reg(ctx, pattern20_1);
let expr2_0 = RegMem::Reg {
@@ -3296,7 +3296,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
}
}
}
// Rule at src/isa/x64/lower.isle line 726.
// Rule at src/isa/x64/lower.isle line 746.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_pmulld(ctx, expr0_0, &expr1_0)?;
@@ -3330,7 +3330,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Isub => {
let (pattern9_0, pattern9_1) =
C::unpack_value_array_2(ctx, &pattern7_1);
// Rule at src/isa/x64/lower.isle line 260.
// Rule at src/isa/x64/lower.isle line 280.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
let expr2_0 = constructor_psubq(ctx, expr0_0, &expr1_0)?;
@@ -3383,7 +3383,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 32 {
if pattern23_1 == 4
{
// Rule at src/isa/x64/lower.isle line 841.
// Rule at src/isa/x64/lower.isle line 861.
let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
let expr1_0: u8 = 80;
let expr2_0 = OperandSize::Size32;
@@ -3445,7 +3445,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 32 {
if pattern23_1 == 4
{
// Rule at src/isa/x64/lower.isle line 805.
// Rule at src/isa/x64/lower.isle line 825.
let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
let expr1_0: u8 = 250;
let expr2_0 = OperandSize::Size32;
@@ -3507,7 +3507,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 32 {
if pattern23_1 == 4
{
// Rule at src/isa/x64/lower.isle line 917.
// Rule at src/isa/x64/lower.isle line 937.
let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
let expr1_0: u8 = 80;
let expr2_0 = OperandSize::Size32;
@@ -3569,7 +3569,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
if pattern23_0 == 32 {
if pattern23_1 == 4
{
// Rule at src/isa/x64/lower.isle line 881.
// Rule at src/isa/x64/lower.isle line 901.
let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
let expr1_0: u8 = 250;
let expr2_0 = OperandSize::Size32;
@@ -3600,7 +3600,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
}
}
}
// Rule at src/isa/x64/lower.isle line 757.
// Rule at src/isa/x64/lower.isle line 777.
let expr0_0 = C::put_in_reg(ctx, pattern9_0);
let expr1_0 = C::put_in_reg(ctx, pattern9_1);
let expr2_0: u32 = 32;
@@ -3640,7 +3640,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Band => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 351.
// Rule at src/isa/x64/lower.isle line 371.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr2_0 = constructor_pand(ctx, expr0_0, &expr1_0)?;
@@ -3650,7 +3650,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 427.
// Rule at src/isa/x64/lower.isle line 447.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr2_0 = constructor_por(ctx, expr0_0, &expr1_0)?;
@@ -3660,7 +3660,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::Bxor => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 497.
// Rule at src/isa/x64/lower.isle line 517.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
let expr2_0 = constructor_sse_xor(ctx, pattern2_0, expr0_0, &expr1_0)?;
@@ -3670,7 +3670,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
&Opcode::BandNot => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
// Rule at src/isa/x64/lower.isle line 946.
// Rule at src/isa/x64/lower.isle line 966.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0);
let expr2_0 = constructor_pandn(ctx, expr0_0, &expr1_0)?;
@@ -3685,7 +3685,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
arg: pattern5_1,
} => {
if let &Opcode::Bnot = &pattern5_0 {
// Rule at src/isa/x64/lower.isle line 992.
// Rule at src/isa/x64/lower.isle line 1012.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?;
let expr2_0 = RegMem::Reg { reg: expr1_0 };
@@ -3785,7 +3785,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 232.
// Rule at src/isa/x64/lower.isle line 252.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_sub(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3793,14 +3793,14 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 237.
// Rule at src/isa/x64/lower.isle line 257.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_sub(ctx, pattern3_0, expr0_0, &expr1_0)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
// Rule at src/isa/x64/lower.isle line 225.
// Rule at src/isa/x64/lower.isle line 245.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg(ctx, pattern7_1);
let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
@@ -3812,7 +3812,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 663.
// Rule at src/isa/x64/lower.isle line 683.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 =
constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3820,7 +3820,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 675.
// Rule at src/isa/x64/lower.isle line 695.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?;
@@ -3828,7 +3828,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr3_0);
}
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 659.
// Rule at src/isa/x64/lower.isle line 679.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3836,14 +3836,14 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 669.
// Rule at src/isa/x64/lower.isle line 689.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
// Rule at src/isa/x64/lower.isle line 652.
// Rule at src/isa/x64/lower.isle line 672.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg(ctx, pattern7_1);
let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
@@ -3855,50 +3855,65 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 169.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 =
constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
return Some(expr2_0);
// Rule at src/isa/x64/lower.isle line 183.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
let expr3_0 = C::put_in_reg(ctx, pattern7_1);
let expr4_0 =
constructor_add(ctx, pattern3_0, expr3_0, &pattern8_0)?;
let expr5_0 = C::value_regs(ctx, expr4_0, expr2_0);
return Some(expr5_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 181.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
// Rule at src/isa/x64/lower.isle line 199.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
let expr3_0 = C::put_in_reg(ctx, pattern7_1);
let expr4_0 = C::sink_load(ctx, &pattern8_0);
let expr5_0 = constructor_add(ctx, pattern3_0, expr3_0, &expr4_0)?;
let expr6_0 = C::value_regs(ctx, expr5_0, expr2_0);
return Some(expr6_0);
}
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 165.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
return Some(expr2_0);
// Rule at src/isa/x64/lower.isle line 177.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
let expr3_0 = C::put_in_reg(ctx, pattern7_0);
let expr4_0 =
constructor_add(ctx, pattern3_0, expr3_0, &pattern8_0)?;
let expr5_0 = C::value_regs(ctx, expr4_0, expr2_0);
return Some(expr5_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 175.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
// Rule at src/isa/x64/lower.isle line 191.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
let expr3_0 = C::put_in_reg(ctx, pattern7_0);
let expr4_0 = C::sink_load(ctx, &pattern8_0);
let expr5_0 = constructor_add(ctx, pattern3_0, expr3_0, &expr4_0)?;
let expr6_0 = C::value_regs(ctx, expr5_0, expr2_0);
return Some(expr6_0);
}
// Rule at src/isa/x64/lower.isle line 157.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg(ctx, pattern7_1);
let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
let expr3_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr2_0)?;
let expr4_0 = C::value_reg(ctx, expr3_0);
return Some(expr4_0);
// Rule at src/isa/x64/lower.isle line 167.
let expr0_0: Type = I64;
let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
let expr3_0 = C::put_in_reg(ctx, pattern7_0);
let expr4_0 = C::put_in_reg(ctx, pattern7_1);
let expr5_0 = RegMemImm::Reg { reg: expr4_0 };
let expr6_0 = constructor_add(ctx, pattern3_0, expr3_0, &expr5_0)?;
let expr7_0 = C::value_regs(ctx, expr6_0, expr2_0);
return Some(expr7_0);
}
&Opcode::Band => {
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 335.
// Rule at src/isa/x64/lower.isle line 355.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 =
constructor_m_and(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3906,7 +3921,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 321.
// Rule at src/isa/x64/lower.isle line 341.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 =
@@ -3915,7 +3930,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr3_0);
}
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 329.
// Rule at src/isa/x64/lower.isle line 349.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_m_and(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3923,7 +3938,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 315.
// Rule at src/isa/x64/lower.isle line 335.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 =
@@ -3931,7 +3946,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
// Rule at src/isa/x64/lower.isle line 308.
// Rule at src/isa/x64/lower.isle line 328.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg(ctx, pattern7_1);
let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
@@ -3943,7 +3958,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 411.
// Rule at src/isa/x64/lower.isle line 431.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 =
constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3951,7 +3966,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 397.
// Rule at src/isa/x64/lower.isle line 417.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?;
@@ -3959,7 +3974,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr3_0);
}
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 405.
// Rule at src/isa/x64/lower.isle line 425.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3967,14 +3982,14 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 391.
// Rule at src/isa/x64/lower.isle line 411.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
// Rule at src/isa/x64/lower.isle line 384.
// Rule at src/isa/x64/lower.isle line 404.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg(ctx, pattern7_1);
let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
@@ -3986,7 +4001,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 489.
// Rule at src/isa/x64/lower.isle line 509.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 =
constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -3994,7 +4009,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 475.
// Rule at src/isa/x64/lower.isle line 495.
let expr0_0 = C::put_in_reg(ctx, pattern7_1);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?;
@@ -4002,7 +4017,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr3_0);
}
if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 483.
// Rule at src/isa/x64/lower.isle line 503.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?;
@@ -4010,14 +4025,14 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
return Some(expr2_0);
}
if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 469.
// Rule at src/isa/x64/lower.isle line 489.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::sink_load(ctx, &pattern8_0);
let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?;
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
// Rule at src/isa/x64/lower.isle line 462.
// Rule at src/isa/x64/lower.isle line 482.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 = C::put_in_reg(ctx, pattern7_1);
let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
@@ -4029,14 +4044,14 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 624.
// Rule at src/isa/x64/lower.isle line 644.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_m_rotl(ctx, pattern3_0, expr0_0, &pattern8_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
return Some(expr2_0);
}
// Rule at src/isa/x64/lower.isle line 618.
// Rule at src/isa/x64/lower.isle line 638.
let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
let expr1_0 = C::put_in_reg(ctx, pattern7_0);
let expr2_0 = Imm8Reg::Reg { reg: expr0_0 };
@@ -4048,14 +4063,14 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 533.
// Rule at src/isa/x64/lower.isle line 553.
let expr0_0 = C::put_in_reg(ctx, pattern7_0);
let expr1_0 =
constructor_shl(ctx, pattern3_0, expr0_0, &pattern8_0)?;
let expr2_0 = C::value_reg(ctx, expr1_0);
return Some(expr2_0);
}
// Rule at src/isa/x64/lower.isle line 527.
// Rule at src/isa/x64/lower.isle line 547.
let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
let expr1_0 = C::put_in_reg(ctx, pattern7_0);
let expr2_0 = Imm8Reg::Reg { reg: expr0_0 };
@@ -4067,7 +4082,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let (pattern7_0, pattern7_1) =
C::unpack_value_array_2(ctx, &pattern5_1);
if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) {
// Rule at src/isa/x64/lower.isle line 579.
// Rule at src/isa/x64/lower.isle line 599.
let expr0_0 = ExtendKind::Zero;
let expr1_0 = constructor_extend_to_reg(
ctx, pattern7_0, pattern3_0, &expr0_0,
@@ -4077,7 +4092,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
let expr3_0 = C::value_reg(ctx, expr2_0);
return Some(expr3_0);
}
// Rule at src/isa/x64/lower.isle line 572.
// Rule at src/isa/x64/lower.isle line 592.
let expr0_0 = ExtendKind::Zero;
let expr1_0 =
constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?;
@@ -4097,7 +4112,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
} => {
if let &Opcode::IaddImm = &pattern5_0 {
let pattern7_0 = C::u64_from_imm64(ctx, pattern5_2);
// Rule at src/isa/x64/lower.isle line 198.
// Rule at src/isa/x64/lower.isle line 218.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0 = constructor_imm(ctx, pattern3_0, pattern7_0)?;
let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
@@ -4121,7 +4136,7 @@ pub fn constructor_or_i128<C: Context>(
) -> Option<ValueRegs> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/lower.isle line 435.
// Rule at src/isa/x64/lower.isle line 455.
let expr0_0: usize = 0;
let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
let expr2_0: usize = 1;
@@ -4148,7 +4163,7 @@ pub fn constructor_shl_i128<C: Context>(
) -> Option<ValueRegs> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/lower.isle line 539.
// Rule at src/isa/x64/lower.isle line 559.
let expr0_0: usize = 0;
let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
let expr2_0: usize = 1;
@@ -4207,7 +4222,7 @@ pub fn constructor_shr_i128<C: Context>(
) -> Option<ValueRegs> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/isa/x64/lower.isle line 586.
// Rule at src/isa/x64/lower.isle line 606.
let expr0_0: usize = 0;
let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
let expr2_0: usize = 1;
@@ -4274,7 +4289,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
let pattern2_0 = arg1;
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/x64/lower.isle line 1008.
// Rule at src/isa/x64/lower.isle line 1028.
let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?;
return Some(expr0_0);
}
@@ -4282,7 +4297,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
let pattern2_0 = arg1;
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/x64/lower.isle line 1011.
// Rule at src/isa/x64/lower.isle line 1031.
let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?;
return Some(expr0_0);
}
@@ -4290,7 +4305,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
let pattern2_0 = arg1;
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/x64/lower.isle line 1014.
// Rule at src/isa/x64/lower.isle line 1034.
let expr0_0 = OperandSize::Size32;
let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?;
return Some(expr1_0);
@@ -4299,7 +4314,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
let pattern2_0 = arg1;
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/x64/lower.isle line 1017.
// Rule at src/isa/x64/lower.isle line 1037.
let expr0_0 = OperandSize::Size64;
let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?;
return Some(expr1_0);
@@ -4308,7 +4323,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
let pattern2_0 = arg1;
let pattern3_0 = arg2;
let pattern4_0 = arg3;
// Rule at src/isa/x64/lower.isle line 1020.
// Rule at src/isa/x64/lower.isle line 1040.
let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0);
let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?;
return Some(expr1_0);
@@ -4319,7 +4334,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 {
let pattern5_0 = arg3;
if pattern5_0 == 0 {
// Rule at src/isa/x64/lower.isle line 1041.
// Rule at src/isa/x64/lower.isle line 1061.
let expr0_0 = RegMem::Reg { reg: pattern4_0 };
let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?;
return Some(expr1_0);
@@ -4327,7 +4342,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
}
let pattern4_0 = arg3;
if pattern4_0 == 0 {
// Rule at src/isa/x64/lower.isle line 1042.
// Rule at src/isa/x64/lower.isle line 1062.
let expr0_0 = SseOpcode::Movsd;
let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?;
let expr2_0 = RegMem::Reg { reg: expr1_0 };
@@ -4335,7 +4350,7 @@ pub fn constructor_vec_insert_lane<C: Context>(
return Some(expr3_0);
}
if pattern4_0 == 1 {
// Rule at src/isa/x64/lower.isle line 1050.
// Rule at src/isa/x64/lower.isle line 1070.
let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?;
return Some(expr0_0);
}

9
cranelift/codegen/src/machinst/lower.rs
View File

@@ -11,6 +11,7 @@ use crate::fx::{FxHashMap, FxHashSet};
use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
use crate::ir::instructions::BranchInfo;
use crate::ir::{
types::{FFLAGS, IFLAGS},
ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function,
GlobalValueData, Inst, InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value,
ValueDef, ValueLabelAssignments, ValueLabelStart,
@@ -1211,6 +1212,12 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
let val = self.f.dfg.resolve_aliases(val);
log::trace!("put_value_in_regs: val {}", val);
// Assert that the value is not `iflags`/`fflags`-typed; these
// cannot be reified into normal registers. TODO(#3249)
// eventually remove the `iflags` type altogether!
let ty = self.f.dfg.value_type(val);
assert!(ty != IFLAGS && ty != FFLAGS);
// If the value is a constant, then (re)materialize it at each use. This
// lowers register pressure.
if let Some(c) = self
@@ -1220,8 +1227,6 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
.inst()
.and_then(|inst| self.get_constant(inst))
{
let ty = self.f.dfg.value_type(val);
let regs = self.alloc_tmp(ty);
log::trace!(" -> regs {:?}", regs);
assert!(regs.is_valid());