x64 backend: add lowerings with load-op-store fusion. (#4071)

x64 backend: add lowerings with load-op-store fusion.

These lowerings use the `OP [mem], reg` forms (or in AT&T syntax, `OP
%reg, (mem)`) -- i.e., x86 instructions that load from memory, perform
an ALU operation, and store the result, all in one instruction. Using
these instruction forms, we can merge three CLIF ops together: a load,
an arithmetic operation, and a store.

cranelift/codegen/src/isa/x64/inst.isle

@@ -20,6 +20,12 @@
                 (src2 GprMemImm)
                 (dst WritableGpr))
 
+       ;; Integer arithmetic read-modify-write on memory.
+       (AluRM (size OperandSize) ;; 4 or 8
+              (op AluRmiROpcode)
+              (src1_dst SyntheticAmode)
+              (src2 Gpr))
+
        ;; Instructions on general-purpose registers that only read src and
        ;; defines dst (dst is not modified). `bsr`, etc.
        (UnaryRmR (size OperandSize) ;; 2, 4, or 8
@@ -2695,6 +2701,32 @@
 (decl x64_pcmpgtq (Xmm XmmMem) Xmm)
 (rule (x64_pcmpgtq x y) (xmm_rm_r $I64X2 (SseOpcode.Pcmpgtq) x y))
 
+;; Helpers for read-modify-write ALU form (AluRM).
+(decl alu_rm (Type AluRmiROpcode Amode Gpr) SideEffectNoResult)
+(rule (alu_rm ty opcode src1_dst src2)
+      (let ((size OperandSize (operand_size_of_type_32_64 ty)))
+        (SideEffectNoResult.Inst (MInst.AluRM size opcode src1_dst src2))))
+
+(decl x64_add_mem (Type Amode Gpr) SideEffectNoResult)
+(rule (x64_add_mem ty addr val)
+      (alu_rm ty (AluRmiROpcode.Add) addr val))
+
+(decl x64_sub_mem (Type Amode Gpr) SideEffectNoResult)
+(rule (x64_sub_mem ty addr val)
+      (alu_rm ty (AluRmiROpcode.Sub) addr val))
+
+(decl x64_and_mem (Type Amode Gpr) SideEffectNoResult)
+(rule (x64_and_mem ty addr val)
+      (alu_rm ty (AluRmiROpcode.And) addr val))
+
+(decl x64_or_mem (Type Amode Gpr) SideEffectNoResult)
+(rule (x64_or_mem ty addr val)
+      (alu_rm ty (AluRmiROpcode.Or) addr val))
+
+(decl x64_xor_mem (Type Amode Gpr) SideEffectNoResult)
+(rule (x64_xor_mem ty addr val)
+      (alu_rm ty (AluRmiROpcode.Xor) addr val))
+
 ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (convert Gpr InstOutput output_gpr)

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -270,6 +270,38 @@ pub(crate) fn emit(
             }
         }
 
+        Inst::AluRM {
+            size,
+            src1_dst,
+            src2,
+            op,
+        } => {
+            let src2 = allocs.next(src2.to_reg());
+            let src1_dst = src1_dst.finalize(state, sink).with_allocs(allocs);
+
+            assert!(*size == OperandSize::Size32 || *size == OperandSize::Size64);
+            let opcode = match op {
+                AluRmiROpcode::Add => 0x01,
+                AluRmiROpcode::Sub => 0x29,
+                AluRmiROpcode::And => 0x21,
+                AluRmiROpcode::Or => 0x09,
+                AluRmiROpcode::Xor => 0x31,
+                _ => panic!("Unsupported read-modify-write ALU opcode"),
+            };
+            let enc_g = int_reg_enc(src2);
+            emit_std_enc_mem(
+                sink,
+                state,
+                info,
+                LegacyPrefixes::None,
+                opcode,
+                1,
+                enc_g,
+                &src1_dst,
+                RexFlags::from(*size),
+            );
+        }
+
         Inst::UnaryRmR { size, op, src, dst } => {
             let dst = allocs.next(dst.to_reg().to_reg());
             let rex_flags = RexFlags::from(*size);

cranelift/codegen/src/isa/x64/inst/emit_tests.rs

@@ -1500,6 +1500,125 @@ fn test_x64_emit() {
         "imull   %esi, $76543210, %esi",
     ));
 
+    // ========================================================
+    // AluRM
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size32,
+            op: AluRmiROpcode::Add,
+            src1_dst: Amode::imm_reg(99, rdi).into(),
+            src2: Gpr::new(r12).unwrap(),
+        },
+        "44016763",
+        "addl    %r12d, 99(%rdi)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size64,
+            op: AluRmiROpcode::Add,
+            src1_dst: Amode::imm_reg_reg_shift(
+                0,
+                Gpr::new(rbp).unwrap(),
+                Gpr::new(rax).unwrap(),
+                3,
+            )
+            .into(),
+            src2: Gpr::new(rax).unwrap(),
+        },
+        "480144C500",
+        "addq    %rax, 0(%rbp,%rax,8)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size32,
+            op: AluRmiROpcode::Sub,
+            src1_dst: Amode::imm_reg(0, rsp).into(),
+            src2: Gpr::new(rcx).unwrap(),
+        },
+        "290C24",
+        "subl    %ecx, 0(%rsp)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size64,
+            op: AluRmiROpcode::Sub,
+            src1_dst: Amode::imm_reg(0, rbp).into(),
+            src2: Gpr::new(rax).unwrap(),
+        },
+        "48294500",
+        "subq    %rax, 0(%rbp)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size32,
+            op: AluRmiROpcode::And,
+            src1_dst: Amode::imm_reg(0, rsp).into(),
+            src2: Gpr::new(rcx).unwrap(),
+        },
+        "210C24",
+        "andl    %ecx, 0(%rsp)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size64,
+            op: AluRmiROpcode::And,
+            src1_dst: Amode::imm_reg(0, rbp).into(),
+            src2: Gpr::new(rax).unwrap(),
+        },
+        "48214500",
+        "andq    %rax, 0(%rbp)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size32,
+            op: AluRmiROpcode::Or,
+            src1_dst: Amode::imm_reg(0, rsp).into(),
+            src2: Gpr::new(rcx).unwrap(),
+        },
+        "090C24",
+        "orl     %ecx, 0(%rsp)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size64,
+            op: AluRmiROpcode::Or,
+            src1_dst: Amode::imm_reg(0, rbp).into(),
+            src2: Gpr::new(rax).unwrap(),
+        },
+        "48094500",
+        "orq     %rax, 0(%rbp)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size32,
+            op: AluRmiROpcode::Xor,
+            src1_dst: Amode::imm_reg(0, rsp).into(),
+            src2: Gpr::new(rcx).unwrap(),
+        },
+        "310C24",
+        "xorl    %ecx, 0(%rsp)",
+    ));
+
+    insns.push((
+        Inst::AluRM {
+            size: OperandSize::Size64,
+            op: AluRmiROpcode::Xor,
+            src1_dst: Amode::imm_reg(0, rbp).into(),
+            src2: Gpr::new(rax).unwrap(),
+        },
+        "48314500",
+        "xorq    %rax, 0(%rbp)",
+    ));
+
     // ========================================================
     // UnaryRmR
 

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -44,6 +44,7 @@ impl Inst {
             // These instructions are part of SSE2, which is a basic requirement in Cranelift, and
             // don't have to be checked.
             Inst::AluRmiR { .. }
+            | Inst::AluRM { .. }
             | Inst::AtomicRmwSeq { .. }
             | Inst::CallKnown { .. }
             | Inst::CallUnknown { .. }
@@ -917,6 +918,22 @@ impl PrettyPrint for Inst {
                     dst
                 )
             }
+            Inst::AluRM {
+                size,
+                op,
+                src1_dst,
+                src2,
+            } => {
+                let size_bytes = size_lqb(*size, op.is_8bit());
+                let src2 = pretty_print_reg(src2.to_reg(), size_bytes, allocs);
+                let src1_dst = src1_dst.pretty_print(size_bytes, allocs);
+                format!(
+                    "{} {}, {}",
+                    ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())),
+                    src2,
+                    src1_dst,
+                )
+            }
             Inst::UnaryRmR { src, dst, op, size } => {
                 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
                 let src = src.pretty_print(size.to_bytes(), allocs);
@@ -1691,6 +1708,10 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
                 src2.get_operands(collector);
             }
         }
+        Inst::AluRM { src1_dst, src2, .. } => {
+            collector.reg_use(src2.to_reg());
+            src1_dst.get_operands(collector);
+        }
         Inst::Not { src, dst, .. } => {
             collector.reg_use(src.to_reg());
             collector.reg_reuse_def(dst.to_writable_reg(), 0);

cranelift/codegen/src/isa/x64/lower.isle

@@ -2655,3 +2655,131 @@
        (side_effect_concat
         (x64_movrm $I64 addr_lo value_lo)
         (x64_movrm $I64 addr_hi value_hi)))))
+
+;; Rules for `load*` + ALU op + `store*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add mem, reg
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (iadd (and
+                               (sinkable_load sink)
+                               (load flags addr offset))
+                              src2))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_add_mem ty (to_amode flags addr offset) src2))))
+
+;; Add mem, reg with args swapped
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (iadd src2
+                              (and
+                               (sinkable_load sink)
+                               (load flags addr offset))))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_add_mem ty (to_amode flags addr offset) src2))))
+
+;; Sub mem, reg
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (isub (and
+                               (sinkable_load sink)
+                               (load flags addr offset))
+                              src2))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_sub_mem ty (to_amode flags addr offset) src2))))
+
+;; And mem, reg
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (band (and
+                               (sinkable_load sink)
+                               (load flags addr offset))
+                              src2))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_and_mem ty (to_amode flags addr offset) src2))))
+
+;; And mem, reg with args swapped
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (band src2
+                              (and
+                               (sinkable_load sink)
+                               (load flags addr offset))))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_and_mem ty (to_amode flags addr offset) src2))))
+
+;; Or mem, reg
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (bor (and
+                               (sinkable_load sink)
+                               (load flags addr offset))
+                              src2))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_or_mem ty (to_amode flags addr offset) src2))))
+
+;; Or mem, reg with args swapped
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (bor src2
+                              (and
+                               (sinkable_load sink)
+                               (load flags addr offset))))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_or_mem ty (to_amode flags addr offset) src2))))
+
+;; Xor mem, reg
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (bxor (and
+                               (sinkable_load sink)
+                               (load flags addr offset))
+                              src2))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_xor_mem ty (to_amode flags addr offset) src2))))
+
+;; Xor mem, reg with args swapped
+(rule (lower
+       (store =flags
+              (has_type (ty_32_or_64 ty)
+                        (bxor src2
+                              (and
+                               (sinkable_load sink)
+                               (load flags addr offset))))
+              =addr
+              =offset))
+      (let ((_ RegMemImm (sink_load sink)))
+        (side_effect
+         (x64_xor_mem ty (to_amode flags addr offset) src2))))

cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 443b34b797fc8ace
 src/prelude.isle d8a93eb727abd7f4
-src/isa/x64/inst.isle 2fa48b8183f9d5cb
-src/isa/x64/lower.isle b7fe1c95c21edbe4
+src/isa/x64/inst.isle 6dcba190988a695
+src/isa/x64/lower.isle b95161bdf07b9365

cranelift/filetests/filetests/isa/x64/load-op-store.clif

@@ -0,0 +1,147 @@
+test compile precise-output
+target x86_64
+
+function %f0(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = iadd v2, v1
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addl    %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f1(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = iadd v1, v2
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addl    %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f2(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = isub v2, v1
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   subl    %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f3(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = band v2, v1
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   andl    %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f4(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = band v1, v2
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   andl    %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f5(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = bor v2, v1
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   orl     %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f6(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = bor v1, v2
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   orl     %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f7(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = bxor v2, v1
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   xorl    %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+
+function %f8(i64, i32) {
+block0(v0: i64, v1: i32):
+  v2 = load.i32 v0+32
+  v3 = bxor v1, v2
+  store v3, v0+32
+  return
+}
+
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   xorl    %esi, 32(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+

cranelift/filetests/filetests/runtests/load-op-store.clif

@@ -0,0 +1,95 @@
+test run
+target x86_64
+target aarch64
+
+function %load_op_store_iadd_i64(i64 vmctx, i64, i64) -> i64 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    heap0 = static gv1, min 0x1000, bound 0x1_0000_0000, offset_guard 0, index_type i64
+
+block0(v0: i64, v1: i64, v2: i64):
+    v3 = heap_addr.i64 heap0, v1, 8
+    v4 = iconst.i64 42
+    store.i64 v4, v3
+    v5 = load.i64 v3
+    v6 = iadd.i64 v5, v2
+    store.i64 v6, v3
+    v7 = load.i64 v3
+    return v7
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %static_heap_i64_load_store(0, 1) == 43
+; run: %static_heap_i64_load_store(0, -1) == 41
+
+function %load_op_store_iadd_i32(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    heap0 = static gv1, min 0x1000, bound 0x1_0000_0000, offset_guard 0, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    v3 = heap_addr.i64 heap0, v1, 4
+    v4 = iconst.i64 42
+    store.i32 v4, v3
+    v5 = load.i32 v3
+    v6 = iadd.i32 v5, v2
+    store.i32 v6, v3
+    v7 = load.i32 v3
+    return v7
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %static_heap_i64_load_store(0, 1) == 43
+; run: %static_heap_i64_load_store(0, -1) == 41
+
+function %load_op_store_iadd_i8(i64 vmctx, i64, i8) -> i8 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    heap0 = static gv1, min 0x1000, bound 0x1_0000_0000, offset_guard 0, index_type i64
+
+block0(v0: i64, v1: i64, v2: i8):
+    v3 = heap_addr.i64 heap0, v1, 4
+    v4 = iconst.i8 42
+    store.i8 v4, v3
+    v5 = load.i8 v3
+    v6 = iadd.i8 v5, v2
+    store.i8 v6, v3
+    v7 = load.i8 v3
+    return v7
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %static_heap_i64_load_store(0, 1) == 43
+; run: %static_heap_i64_load_store(0, -1) == 41
+
+function %load_op_store_iadd_isub_iand_ior_ixor_i64(i64 vmctx, i64, i64) -> i64 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    heap0 = static gv1, min 0x1000, bound 0x1_0000_0000, offset_guard 0, index_type i64
+
+block0(v0: i64, v1: i64, v2: i64):
+    v3 = heap_addr.i64 heap0, v1, 8
+    store.i64 v2, v3
+    v4 = load.i64 v3
+    v5 = iconst.i64 1
+    v6 = iadd.i64 v5, v4
+    store.i64 v6, v3
+    v7 = load.i64 v3
+    v8 = iconst.i64 2
+    v9 = load.i64 v3
+    v10 = isub.i64 v9, v8
+    store.i64 v10, v3
+    v11 = load.i64 v3
+    v12 = iconst.i64 0xf
+    v13 = band.i64 v12, v11
+    store.i64 v13, v3
+    v14 = iconst.i64 0x10
+    v15 = load.i64 v3
+    v16 = bor.i64 v15, v14
+    store.i64 v16, v3
+    v17 = load.i64 v3
+    v18 = iconst.i64 0xff
+    v19 = bxor.i64 v17, v18
+    store.i64 v19, v3
+    v20 = load.i64 v3
+    return v20
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %static_heap_i64_load_store(0, 0x1234) == 236