aarch64: Migrate some bit-ops to ISLE (#3602)

* aarch64: Migrate some bit-ops to ISLE

This commit migrates these instructions to ISLE:

* `bnot`
* `band`
* `bor`
* `bxor`
* `band_not`
* `bor_not`
* `bxor_not`

The translations were relatively straightforward but the interesting
part here was trying to reduce the duplication between all these
instructions. I opted for a route that's similar to what the lowering
does today, having a `decl` which takes the `ALUOp` and then performs
further pattern matching internally. This enabled each instruction's
lowering to be pretty simple while we still get to handle all the fancy
cases of shifts, constants, etc, for each instruction.

* Actually delete previous lowerings

* Remove dead code

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -1298,8 +1298,11 @@
 (decl move_wide_const_from_negated_u64 (MoveWideConst) u64)
 (extern extractor move_wide_const_from_negated_u64 move_wide_const_from_negated_u64)
 
-(decl imm_logic_from_u64 (ImmLogic) u64)
+(decl imm_logic_from_u64 (Type ImmLogic) u64)
-(extern extractor imm_logic_from_u64 imm_logic_from_u64)
+(extern extractor imm_logic_from_u64 imm_logic_from_u64 (in out))
+
+(decl imm_logic_from_imm64 (Type ImmLogic) Imm64)
+(extern extractor imm_logic_from_imm64 imm_logic_from_imm64 (in out))
 
 (decl imm_shift_from_u8 (u8) ImmShift)
 (extern constructor imm_shift_from_u8 imm_shift_from_u8)
@@ -1556,7 +1559,7 @@
       (movn n (OperandSize.Size64)))
 
 ;; Weird logical-instruction immediate in ORI using zero register
-(rule (imm (integral_ty _ty) (imm_logic_from_u64 n))
+(rule (imm (integral_ty _ty) (imm_logic_from_u64 <$I64 n))
       (alu_rr_imm_logic (ALUOp.Orr64) (zero_reg) n))
 
 (decl load_constant64_full (u64) Reg)
@@ -1636,3 +1639,52 @@
 ;; instruction, and no longer needs to be lowered.
 (decl sink_atomic_load (SinkableAtomicLoad) Reg)
 (extern constructor sink_atomic_load sink_atomic_load)
+
+;; Helper for generating either an `AluRRR`, `AluRRRShift`, or `AluRRImmLogic`
+;; instruction depending on the input. Note that this requires that the `ALUOp`
+;; specified is commutative.
+(decl alu_rs_imm_logic_commutative (ALUOp Type Value Value) Reg)
+
+;; Base case of operating on registers.
+(rule (alu_rs_imm_logic_commutative op ty x y)
+      (alu_rrr op (put_in_reg x) (put_in_reg y)))
+
+;; Special cases for when one operand is a constant.
+(rule (alu_rs_imm_logic_commutative op ty x (def_inst (iconst (imm_logic_from_imm64 <ty imm))))
+      (alu_rr_imm_logic op (put_in_reg x) imm))
+(rule (alu_rs_imm_logic_commutative op ty (def_inst (iconst (imm_logic_from_imm64 <ty imm))) x)
+      (alu_rr_imm_logic op (put_in_reg x) imm))
+
+;; Special cases for when one operand is shifted left by a constant.
+(rule (alu_rs_imm_logic_commutative op ty x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 <ty amt))))))
+      (alu_rrr_shift op (put_in_reg x) (put_in_reg y) amt))
+(rule (alu_rs_imm_logic_commutative op ty (def_inst (ishl x (def_inst (iconst (lshl_from_imm64 <ty amt))))) y)
+      (alu_rrr_shift op (put_in_reg y) (put_in_reg x) amt))
+
+;; Same as `alu_rs_imm_logic_commutative` above, except that it doesn't require
+;; that the operation is commutative.
+(decl alu_rs_imm_logic (ALUOp Type Value Value) Reg)
+(rule (alu_rs_imm_logic op ty x y)
+      (alu_rrr op (put_in_reg x) (put_in_reg y)))
+(rule (alu_rs_imm_logic op ty x (def_inst (iconst (imm_logic_from_imm64 <ty imm))))
+      (alu_rr_imm_logic op (put_in_reg x) imm))
+(rule (alu_rs_imm_logic op ty x (def_inst (ishl y (def_inst (iconst (lshl_from_imm64 <ty amt))))))
+      (alu_rrr_shift op (put_in_reg x) (put_in_reg y) amt))
+
+;; Helper for generating i128 bitops which simply do the same operation to the
+;; hi/lo registers.
+;;
+;; TODO: Support immlogic here
+(decl i128_alu_bitop (ALUOp Value Value) ValueRegs)
+(rule (i128_alu_bitop op x y)
+      (let (
+          (x_regs ValueRegs (put_in_regs x))
+          (x_lo Reg (value_regs_get x_regs 0))
+          (x_hi Reg (value_regs_get x_regs 1))
+          (y_regs ValueRegs (put_in_regs y))
+          (y_lo Reg (value_regs_get y_regs 0))
+          (y_hi Reg (value_regs_get y_regs 1))
+        )
+        (value_regs
+          (alu_rrr op x_lo y_lo)
+          (alu_rrr op x_hi y_hi))))

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -588,3 +588,110 @@
           (hi Reg (alu_rr_imm_shift (ALUOp.Asr64) lo (imm_shift_from_u8 63)))
         )
         (value_regs lo hi)))
+
+;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl orr_not_op (Type) ALUOp)
+(rule (orr_not_op (fits_in_32 _ty)) (ALUOp.OrrNot32))
+(rule (orr_not_op $I64) (ALUOp.OrrNot64))
+
+;; Base case using `orn` between two registers.
+;;
+;; Note that bitwise negation is implemented here as
+;;
+;;      NOT rd, rm ==> ORR_NOT rd, zero, rm
+(rule (lower (has_type (fits_in_64 ty) (bnot x)))
+      (value_reg (alu_rrr (orr_not_op ty) (zero_reg) (put_in_reg x))))
+
+;; Special case to use `AluRRRShift` if it's a `bnot` of a const-left-shifted
+;; value.
+(rule (lower (has_type (fits_in_64 ty)
+                       (bnot (def_inst (ishl x (def_inst (iconst (lshl_from_imm64 <ty amt))))))))
+      (value_reg (alu_rrr_shift (orr_not_op ty) (zero_reg) (put_in_reg x) amt)))
+
+;; Implementation of `bnot` for `i128`.
+(rule (lower (has_type $I128 (bnot x)))
+      (let (
+          (x_regs ValueRegs (put_in_regs x))
+          (x_lo Reg (value_regs_get x_regs 0))
+          (x_hi Reg (value_regs_get x_regs 1))
+          (new_lo Reg (alu_rrr (ALUOp.OrrNot64) (zero_reg) x_lo))
+          (new_hi Reg (alu_rrr (ALUOp.OrrNot64) (zero_reg) x_hi))
+        )
+        (value_regs new_lo new_hi)))
+
+;; Implementation of `bnot` for vector types.
+(rule (lower (has_type (vec128 ty) (bnot x)))
+      (value_reg (vec_misc (VecMisc2.Not) (put_in_reg x) (vector_size ty))))
+
+;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_32 ty) (band x y)))
+      (value_reg (alu_rs_imm_logic_commutative (ALUOp.And32) ty x y)))
+
+(rule (lower (has_type $I64 (band x y)))
+      (value_reg (alu_rs_imm_logic_commutative (ALUOp.And64) $I64 x y)))
+
+(rule (lower (has_type $I128 (band x y))) (i128_alu_bitop (ALUOp.And64) x y))
+
+(rule (lower (has_type (vec128 ty) (band x y)))
+      (value_reg (vec_rrr (VecALUOp.And) (put_in_reg x) (put_in_reg y) (vector_size ty))))
+
+;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_32 ty) (bor x y)))
+      (value_reg (alu_rs_imm_logic_commutative (ALUOp.Orr32) ty x y)))
+
+(rule (lower (has_type $I64 (bor x y)))
+      (value_reg (alu_rs_imm_logic_commutative (ALUOp.Orr64) $I64 x y)))
+
+(rule (lower (has_type $I128 (bor x y))) (i128_alu_bitop (ALUOp.Orr64) x y))
+
+(rule (lower (has_type (vec128 ty) (bor x y)))
+      (value_reg (vec_rrr (VecALUOp.Orr) (put_in_reg x) (put_in_reg y) (vector_size ty))))
+
+;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_32 ty) (bxor x y)))
+      (value_reg (alu_rs_imm_logic_commutative (ALUOp.Eor32) ty x y)))
+
+(rule (lower (has_type $I64 (bxor x y)))
+      (value_reg (alu_rs_imm_logic_commutative (ALUOp.Eor64) $I64 x y)))
+
+(rule (lower (has_type $I128 (bxor x y))) (i128_alu_bitop (ALUOp.Eor64) x y))
+
+(rule (lower (has_type (vec128 ty) (bxor x y)))
+      (value_reg (vec_rrr (VecALUOp.Eor) (put_in_reg x) (put_in_reg y) (vector_size ty))))
+
+;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_32 ty) (band_not x y)))
+      (value_reg (alu_rs_imm_logic (ALUOp.AndNot32) ty x y)))
+
+(rule (lower (has_type $I64 (band_not x y)))
+      (value_reg (alu_rs_imm_logic (ALUOp.AndNot64) $I64 x y)))
+
+(rule (lower (has_type $I128 (band_not x y))) (i128_alu_bitop (ALUOp.AndNot64) x y))
+
+(rule (lower (has_type (vec128 ty) (band_not x y)))
+      (value_reg (vec_rrr (VecALUOp.Bic) (put_in_reg x) (put_in_reg y) (vector_size ty))))
+
+;;;; Rules for `bor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_32 ty) (bor_not x y)))
+      (value_reg (alu_rs_imm_logic (ALUOp.OrrNot32) ty x y)))
+
+(rule (lower (has_type $I64 (bor_not x y)))
+      (value_reg (alu_rs_imm_logic (ALUOp.OrrNot64) $I64 x y)))
+
+(rule (lower (has_type $I128 (bor_not x y))) (i128_alu_bitop (ALUOp.OrrNot64) x y))
+
+;;;; Rules for `bxor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (fits_in_32 ty) (bxor_not x y)))
+      (value_reg (alu_rs_imm_logic (ALUOp.EorNot32) ty x y)))
+
+(rule (lower (has_type $I64 (bxor_not x y)))
+      (value_reg (alu_rs_imm_logic (ALUOp.EorNot64) $I64 x y)))
+
+(rule (lower (has_type $I128 (bxor_not x y))) (i128_alu_bitop (ALUOp.EorNot64) x y))

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -80,25 +80,6 @@ impl ResultRSEImm12 {
     }
 }
 
-/// A lowering result: register, register-shift, or logical immediate form.
-/// An SSA value can always be lowered into one of these options; the register form is the
-/// fallback.
-#[derive(Clone, Debug)]
-pub(crate) enum ResultRSImmLogic {
-    Reg(Reg),
-    RegShift(Reg, ShiftOpAndAmt),
-    ImmLogic(ImmLogic),
-}
-
-impl ResultRSImmLogic {
-    fn from_rs(rse: ResultRS) -> ResultRSImmLogic {
-        match rse {
-            ResultRS::Reg(r) => ResultRSImmLogic::Reg(r),
-            ResultRS::RegShift(r, s) => ResultRSImmLogic::RegShift(r, s),
-        }
-    }
-}
-
 /// A lowering result: register or immediate shift amount (arg to a shift op).
 /// An SSA value can always be lowered into one of these options; the register form is the
 /// fallback.
@@ -488,22 +469,6 @@ pub(crate) fn put_input_in_rse_imm12<C: LowerCtx<I = Inst>>(
     ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode))
 }
 
-pub(crate) fn put_input_in_rs_immlogic<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-    narrow_mode: NarrowValueMode,
-) -> ResultRSImmLogic {
-    if let Some(imm_value) = input_to_const(ctx, input) {
-        let ty = ctx.input_ty(input.insn, input.input);
-        let ty = if ty_bits(ty) < 32 { I32 } else { ty };
-        if let Some(i) = ImmLogic::maybe_from_u64(imm_value, ty) {
-            return ResultRSImmLogic::ImmLogic(i);
-        }
-    }
-
-    ResultRSImmLogic::from_rs(put_input_in_rs(ctx, input, narrow_mode))
-}
-
 pub(crate) fn put_input_in_reg_immshift<C: LowerCtx<I = Inst>>(
     ctx: &mut C,
     input: InsnInput,
@@ -553,35 +518,6 @@ pub(crate) fn alu_inst_imm12(op: ALUOp, rd: Writable<Reg>, rn: Reg, rm: ResultRS
     }
 }
 
-pub(crate) fn alu_inst_immlogic(
-    op: ALUOp,
-    rd: Writable<Reg>,
-    rn: Reg,
-    rm: ResultRSImmLogic,
-) -> Inst {
-    match rm {
-        ResultRSImmLogic::ImmLogic(imml) => Inst::AluRRImmLogic {
-            alu_op: op,
-            rd,
-            rn,
-            imml,
-        },
-        ResultRSImmLogic::Reg(rm) => Inst::AluRRR {
-            alu_op: op,
-            rd,
-            rn,
-            rm,
-        },
-        ResultRSImmLogic::RegShift(rm, shiftop) => Inst::AluRRRShift {
-            alu_op: op,
-            rd,
-            rn,
-            rm,
-            shiftop,
-        },
-    }
-}
-
 pub(crate) fn alu_inst_immshift(
     op: ALUOp,
     rd: Writable<Reg>,

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -76,8 +76,13 @@ where
         MoveWideConst::maybe_from_u64(!n)
     }
 
-    fn imm_logic_from_u64(&mut self, n: u64) -> Option<ImmLogic> {
+    fn imm_logic_from_u64(&mut self, n: u64, ty: Type) -> Option<ImmLogic> {
-        ImmLogic::maybe_from_u64(n, I64)
+        let ty = if ty.bits() < 32 { I32 } else { ty };
+        ImmLogic::maybe_from_u64(n, ty)
+    }
+
+    fn imm_logic_from_imm64(&mut self, n: Imm64, ty: Type) -> Option<ImmLogic> {
+        self.imm_logic_from_u64(n.bits() as u64, ty)
     }
 
     fn imm12_from_u64(&mut self, n: u64) -> Option<Imm12> {

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle be1359b4b6b153f378517c1dd95cd80f4a6bed0c7b86eaba11c088fd71b7bfe80a3c868ace245b2da0bfbbd6ded262ea9576c8e0eeacbf89d03c34a17a709602
 src/prelude.isle d3d2a6a42fb778231a4cdca30995324e1293a9ca8073c5a27a501535759eb51f84a6718322a93dfba4b66ee4f0c9afce7dcec0428516ef0c5bc96e8c8b76925d
-src/isa/aarch64/inst.isle 70d7b319ba0b28173d2ef1820bd0e9c4b8cf7a5ab34475a43f03bdc5a6b945a7faf40d7b539a12050ddd8ebc4c6b0fe82df5940eaf966420bb4d58e7420d4206
+src/isa/aarch64/inst.isle 8e4b8e452cf06a368c2e1d930042027a5d3bd690ab46d498d959257e9b4461d17abf244838395cd80da1fe5e2e86fc43855fb5753ca4f1643538c2ae4b3b4a1e
-src/isa/aarch64/lower.isle dfc622b2fecea98079fff182ce3443ada5448256662f598ea009caed3d9bcf6b4816f736a8c7f70142467febf8fc97230c57287f06e80e6101f3b401208c599c
+src/isa/aarch64/lower.isle bc3db9c1e6ac186b918cc04f4d26af398f99ec36c8cdc20ec4d02d18dd57dba12e3184fea031b4ac97051c5e194a69666afb5e204807c818e6688c177f9c1b91

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs

@@ -55,7 +55,8 @@ pub trait Context {
     fn trap_code_integer_overflow(&mut self) -> TrapCode;
     fn move_wide_const_from_u64(&mut self, arg0: u64) -> Option<MoveWideConst>;
     fn move_wide_const_from_negated_u64(&mut self, arg0: u64) -> Option<MoveWideConst>;
-    fn imm_logic_from_u64(&mut self, arg0: u64) -> Option<ImmLogic>;
+    fn imm_logic_from_u64(&mut self, arg0: u64, arg1: Type) -> Option<ImmLogic>;
+    fn imm_logic_from_imm64(&mut self, arg0: Imm64, arg1: Type) -> Option<ImmLogic>;
     fn imm_shift_from_u8(&mut self, arg0: u8) -> ImmShift;
     fn imm12_from_u64(&mut self, arg0: u64) -> Option<Imm12>;
     fn u8_into_uimm5(&mut self, arg0: u8) -> UImm5;
@@ -1116,7 +1117,7 @@ pub fn constructor_movz<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1375.
+    // Rule at src/isa/aarch64/inst.isle line 1378.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::MovZ {
@@ -1137,7 +1138,7 @@ pub fn constructor_movn<C: Context>(
 ) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1382.
+    // Rule at src/isa/aarch64/inst.isle line 1385.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::MovN {
@@ -1160,7 +1161,7 @@ pub fn constructor_alu_rr_imm_logic<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1389.
+    // Rule at src/isa/aarch64/inst.isle line 1392.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::AluRRImmLogic {
@@ -1184,7 +1185,7 @@ pub fn constructor_alu_rr_imm_shift<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1396.
+    // Rule at src/isa/aarch64/inst.isle line 1399.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::AluRRImmShift {
@@ -1208,7 +1209,7 @@ pub fn constructor_alu_rrr<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1403.
+    // Rule at src/isa/aarch64/inst.isle line 1406.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::AluRRR {
@@ -1234,7 +1235,7 @@ pub fn constructor_vec_rrr<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1410.
+    // Rule at src/isa/aarch64/inst.isle line 1413.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRR {
@@ -1259,7 +1260,7 @@ pub fn constructor_alu_rr_imm12<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1417.
+    // Rule at src/isa/aarch64/inst.isle line 1420.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::AluRRImm12 {
@@ -1285,7 +1286,7 @@ pub fn constructor_alu_rrr_shift<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1424.
+    // Rule at src/isa/aarch64/inst.isle line 1427.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::AluRRRShift {
@@ -1312,7 +1313,7 @@ pub fn constructor_alu_rrr_extend<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1431.
+    // Rule at src/isa/aarch64/inst.isle line 1434.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::AluRRRExtend {
@@ -1337,7 +1338,7 @@ pub fn constructor_alu_rr_extend_reg<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1439.
+    // Rule at src/isa/aarch64/inst.isle line 1442.
     let expr0_0 = C::put_extended_in_reg(ctx, pattern2_0);
     let expr1_0 = C::get_extended_op(ctx, pattern2_0);
     let expr2_0 = constructor_alu_rrr_extend(ctx, pattern0_0, pattern1_0, expr0_0, &expr1_0)?;
@@ -1356,7 +1357,7 @@ pub fn constructor_alu_rrrr<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1446.
+    // Rule at src/isa/aarch64/inst.isle line 1449.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::AluRRRR {
@@ -1379,7 +1380,7 @@ pub fn constructor_add64_with_flags<C: Context>(
 ) -> Option<ProducesFlags> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1453.
+    // Rule at src/isa/aarch64/inst.isle line 1456.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::AddS64;
@@ -1401,7 +1402,7 @@ pub fn constructor_add64_with_flags<C: Context>(
 pub fn constructor_adc64<C: Context>(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option<ConsumesFlags> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1460.
+    // Rule at src/isa/aarch64/inst.isle line 1463.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::Adc64;
@@ -1427,7 +1428,7 @@ pub fn constructor_sub64_with_flags<C: Context>(
 ) -> Option<ProducesFlags> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1467.
+    // Rule at src/isa/aarch64/inst.isle line 1470.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::SubS64;
@@ -1449,7 +1450,7 @@ pub fn constructor_sub64_with_flags<C: Context>(
 pub fn constructor_sbc64<C: Context>(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option<ConsumesFlags> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1474.
+    // Rule at src/isa/aarch64/inst.isle line 1477.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = ALUOp::Sbc64;
@@ -1477,7 +1478,7 @@ pub fn constructor_vec_misc<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1481.
+    // Rule at src/isa/aarch64/inst.isle line 1484.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecMisc {
@@ -1503,7 +1504,7 @@ pub fn constructor_vec_rrr_long<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1488.
+    // Rule at src/isa/aarch64/inst.isle line 1491.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRRLong {
@@ -1532,7 +1533,7 @@ pub fn constructor_vec_rrrr_long<C: Context>(
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
     let pattern4_0 = arg4;
-    // Rule at src/isa/aarch64/inst.isle line 1498.
+    // Rule at src/isa/aarch64/inst.isle line 1501.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::FpuMove128 {
@@ -1562,7 +1563,7 @@ pub fn constructor_vec_rr_narrow<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1506.
+    // Rule at src/isa/aarch64/inst.isle line 1509.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRNarrow {
@@ -1586,7 +1587,7 @@ pub fn constructor_vec_rr_long<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1513.
+    // Rule at src/isa/aarch64/inst.isle line 1516.
     let expr0_0: Type = I8X16;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::VecRRLong {
@@ -1610,7 +1611,7 @@ pub fn constructor_mov_from_vec<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1520.
+    // Rule at src/isa/aarch64/inst.isle line 1523.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::MovFromVec {
@@ -1636,7 +1637,7 @@ pub fn constructor_mov_from_vec_signed<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1527.
+    // Rule at src/isa/aarch64/inst.isle line 1530.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::MovFromVecSigned {
@@ -1663,7 +1664,7 @@ pub fn constructor_extend<C: Context>(
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
     let pattern3_0 = arg3;
-    // Rule at src/isa/aarch64/inst.isle line 1534.
+    // Rule at src/isa/aarch64/inst.isle line 1537.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::Extend {
@@ -1682,7 +1683,7 @@ pub fn constructor_extend<C: Context>(
 pub fn constructor_load_acquire<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
-    // Rule at src/isa/aarch64/inst.isle line 1541.
+    // Rule at src/isa/aarch64/inst.isle line 1544.
     let expr0_0: Type = I64;
     let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
     let expr2_0 = MInst::LoadAcquire {
@@ -1700,26 +1701,32 @@ pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option
     let pattern0_0 = arg0;
     if let Some(pattern1_0) = C::integral_ty(ctx, pattern0_0) {
         let pattern2_0 = arg1;
-        if let Some(pattern3_0) = C::imm_logic_from_u64(ctx, pattern2_0) {
+        let closure3 = || {
-            // Rule at src/isa/aarch64/inst.isle line 1559.
+            let expr0_0: Type = I64;
-            let expr0_0 = ALUOp::Orr64;
+            return Some(expr0_0);
-            let expr1_0 = C::zero_reg(ctx);
+        };
-            let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern3_0)?;
+        if let Some(pattern3_0) = closure3() {
-            return Some(expr2_0);
+            if let Some(pattern4_0) = C::imm_logic_from_u64(ctx, pattern2_0, pattern3_0) {
+                // Rule at src/isa/aarch64/inst.isle line 1562.
+                let expr0_0 = ALUOp::Orr64;
+                let expr1_0 = C::zero_reg(ctx);
+                let expr2_0 = constructor_alu_rr_imm_logic(ctx, &expr0_0, expr1_0, pattern4_0)?;
+                return Some(expr2_0);
+            }
         }
         if let Some(pattern3_0) = C::move_wide_const_from_u64(ctx, pattern2_0) {
-            // Rule at src/isa/aarch64/inst.isle line 1551.
+            // Rule at src/isa/aarch64/inst.isle line 1554.
             let expr0_0 = OperandSize::Size64;
             let expr1_0 = constructor_movz(ctx, pattern3_0, &expr0_0)?;
             return Some(expr1_0);
         }
         if let Some(pattern3_0) = C::move_wide_const_from_negated_u64(ctx, pattern2_0) {
-            // Rule at src/isa/aarch64/inst.isle line 1555.
+            // Rule at src/isa/aarch64/inst.isle line 1558.
             let expr0_0 = OperandSize::Size64;
             let expr1_0 = constructor_movn(ctx, pattern3_0, &expr0_0)?;
             return Some(expr1_0);
         }
-        // Rule at src/isa/aarch64/inst.isle line 1566.
+        // Rule at src/isa/aarch64/inst.isle line 1569.
         let expr0_0 = C::load_constant64_full(ctx, pattern2_0);
         return Some(expr0_0);
     }
@@ -1731,12 +1738,12 @@ pub fn constructor_put_in_reg_sext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
     let pattern0_0 = arg0;
     let pattern1_0 = C::value_type(ctx, pattern0_0);
     if pattern1_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1577.
+        // Rule at src/isa/aarch64/inst.isle line 1580.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1573.
+        // Rule at src/isa/aarch64/inst.isle line 1576.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         let expr1_0: bool = true;
         let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -1752,12 +1759,12 @@ pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
     let pattern0_0 = arg0;
     let pattern1_0 = C::value_type(ctx, pattern0_0);
     if pattern1_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1585.
+        // Rule at src/isa/aarch64/inst.isle line 1588.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         return Some(expr0_0);
     }
     if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1581.
+        // Rule at src/isa/aarch64/inst.isle line 1584.
         let expr0_0 = C::put_in_reg(ctx, pattern0_0);
         let expr1_0: bool = false;
         let expr2_0 = C::ty_bits(ctx, pattern2_0);
@@ -1771,7 +1778,7 @@ pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
 // Generated as internal constructor for term trap_if_zero_divisor.
 pub fn constructor_trap_if_zero_divisor<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Reg> {
     let pattern0_0 = arg0;
-    // Rule at src/isa/aarch64/inst.isle line 1590.
+    // Rule at src/isa/aarch64/inst.isle line 1593.
     let expr0_0 = C::cond_br_zero(ctx, pattern0_0);
     let expr1_0 = C::trap_code_division_by_zero(ctx);
     let expr2_0 = MInst::TrapIf {
@@ -1786,12 +1793,12 @@ pub fn constructor_trap_if_zero_divisor<C: Context>(ctx: &mut C, arg0: Reg) -> O
 pub fn constructor_size_from_ty<C: Context>(ctx: &mut C, arg0: Type) -> Option<OperandSize> {
     let pattern0_0 = arg0;
     if pattern0_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1596.
+        // Rule at src/isa/aarch64/inst.isle line 1599.
         let expr0_0 = OperandSize::Size64;
         return Some(expr0_0);
     }
     if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1595.
+        // Rule at src/isa/aarch64/inst.isle line 1598.
         let expr0_0 = OperandSize::Size32;
         return Some(expr0_0);
     }
@@ -1808,7 +1815,7 @@ pub fn constructor_trap_if_div_overflow<C: Context>(
     let pattern0_0 = arg0;
     let pattern1_0 = arg1;
     let pattern2_0 = arg2;
-    // Rule at src/isa/aarch64/inst.isle line 1602.
+    // Rule at src/isa/aarch64/inst.isle line 1605.
     let expr0_0 = constructor_adds_op(ctx, pattern0_0)?;
     let expr1_0 = C::writable_zero_reg(ctx);
     let expr2_0: u8 = 1;
@@ -1852,18 +1859,283 @@ pub fn constructor_trap_if_div_overflow<C: Context>(
 pub fn constructor_adds_op<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
     let pattern0_0 = arg0;
     if pattern0_0 == I64 {
-        // Rule at src/isa/aarch64/inst.isle line 1622.
+        // Rule at src/isa/aarch64/inst.isle line 1625.
         let expr0_0 = ALUOp::AddS64;
         return Some(expr0_0);
     }
     if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
-        // Rule at src/isa/aarch64/inst.isle line 1621.
+        // Rule at src/isa/aarch64/inst.isle line 1624.
         let expr0_0 = ALUOp::AddS32;
         return Some(expr0_0);
     }
     return None;
 }
 
+// Generated as internal constructor for term alu_rs_imm_logic_commutative.
+pub fn constructor_alu_rs_imm_logic_commutative<C: Context>(
+    ctx: &mut C,
+    arg0: &ALUOp,
+    arg1: Type,
+    arg2: Value,
+    arg3: Value,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    if let Some(pattern3_0) = C::def_inst(ctx, pattern2_0) {
+        let pattern4_0 = C::inst_data(ctx, pattern3_0);
+        match &pattern4_0 {
+            &InstructionData::UnaryImm {
+                opcode: ref pattern5_0,
+                imm: pattern5_1,
+            } => {
+                if let &Opcode::Iconst = &pattern5_0 {
+                    let closure7 = || {
+                        return Some(pattern1_0);
+                    };
+                    if let Some(pattern7_0) = closure7() {
+                        if let Some(pattern8_0) =
+                            C::imm_logic_from_imm64(ctx, pattern5_1, pattern7_0)
+                        {
+                            let pattern9_0 = arg3;
+                            // Rule at src/isa/aarch64/inst.isle line 1655.
+                            let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                            let expr1_0 =
+                                constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern8_0)?;
+                            return Some(expr1_0);
+                        }
+                    }
+                }
+            }
+            &InstructionData::Binary {
+                opcode: ref pattern5_0,
+                args: ref pattern5_1,
+            } => {
+                if let &Opcode::Ishl = &pattern5_0 {
+                    let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                    if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) {
+                        let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                        if let &InstructionData::UnaryImm {
+                            opcode: ref pattern10_0,
+                            imm: pattern10_1,
+                        } = &pattern9_0
+                        {
+                            if let &Opcode::Iconst = &pattern10_0 {
+                                let closure12 = || {
+                                    return Some(pattern1_0);
+                                };
+                                if let Some(pattern12_0) = closure12() {
+                                    if let Some(pattern13_0) =
+                                        C::lshl_from_imm64(ctx, pattern10_1, pattern12_0)
+                                    {
+                                        let pattern14_0 = arg3;
+                                        // Rule at src/isa/aarch64/inst.isle line 1661.
+                                        let expr0_0 = C::put_in_reg(ctx, pattern14_0);
+                                        let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                                        let expr2_0 = constructor_alu_rrr_shift(
+                                            ctx,
+                                            pattern0_0,
+                                            expr0_0,
+                                            expr1_0,
+                                            pattern13_0,
+                                        )?;
+                                        return Some(expr2_0);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+    let pattern3_0 = arg3;
+    if let Some(pattern4_0) = C::def_inst(ctx, pattern3_0) {
+        let pattern5_0 = C::inst_data(ctx, pattern4_0);
+        match &pattern5_0 {
+            &InstructionData::UnaryImm {
+                opcode: ref pattern6_0,
+                imm: pattern6_1,
+            } => {
+                if let &Opcode::Iconst = &pattern6_0 {
+                    let closure8 = || {
+                        return Some(pattern1_0);
+                    };
+                    if let Some(pattern8_0) = closure8() {
+                        if let Some(pattern9_0) =
+                            C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0)
+                        {
+                            // Rule at src/isa/aarch64/inst.isle line 1653.
+                            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                            let expr1_0 =
+                                constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?;
+                            return Some(expr1_0);
+                        }
+                    }
+                }
+            }
+            &InstructionData::Binary {
+                opcode: ref pattern6_0,
+                args: ref pattern6_1,
+            } => {
+                if let &Opcode::Ishl = &pattern6_0 {
+                    let (pattern8_0, pattern8_1) = C::unpack_value_array_2(ctx, &pattern6_1);
+                    if let Some(pattern9_0) = C::def_inst(ctx, pattern8_1) {
+                        let pattern10_0 = C::inst_data(ctx, pattern9_0);
+                        if let &InstructionData::UnaryImm {
+                            opcode: ref pattern11_0,
+                            imm: pattern11_1,
+                        } = &pattern10_0
+                        {
+                            if let &Opcode::Iconst = &pattern11_0 {
+                                let closure13 = || {
+                                    return Some(pattern1_0);
+                                };
+                                if let Some(pattern13_0) = closure13() {
+                                    if let Some(pattern14_0) =
+                                        C::lshl_from_imm64(ctx, pattern11_1, pattern13_0)
+                                    {
+                                        // Rule at src/isa/aarch64/inst.isle line 1659.
+                                        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                                        let expr1_0 = C::put_in_reg(ctx, pattern8_0);
+                                        let expr2_0 = constructor_alu_rrr_shift(
+                                            ctx,
+                                            pattern0_0,
+                                            expr0_0,
+                                            expr1_0,
+                                            pattern14_0,
+                                        )?;
+                                        return Some(expr2_0);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+    // Rule at src/isa/aarch64/inst.isle line 1649.
+    let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+    let expr1_0 = C::put_in_reg(ctx, pattern3_0);
+    let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term alu_rs_imm_logic.
+pub fn constructor_alu_rs_imm_logic<C: Context>(
+    ctx: &mut C,
+    arg0: &ALUOp,
+    arg1: Type,
+    arg2: Value,
+    arg3: Value,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    if let Some(pattern4_0) = C::def_inst(ctx, pattern3_0) {
+        let pattern5_0 = C::inst_data(ctx, pattern4_0);
+        match &pattern5_0 {
+            &InstructionData::UnaryImm {
+                opcode: ref pattern6_0,
+                imm: pattern6_1,
+            } => {
+                if let &Opcode::Iconst = &pattern6_0 {
+                    let closure8 = || {
+                        return Some(pattern1_0);
+                    };
+                    if let Some(pattern8_0) = closure8() {
+                        if let Some(pattern9_0) =
+                            C::imm_logic_from_imm64(ctx, pattern6_1, pattern8_0)
+                        {
+                            // Rule at src/isa/aarch64/inst.isle line 1669.
+                            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                            let expr1_0 =
+                                constructor_alu_rr_imm_logic(ctx, pattern0_0, expr0_0, pattern9_0)?;
+                            return Some(expr1_0);
+                        }
+                    }
+                }
+            }
+            &InstructionData::Binary {
+                opcode: ref pattern6_0,
+                args: ref pattern6_1,
+            } => {
+                if let &Opcode::Ishl = &pattern6_0 {
+                    let (pattern8_0, pattern8_1) = C::unpack_value_array_2(ctx, &pattern6_1);
+                    if let Some(pattern9_0) = C::def_inst(ctx, pattern8_1) {
+                        let pattern10_0 = C::inst_data(ctx, pattern9_0);
+                        if let &InstructionData::UnaryImm {
+                            opcode: ref pattern11_0,
+                            imm: pattern11_1,
+                        } = &pattern10_0
+                        {
+                            if let &Opcode::Iconst = &pattern11_0 {
+                                let closure13 = || {
+                                    return Some(pattern1_0);
+                                };
+                                if let Some(pattern13_0) = closure13() {
+                                    if let Some(pattern14_0) =
+                                        C::lshl_from_imm64(ctx, pattern11_1, pattern13_0)
+                                    {
+                                        // Rule at src/isa/aarch64/inst.isle line 1671.
+                                        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                                        let expr1_0 = C::put_in_reg(ctx, pattern8_0);
+                                        let expr2_0 = constructor_alu_rrr_shift(
+                                            ctx,
+                                            pattern0_0,
+                                            expr0_0,
+                                            expr1_0,
+                                            pattern14_0,
+                                        )?;
+                                        return Some(expr2_0);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+    // Rule at src/isa/aarch64/inst.isle line 1667.
+    let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+    let expr1_0 = C::put_in_reg(ctx, pattern3_0);
+    let expr2_0 = constructor_alu_rrr(ctx, pattern0_0, expr0_0, expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term i128_alu_bitop.
+pub fn constructor_i128_alu_bitop<C: Context>(
+    ctx: &mut C,
+    arg0: &ALUOp,
+    arg1: Value,
+    arg2: Value,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/aarch64/inst.isle line 1679.
+    let expr0_0 = C::put_in_regs(ctx, pattern1_0);
+    let expr1_0: usize = 0;
+    let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+    let expr3_0: usize = 1;
+    let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+    let expr5_0 = C::put_in_regs(ctx, pattern2_0);
+    let expr6_0: usize = 0;
+    let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0);
+    let expr8_0: usize = 1;
+    let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0);
+    let expr10_0 = constructor_alu_rrr(ctx, pattern0_0, expr2_0, expr7_0)?;
+    let expr11_0 = constructor_alu_rrr(ctx, pattern0_0, expr4_0, expr9_0)?;
+    let expr12_0 = C::value_regs(ctx, expr10_0, expr11_0);
+    return Some(expr12_0);
+}
+
 // Generated as internal constructor for term lower.
 pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueRegs> {
     let pattern0_0 = arg0;
@@ -1897,6 +2169,72 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
                         let expr4_0 = C::value_reg(ctx, expr3_0);
                         return Some(expr4_0);
                     }
+                    &Opcode::Band => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 632.
+                        let expr0_0 = ALUOp::And64;
+                        let expr1_0: Type = I64;
+                        let expr2_0 = constructor_alu_rs_imm_logic_commutative(
+                            ctx, &expr0_0, expr1_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 645.
+                        let expr0_0 = ALUOp::Orr64;
+                        let expr1_0: Type = I64;
+                        let expr2_0 = constructor_alu_rs_imm_logic_commutative(
+                            ctx, &expr0_0, expr1_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bxor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 658.
+                        let expr0_0 = ALUOp::Eor64;
+                        let expr1_0: Type = I64;
+                        let expr2_0 = constructor_alu_rs_imm_logic_commutative(
+                            ctx, &expr0_0, expr1_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::BandNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 671.
+                        let expr0_0 = ALUOp::AndNot64;
+                        let expr1_0: Type = I64;
+                        let expr2_0 = constructor_alu_rs_imm_logic(
+                            ctx, &expr0_0, expr1_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::BorNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 684.
+                        let expr0_0 = ALUOp::OrrNot64;
+                        let expr1_0: Type = I64;
+                        let expr2_0 = constructor_alu_rs_imm_logic(
+                            ctx, &expr0_0, expr1_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::BxorNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 694.
+                        let expr0_0 = ALUOp::EorNot64;
+                        let expr1_0: Type = I64;
+                        let expr2_0 = constructor_alu_rs_imm_logic(
+                            ctx, &expr0_0, expr1_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
                     _ => {}
                 }
             }
@@ -1976,6 +2314,60 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
                             let expr19_0 = C::value_regs(ctx, expr18_0, expr15_0);
                             return Some(expr19_0);
                         }
+                        &Opcode::Band => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 635.
+                            let expr0_0 = ALUOp::And64;
+                            let expr1_0 =
+                                constructor_i128_alu_bitop(ctx, &expr0_0, pattern7_0, pattern7_1)?;
+                            return Some(expr1_0);
+                        }
+                        &Opcode::Bor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 648.
+                            let expr0_0 = ALUOp::Orr64;
+                            let expr1_0 =
+                                constructor_i128_alu_bitop(ctx, &expr0_0, pattern7_0, pattern7_1)?;
+                            return Some(expr1_0);
+                        }
+                        &Opcode::Bxor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 661.
+                            let expr0_0 = ALUOp::Eor64;
+                            let expr1_0 =
+                                constructor_i128_alu_bitop(ctx, &expr0_0, pattern7_0, pattern7_1)?;
+                            return Some(expr1_0);
+                        }
+                        &Opcode::BandNot => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 674.
+                            let expr0_0 = ALUOp::AndNot64;
+                            let expr1_0 =
+                                constructor_i128_alu_bitop(ctx, &expr0_0, pattern7_0, pattern7_1)?;
+                            return Some(expr1_0);
+                        }
+                        &Opcode::BorNot => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 687.
+                            let expr0_0 = ALUOp::OrrNot64;
+                            let expr1_0 =
+                                constructor_i128_alu_bitop(ctx, &expr0_0, pattern7_0, pattern7_1)?;
+                            return Some(expr1_0);
+                        }
+                        &Opcode::BxorNot => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 697.
+                            let expr0_0 = ALUOp::EorNot64;
+                            let expr1_0 =
+                                constructor_i128_alu_bitop(ctx, &expr0_0, pattern7_0, pattern7_1)?;
+                            return Some(expr1_0);
+                        }
                         _ => {}
                     }
                 }
@@ -1984,6 +2376,22 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
                     arg: pattern5_1,
                 } => {
                     match &pattern5_0 {
+                        &Opcode::Bnot => {
+                            // Rule at src/isa/aarch64/lower.isle line 613.
+                            let expr0_0 = C::put_in_regs(ctx, pattern5_1);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = ALUOp::OrrNot64;
+                            let expr6_0 = C::zero_reg(ctx);
+                            let expr7_0 = constructor_alu_rrr(ctx, &expr5_0, expr6_0, expr2_0)?;
+                            let expr8_0 = ALUOp::OrrNot64;
+                            let expr9_0 = C::zero_reg(ctx);
+                            let expr10_0 = constructor_alu_rrr(ctx, &expr8_0, expr9_0, expr4_0)?;
+                            let expr11_0 = C::value_regs(ctx, expr7_0, expr10_0);
+                            return Some(expr11_0);
+                        }
                         &Opcode::Uextend => {
                             if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) {
                                 let pattern8_0 = C::inst_data(ctx, pattern7_0);
@@ -2718,6 +3126,66 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
                         let expr9_0 = C::value_reg(ctx, expr8_0);
                         return Some(expr9_0);
                     }
+                    &Opcode::Band => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 629.
+                        let expr0_0 = ALUOp::And32;
+                        let expr1_0 = constructor_alu_rs_imm_logic_commutative(
+                            ctx, &expr0_0, pattern3_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Bor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 642.
+                        let expr0_0 = ALUOp::Orr32;
+                        let expr1_0 = constructor_alu_rs_imm_logic_commutative(
+                            ctx, &expr0_0, pattern3_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Bxor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 655.
+                        let expr0_0 = ALUOp::Eor32;
+                        let expr1_0 = constructor_alu_rs_imm_logic_commutative(
+                            ctx, &expr0_0, pattern3_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::BandNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 668.
+                        let expr0_0 = ALUOp::AndNot32;
+                        let expr1_0 = constructor_alu_rs_imm_logic(
+                            ctx, &expr0_0, pattern3_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::BorNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 681.
+                        let expr0_0 = ALUOp::OrrNot32;
+                        let expr1_0 = constructor_alu_rs_imm_logic(
+                            ctx, &expr0_0, pattern3_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::BxorNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/aarch64/lower.isle line 691.
+                        let expr0_0 = ALUOp::EorNot32;
+                        let expr1_0 = constructor_alu_rs_imm_logic(
+                            ctx, &expr0_0, pattern3_0, pattern7_0, pattern7_1,
+                        )?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
                     _ => {}
                 }
             }
@@ -3240,6 +3708,70 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
                             let expr4_0 = C::value_reg(ctx, expr3_0);
                             return Some(expr4_0);
                         }
+                        &Opcode::Bnot => {
+                            if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) {
+                                let pattern8_0 = C::inst_data(ctx, pattern7_0);
+                                if let &InstructionData::Binary {
+                                    opcode: ref pattern9_0,
+                                    args: ref pattern9_1,
+                                } = &pattern8_0
+                                {
+                                    if let &Opcode::Ishl = &pattern9_0 {
+                                        let (pattern11_0, pattern11_1) =
+                                            C::unpack_value_array_2(ctx, &pattern9_1);
+                                        if let Some(pattern12_0) = C::def_inst(ctx, pattern11_1) {
+                                            let pattern13_0 = C::inst_data(ctx, pattern12_0);
+                                            if let &InstructionData::UnaryImm {
+                                                opcode: ref pattern14_0,
+                                                imm: pattern14_1,
+                                            } = &pattern13_0
+                                            {
+                                                if let &Opcode::Iconst = &pattern14_0 {
+                                                    let closure16 = || {
+                                                        return Some(pattern3_0);
+                                                    };
+                                                    if let Some(pattern16_0) = closure16() {
+                                                        if let Some(pattern17_0) =
+                                                            C::lshl_from_imm64(
+                                                                ctx,
+                                                                pattern14_1,
+                                                                pattern16_0,
+                                                            )
+                                                        {
+                                                            // Rule at src/isa/aarch64/lower.isle line 608.
+                                                            let expr0_0 = constructor_orr_not_op(
+                                                                ctx, pattern3_0,
+                                                            )?;
+                                                            let expr1_0 = C::zero_reg(ctx);
+                                                            let expr2_0 =
+                                                                C::put_in_reg(ctx, pattern11_0);
+                                                            let expr3_0 =
+                                                                constructor_alu_rrr_shift(
+                                                                    ctx,
+                                                                    &expr0_0,
+                                                                    expr1_0,
+                                                                    expr2_0,
+                                                                    pattern17_0,
+                                                                )?;
+                                                            let expr4_0 =
+                                                                C::value_reg(ctx, expr3_0);
+                                                            return Some(expr4_0);
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            // Rule at src/isa/aarch64/lower.isle line 603.
+                            let expr0_0 = constructor_orr_not_op(ctx, pattern3_0)?;
+                            let expr1_0 = C::zero_reg(ctx);
+                            let expr2_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr3_0 = constructor_alu_rrr(ctx, &expr0_0, expr1_0, expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
                         &Opcode::Uextend => {
                             if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) {
                                 let pattern8_0 = C::inst_data(ctx, pattern7_0);
@@ -3389,6 +3921,58 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
                             let expr5_0 = C::value_reg(ctx, expr4_0);
                             return Some(expr5_0);
                         }
+                        &Opcode::Band => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 637.
+                            let expr0_0 = VecALUOp::And;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr2_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr3_0 = constructor_vector_size(ctx, pattern3_0)?;
+                            let expr4_0 =
+                                constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                        &Opcode::Bor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 650.
+                            let expr0_0 = VecALUOp::Orr;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr2_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr3_0 = constructor_vector_size(ctx, pattern3_0)?;
+                            let expr4_0 =
+                                constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                        &Opcode::Bxor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 663.
+                            let expr0_0 = VecALUOp::Eor;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr2_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr3_0 = constructor_vector_size(ctx, pattern3_0)?;
+                            let expr4_0 =
+                                constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                        &Opcode::BandNot => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/aarch64/lower.isle line 676.
+                            let expr0_0 = VecALUOp::Bic;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr2_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr3_0 = constructor_vector_size(ctx, pattern3_0)?;
+                            let expr4_0 =
+                                constructor_vec_rrr(ctx, &expr0_0, expr1_0, expr2_0, &expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
                         _ => {}
                     }
                 }
@@ -3396,14 +3980,26 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueReg
                     opcode: ref pattern5_0,
                     arg: pattern5_1,
                 } => {
-                    if let &Opcode::Ineg = &pattern5_0 {
+                    match &pattern5_0 {
-                        // Rule at src/isa/aarch64/lower.isle line 190.
+                        &Opcode::Ineg => {
-                        let expr0_0 = VecMisc2::Neg;
+                            // Rule at src/isa/aarch64/lower.isle line 190.
-                        let expr1_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr0_0 = VecMisc2::Neg;
-                        let expr2_0 = constructor_vector_size(ctx, pattern3_0)?;
+                            let expr1_0 = C::put_in_reg(ctx, pattern5_1);
-                        let expr3_0 = constructor_vec_misc(ctx, &expr0_0, expr1_0, &expr2_0)?;
+                            let expr2_0 = constructor_vector_size(ctx, pattern3_0)?;
-                        let expr4_0 = C::value_reg(ctx, expr3_0);
+                            let expr3_0 = constructor_vec_misc(ctx, &expr0_0, expr1_0, &expr2_0)?;
-                        return Some(expr4_0);
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Bnot => {
+                            // Rule at src/isa/aarch64/lower.isle line 624.
+                            let expr0_0 = VecMisc2::Not;
+                            let expr1_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr2_0 = constructor_vector_size(ctx, pattern3_0)?;
+                            let expr3_0 = constructor_vec_misc(ctx, &expr0_0, expr1_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        _ => {}
                     }
                 }
                 _ => {}
@@ -3533,3 +4129,19 @@ pub fn constructor_put_nonzero_in_reg_sext64<C: Context>(ctx: &mut C, arg0: Valu
     let expr1_0 = constructor_trap_if_zero_divisor(ctx, expr0_0)?;
     return Some(expr1_0);
 }
+
+// Generated as internal constructor for term orr_not_op.
+pub fn constructor_orr_not_op<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/aarch64/lower.isle line 596.
+        let expr0_0 = ALUOp::OrrNot64;
+        return Some(expr0_0);
+    }
+    if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
+        // Rule at src/isa/aarch64/lower.isle line 595.
+        let expr0_0 = ALUOp::OrrNot32;
+        return Some(expr0_0);
+    }
+    return None;
+}

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -79,112 +79,14 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::Uextend | Opcode::Sextend => implemented_in_isle(ctx),
 
-        Opcode::Bnot => {
+        Opcode::Bnot => implemented_in_isle(ctx),
-            let out_regs = get_output_reg(ctx, outputs[0]);
-            let ty = ty.unwrap();
-            if ty == I128 {
-                // TODO: We can merge this block with the one below once we support immlogic here
-                let in_regs = put_input_in_regs(ctx, inputs[0]);
-                ctx.emit(Inst::AluRRR {
-                    alu_op: ALUOp::OrrNot64,
-                    rd: out_regs.regs()[0],
-                    rn: zero_reg(),
-                    rm: in_regs.regs()[0],
-                });
-                ctx.emit(Inst::AluRRR {
-                    alu_op: ALUOp::OrrNot64,
-                    rd: out_regs.regs()[1],
-                    rn: zero_reg(),
-                    rm: in_regs.regs()[1],
-                });
-            } else if !ty.is_vector() {
-                let rd = out_regs.only_reg().unwrap();
-                let rm = put_input_in_rs_immlogic(ctx, inputs[0], NarrowValueMode::None);
-                let alu_op = choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64);
-                // NOT rd, rm ==> ORR_NOT rd, zero, rm
-                ctx.emit(alu_inst_immlogic(alu_op, rd, zero_reg(), rm));
-            } else {
-                let rd = out_regs.only_reg().unwrap();
-                let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                ctx.emit(Inst::VecMisc {
-                    op: VecMisc2::Not,
-                    rd,
-                    rn: rm,
-                    size: VectorSize::from_ty(ty),
-                });
-            }
-        }
 
         Opcode::Band
         | Opcode::Bor
         | Opcode::Bxor
         | Opcode::BandNot
         | Opcode::BorNot
-        | Opcode::BxorNot => {
+        | Opcode::BxorNot => implemented_in_isle(ctx),
-            let out_regs = get_output_reg(ctx, outputs[0]);
-            let ty = ty.unwrap();
-            if ty == I128 {
-                // TODO: Support immlogic here
-                let lhs = put_input_in_regs(ctx, inputs[0]);
-                let rhs = put_input_in_regs(ctx, inputs[1]);
-                let alu_op = match op {
-                    Opcode::Band => ALUOp::And64,
-                    Opcode::Bor => ALUOp::Orr64,
-                    Opcode::Bxor => ALUOp::Eor64,
-                    Opcode::BandNot => ALUOp::AndNot64,
-                    Opcode::BorNot => ALUOp::OrrNot64,
-                    Opcode::BxorNot => ALUOp::EorNot64,
-                    _ => unreachable!(),
-                };
-
-                ctx.emit(Inst::AluRRR {
-                    alu_op,
-                    rd: out_regs.regs()[0],
-                    rn: lhs.regs()[0],
-                    rm: rhs.regs()[0],
-                });
-                ctx.emit(Inst::AluRRR {
-                    alu_op,
-                    rd: out_regs.regs()[1],
-                    rn: lhs.regs()[1],
-                    rm: rhs.regs()[1],
-                });
-            } else if !ty.is_vector() {
-                let rd = out_regs.only_reg().unwrap();
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                let rm = put_input_in_rs_immlogic(ctx, inputs[1], NarrowValueMode::None);
-                let alu_op = match op {
-                    Opcode::Band => choose_32_64(ty, ALUOp::And32, ALUOp::And64),
-                    Opcode::Bor => choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64),
-                    Opcode::Bxor => choose_32_64(ty, ALUOp::Eor32, ALUOp::Eor64),
-                    Opcode::BandNot => choose_32_64(ty, ALUOp::AndNot32, ALUOp::AndNot64),
-                    Opcode::BorNot => choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64),
-                    Opcode::BxorNot => choose_32_64(ty, ALUOp::EorNot32, ALUOp::EorNot64),
-                    _ => unreachable!(),
-                };
-                ctx.emit(alu_inst_immlogic(alu_op, rd, rn, rm));
-            } else {
-                let alu_op = match op {
-                    Opcode::Band => VecALUOp::And,
-                    Opcode::BandNot => VecALUOp::Bic,
-                    Opcode::Bor => VecALUOp::Orr,
-                    Opcode::Bxor => VecALUOp::Eor,
-                    _ => unreachable!(),
-                };
-
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                let rd = out_regs.only_reg().unwrap();
-
-                ctx.emit(Inst::VecRRR {
-                    alu_op,
-                    rd,
-                    rn,
-                    rm,
-                    size: VectorSize::from_ty(ty),
-                });
-            }
-        }
 
         Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => {
             let out_regs = get_output_reg(ctx, outputs[0]);

cranelift/filetests/filetests/isa/aarch64/bitops.clif

@@ -298,6 +298,35 @@ block0:
 ; nextln: sbfx w0, w0, #0, #1
 ; nextln: ret
 
+function %bnot_i32(i32) -> i32 {
+block0(v0: i32):
+    v1 = bnot v0
+    return v1
+}
+
+; check: orn w0, wzr, w0
+; nextln: ret
+
+function %bnot_i64(i64) -> i64 {
+block0(v0: i64):
+    v1 = bnot v0
+    return v1
+}
+
+; check: orn x0, xzr, x0
+; nextln: ret
+
+function %bnot_i64_with_shift(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 3
+    v2 = ishl.i64 v0, v1
+    v3 = bnot v2
+    return v3
+}
+
+; check:  orn x0, xzr, x0, LSL 3
+; nextln: ret
+
 function %bnot_i128(i128) -> i128 {
 block0(v0: i128):
     v1 = bnot v0
@@ -308,6 +337,33 @@ block0(v0: i128):
 ; nextln: orn x1, xzr, x1
 ; nextln: ret
 
+function %bnot_i8x16(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = bnot v0
+    return v1
+}
+
+; check: mvn v0.16b, v0.16b
+; nextln: ret
+
+function %band_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    v2 = band v0, v1
+    return v2
+}
+
+; check: and w0, w0, w1
+; nextln: ret
+
+function %band_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = band v0, v1
+    return v2
+}
+
+; check: and x0, x0, x1
+; nextln: ret
+
 function %band_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):
     v2 = band v0, v1
@@ -318,6 +374,72 @@ block0(v0: i128, v1: i128):
 ; nextln: and x1, x1, x3
 ; nextln: ret
 
+function %band_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = band v0, v1
+    return v2
+}
+
+; check: and v0.16b, v0.16b, v1.16b
+; nextln: ret
+
+function %band_i64_constant(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 3
+    v2 = band v0, v1
+    return v2
+}
+
+; check: and x0, x0, #3
+; nextln: ret
+
+function %band_i64_constant2(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 3
+    v2 = band v1, v0
+    return v2
+}
+
+; check: and x0, x0, #3
+; nextln: ret
+
+function %band_i64_constant_shift(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 3
+    v3 = ishl.i64 v1, v2
+    v4 = band v0, v3
+    return v4
+}
+
+function %band_i64_constant_shift2(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 3
+    v3 = ishl.i64 v1, v2
+    v4 = band v3, v0
+    return v4
+}
+
+; check: and x0, x0, x1, LSL 3
+; nextln: ret
+
+function %bor_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    v2 = bor v0, v1
+    return v2
+}
+
+; check: orr w0, w0, w1
+; nextln: ret
+
+function %bor_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = bor v0, v1
+    return v2
+}
+
+; check: orr x0, x0, x1
+; nextln: ret
+
 function %bor_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):
     v2 = bor v0, v1
@@ -328,6 +450,75 @@ block0(v0: i128, v1: i128):
 ; nextln: orr x1, x1, x3
 ; nextln: ret
 
+function %bor_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = bor v0, v1
+    return v2
+}
+
+; check: orr v0.16b, v0.16b, v1.16b
+; nextln: ret
+
+function %bor_i64_constant(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 3
+    v2 = bor v0, v1
+    return v2
+}
+
+; check: orr x0, x0, #3
+; nextln: ret
+
+function %bor_i64_constant2(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 3
+    v2 = bor v1, v0
+    return v2
+}
+
+; check: orr x0, x0, #3
+; nextln: ret
+
+function %bor_i64_constant_shift(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 3
+    v3 = ishl.i64 v1, v2
+    v4 = bor v0, v3
+    return v4
+}
+
+; check: orr x0, x0, x1, LSL 3
+; nextln: ret
+
+function %bor_i64_constant_shift2(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 3
+    v3 = ishl.i64 v1, v2
+    v4 = bor v3, v0
+    return v4
+}
+
+; check: orr x0, x0, x1, LSL 3
+; nextln: ret
+
+function %bxor_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    v2 = bxor v0, v1
+    return v2
+}
+
+; check: eor w0, w0, w1
+; nextln: ret
+
+function %bxor_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = bxor v0, v1
+    return v2
+}
+
+; check: eor x0, x0, x1
+; nextln: ret
+
 function %bxor_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):
     v2 = bxor v0, v1
@@ -338,6 +529,75 @@ block0(v0: i128, v1: i128):
 ; nextln: eor x1, x1, x3
 ; nextln: ret
 
+function %bxor_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = bxor v0, v1
+    return v2
+}
+
+; check: eor v0.16b, v0.16b, v1.16b
+; nextln: ret
+
+function %bxor_i64_constant(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 3
+    v2 = bxor v0, v1
+    return v2
+}
+
+; check: eor x0, x0, #3
+; nextln: ret
+
+function %bxor_i64_constant2(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 3
+    v2 = bxor v1, v0
+    return v2
+}
+
+; check: eor x0, x0, #3
+; nextln: ret
+
+function %bxor_i64_constant_shift(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 3
+    v3 = ishl.i64 v1, v2
+    v4 = bxor v0, v3
+    return v4
+}
+
+; check: eor x0, x0, x1, LSL 3
+; nextln: ret
+
+function %bxor_i64_constant_shift2(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 3
+    v3 = ishl.i64 v1, v2
+    v4 = bxor v3, v0
+    return v4
+}
+
+; check: eor x0, x0, x1, LSL 3
+; nextln: ret
+
+function %band_not_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    v2 = band_not v0, v1
+    return v2
+}
+
+; check: bic w0, w0, w1
+; nextln: ret
+
+function %band_not_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = band_not v0, v1
+    return v2
+}
+
+; check: bic x0, x0, x1
+; nextln: ret
+
 function %band_not_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):
     v2 = band_not v0, v1
@@ -348,6 +608,54 @@ block0(v0: i128, v1: i128):
 ; nextln: bic x1, x1, x3
 ; nextln: ret
 
+function %band_not_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = band_not v0, v1
+    return v2
+}
+
+; check: bic v0.16b, v0.16b, v1.16b
+; nextln: ret
+
+function %band_not_i64_constant(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 4
+    v2 = band_not v0, v1
+    return v2
+}
+
+; check: bic x0, x0, #4
+; nextln: ret
+
+function %band_not_i64_constant_shift(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 4
+    v3 = ishl.i64 v1, v2
+    v4 = band_not v0, v3
+    return v4
+}
+
+; check: bic x0, x0, x1, LSL 4
+; nextln: ret
+
+function %bor_not_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    v2 = bor_not v0, v1
+    return v2
+}
+
+; check: orn w0, w0, w1
+; nextln: ret
+
+function %bor_not_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = bor_not v0, v1
+    return v2
+}
+
+; check: orn x0, x0, x1
+; nextln: ret
+
 function %bor_not_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):
     v2 = bor_not v0, v1
@@ -358,6 +666,45 @@ block0(v0: i128, v1: i128):
 ; nextln: orn x1, x1, x3
 ; nextln: ret
 
+function %bor_not_i64_constant(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 4
+    v2 = bor_not v0, v1
+    return v2
+}
+
+; check: orn x0, x0, #4
+; nextln: ret
+
+function %bor_not_i64_constant_shift(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 4
+    v3 = ishl.i64 v1, v2
+    v4 = bor_not v0, v3
+    return v4
+}
+
+; check: orn x0, x0, x1, LSL 4
+; nextln: ret
+
+function %bxor_not_i32(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    v2 = bxor_not v0, v1
+    return v2
+}
+
+; check: eon w0, w0, w1
+; nextln: ret
+
+function %bxor_not_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = bxor_not v0, v1
+    return v2
+}
+
+; check: eon x0, x0, x1
+; nextln: ret
+
 function %bxor_not_i128(i128, i128) -> i128 {
 block0(v0: i128, v1: i128):
     v2 = bxor_not v0, v1
@@ -368,6 +715,26 @@ block0(v0: i128, v1: i128):
 ; nextln: eon x1, x1, x3
 ; nextln: ret
 
+function %bxor_not_i64_constant(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 4
+    v2 = bxor_not v0, v1
+    return v2
+}
+
+; check: eon x0, x0, #4
+; nextln: ret
+
+function %bxor_not_i64_constant_shift(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+    v2 = iconst.i64 4
+    v3 = ishl.i64 v1, v2
+    v4 = bxor_not v0, v3
+    return v4
+}
+
+; check: eon x0, x0, x1, LSL 4
+; nextln: ret
 
 function %ishl_i128_i8(i128, i8) -> i128 {
 block0(v0: i128, v1: i8):