x64: Migrate {s,u}{div,rem} to ISLE (#6008)

* x64: Add precise-output tests for div traps

This adds a suite of `*.clif` files which are intended to test the
`avoid_div_traps=true` compilation of the `{s,u}{div,rem}` instructions.

* x64: Remove conditional regalloc in `Div` instruction

Move the 8-bit `Div` logic into a dedicated `Div8` instruction to avoid
having conditionally-used registers with respect to regalloc.

* x64: Migrate non-trapping, `udiv`/`urem` to ISLE

* x64: Port checked `udiv` to ISLE

* x64: Migrate urem entirely to ISLE

* x64: Use `test` instead of `cmp` to compare-to-zero

* x64: Port `sdiv` lowering to ISLE

* x64: Port `srem` lowering to ISLE

* Tidy up regalloc behavior and fix tests

* Update docs and winch

* Review comments

* Reword again

* More refactoring test fixes

* More test fixes

winch/codegen/src/isa/x64/asm.rs

@@ -5,10 +5,11 @@ use crate::{
     masm::{DivKind, OperandSize, RemKind},
 };
 use cranelift_codegen::{
+    ir::TrapCode,
     isa::x64::{
         args::{
-            self, AluRmiROpcode, Amode, DivOrRemKind, ExtMode, FromWritableReg, Gpr, GprMem,
-            GprMemImm, RegMem, RegMemImm, SyntheticAmode, WritableGpr,
+            self, AluRmiROpcode, Amode, CmpOpcode, DivSignedness, ExtMode, FromWritableReg, Gpr,
+            GprMem, GprMemImm, RegMem, RegMemImm, SyntheticAmode, WritableGpr, CC,
         },
         settings as x64_settings, EmitInfo, EmitState, Inst,
     },
@@ -64,20 +65,11 @@ impl From<OperandSize> for args::OperandSize {
     }
 }
 
-impl From<DivKind> for DivOrRemKind {
-    fn from(kind: DivKind) -> Self {
+impl From<DivKind> for DivSignedness {
+    fn from(kind: DivKind) -> DivSignedness {
         match kind {
-            DivKind::Signed => DivOrRemKind::SignedDiv,
-            DivKind::Unsigned => DivOrRemKind::UnsignedDiv,
-        }
-    }
-}
-
-impl From<RemKind> for DivOrRemKind {
-    fn from(kind: RemKind) -> Self {
-        match kind {
-            RemKind::Signed => DivOrRemKind::SignedRem,
-            RemKind::Unsigned => DivOrRemKind::UnsignedRem,
+            DivKind::Signed => DivSignedness::Signed,
+            DivKind::Unsigned => DivSignedness::Unsigned,
         }
     }
 }
@@ -290,21 +282,61 @@ impl Assembler {
     /// caller has correctly allocated the dividend as `(rdx:rax)` and
     /// accounted for the quotient to be stored in `rax`.
     pub fn div(&mut self, divisor: Reg, dst: (Reg, Reg), kind: DivKind, size: OperandSize) {
-        let tmp = if size == OperandSize::S64 && kind == DivKind::Signed {
-            Some(regs::scratch())
-        } else {
-            None
-        };
+        match kind {
+            // Signed division goes through a pseudo-instruction to validate
+            // the divisor followed by a sign extension to initialize `rdx`.
+            DivKind::Signed => {
+                if size == OperandSize::S64 {
+                    self.emit(Inst::ValidateSdivDivisor64 {
+                        dividend: dst.0.into(),
+                        divisor: divisor.into(),
+                        tmp: regs::scratch().into(),
+                    });
+                } else {
+                    self.emit(Inst::ValidateSdivDivisor {
+                        dividend: dst.0.into(),
+                        divisor: divisor.into(),
+                        size: size.into(),
+                    });
+                }
+                self.emit(Inst::SignExtendData {
+                    size: size.into(),
+                    src: dst.0.into(),
+                    dst: dst.1.into(),
+                });
+            }
 
-        self.emit(Inst::CheckedDivOrRemSeq {
-            kind: kind.into(),
+            // Unsigned division only needs to check for 0 and then the `rdx`
+            // divisor_hi is initialized with zero through an xor-against-itself
+            // op.
+            DivKind::Unsigned => {
+                self.emit(Inst::CmpRmiR {
+                    size: size.into(),
+                    src: GprMemImm::new(RegMemImm::imm(0)).unwrap(),
+                    dst: divisor.into(),
+                    opcode: CmpOpcode::Cmp,
+                });
+                self.emit(Inst::TrapIf {
+                    cc: CC::Z,
+                    trap_code: TrapCode::IntegerDivisionByZero,
+                });
+                self.emit(Inst::AluRmiR {
+                    size: size.into(),
+                    op: AluRmiROpcode::Xor,
+                    src1: dst.1.into(),
+                    src2: dst.1.into(),
+                    dst: dst.1.into(),
+                });
+            }
+        }
+        self.emit(Inst::Div {
+            sign: kind.into(),
             size: size.into(),
-            divisor: divisor.into(),
+            divisor: GprMem::new(RegMem::reg(divisor.into())).unwrap(),
             dividend_lo: dst.0.into(),
             dividend_hi: dst.1.into(),
             dst_quotient: dst.0.into(),
             dst_remainder: dst.1.into(),
-            tmp: tmp.map(|reg| reg.into()),
         });
     }
 
@@ -316,16 +348,58 @@ impl Assembler {
     /// caller has correctly allocated the dividend as `(rdx:rax)` and
     /// accounted for the remainder to be stored in `rdx`.
     pub fn rem(&mut self, divisor: Reg, dst: (Reg, Reg), kind: RemKind, size: OperandSize) {
-        self.emit(Inst::CheckedDivOrRemSeq {
-            kind: kind.into(),
+        // First check for zero and explicitly trap.
+        self.emit(Inst::CmpRmiR {
             size: size.into(),
-            divisor: divisor.into(),
-            dividend_lo: dst.0.into(),
-            dividend_hi: dst.1.into(),
-            dst_quotient: dst.0.into(),
-            dst_remainder: dst.1.into(),
-            tmp: None,
+            src: GprMemImm::new(RegMemImm::imm(0)).unwrap(),
+            dst: divisor.into(),
+            opcode: CmpOpcode::Cmp,
         });
+        self.emit(Inst::TrapIf {
+            cc: CC::Z,
+            trap_code: TrapCode::IntegerDivisionByZero,
+        });
+        match kind {
+            // Signed remainder goes through a pseudo-instruction which has
+            // some internal branching. The `dividend_hi`, or `rdx`, is
+            // initialized here with a `SignExtendData` instruction.
+            RemKind::Signed => {
+                self.emit(Inst::SignExtendData {
+                    size: size.into(),
+                    src: dst.0.into(),
+                    dst: dst.1.into(),
+                });
+                self.emit(Inst::CheckedSRemSeq {
+                    size: size.into(),
+                    divisor: divisor.into(),
+                    dividend_lo: dst.0.into(),
+                    dividend_hi: dst.1.into(),
+                    dst_quotient: dst.0.into(),
+                    dst_remainder: dst.1.into(),
+                });
+            }
+
+            // Unsigned remainder initializes `dividend_hi` with zero and
+            // then executes a normal `div` instruction.
+            RemKind::Unsigned => {
+                self.emit(Inst::AluRmiR {
+                    size: size.into(),
+                    op: AluRmiROpcode::Xor,
+                    src1: dst.1.into(),
+                    src2: dst.1.into(),
+                    dst: dst.1.into(),
+                });
+                self.emit(Inst::Div {
+                    sign: DivSignedness::Unsigned,
+                    size: size.into(),
+                    divisor: GprMem::new(RegMem::reg(divisor.into())).unwrap(),
+                    dividend_lo: dst.0.into(),
+                    dividend_hi: dst.1.into(),
+                    dst_quotient: dst.0.into(),
+                    dst_remainder: dst.1.into(),
+                });
+            }
+        }
     }
 
     /// Multiply immediate and register.