machinst x64: optimize select/brz/brnz when the input is a comparison;

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -849,7 +849,7 @@ impl CC {
             FloatCC::Ordered => CC::NP,
             FloatCC::Unordered => CC::P,
             // Alias for NE
-            FloatCC::NotEqual | FloatCC::OrderedNotEqual => CC::NZ,
+            FloatCC::OrderedNotEqual => CC::NZ,
             // Alias for E
             FloatCC::UnorderedOrEqual => CC::Z,
             // Alias for A
@@ -859,12 +859,14 @@ impl CC {
             FloatCC::UnorderedOrLessThan => CC::B,
             FloatCC::UnorderedOrLessThanOrEqual => CC::BE,
             FloatCC::Equal
+            | FloatCC::NotEqual
             | FloatCC::LessThan
             | FloatCC::LessThanOrEqual
             | FloatCC::UnorderedOrGreaterThan
-            | FloatCC::UnorderedOrGreaterThanOrEqual => {
-                panic!("No single condition code to guarantee ordered. Treat as special case.")
-            }
+            | FloatCC::UnorderedOrGreaterThanOrEqual => panic!(
+                "{:?} can't be lowered to a CC code; treat as special case.",
+                floatcc
+            ),
         }
     }
 

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -1298,6 +1298,8 @@ pub(crate) fn emit(
             src,
             dst,
         } => {
+            // Lowering of the Select IR opcode when the input is an fcmp relies on the fact that
+            // this doesn't clobber flags. Make sure to not do so here.
             let next = sink.get_label();
 
             // Jump if cc is *not* set.
@@ -1432,6 +1434,21 @@ pub(crate) fn emit(
             sink.put4(disp);
         }
 
+        Inst::JmpIf { cc, taken } => {
+            let cond_start = sink.cur_offset();
+            let cond_disp_off = cond_start + 2;
+            if let Some(l) = taken.as_label() {
+                sink.use_label_at_offset(cond_disp_off, l, LabelUse::JmpRel32);
+                // Since this is not a terminator, don't enroll in the branch inversion mechanism.
+            }
+
+            let taken_disp = taken.as_offset32_or_zero();
+            let taken_disp = taken_disp as u32;
+            sink.put1(0x0F);
+            sink.put1(0x80 + cc.get_enc());
+            sink.put4(taken_disp);
+        }
+
         Inst::JmpCond {
             cc,
             taken,

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -359,6 +359,9 @@ pub enum Inst {
     /// Jump to a known target: jmp simm32.
     JmpKnown { dst: BranchTarget },
 
+    /// One-way conditional branch: jcond cond target.
+    JmpIf { cc: CC, taken: BranchTarget },
+
     /// Two-way conditional branch: jcond cond target target.
     /// Emitted as a compound sequence; the MachBuffer will shrink it as appropriate.
     JmpCond {
@@ -966,6 +969,10 @@ impl Inst {
         Inst::JmpKnown { dst }
     }
 
+    pub(crate) fn jmp_if(cc: CC, taken: BranchTarget) -> Inst {
+        Inst::JmpIf { cc, taken }
+    }
+
     pub(crate) fn jmp_cond(cc: CC, taken: BranchTarget, not_taken: BranchTarget) -> Inst {
         Inst::JmpCond {
             cc,
@@ -1536,12 +1543,18 @@ impl ShowWithRRU for Inst {
                 format!("{} {}", ljustify("jmp".to_string()), dst.show_rru(mb_rru))
             }
 
+            Inst::JmpIf { cc, taken } => format!(
+                "{} {}",
+                ljustify2("j".to_string(), cc.to_string()),
+                taken.show_rru(mb_rru),
+            ),
+
             Inst::JmpCond {
                 cc,
                 taken,
                 not_taken,
             } => format!(
-                "{} taken={} not_taken={}",
+                "{} {}; j {}",
                 ljustify2("j".to_string(), cc.to_string()),
                 taken.show_rru(mb_rru),
                 not_taken.show_rru(mb_rru)
@@ -1823,6 +1836,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         Inst::Ret
         | Inst::EpiloguePlaceholder
         | Inst::JmpKnown { .. }
+        | Inst::JmpIf { .. }
         | Inst::JmpCond { .. }
         | Inst::Nop { .. }
         | Inst::TrapIf { .. }
@@ -2188,6 +2202,7 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         | Inst::EpiloguePlaceholder
         | Inst::JmpKnown { .. }
         | Inst::JmpCond { .. }
+        | Inst::JmpIf { .. }
         | Inst::Nop { .. }
         | Inst::TrapIf { .. }
         | Inst::VirtualSPOffsetAdj { .. }