Merge pull request #3698 from cfallin/cold-blocks

Cranelift: add support for cold blocks.

cranelift/codegen/src/ir/layout.rs

@@ -469,6 +469,19 @@ impl Layout {
     pub fn next_block(&self, block: Block) -> Option<Block> {
         self.blocks[block].next.expand()
     }
+
+    /// Mark a block as "cold".
+    ///
+    /// This will try to move it out of the ordinary path of execution
+    /// when lowered to machine code.
+    pub fn set_cold(&mut self, block: Block) {
+        self.blocks[block].cold = true;
+    }
+
+    /// Is the given block cold?
+    pub fn is_cold(&self, block: Block) -> bool {
+        self.blocks[block].cold
+    }
 }
 
 #[derive(Clone, Debug, Default)]
@@ -478,6 +491,7 @@ struct BlockNode {
     first_inst: PackedOption<Inst>,
     last_inst: PackedOption<Inst>,
     seq: SequenceNumber,
+    cold: bool,
 }
 
 /// Iterate over blocks in layout order. See [crate::ir::layout::Layout::blocks].

cranelift/codegen/src/machinst/blockorder.rs

@@ -378,11 +378,24 @@ impl BlockLoweringOrder {
 
         postorder.reverse();
         let mut rpo = postorder;
+
+        // Step 3: sink any cold blocks to the end of the
+        // function. Put the "deferred last" block truly at the end;
+        // this is a correctness requirement (for fallthrough
+        // returns).
+        rpo.sort_by_key(|block| {
+            block
+                .0
+                .orig_block()
+                .map(|block| f.layout.is_cold(block))
+                .unwrap_or(false)
+        });
+
         if let Some(d) = deferred_last {
             rpo.push(d);
         }
 
-        // Step 3: now that we have RPO, build the BlockIndex/BB fwd/rev maps.
+        // Step 4: now that we have RPO, build the BlockIndex/BB fwd/rev maps.
         let mut lowered_order = vec![];
         let mut lowered_succ_ranges = vec![];
         let mut lb_to_bindex = FxHashMap::default();

cranelift/frontend/src/frontend.rs

@@ -229,6 +229,14 @@ impl<'a> FunctionBuilder<'a> {
         block
     }
 
+    /// Mark a block as "cold".
+    ///
+    /// This will try to move it out of the ordinary path of execution
+    /// when lowered to machine code.
+    pub fn set_cold_block(&mut self, block: Block) {
+        self.func.layout.set_cold(block);
+    }
+
     /// Insert `block` in the layout *after* the existing block `after`.
     pub fn insert_block_after(&mut self, block: Block, after: Block) {
         self.func.layout.insert_block_after(block, after);