wasmtime: Implement table.get and table.set

These instructions have fast, inline JIT paths for the common cases, and only
call out to host VM functions for the slow paths. This required some changes to
`cranelift-wasm`'s `FuncEnvironment`: instead of taking a `FuncCursor` to insert
an instruction sequence within the current basic block,
`FuncEnvironment::translate_table_{get,set}` now take a `&mut FunctionBuilder`
so that they can create whole new basic blocks. This is necessary for
implementing GC read/write barriers that involve branching (e.g. checking for
null, or whether a store buffer is at capacity).

Furthermore, it required that the `load`, `load_complex`, and `store`
instructions handle loading and storing through an `r{32,64}` rather than just
`i{32,64}` addresses. This involved making `r{32,64}` types acceptable
instantiations of the `iAddr` type variable, plus a few new instruction
encodings.

Part of #929

cranelift/codegen/meta/src/isa/x86/encodings.rs

@@ -231,6 +231,32 @@ impl PerCpuModeEncodings {
         });
     }
 
+    /// Add encodings for `inst.r32` to X86_32.
+    /// Add encodings for `inst.r32` to X86_64 with and without REX.
+    /// Add encodings for `inst.r64` to X86_64 with a REX.W prefix.
+    fn enc_r32_r64_instp(
+        &mut self,
+        inst: &Instruction,
+        template: Template,
+        instp: InstructionPredicateNode,
+    ) {
+        self.enc32_func(inst.bind(R32), template.nonrex(), |builder| {
+            builder.inst_predicate(instp.clone())
+        });
+
+        // REX-less encoding must come after REX encoding so we don't use it by default. Otherwise
+        // reg-alloc would never use r8 and up.
+        self.enc64_func(inst.bind(R32), template.rex(), |builder| {
+            builder.inst_predicate(instp.clone())
+        });
+        self.enc64_func(inst.bind(R32), template.nonrex(), |builder| {
+            builder.inst_predicate(instp.clone())
+        });
+        self.enc64_func(inst.bind(R64), template.rex().w(), |builder| {
+            builder.inst_predicate(instp)
+        });
+    }
+
     /// Add encodings for `inst.r32` to X86_32.
     /// Add encodings for `inst.r64` to X86_64 with a REX.W prefix.
     fn enc_r32_r64_rex_only(&mut self, inst: impl Into<InstSpec>, template: Template) {
@@ -810,6 +836,11 @@ fn define_memory(
             recipe.opcodes(&MOV_LOAD),
             is_load_complex_length_two.clone(),
         );
+        e.enc_r32_r64_instp(
+            load_complex,
+            recipe.opcodes(&MOV_LOAD),
+            is_load_complex_length_two.clone(),
+        );
         e.enc_x86_64_instp(
             uload32_complex,
             recipe.opcodes(&MOV_LOAD),
@@ -855,6 +886,11 @@ fn define_memory(
             recipe.opcodes(&MOV_STORE),
             is_store_complex_length_three.clone(),
         );
+        e.enc_r32_r64_instp(
+            store_complex,
+            recipe.opcodes(&MOV_STORE),
+            is_store_complex_length_three.clone(),
+        );
         e.enc_x86_64_instp(
             istore32_complex,
             recipe.opcodes(&MOV_STORE),
@@ -948,6 +984,10 @@ fn define_memory(
         e.enc64_rec(fill_nop.bind(ty), rec_ffillnull, 0);
         e.enc32_rec(fill_nop.bind(ty), rec_ffillnull, 0);
     }
+    for &ty in &[R64, R32] {
+        e.enc64_rec(fill_nop.bind(ty), rec_fillnull, 0);
+        e.enc32_rec(fill_nop.bind(ty), rec_fillnull, 0);
+    }
 
     // Load 32 bits from `b1`, `i8` and `i16` spill slots. See `spill.b1` above.
 

cranelift/codegen/meta/src/shared/instructions.rs

@@ -211,7 +211,7 @@ fn define_control_flow(
     let iAddr = &TypeVar::new(
         "iAddr",
         "An integer address type",
-        TypeSetBuilder::new().ints(32..64).build(),
+        TypeSetBuilder::new().ints(32..64).refs(32..64).build(),
     );
 
     {
@@ -744,7 +744,7 @@ pub(crate) fn define(
     let iAddr = &TypeVar::new(
         "iAddr",
         "An integer address type",
-        TypeSetBuilder::new().ints(32..64).build(),
+        TypeSetBuilder::new().ints(32..64).refs(32..64).build(),
     );
 
     let Ref = &TypeVar::new(

cranelift/codegen/src/binemit/relaxation.rs

@@ -302,7 +302,11 @@ fn fallthroughs(func: &mut Function) {
                 Opcode::Fallthrough => {
                     // Somebody used a fall-through instruction before the branch relaxation pass.
                     // Make sure it is correct, i.e. the destination is the layout successor.
-                    debug_assert_eq!(destination, succ, "Illegal fall-through in {}", block)
+                    debug_assert_eq!(
+                        destination, succ,
+                        "Illegal fallthrough from {} to {}, but {}'s successor is {}",
+                        block, destination, block, succ
+                    )
                 }
                 Opcode::Jump => {
                     // If this is a jump to the successor block, change it to a fall-through.

cranelift/codegen/src/ir/valueloc.rs

@@ -41,7 +41,7 @@ impl ValueLoc {
     pub fn unwrap_reg(self) -> RegUnit {
         match self {
             Self::Reg(ru) => ru,
-            _ => panic!("Expected register: {:?}", self),
+            _ => panic!("unwrap_reg expected register, found {:?}", self),
         }
     }
 
@@ -49,7 +49,7 @@ impl ValueLoc {
     pub fn unwrap_stack(self) -> StackSlot {
         match self {
             Self::Stack(ss) => ss,
-            _ => panic!("Expected stack slot: {:?}", self),
+            _ => panic!("unwrap_stack expected stack slot, found {:?}", self),
         }
     }
 

cranelift/codegen/src/postopt.rs

@@ -386,7 +386,11 @@ fn optimize_complex_addresses(pos: &mut EncCursor, inst: Inst, isa: &dyn TargetI
     }
 
     let ok = pos.func.update_encoding(inst, isa).is_ok();
-    debug_assert!(ok);
+    debug_assert!(
+        ok,
+        "failed to update encoding for `{}`",
+        pos.func.dfg.display_inst(inst, isa)
+    );
 }
 
 //----------------------------------------------------------------------

cranelift/codegen/src/redundant_reload_remover.rs

@@ -635,7 +635,11 @@ impl RedundantReloadRemover {
                 // Load is completely redundant.  Convert it to a no-op.
                 dfg.replace(inst).fill_nop(arg);
                 let ok = func.update_encoding(inst, isa).is_ok();
-                debug_assert!(ok, "fill_nop encoding missing for this type");
+                debug_assert!(
+                    ok,
+                    "fill_nop encoding missing for this type: `{}`",
+                    func.dfg.display_inst(inst, isa)
+                );
             }
             Transform::ChangeToCopyToSSA(ty, reg) => {
                 // We already have the relevant value in some other register.  Convert the