Codegen: Allow encoding of (r32|r64).(load|store)

Accessing Wasm reference globals that are reference types will
want to use the plain load/store instructions. This commit adds
encodings for these instructions to match loading a i32/i64.
Producers of IR are required to insert the appropriate barriers
around the loads/stores.

cranelift/codegen/meta/src/cdsl/cpu_modes.rs

@@ -37,6 +37,12 @@ impl CpuMode {
         assert!(self.default_legalize.is_none());
         self.default_legalize = Some(group.id);
     }
+    pub fn legalize_value_type(&mut self, lane_type: impl Into<ValueType>, group: &TransformGroup) {
+        assert!(self
+            .typed_legalize
+            .insert(lane_type.into(), group.id)
+            .is_none());
+    }
     pub fn legalize_type(&mut self, lane_type: impl Into<LaneType>, group: &TransformGroup) {
         assert!(self
             .typed_legalize

cranelift/codegen/meta/src/isa/x86/encodings.rs

@@ -239,6 +239,22 @@ impl PerCpuModeEncodings {
         self.enc64(inst.bind(R64), template.rex().w());
     }
 
+    fn enc_r32_r64_ld_st(&mut self, inst: &Instruction, w_bit: bool, template: Template) {
+        self.enc32(inst.clone().bind(R32).bind(Any), template.clone());
+
+        // REX-less encoding must come after REX encoding so we don't use it by
+        // default. Otherwise reg-alloc would never use r8 and up.
+        self.enc64(inst.clone().bind(R32).bind(Any), template.clone().rex());
+        self.enc64(inst.clone().bind(R32).bind(Any), template.clone());
+
+        if w_bit {
+            self.enc64(inst.clone().bind(R64).bind(Any), template.rex().w());
+        } else {
+            self.enc64(inst.clone().bind(R64).bind(Any), template.clone().rex());
+            self.enc64(inst.clone().bind(R64).bind(Any), template);
+        }
+    }
+
     /// Add encodings for `inst` to X86_64 with and without a REX prefix.
     fn enc_x86_64(&mut self, inst: impl Into<InstSpec> + Clone, template: Template) {
         // See above comment about the ordering of rex vs non-rex encodings.
@@ -858,6 +874,7 @@ fn define_memory(
 
     for recipe in &[rec_st, rec_stDisp8, rec_stDisp32] {
         e.enc_i32_i64_ld_st(store, true, recipe.opcodes(&MOV_STORE));
+        e.enc_r32_r64_ld_st(store, true, recipe.opcodes(&MOV_STORE));
         e.enc_x86_64(istore32.bind(I64).bind(Any), recipe.opcodes(&MOV_STORE));
         e.enc_i32_i64_ld_st(istore16, false, recipe.opcodes(&MOV_STORE_16));
     }
@@ -889,6 +906,7 @@ fn define_memory(
 
     for recipe in &[rec_ld, rec_ldDisp8, rec_ldDisp32] {
         e.enc_i32_i64_ld_st(load, true, recipe.opcodes(&MOV_LOAD));
+        e.enc_r32_r64_ld_st(load, true, recipe.opcodes(&MOV_LOAD));
         e.enc_x86_64(uload32.bind(I64), recipe.opcodes(&MOV_LOAD));
         e.enc64(sload32.bind(I64), recipe.opcodes(&MOVSXD).rex().w());
         e.enc_i32_i64_ld_st(uload16, true, recipe.opcodes(&MOVZX_WORD));

cranelift/codegen/meta/src/isa/x86/mod.rs

@@ -1,9 +1,11 @@
 use crate::cdsl::cpu_modes::CpuMode;
 use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::types::ReferenceType;
 
 use crate::shared::types::Bool::B1;
 use crate::shared::types::Float::{F32, F64};
 use crate::shared::types::Int::{I16, I32, I64, I8};
+use crate::shared::types::Reference::{R32, R64};
 use crate::shared::Definitions as SharedDefinitions;
 
 mod encodings;
@@ -41,6 +43,7 @@ pub(crate) fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
     x86_32.legalize_type(I8, widen);
     x86_32.legalize_type(I16, widen);
     x86_32.legalize_type(I32, x86_expand);
+    x86_32.legalize_value_type(ReferenceType(R32), x86_expand);
     x86_32.legalize_type(F32, x86_expand);
     x86_32.legalize_type(F64, x86_expand);
 
@@ -51,6 +54,7 @@ pub(crate) fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
     x86_64.legalize_type(I16, widen);
     x86_64.legalize_type(I32, x86_expand);
     x86_64.legalize_type(I64, x86_expand);
+    x86_64.legalize_value_type(ReferenceType(R64), x86_expand);
     x86_64.legalize_type(F32, x86_expand);
     x86_64.legalize_type(F64, x86_expand);
 

cranelift/codegen/meta/src/shared/instructions.rs

@@ -580,6 +580,7 @@ pub(crate) fn define(
             .ints(Interval::All)
             .floats(Interval::All)
             .simd_lanes(Interval::All)
+            .refs(Interval::All)
             .build(),
     );