Fixes #600: Add a SIB byte when encoding a non-indexed load/store into r12/rsp;

Memory access instructions which took the GPR_ZERO_DEREF_SAFE register
class (that was removed in #600) should check for the need of either an
offset (r13/rbp) or the SIB byte (r12/rsp). Some load/store instructions
would already take an index, thus already contain the SIB byte in this
case (see instructions which have a comment telling that the else branch
already contains an SIB byte). Non-indexed memory accesses lacked the
SIB byte check, which this patch adds.

lib/codegen/meta-python/isa/x86/recipes.py

@@ -806,13 +806,16 @@ st = TailRecipe(
         'st', Store, base_size=1, ins=(GPR, GPR), outs=(),
         instp=IsEqual(Store.offset, 0),
         clobbers_flags=False,
-        compute_size="size_plus_maybe_offset_for_in_reg_1",
+        compute_size="size_plus_maybe_sib_or_offset_for_in_reg_1",
         emit='''
         if !flags.notrap() {
             sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
         }
         PUT_OP(bits, rex2(in_reg1, in_reg0), sink);
-        if needs_offset(in_reg1) {
+        if needs_sib_byte(in_reg1) {
+            modrm_sib(in_reg0, sink);
+            sib_noindex(in_reg1, sink);
+        } else if needs_offset(in_reg1) {
             modrm_disp8(in_reg1, in_reg0, sink);
             sink.put1(0);
         } else {
@@ -833,6 +836,7 @@ stWithIndex = TailRecipe(
         sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
     }
     PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink);
+    // The else branch always inserts an SIB byte.
     if needs_offset(in_reg1) {
         modrm_sib_disp8(in_reg0, sink);
         sib(0, in_reg2, in_reg1, sink);
@@ -872,6 +876,7 @@ stWithIndex_abcd = TailRecipe(
         sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
     }
     PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink);
+    // The else branch always inserts an SIB byte.
     if needs_offset(in_reg1) {
         modrm_sib_disp8(in_reg0, sink);
         sib(0, in_reg2, in_reg1, sink);
@@ -887,13 +892,16 @@ fst = TailRecipe(
         'fst', Store, base_size=1, ins=(FPR, GPR), outs=(),
         instp=IsEqual(Store.offset, 0),
         clobbers_flags=False,
-        compute_size="size_plus_maybe_offset_for_in_reg_1",
+        compute_size="size_plus_maybe_sib_or_offset_for_in_reg_1",
         emit='''
         if !flags.notrap() {
             sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
         }
         PUT_OP(bits, rex2(in_reg1, in_reg0), sink);
-        if needs_offset(in_reg1) {
+        if needs_sib_byte(in_reg1) {
+            modrm_sib(in_reg0, sink);
+            sib_noindex(in_reg1, sink);
+        } else if needs_offset(in_reg1) {
             modrm_disp8(in_reg1, in_reg0, sink);
             sink.put1(0);
         } else {
@@ -912,6 +920,7 @@ fstWithIndex = TailRecipe(
             sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
         }
         PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink);
+        // The else branch always inserts an SIB byte.
         if needs_offset(in_reg1) {
             modrm_sib_disp8(in_reg0, sink);
             sib(0, in_reg2, in_reg1, sink);
@@ -1210,13 +1219,16 @@ ld = TailRecipe(
         'ld', Load, base_size=1, ins=(GPR), outs=(GPR),
         instp=IsEqual(Load.offset, 0),
         clobbers_flags=False,
-        compute_size="size_plus_maybe_offset_for_in_reg_0",
+        compute_size="size_plus_maybe_sib_or_offset_for_in_reg_0",
         emit='''
         if !flags.notrap() {
             sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
         }
         PUT_OP(bits, rex2(in_reg0, out_reg0), sink);
-        if needs_offset(in_reg0) {
+        if needs_sib_byte(in_reg0) {
+            modrm_sib(out_reg0, sink);
+            sib_noindex(in_reg0, sink);
+        } else if needs_offset(in_reg0) {
             modrm_disp8(in_reg0, out_reg0, sink);
             sink.put1(0);
         } else {
@@ -1237,6 +1249,7 @@ ldWithIndex = TailRecipe(
         sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
     }
     PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink);
+    // The else branch always inserts an SIB byte.
     if needs_offset(in_reg0) {
         modrm_sib_disp8(out_reg0, sink);
         sib(0, in_reg1, in_reg0, sink);
@@ -1252,13 +1265,16 @@ fld = TailRecipe(
         'fld', Load, base_size=1, ins=(GPR), outs=(FPR),
         instp=IsEqual(Load.offset, 0),
         clobbers_flags=False,
-        compute_size="size_plus_maybe_offset_for_in_reg_0",
+        compute_size="size_plus_maybe_sib_or_offset_for_in_reg_0",
         emit='''
         if !flags.notrap() {
             sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
         }
         PUT_OP(bits, rex2(in_reg0, out_reg0), sink);
-        if needs_offset(in_reg0) {
+        if needs_sib_byte(in_reg0) {
+            modrm_sib(out_reg0, sink);
+            sib_noindex(in_reg0, sink);
+        } else if needs_offset(in_reg0) {
             modrm_disp8(in_reg0, out_reg0, sink);
             sink.put1(0);
         } else {
@@ -1279,6 +1295,7 @@ fldWithIndex = TailRecipe(
         sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]);
     }
     PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink);
+    // The else branch always inserts an SIB byte.
     if needs_offset(in_reg0) {
         modrm_sib_disp8(out_reg0, sink);
         sib(0, in_reg1, in_reg0, sink);

lib/codegen/src/isa/x86/enc_tables.rs

@@ -23,6 +23,9 @@ pub fn needs_sib_byte(reg: RegUnit) -> bool {
 pub fn needs_offset(reg: RegUnit) -> bool {
     reg == RU::r13 as RegUnit || reg == RU::rbp as RegUnit
 }
+pub fn needs_sib_byte_or_offset(reg: RegUnit) -> bool {
+    needs_sib_byte(reg) || needs_offset(reg)
+}
 
 fn additional_size_if(
     op_index: usize,
@@ -71,6 +74,22 @@ fn size_plus_maybe_sib_for_in_reg_1(
 ) -> u8 {
     sizing.base_size + additional_size_if(1, inst, divert, func, needs_sib_byte)
 }
+fn size_plus_maybe_sib_or_offset_for_in_reg_0(
+    sizing: &RecipeSizing,
+    inst: Inst,
+    divert: &RegDiversions,
+    func: &Function,
+) -> u8 {
+    sizing.base_size + additional_size_if(0, inst, divert, func, needs_sib_byte_or_offset)
+}
+fn size_plus_maybe_sib_or_offset_for_in_reg_1(
+    sizing: &RecipeSizing,
+    inst: Inst,
+    divert: &RegDiversions,
+    func: &Function,
+) -> u8 {
+    sizing.base_size + additional_size_if(1, inst, divert, func, needs_sib_byte_or_offset)
+}
 
 /// Expand the `sdiv` and `srem` instructions using `x86_sdivmodx`.
 fn expand_sdivrem(