diff --git a/lib/codegen/meta-python/isa/x86/recipes.py b/lib/codegen/meta-python/isa/x86/recipes.py index 4c495807b7..9ed9e09e3a 100644 --- a/lib/codegen/meta-python/isa/x86/recipes.py +++ b/lib/codegen/meta-python/isa/x86/recipes.py @@ -806,13 +806,16 @@ st = TailRecipe( 'st', Store, base_size=1, ins=(GPR, GPR), outs=(), instp=IsEqual(Store.offset, 0), clobbers_flags=False, - compute_size="size_plus_maybe_offset_for_in_reg_1", + compute_size="size_plus_maybe_sib_or_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); - if needs_offset(in_reg1) { + if needs_sib_byte(in_reg1) { + modrm_sib(in_reg0, sink); + sib_noindex(in_reg1, sink); + } else if needs_offset(in_reg1) { modrm_disp8(in_reg1, in_reg0, sink); sink.put1(0); } else { @@ -833,6 +836,7 @@ stWithIndex = TailRecipe( sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); + // The else branch always inserts an SIB byte. if needs_offset(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); @@ -872,6 +876,7 @@ stWithIndex_abcd = TailRecipe( sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); + // The else branch always inserts an SIB byte. if needs_offset(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); @@ -887,13 +892,16 @@ fst = TailRecipe( 'fst', Store, base_size=1, ins=(FPR, GPR), outs=(), instp=IsEqual(Store.offset, 0), clobbers_flags=False, - compute_size="size_plus_maybe_offset_for_in_reg_1", + compute_size="size_plus_maybe_sib_or_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); - if needs_offset(in_reg1) { + if needs_sib_byte(in_reg1) { + modrm_sib(in_reg0, sink); + sib_noindex(in_reg1, sink); + } else if needs_offset(in_reg1) { modrm_disp8(in_reg1, in_reg0, sink); sink.put1(0); } else { @@ -912,6 +920,7 @@ fstWithIndex = TailRecipe( sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); + // The else branch always inserts an SIB byte. if needs_offset(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); @@ -1210,13 +1219,16 @@ ld = TailRecipe( 'ld', Load, base_size=1, ins=(GPR), outs=(GPR), instp=IsEqual(Load.offset, 0), clobbers_flags=False, - compute_size="size_plus_maybe_offset_for_in_reg_0", + compute_size="size_plus_maybe_sib_or_offset_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); - if needs_offset(in_reg0) { + if needs_sib_byte(in_reg0) { + modrm_sib(out_reg0, sink); + sib_noindex(in_reg0, sink); + } else if needs_offset(in_reg0) { modrm_disp8(in_reg0, out_reg0, sink); sink.put1(0); } else { @@ -1237,6 +1249,7 @@ ldWithIndex = TailRecipe( sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); + // The else branch always inserts an SIB byte. if needs_offset(in_reg0) { modrm_sib_disp8(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); @@ -1252,13 +1265,16 @@ fld = TailRecipe( 'fld', Load, base_size=1, ins=(GPR), outs=(FPR), instp=IsEqual(Load.offset, 0), clobbers_flags=False, - compute_size="size_plus_maybe_offset_for_in_reg_0", + compute_size="size_plus_maybe_sib_or_offset_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); - if needs_offset(in_reg0) { + if needs_sib_byte(in_reg0) { + modrm_sib(out_reg0, sink); + sib_noindex(in_reg0, sink); + } else if needs_offset(in_reg0) { modrm_disp8(in_reg0, out_reg0, sink); sink.put1(0); } else { @@ -1279,6 +1295,7 @@ fldWithIndex = TailRecipe( sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); + // The else branch always inserts an SIB byte. if needs_offset(in_reg0) { modrm_sib_disp8(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); diff --git a/lib/codegen/src/isa/x86/enc_tables.rs b/lib/codegen/src/isa/x86/enc_tables.rs index 9c52563a27..a709d03b76 100644 --- a/lib/codegen/src/isa/x86/enc_tables.rs +++ b/lib/codegen/src/isa/x86/enc_tables.rs @@ -23,6 +23,9 @@ pub fn needs_sib_byte(reg: RegUnit) -> bool { pub fn needs_offset(reg: RegUnit) -> bool { reg == RU::r13 as RegUnit || reg == RU::rbp as RegUnit } +pub fn needs_sib_byte_or_offset(reg: RegUnit) -> bool { + needs_sib_byte(reg) || needs_offset(reg) +} fn additional_size_if( op_index: usize, @@ -71,6 +74,22 @@ fn size_plus_maybe_sib_for_in_reg_1( ) -> u8 { sizing.base_size + additional_size_if(1, inst, divert, func, needs_sib_byte) } +fn size_plus_maybe_sib_or_offset_for_in_reg_0( + sizing: &RecipeSizing, + inst: Inst, + divert: &RegDiversions, + func: &Function, +) -> u8 { + sizing.base_size + additional_size_if(0, inst, divert, func, needs_sib_byte_or_offset) +} +fn size_plus_maybe_sib_or_offset_for_in_reg_1( + sizing: &RecipeSizing, + inst: Inst, + divert: &RegDiversions, + func: &Function, +) -> u8 { + sizing.base_size + additional_size_if(1, inst, divert, func, needs_sib_byte_or_offset) +} /// Expand the `sdiv` and `srem` instructions using `x86_sdivmodx`. fn expand_sdivrem(