T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2654 from kaseyc/rex

Browse Source 
Add methods to construct RexFlags from OperandSizes.
This commit is contained in:
Chris Fallin
2021-02-17 19:27:18 -08:00
committed by GitHub
parent 8d90ea0390 9c3edee9d0
commit 7b4652bb82
1 changed files with 129 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

320
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -85,8 +85,9 @@ impl RexFlags {
} }
#[inline(always)] #[inline(always)]
fn always_emit_if_8bit_needed(&mut self, reg: u8) -> &mut Self { fn always_emit_if_8bit_needed(&mut self, reg: Reg) -> &mut Self {
if reg >= 4 && reg <= 7 { let enc_reg = int_reg_enc(reg);
if enc_reg >= 4 && enc_reg <= 7 {
self.always_emit(); self.always_emit();
} }
self self
@@ -126,6 +127,26 @@ impl RexFlags {
} }
} }
/// Generate the proper Rex flags for the given operand size.
impl From<OperandSize> for RexFlags {
fn from(size: OperandSize) -> Self {
match size {
OperandSize::Size64 => RexFlags::set_w(),
_ => RexFlags::clear_w(),
}
}
}
/// Generate Rex flags for an OperandSize/register tuple.
impl From<(OperandSize, Reg)> for RexFlags {
fn from((size, reg): (OperandSize, Reg)) -> Self {
let mut rex = RexFlags::from(size);
if size == OperandSize::Size8 {
rex.always_emit_if_8bit_needed(reg);
}
rex
}
}
/// We may need to include one or more legacy prefix bytes before the REX prefix. This enum /// We may need to include one or more legacy prefix bytes before the REX prefix. This enum
/// covers only the small set of possibilities that we actually need. /// covers only the small set of possibilities that we actually need.
enum LegacyPrefixes { enum LegacyPrefixes {
@@ -546,12 +567,7 @@ pub(crate) fn emit(
src, src,
dst: reg_g, dst: reg_g,
} => { } => {
let mut rex = if *size == OperandSize::Size64 { let mut rex = RexFlags::from(*size);
RexFlags::set_w()
} else {
RexFlags::clear_w()
};
if *op == AluRmiROpcode::Mul { if *op == AluRmiROpcode::Mul {
// We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so // We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so
// we have to special-case it. // we have to special-case it.
@@ -617,8 +633,8 @@ pub(crate) fn emit(
match src { match src {
RegMemImm::Reg { reg: reg_e } => { RegMemImm::Reg { reg: reg_e } => {
if is_8bit { if is_8bit {
rex.always_emit_if_8bit_needed(int_reg_enc(*reg_e)); rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg())); rex.always_emit_if_8bit_needed(reg_g.to_reg());
} }
// GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R
// duality). Do this too, so as to be able to compare generated machine // duality). Do this too, so as to be able to compare generated machine
@@ -636,7 +652,7 @@ pub(crate) fn emit(
RegMemImm::Mem { addr } => { RegMemImm::Mem { addr } => {
if is_8bit { if is_8bit {
rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg())); rex.always_emit_if_8bit_needed(reg_g.to_reg());
} }
// Here we revert to the "normal" G-E ordering. // Here we revert to the "normal" G-E ordering.
let amode = addr.finalize(state, sink); let amode = addr.finalize(state, sink);
@@ -675,12 +691,7 @@ pub(crate) fn emit(
} }
Inst::UnaryRmR { size, op, src, dst } => { Inst::UnaryRmR { size, op, src, dst } => {
let rex_flags = match size { let rex_flags = RexFlags::from(*size);
OperandSize::Size16 | OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size64 => RexFlags::set_w(),
_ => unreachable!(),
};
use UnaryRmROpcode::*; use UnaryRmROpcode::*;
let prefix = match size { let prefix = match size {
OperandSize::Size16 => match op { OperandSize::Size16 => match op {
@@ -730,37 +741,31 @@ pub(crate) fn emit(
} }
Inst::Not { size, src } => { Inst::Not { size, src } => {
let src = int_reg_enc(src.to_reg()); let rex_flags = RexFlags::from((*size, src.to_reg()));
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
0xF6, OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xF7, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(src), OperandSize::Size64 => (0xF7, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
}; };
let subopcode = 2; let subopcode = 2;
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) let enc_src = int_reg_enc(src.to_reg());
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags)
} }
Inst::Neg { size, src } => { Inst::Neg { size, src } => {
let src = int_reg_enc(src.to_reg()); let rex_flags = RexFlags::from((*size, src.to_reg()));
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
0xF6, OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xF7, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(src), OperandSize::Size64 => (0xF7, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
}; };
let subopcode = 3; let subopcode = 3;
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) let enc_src = int_reg_enc(src.to_reg());
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags)
} }
Inst::Div { Inst::Div {
@@ -768,11 +773,11 @@ pub(crate) fn emit(
signed, signed,
divisor, divisor,
} => { } => {
let (opcode, prefix, mut rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
OperandSize::Size16 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
OperandSize::Size32 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size32 => (0xF7, LegacyPrefixes::None),
OperandSize::Size64 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()), OperandSize::Size64 => (0xF7, LegacyPrefixes::None),
}; };
let loc = state.cur_srcloc(); let loc = state.cur_srcloc();
@@ -782,25 +787,39 @@ pub(crate) fn emit(
match divisor { match divisor {
RegMem::Reg { reg } => { RegMem::Reg { reg } => {
let src = int_reg_enc(*reg); let src = int_reg_enc(*reg);
if *size == OperandSize::Size8 { emit_std_enc_enc(
rex_flags.always_emit_if_8bit_needed(src); sink,
} prefix,
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) opcode,
1,
subopcode,
src,
RexFlags::from((*size, *reg)),
)
} }
RegMem::Mem { addr: src } => { RegMem::Mem { addr: src } => {
let amode = src.finalize(state, sink); let amode = src.finalize(state, sink);
emit_std_enc_mem( emit_std_enc_mem(
sink, state, info, prefix, opcode, 1, subopcode, &amode, rex_flags, sink,
state,
info,
prefix,
opcode,
1,
subopcode,
&amode,
RexFlags::from(*size),
); );
} }
} }
} }
Inst::MulHi { size, signed, rhs } => { Inst::MulHi { size, signed, rhs } => {
let (prefix, rex_flags) = match size { let rex_flags = RexFlags::from(*size);
OperandSize::Size16 => (LegacyPrefixes::_66, RexFlags::clear_w()), let prefix = match size {
OperandSize::Size32 => (LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size16 => LegacyPrefixes::_66,
OperandSize::Size64 => (LegacyPrefixes::None, RexFlags::set_w()), OperandSize::Size32 => LegacyPrefixes::None,
OperandSize::Size64 => LegacyPrefixes::None,
_ => unreachable!(), _ => unreachable!(),
}; };
@@ -993,12 +1012,15 @@ pub(crate) fn emit(
} }
Inst::MovRR { size, src, dst } => { Inst::MovRR { size, src, dst } => {
let rex = if *size == OperandSize::Size64 { emit_std_reg_reg(
RexFlags::set_w() sink,
} else { LegacyPrefixes::None,
RexFlags::clear_w() 0x89,
}; 1,
emit_std_reg_reg(sink, LegacyPrefixes::None, 0x89, 1, *src, dst.to_reg(), rex); *src,
dst.to_reg(),
RexFlags::from(*size),
);
} }
Inst::MovzxRmR { ext_mode, src, dst } => { Inst::MovzxRmR { ext_mode, src, dst } => {
@@ -1038,8 +1060,7 @@ pub(crate) fn emit(
match ext_mode { match ext_mode {
ExtMode::BL | ExtMode::BQ => { ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs. // A redundant REX prefix must be emitted for certain register inputs.
let enc_src = int_reg_enc(*src); rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(enc_src);
} }
_ => {} _ => {}
} }
@@ -1133,8 +1154,7 @@ pub(crate) fn emit(
match ext_mode { match ext_mode {
ExtMode::BL | ExtMode::BQ => { ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs. // A redundant REX prefix must be emitted for certain register inputs.
let enc_src = int_reg_enc(*src); rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(enc_src);
} }
_ => {} _ => {}
} }
@@ -1170,75 +1190,26 @@ pub(crate) fn emit(
Inst::MovRM { size, src, dst } => { Inst::MovRM { size, src, dst } => {
let dst = &dst.finalize(state, sink); let dst = &dst.finalize(state, sink);
match size { let prefix = match size {
OperandSize::Size8 => { OperandSize::Size16 => LegacyPrefixes::_66,
_ => LegacyPrefixes::None,
};
let opcode = match size {
OperandSize::Size8 => 0x88,
_ => 0x89,
};
// This is one of the few places where the presence of a // This is one of the few places where the presence of a
// redundant REX prefix changes the meaning of the // redundant REX prefix changes the meaning of the
// instruction. // instruction.
let mut rex = RexFlags::clear_w(); let rex = RexFlags::from((*size, *src));
let enc_src = int_reg_enc(*src); // 8-bit: MOV r8, r/m8 is (REX.W==0) 88 /r
rex.always_emit_if_8bit_needed(enc_src); // 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
// 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
// MOV r8, r/m8 is (REX.W==0) 88 /r // 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem( emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *src, dst, rex);
sink,
state,
info,
LegacyPrefixes::None,
0x88,
1,
*src,
dst,
rex,
)
}
OperandSize::Size16 => {
// MOV r16, r/m16 is 66 (REX.W==0) 89 /r
emit_std_reg_mem(
sink,
state,
info,
LegacyPrefixes::_66,
0x89,
1,
*src,
dst,
RexFlags::clear_w(),
)
}
OperandSize::Size32 => {
// MOV r32, r/m32 is (REX.W==0) 89 /r
emit_std_reg_mem(
sink,
state,
info,
LegacyPrefixes::None,
0x89,
1,
*src,
dst,
RexFlags::clear_w(),
)
}
OperandSize::Size64 => {
// MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem(
sink,
state,
info,
LegacyPrefixes::None,
0x89,
1,
*src,
dst,
RexFlags::set_w(),
)
}
}
} }
Inst::ShiftR { Inst::ShiftR {
@@ -1247,7 +1218,6 @@ pub(crate) fn emit(
num_bits, num_bits,
dst, dst,
} => { } => {
let enc_dst = int_reg_enc(dst.to_reg());
let subopcode = match kind { let subopcode = match kind {
ShiftKind::RotateLeft => 0, ShiftKind::RotateLeft => 0,
ShiftKind::RotateRight => 1, ShiftKind::RotateRight => 1,
@@ -1255,18 +1225,15 @@ pub(crate) fn emit(
ShiftKind::ShiftRightLogical => 5, ShiftKind::ShiftRightLogical => 5,
ShiftKind::ShiftRightArithmetic => 7, ShiftKind::ShiftRightArithmetic => 7,
}; };
let enc_dst = int_reg_enc(dst.to_reg());
let rex_flags = RexFlags::from((*size, dst.to_reg()));
match num_bits { match num_bits {
None => { None => {
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xD2, LegacyPrefixes::None),
0xD2, OperandSize::Size16 => (0xD3, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xD3, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst), OperandSize::Size64 => (0xD3, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xD3, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xD3, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xD3, LegacyPrefixes::None, RexFlags::set_w()),
}; };
// SHL/SHR/SAR %cl, reg8 is (REX.W==0) D2 /subopcode // SHL/SHR/SAR %cl, reg8 is (REX.W==0) D2 /subopcode
@@ -1277,15 +1244,11 @@ pub(crate) fn emit(
} }
Some(num_bits) => { Some(num_bits) => {
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xC0, LegacyPrefixes::None),
0xC0, OperandSize::Size16 => (0xC1, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xC1, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst), OperandSize::Size64 => (0xC1, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xC1, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xC1, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xC1, LegacyPrefixes::None, RexFlags::set_w()),
}; };
// SHL/SHR/SAR $ib, reg8 is (REX.W==0) C0 /subopcode // SHL/SHR/SAR $ib, reg8 is (REX.W==0) C0 /subopcode
@@ -1372,25 +1335,14 @@ pub(crate) fn emit(
if *size == OperandSize::Size16 { if *size == OperandSize::Size16 {
prefix = LegacyPrefixes::_66; prefix = LegacyPrefixes::_66;
} }
// A redundant REX prefix can change the meaning of this instruction.
let mut rex = match size { let mut rex = RexFlags::from((*size, *reg_g));
OperandSize::Size64 => RexFlags::set_w(),
OperandSize::Size16 | OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size8 => {
let mut rex = RexFlags::clear_w();
// Here, a redundant REX prefix changes the meaning of the instruction.
let enc_g = int_reg_enc(*reg_g);
rex.always_emit_if_8bit_needed(enc_g);
rex
}
};
match src_e { match src_e {
RegMemImm::Reg { reg: reg_e } => { RegMemImm::Reg { reg: reg_e } => {
if *size == OperandSize::Size8 { if *size == OperandSize::Size8 {
// Check whether the E register forces the use of a redundant REX. // Check whether the E register forces the use of a redundant REX.
let enc_e = int_reg_enc(*reg_e); rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(enc_e);
} }
// Use the swapped operands encoding for CMP, to stay consistent with the output of // Use the swapped operands encoding for CMP, to stay consistent with the output of
@@ -1467,10 +1419,11 @@ pub(crate) fn emit(
src, src,
dst: reg_g, dst: reg_g,
} => { } => {
let (prefix, rex_flags) = match size { let rex_flags = RexFlags::from(*size);
OperandSize::Size16 => (LegacyPrefixes::_66, RexFlags::clear_w()), let prefix = match size {
OperandSize::Size32 => (LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size16 => LegacyPrefixes::_66,
OperandSize::Size64 => (LegacyPrefixes::None, RexFlags::set_w()), OperandSize::Size32 => LegacyPrefixes::None,
OperandSize::Size64 => LegacyPrefixes::None,
_ => unreachable!("invalid size spec for cmove"), _ => unreachable!("invalid size spec for cmove"),
}; };
let opcode = 0x0F40 + cc.get_enc() as u32; let opcode = 0x0F40 + cc.get_enc() as u32;
@@ -2111,11 +2064,7 @@ pub(crate) fn emit(
SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3), SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3),
_ => unimplemented!("Opcode {:?} not implemented", op), _ => unimplemented!("Opcode {:?} not implemented", op),
}; };
let rex = if *size == OperandSize::Size64 { let rex = RexFlags::from(*size);
RexFlags::set_w()
} else {
RexFlags::clear_w()
};
let regs_swapped = match *op { let regs_swapped = match *op {
// These opcodes (and not the SSE2 version of PEXTRW) flip the operand // These opcodes (and not the SSE2 version of PEXTRW) flip the operand
// encoding: `dst` in ModRM's r/m, `src` in ModRM's reg field. // encoding: `dst` in ModRM's r/m, `src` in ModRM's reg field.
@@ -2208,12 +2157,7 @@ pub(crate) fn emit(
SseOpcode::Pmovmskb => (LegacyPrefixes::_66, 0x0FD7, true), SseOpcode::Pmovmskb => (LegacyPrefixes::_66, 0x0FD7, true),
_ => panic!("unexpected opcode {:?}", op), _ => panic!("unexpected opcode {:?}", op),
}; };
let rex = match dst_size { let rex = RexFlags::from(*dst_size);
OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size64 => RexFlags::set_w(),
_ => unreachable!(),
};
let (src, dst) = if dst_first { let (src, dst) = if dst_first {
(dst.to_reg(), *src) (dst.to_reg(), *src)
} else { } else {
@@ -2237,11 +2181,7 @@ pub(crate) fn emit(
SseOpcode::Cvtsi2sd => (LegacyPrefixes::_F2, 0x0F2A), SseOpcode::Cvtsi2sd => (LegacyPrefixes::_F2, 0x0F2A),
_ => panic!("unexpected opcode {:?}", op), _ => panic!("unexpected opcode {:?}", op),
}; };
let rex = match *src_size { let rex = RexFlags::from(*src_size);
OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size64 => RexFlags::set_w(),
_ => unreachable!(),
};
match src_e { match src_e {
RegMem::Reg { reg: reg_e } => { RegMem::Reg { reg: reg_e } => {
emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex);
@@ -2827,18 +2767,14 @@ pub(crate) fn emit(
Inst::LockCmpxchg { ty, src, dst } => { Inst::LockCmpxchg { ty, src, dst } => {
// lock cmpxchg{b,w,l,q} %src, (dst) // lock cmpxchg{b,w,l,q} %src, (dst)
// Note that 0xF0 is the Lock prefix. // Note that 0xF0 is the Lock prefix.
let (prefix, rex, opcodes) = match *ty { let (prefix, opcodes) = match *ty {
types::I8 => { types::I8 => (LegacyPrefixes::_F0, 0x0FB0),
let mut rex_flags = RexFlags::clear_w(); types::I16 => (LegacyPrefixes::_66F0, 0x0FB1),
let enc_src = int_reg_enc(*src); types::I32 => (LegacyPrefixes::_F0, 0x0FB1),
rex_flags.always_emit_if_8bit_needed(enc_src); types::I64 => (LegacyPrefixes::_F0, 0x0FB1),
(LegacyPrefixes::_F0, rex_flags, 0x0FB0)
}
types::I16 => (LegacyPrefixes::_66F0, RexFlags::clear_w(), 0x0FB1),
types::I32 => (LegacyPrefixes::_F0, RexFlags::clear_w(), 0x0FB1),
types::I64 => (LegacyPrefixes::_F0, RexFlags::set_w(), 0x0FB1),
_ => unreachable!(), _ => unreachable!(),
}; };
let rex = RexFlags::from((OperandSize::from_ty(*ty), *src));
let amode = dst.finalize(state, sink); let amode = dst.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex); emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex);
} }