T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cranelift: Use GPR newtypes extensively in x64 lowering (#3798)

Browse Source 
We already defined the `Gpr` newtype and used it in a few places, and we already
defined the `Xmm` newtype and used it extensively. This finishes the transition
to using the newtypes extensively in lowering by making use of `Gpr` in more
places.

Fixes #3685
This commit is contained in:
Nick Fitzgerald
2022-02-14 12:54:41 -08:00
committed by GitHub
parent 84b9c7bb8a
commit dc86e7a6dc
9 changed files with 1804 additions and 1482 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -48,8 +48,10 @@ macro_rules! newtype_of_reg {
(
$newtype_reg:ident,
$newtype_writable_reg:ident,
$newtype_option_writable_reg:ident,
$newtype_reg_mem:ident,
$newtype_reg_mem_imm:ident,
$newtype_imm8_reg:ident,
|$check_reg:ident| $check:expr
) => {
/// A newtype wrapper around `Reg`.
@@ -122,6 +124,9 @@ macro_rules! newtype_of_reg {
pub type $newtype_writable_reg = Writable<$newtype_reg>;
#[allow(dead_code)] // Used by some newtypes and not others.
pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
impl ToWritableReg for $newtype_writable_reg {
fn to_writable_reg(&self) -> Writable<Reg> {
Writable::from_reg(self.to_reg().to_reg())
@@ -218,6 +223,11 @@ macro_rules! newtype_of_reg {
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
self.0.get_regs_as_uses(collector);
}
}
impl PrettyPrint for $newtype_reg_mem {
@@ -290,6 +300,11 @@ macro_rules! newtype_of_reg {
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
self.0.get_regs_as_uses(collector);
}
}
impl PrettyPrint for $newtype_reg_mem_imm {
@@ -303,18 +318,60 @@ macro_rules! newtype_of_reg {
self.0.show_rru_sized(mb_rru, size)
}
}
/// A newtype wrapper around `Imm8Reg`.
#[derive(Clone, Debug)]
#[allow(dead_code)] // Used by some newtypes and not others.
pub struct $newtype_imm8_reg(Imm8Reg);
impl From<$newtype_reg> for $newtype_imm8_reg {
fn from(r: $newtype_reg) -> Self {
Self(Imm8Reg::Reg { reg: r.to_reg() })
}
}
impl $newtype_imm8_reg {
/// Construct this newtype from the given `Imm8Reg`, or return
/// `None` if the `Imm8Reg` is not a valid instance of this newtype.
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
match imm8_reg {
Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
Imm8Reg::Reg { reg: $check_reg } if $check => Some(Self(imm8_reg)),
Imm8Reg::Reg { reg: _ } => None,
}
}
/// Convert this newtype into its underlying `Imm8Reg`.
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn to_imm8_reg(self) -> Imm8Reg {
self.0
}
}
};
}
// Define a newtype of `Reg` for general-purpose registers.
newtype_of_reg!(Gpr, WritableGpr, GprMem, GprMemImm, |reg| {
reg.get_class() == RegClass::I64
});
newtype_of_reg!(
Gpr,
WritableGpr,
OptionWritableGpr,
GprMem,
GprMemImm,
Imm8Gpr,
|reg| reg.get_class() == RegClass::I64
);
// Define a newtype of `Reg` for XMM registers.
newtype_of_reg!(Xmm, WritableXmm, XmmMem, XmmMemImm, |reg| {
reg.get_class() == RegClass::V128
});
newtype_of_reg!(
Xmm,
WritableXmm,
OptionWritableXmm,
XmmMem,
XmmMemImm,
Imm8Xmm,
|reg| reg.get_class() == RegClass::V128
);
/// A possible addressing mode (amode) that can be used in instructions.
/// These denote a 64-bit value only.

196
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -156,15 +156,15 @@ pub(crate) fn emit(
if *op == AluRmiROpcode::Mul {
// We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so
// we have to special-case it.
match src2 {
match src2.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg: reg_e } => {
emit_std_reg_reg(
sink,
LegacyPrefixes::None,
0x0FAF,
2,
reg_g.to_reg(),
*reg_e,
reg_g.to_reg().to_reg(),
reg_e,
rex,
);
}
@@ -178,14 +178,14 @@ pub(crate) fn emit(
LegacyPrefixes::None,
0x0FAF,
2,
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
&amode,
rex,
);
}
RegMemImm::Imm { simm32 } => {
let use_imm8 = low8_will_sign_extend_to_32(*simm32);
let use_imm8 = low8_will_sign_extend_to_32(simm32);
let opcode = if use_imm8 { 0x6B } else { 0x69 };
// Yes, really, reg_g twice.
emit_std_reg_reg(
@@ -193,11 +193,11 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcode,
1,
reg_g.to_reg(),
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
reg_g.to_reg().to_reg(),
rex,
);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32);
}
}
} else {
@@ -215,11 +215,11 @@ pub(crate) fn emit(
};
assert!(!(is_8bit && *size == OperandSize::Size64));
match src2 {
match src2.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg: reg_e } => {
if is_8bit {
rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(reg_g.to_reg());
rex.always_emit_if_8bit_needed(reg_e);
rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg());
}
// GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R
// duality). Do this too, so as to be able to compare generated machine
@@ -229,15 +229,15 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcode_r,
1,
*reg_e,
reg_g.to_reg(),
reg_e,
reg_g.to_reg().to_reg(),
rex,
);
}
RegMemImm::Mem { addr } => {
if is_8bit {
rex.always_emit_if_8bit_needed(reg_g.to_reg());
rex.always_emit_if_8bit_needed(reg_g.to_reg().to_reg());
}
// Here we revert to the "normal" G-E ordering.
let amode = addr.finalize(state, sink);
@@ -248,7 +248,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcode_m,
1,
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
&amode,
rex,
);
@@ -256,10 +256,10 @@ pub(crate) fn emit(
RegMemImm::Imm { simm32 } => {
assert!(!is_8bit);
let use_imm8 = low8_will_sign_extend_to_32(*simm32);
let use_imm8 = low8_will_sign_extend_to_32(simm32);
let opcode = if use_imm8 { 0x83 } else { 0x81 };
// And also here we use the "normal" G-E ordering.
let enc_g = int_reg_enc(reg_g.to_reg());
let enc_g = int_reg_enc(reg_g.to_reg().to_reg());
emit_std_enc_enc(
sink,
LegacyPrefixes::None,
@@ -269,7 +269,7 @@ pub(crate) fn emit(
enc_g,
rex,
);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32);
emit_simm(sink, if use_imm8 { 1 } else { 4 }, simm32);
}
}
}
@@ -377,9 +377,9 @@ pub(crate) fn emit(
sink.add_trap(loc, TrapCode::IntegerDivisionByZero);
let subopcode = if *signed { 7 } else { 6 };
match divisor {
match divisor.clone().to_reg_mem() {
RegMem::Reg { reg } => {
let src = int_reg_enc(*reg);
let src = int_reg_enc(reg);
emit_std_enc_enc(
sink,
prefix,
@@ -387,7 +387,7 @@ pub(crate) fn emit(
1,
subopcode,
src,
RexFlags::from((*size, *reg)),
RexFlags::from((*size, reg)),
)
}
RegMem::Mem { addr: src } => {
@@ -428,9 +428,9 @@ pub(crate) fn emit(
};
let subopcode = if *signed { 5 } else { 4 };
match src2 {
match src2.clone().to_reg_mem() {
RegMem::Reg { reg } => {
let src = int_reg_enc(*reg);
let src = int_reg_enc(reg);
emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags)
}
RegMem::Mem { addr: src } => {
@@ -504,7 +504,7 @@ pub(crate) fn emit(
// $done:
// Check if the divisor is zero, first.
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg());
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0), divisor.to_reg().to_reg());
inst.emit(sink, info, state);
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero);
@@ -512,7 +512,8 @@ pub(crate) fn emit(
let (do_op, done_label) = if kind.is_signed() {
// Now check if the divisor is -1.
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg());
let inst =
Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), divisor.to_reg().to_reg());
inst.emit(sink, info, state);
let do_op = sink.get_label();
@@ -537,12 +538,16 @@ pub(crate) fn emit(
if *size == OperandSize::Size64 {
let tmp = tmp.expect("temporary for i64 sdiv");
let inst = Inst::imm(OperandSize::Size64, 0x8000000000000000, tmp);
let inst = Inst::imm(
OperandSize::Size64,
0x8000000000000000,
tmp.to_writable_reg(),
);
inst.emit(sink, info, state);
let inst = Inst::cmp_rmi_r(
OperandSize::Size64,
RegMemImm::reg(tmp.to_reg()),
RegMemImm::reg(tmp.to_reg().to_reg()),
regs::rax(),
);
inst.emit(sink, info, state);
@@ -576,7 +581,11 @@ pub(crate) fn emit(
inst.emit(sink, info, state);
}
let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor.to_reg()));
let inst = Inst::div(
*size,
kind.is_signed(),
RegMem::reg(divisor.to_reg().to_reg()),
);
inst.emit(sink, info, state);
// Lowering takes care of moving the result back into the right register, see comment
@@ -626,8 +635,8 @@ pub(crate) fn emit(
LegacyPrefixes::None,
0x89,
1,
*src,
dst.to_reg(),
src.to_reg(),
dst.to_reg().to_reg(),
RexFlags::from(*size),
);
}
@@ -664,12 +673,12 @@ pub(crate) fn emit(
}
};
match src {
match src.clone().to_reg_mem() {
RegMem::Reg { reg: src } => {
match ext_mode {
ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs.
rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(src);
}
_ => {}
}
@@ -678,8 +687,8 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
*src,
dst.to_reg().to_reg(),
src,
rex_flags,
)
}
@@ -694,7 +703,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
dst.to_reg().to_reg(),
src,
rex_flags,
)
@@ -712,7 +721,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
0x8B,
1,
dst.to_reg(),
dst.to_reg().to_reg(),
src,
RexFlags::set_w(),
)
@@ -758,12 +767,12 @@ pub(crate) fn emit(
}
};
match src {
match src.clone().to_reg_mem() {
RegMem::Reg { reg: src } => {
match ext_mode {
ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs.
rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(src);
}
_ => {}
}
@@ -772,8 +781,8 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
*src,
dst.to_reg().to_reg(),
src,
rex_flags,
)
}
@@ -788,7 +797,7 @@ pub(crate) fn emit(
LegacyPrefixes::None,
opcodes,
num_opcodes,
dst.to_reg(),
dst.to_reg().to_reg(),
src,
rex_flags,
)
@@ -812,13 +821,13 @@ pub(crate) fn emit(
// This is one of the few places where the presence of a
// redundant REX prefix changes the meaning of the
// instruction.
let rex = RexFlags::from((*size, *src));
let rex = RexFlags::from((*size, src.to_reg()));
// 8-bit: MOV r8, r/m8 is (REX.W==0) 88 /r
// 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
// 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
// 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *src, dst, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src.to_reg(), dst, rex);
}
Inst::ShiftR {
@@ -837,10 +846,10 @@ pub(crate) fn emit(
ShiftKind::ShiftRightArithmetic => 7,
};
let enc_dst = int_reg_enc(dst.to_reg());
let rex_flags = RexFlags::from((*size, dst.to_reg()));
match num_bits {
let rex_flags = RexFlags::from((*size, dst.to_reg().to_reg()));
match num_bits.clone().to_imm8_reg() {
Imm8Reg::Reg { reg } => {
debug_assert_eq!(*reg, regs::rcx());
debug_assert_eq!(reg, regs::rcx());
let (opcode, prefix) = match size {
OperandSize::Size8 => (0xD2, LegacyPrefixes::None),
OperandSize::Size16 => (0xD3, LegacyPrefixes::_66),
@@ -870,7 +879,7 @@ pub(crate) fn emit(
// When the shift amount is 1, there's an even shorter encoding, but we don't
// bother with that nicety here.
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags);
sink.put1(*num_bits);
sink.put1(num_bits);
}
}
}
@@ -963,13 +972,13 @@ pub(crate) fn emit(
prefix = LegacyPrefixes::_66;
}
// A redundant REX prefix can change the meaning of this instruction.
let mut rex = RexFlags::from((*size, *reg_g));
let mut rex = RexFlags::from((*size, reg_g.to_reg()));
match src_e {
match src_e.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg: reg_e } => {
if *size == OperandSize::Size8 {
// Check whether the E register forces the use of a redundant REX.
rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(reg_e);
}
// Use the swapped operands encoding for CMP, to stay consistent with the output of
@@ -980,7 +989,7 @@ pub(crate) fn emit(
(OperandSize::Size8, false) => 0x84,
(_, false) => 0x85,
};
emit_std_reg_reg(sink, prefix, opcode, 1, *reg_e, *reg_g, rex);
emit_std_reg_reg(sink, prefix, opcode, 1, reg_e, reg_g.to_reg(), rex);
}
RegMemImm::Mem { addr } => {
@@ -992,13 +1001,23 @@ pub(crate) fn emit(
(OperandSize::Size8, false) => 0x84,
(_, false) => 0x85,
};
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *reg_g, addr, rex);
emit_std_reg_mem(
sink,
state,
info,
prefix,
opcode,
1,
reg_g.to_reg(),
addr,
rex,
);
}
RegMemImm::Imm { simm32 } => {
// FIXME JRS 2020Feb11: there are shorter encodings for
// cmp $imm, rax/eax/ax/al.
let use_imm8 = is_cmp && low8_will_sign_extend_to_32(*simm32);
let use_imm8 = is_cmp && low8_will_sign_extend_to_32(simm32);
// And also here we use the "normal" G-E ordering.
let opcode = if is_cmp {
@@ -1018,9 +1037,9 @@ pub(crate) fn emit(
};
let subopcode = if is_cmp { 7 } else { 0 };
let enc_g = int_reg_enc(*reg_g);
let enc_g = int_reg_enc(reg_g.to_reg());
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_g, rex);
emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, *simm32);
emit_simm(sink, if use_imm8 { 1 } else { size.to_bytes() }, simm32);
}
}
}
@@ -1056,9 +1075,17 @@ pub(crate) fn emit(
_ => unreachable!("invalid size spec for cmove"),
};
let opcode = 0x0F40 + cc.get_enc() as u32;
match consequent {
match consequent.clone().to_reg_mem() {
RegMem::Reg { reg: reg_e } => {
emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex_flags);
emit_std_reg_reg(
sink,
prefix,
opcode,
2,
reg_g.to_reg().to_reg(),
reg_e,
rex_flags,
);
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
@@ -1069,7 +1096,7 @@ pub(crate) fn emit(
prefix,
opcode,
2,
reg_g.to_reg(),
reg_g.to_reg().to_reg(),
addr,
rex_flags,
);
@@ -1090,7 +1117,7 @@ pub(crate) fn emit(
} else {
SseOpcode::Movss
};
let inst = Inst::xmm_unary_rm_r(op, src.clone(), *dst);
let inst = Inst::xmm_unary_rm_r(op, src.clone().to_reg_mem(), dst.to_writable_reg());
inst.emit(sink, info, state);
sink.bind_label(next);
@@ -1101,9 +1128,9 @@ pub(crate) fn emit(
sink.add_trap(state.cur_srcloc(), TrapCode::StackOverflow);
}
match src {
match src.clone().to_reg_mem_imm() {
RegMemImm::Reg { reg } => {
let enc_reg = int_reg_enc(*reg);
let enc_reg = int_reg_enc(reg);
let rex = 0x40 | ((enc_reg >> 3) & 1);
if rex != 0x40 {
sink.put1(rex);
@@ -1127,12 +1154,12 @@ pub(crate) fn emit(
}
RegMemImm::Imm { simm32 } => {
if low8_will_sign_extend_to_64(*simm32) {
if low8_will_sign_extend_to_64(simm32) {
sink.put1(0x6A);
sink.put1(*simm32 as u8);
sink.put1(simm32 as u8);
} else {
sink.put1(0x68);
sink.put4(*simm32);
sink.put4(simm32);
}
}
}
@@ -1711,7 +1738,8 @@ pub(crate) fn emit(
_ => unreachable!(),
};
let inst = Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(*lhs), rhs_dst.to_reg());
let inst =
Inst::xmm_cmp_rm_r(cmp_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_reg().to_reg());
inst.emit(sink, info, state);
one_way_jmp(sink, CC::NZ, do_min_max);
@@ -1721,7 +1749,7 @@ pub(crate) fn emit(
// and negative zero. These instructions merge the sign bits in that
// case, and are no-ops otherwise.
let op = if *is_min { or_op } else { and_op };
let inst = Inst::xmm_rm_r(op, RegMem::reg(*lhs), *rhs_dst);
let inst = Inst::xmm_rm_r(op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg());
inst.emit(sink, info, state);
let inst = Inst::jmp_known(done);
@@ -1731,13 +1759,17 @@ pub(crate) fn emit(
// read-only operand: perform an addition between the two operands, which has the
// desired NaN propagation effects.
sink.bind_label(propagate_nan);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(*lhs), *rhs_dst);
let inst = Inst::xmm_rm_r(add_op, RegMem::reg(lhs.to_reg()), rhs_dst.to_writable_reg());
inst.emit(sink, info, state);
one_way_jmp(sink, CC::P, done);
sink.bind_label(do_min_max);
let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(*lhs), *rhs_dst);
let inst = Inst::xmm_rm_r(
min_max_op,
RegMem::reg(lhs.to_reg()),
rhs_dst.to_writable_reg(),
);
inst.emit(sink, info, state);
sink.bind_label(done);
@@ -1890,17 +1922,9 @@ pub(crate) fn emit(
_ => panic!("unexpected opcode {:?}", op),
};
let rex = RexFlags::from(*src_size);
match src_e {
match src_e.clone().to_reg_mem() {
RegMem::Reg { reg: reg_e } => {
emit_std_reg_reg(
sink,
prefix,
opcode,
2,
reg_g.to_reg().to_reg(),
*reg_e,
rex,
);
emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg().to_reg(), reg_e, rex);
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
@@ -1928,13 +1952,23 @@ pub(crate) fn emit(
_ => unimplemented!("Emit xmm cmp rm r"),
};
match src {
match src.clone().to_reg_mem() {
RegMem::Reg { reg } => {
emit_std_reg_reg(sink, prefix, opcode, len, *dst, *reg, rex);
emit_std_reg_reg(sink, prefix, opcode, len, dst.to_reg(), reg, rex);
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, *dst, addr, rex);
emit_std_reg_mem(
sink,
state,
info,
prefix,
opcode,
len,
dst.to_reg(),
addr,
rex,
);
}
}
}

304
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -133,9 +133,9 @@ impl Inst {
Self::AluRmiR {
size,
op,
src1: dst.to_reg(),
src2: src,
dst,
src1: Gpr::new(dst.to_reg()).unwrap(),
src2: GprMemImm::new(src).unwrap(),
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -174,10 +174,10 @@ impl Inst {
Inst::Div {
size,
signed,
divisor,
dividend: regs::rax(),
dst_quotient: Writable::from_reg(regs::rax()),
dst_remainder: Writable::from_reg(regs::rdx()),
divisor: GprMem::new(divisor).unwrap(),
dividend: Gpr::new(regs::rax()).unwrap(),
dst_quotient: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
}
}
@@ -191,10 +191,10 @@ impl Inst {
Inst::MulHi {
size,
signed,
src1: regs::rax(),
src2: rhs,
dst_lo: Writable::from_reg(regs::rax()),
dst_hi: Writable::from_reg(regs::rdx()),
src1: Gpr::new(regs::rax()).unwrap(),
src2: GprMem::new(rhs).unwrap(),
dst_lo: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_hi: WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
}
}
@@ -211,19 +211,19 @@ impl Inst {
Inst::CheckedDivOrRemSeq {
kind,
size,
divisor,
dividend: regs::rax(),
dst_quotient: Writable::from_reg(regs::rax()),
dst_remainder: Writable::from_reg(regs::rdx()),
tmp,
divisor: WritableGpr::from_writable_reg(divisor).unwrap(),
dividend: Gpr::new(regs::rax()).unwrap(),
dst_quotient: Writable::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
tmp: tmp.map(|tmp| WritableGpr::from_writable_reg(tmp).unwrap()),
}
}
pub(crate) fn sign_extend_data(size: OperandSize) -> Inst {
Inst::SignExtendData {
size,
src: regs::rax(),
dst: Writable::from_reg(regs::rdx()),
src: Gpr::new(regs::rax()).unwrap(),
dst: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
}
}
@@ -239,7 +239,7 @@ impl Inst {
Inst::Imm {
dst_size,
simm64,
dst,
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -247,6 +247,8 @@ impl Inst {
debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
debug_assert!(src.get_class() == RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let src = Gpr::new(src).unwrap();
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::MovRR { size, src, dst }
}
@@ -360,7 +362,7 @@ impl Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::GprToXmm {
op,
src,
src: GprMem::new(src).unwrap(),
dst: WritableXmm::from_writable_reg(dst).unwrap(),
src_size,
}
@@ -369,6 +371,8 @@ impl Inst {
pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst {
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.get_class() == RegClass::V128);
let src = XmmMem::new(src).unwrap();
let dst = Xmm::new(dst).unwrap();
Inst::XmmCmpRmR { op, src, dst }
}
@@ -457,8 +461,8 @@ impl Inst {
Inst::XmmMinMaxSeq {
size,
is_min,
lhs,
rhs_dst,
lhs: Xmm::new(lhs).unwrap(),
rhs_dst: WritableXmm::from_writable_reg(rhs_dst).unwrap(),
}
}
@@ -483,6 +487,8 @@ impl Inst {
pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let src = GprMem::new(src).unwrap();
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::MovzxRmR { ext_mode, src, dst }
}
@@ -500,6 +506,8 @@ impl Inst {
pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::I64);
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let src = GprMem::new(src).unwrap();
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::MovsxRmR { ext_mode, src, dst }
}
@@ -507,7 +515,7 @@ impl Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
Inst::Mov64MR {
src: src.into(),
dst,
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -524,7 +532,7 @@ impl Inst {
debug_assert!(src.get_class() == RegClass::I64);
Inst::MovRM {
size,
src,
src: Gpr::new(src).unwrap(),
dst: dst.into(),
}
}
@@ -552,12 +560,13 @@ impl Inst {
Inst::ShiftR {
size,
kind,
src: dst.to_reg(),
num_bits: match num_bits {
src: Gpr::new(dst.to_reg()).unwrap(),
num_bits: Imm8Gpr::new(match num_bits {
Some(imm) => Imm8Reg::Imm8 { imm },
None => Imm8Reg::Reg { reg: regs::rcx() },
},
dst,
})
.unwrap(),
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -568,8 +577,8 @@ impl Inst {
debug_assert_eq!(dst.get_class(), RegClass::I64);
Inst::CmpRmiR {
size,
src,
dst,
src: GprMemImm::new(src).unwrap(),
dst: Gpr::new(dst).unwrap(),
opcode: CmpOpcode::Cmp,
}
}
@@ -580,8 +589,8 @@ impl Inst {
debug_assert_eq!(dst.get_class(), RegClass::I64);
Inst::CmpRmiR {
size,
src,
dst,
src: GprMemImm::new(src).unwrap(),
dst: Gpr::new(dst).unwrap(),
opcode: CmpOpcode::Test,
}
}
@@ -594,6 +603,7 @@ impl Inst {
pub(crate) fn setcc(cc: CC, dst: Writable<Reg>) -> Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::Setcc { cc, dst }
}
@@ -607,9 +617,9 @@ impl Inst {
Inst::Cmove {
size,
cc,
consequent: src,
alternative: dst.to_reg(),
dst,
consequent: GprMem::new(src).unwrap(),
alternative: Gpr::new(dst.to_reg()).unwrap(),
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -617,16 +627,20 @@ impl Inst {
debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
src.assert_regclass_is(RegClass::V128);
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
let src = XmmMem::new(src).unwrap();
let dst = WritableXmm::from_writable_reg(dst).unwrap();
Inst::XmmCmove { size, cc, src, dst }
}
pub(crate) fn push64(src: RegMemImm) -> Inst {
src.assert_regclass_is(RegClass::I64);
let src = GprMemImm::new(src).unwrap();
Inst::Push64 { src }
}
pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
debug_assert!(dst.to_reg().get_class() == RegClass::I64);
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::Pop64 { dst }
}
@@ -780,7 +794,7 @@ impl Inst {
fn produces_const(&self) -> bool {
match self {
Self::AluRmiR { op, src2, dst, .. } => {
src2.to_reg() == Some(dst.to_reg())
src2.clone().to_reg_mem_imm().to_reg() == Some(dst.to_reg().to_reg())
&& (*op == AluRmiROpcode::Xor || *op == AluRmiROpcode::Sub)
}
@@ -838,7 +852,11 @@ impl Inst {
Inst::AluRmiR { src1, dst, .. } => {
if *src1 != dst.to_reg() {
debug_assert!(src1.is_virtual());
insts.push(Self::gen_move(*dst, *src1, types::I64));
insts.push(Self::gen_move(
dst.to_writable_reg(),
src1.to_reg(),
types::I64,
));
*src1 = dst.to_reg();
}
insts.push(self);
@@ -883,7 +901,11 @@ impl Inst {
} => {
if *alternative != dst.to_reg() {
debug_assert!(alternative.is_virtual());
insts.push(Self::mov_r_r(*size, *alternative, *dst));
insts.push(Self::mov_r_r(
*size,
alternative.to_reg(),
dst.to_writable_reg(),
));
*alternative = dst.to_reg();
}
insts.push(self);
@@ -916,22 +938,30 @@ impl Inst {
debug_assert!(dividend.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rax()),
*dividend,
dividend.to_reg(),
types::I64,
));
*dividend = regs::rax();
*dividend = Gpr::new(regs::rax()).unwrap();
}
let mut quotient_mov = None;
if dst_quotient.to_reg() != regs::rax() {
debug_assert!(dst_quotient.to_reg().is_virtual());
quotient_mov = Some(Self::gen_move(*dst_quotient, regs::rax(), types::I64));
*dst_quotient = Writable::from_reg(regs::rax());
quotient_mov = Some(Self::gen_move(
dst_quotient.to_writable_reg(),
regs::rax(),
types::I64,
));
*dst_quotient = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
}
let mut remainder_mov = None;
if dst_remainder.to_reg() != regs::rdx() {
debug_assert!(dst_remainder.to_reg().is_virtual());
remainder_mov = Some(Self::gen_move(*dst_remainder, regs::rdx(), types::I64));
*dst_remainder = Writable::from_reg(regs::rdx());
remainder_mov = Some(Self::gen_move(
dst_remainder.to_writable_reg(),
regs::rdx(),
types::I64,
));
*dst_remainder = Writable::from_reg(Gpr::new(regs::rdx()).unwrap());
}
insts.push(self);
insts.extend(quotient_mov);
@@ -947,22 +977,30 @@ impl Inst {
debug_assert!(src1.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rax()),
*src1,
src1.to_reg(),
types::I64,
));
*src1 = regs::rax();
*src1 = Gpr::new(regs::rax()).unwrap();
}
let mut dst_lo_mov = None;
if dst_lo.to_reg() != regs::rax() {
debug_assert!(dst_lo.to_reg().is_virtual());
dst_lo_mov = Some(Self::gen_move(*dst_lo, regs::rax(), types::I64));
*dst_lo = Writable::from_reg(regs::rax());
dst_lo_mov = Some(Self::gen_move(
dst_lo.to_writable_reg(),
regs::rax(),
types::I64,
));
*dst_lo = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
}
let mut dst_hi_mov = None;
if dst_hi.to_reg() != regs::rdx() {
debug_assert!(dst_hi.to_reg().is_virtual());
dst_hi_mov = Some(Self::gen_move(*dst_hi, regs::rdx(), types::I64));
*dst_hi = Writable::from_reg(regs::rdx());
dst_hi_mov = Some(Self::gen_move(
dst_hi.to_writable_reg(),
regs::rdx(),
types::I64,
));
*dst_hi = Writable::from_reg(Gpr::new(regs::rdx()).unwrap());
}
insts.push(self);
insts.extend(dst_lo_mov);
@@ -973,16 +1011,20 @@ impl Inst {
debug_assert!(src.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rax()),
*src,
src.to_reg(),
types::I64,
));
*src = regs::rax();
*src = Gpr::new(regs::rax()).unwrap();
}
let mut dst_mov = None;
if dst.to_reg() != regs::rax() {
debug_assert!(dst.to_reg().is_virtual());
dst_mov = Some(Self::gen_move(*dst, dst.to_reg(), types::I64));
*dst = Writable::from_reg(regs::rax());
dst_mov = Some(Self::gen_move(
dst.to_writable_reg(),
dst.to_reg().to_reg(),
types::I64,
));
*dst = Writable::from_reg(Gpr::new(regs::rax()).unwrap());
}
insts.push(self);
insts.extend(dst_mov);
@@ -992,18 +1034,22 @@ impl Inst {
} => {
if *src != dst.to_reg() {
debug_assert!(src.is_virtual());
insts.push(Self::gen_move(*dst, *src, types::I64));
insts.push(Self::gen_move(
dst.to_writable_reg(),
src.to_reg(),
types::I64,
));
*src = dst.to_reg();
}
if let Imm8Reg::Reg { reg } = num_bits {
if *reg != regs::rcx() {
if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() {
if reg != regs::rcx() {
debug_assert!(reg.is_virtual());
insts.push(Self::gen_move(
Writable::from_reg(regs::rcx()),
*reg,
reg,
types::I64,
));
*reg = regs::rcx();
*num_bits = Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap();
}
}
insts.push(self);
@@ -1146,7 +1192,7 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())),
src2.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())),
show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
),
Inst::UnaryRmR { src, dst, op, size } => format!(
@@ -1208,7 +1254,7 @@ impl PrettyPrint for Inst {
DivOrRemKind::SignedRem => "srem",
DivOrRemKind::UnsignedRem => "urem",
},
show_ireg_sized(divisor.to_reg(), mb_rru, size.to_bytes()),
show_ireg_sized(divisor.to_reg().to_reg(), mb_rru, size.to_bytes()),
),
Inst::SignExtendData { size, .. } => match size {
@@ -1276,8 +1322,8 @@ impl PrettyPrint for Inst {
},
format!("f{}", size.to_bits())
),
show_ireg_sized(*lhs, mb_rru, 8),
show_ireg_sized(rhs_dst.to_reg(), mb_rru, 8),
show_ireg_sized(lhs.to_reg(), mb_rru, 8),
show_ireg_sized(rhs_dst.to_reg().to_reg(), mb_rru, 8),
),
Inst::XmmRmRImm {
@@ -1342,7 +1388,7 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify(op.to_string()),
src.show_rru_sized(mb_rru, 8),
show_ireg_sized(*dst, mb_rru, 8),
show_ireg_sized(dst.to_reg(), mb_rru, 8),
),
Inst::CvtUint64ToFloatSeq {
@@ -1405,14 +1451,14 @@ impl PrettyPrint for Inst {
"{} ${}, {}",
ljustify("movabsq".to_string()),
*simm64 as i64,
show_ireg_sized(dst.to_reg(), mb_rru, 8)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 8)
)
} else {
format!(
"{} ${}, {}",
ljustify("movl".to_string()),
(*simm64 as u32) as i32,
show_ireg_sized(dst.to_reg(), mb_rru, 4)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4)
)
}
}
@@ -1420,8 +1466,8 @@ impl PrettyPrint for Inst {
Inst::MovRR { size, src, dst } => format!(
"{} {}, {}",
ljustify2("mov".to_string(), suffix_lq(*size)),
show_ireg_sized(*src, mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
Inst::MovzxRmR {
@@ -1432,14 +1478,14 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify("movl".to_string()),
src.show_rru_sized(mb_rru, ext_mode.src_size()),
show_ireg_sized(dst.to_reg(), mb_rru, 4)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 4)
)
} else {
format!(
"{} {}, {}",
ljustify2("movz".to_string(), ext_mode.to_string()),
src.show_rru_sized(mb_rru, ext_mode.src_size()),
show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size())
)
}
}
@@ -1464,13 +1510,13 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify2("movs".to_string(), ext_mode.to_string()),
src.show_rru_sized(mb_rru, ext_mode.src_size()),
show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, ext_mode.dst_size())
),
Inst::MovRM { size, src, dst, .. } => format!(
"{} {}, {}",
ljustify2("mov".to_string(), suffix_bwlq(*size)),
show_ireg_sized(*src, mb_rru, size.to_bytes()),
show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes()),
dst.show_rru(mb_rru)
),
@@ -1480,19 +1526,19 @@ impl PrettyPrint for Inst {
num_bits,
dst,
..
} => match num_bits {
} => match num_bits.clone().to_imm8_reg() {
Imm8Reg::Reg { reg } => format!(
"{} {}, {}",
ljustify2(kind.to_string(), suffix_bwlq(*size)),
show_ireg_sized(*reg, mb_rru, 1),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(reg, mb_rru, 1),
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
Imm8Reg::Imm8 { imm: num_bits } => format!(
"{} ${}, {}",
ljustify2(kind.to_string(), suffix_bwlq(*size)),
num_bits,
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
},
@@ -1519,14 +1565,14 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify2(op.to_string(), suffix_bwlq(*size)),
src.show_rru_sized(mb_rru, size.to_bytes()),
show_ireg_sized(*dst, mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
)
}
Inst::Setcc { cc, dst } => format!(
"{} {}",
ljustify2("set".to_string(), cc.to_string()),
show_ireg_sized(dst.to_reg(), mb_rru, 1)
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, 1)
),
Inst::Cmove {
@@ -1539,7 +1585,7 @@ impl PrettyPrint for Inst {
"{} {}, {}",
ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))),
src.show_rru_sized(mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
),
Inst::XmmCmove { size, cc, src, dst } => {
@@ -1552,7 +1598,7 @@ impl PrettyPrint for Inst {
"ss"
},
src.show_rru_sized(mb_rru, size.to_bytes()),
show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
show_ireg_sized(dst.to_reg().to_reg(), mb_rru, size.to_bytes())
)
}
@@ -1693,10 +1739,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
debug_assert_eq!(*src1, dst.to_reg());
if inst.produces_const() {
// No need to account for src2, since src2 == dst.
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
} else {
src2.get_regs_as_uses(collector);
collector.add_mod(*dst);
collector.add_mod(dst.to_writable_reg());
}
}
Inst::Not { src, dst, .. } => {
@@ -1760,9 +1806,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
// the rdx register *before* the instruction, which is not too bad.
collector.add_mod(Writable::from_reg(regs::rax()));
collector.add_mod(Writable::from_reg(regs::rdx()));
collector.add_mod(*divisor);
collector.add_mod(divisor.to_writable_reg());
if let Some(tmp) = tmp {
collector.add_def(*tmp);
collector.add_def(tmp.to_writable_reg());
}
}
Inst::SignExtendData { size, src, dst } => {
@@ -1852,8 +1898,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
Inst::XmmUninitializedValue { dst } => collector.add_def(dst.to_writable_reg()),
Inst::XmmLoadConst { dst, .. } => collector.add_def(*dst),
Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => {
collector.add_use(*lhs);
collector.add_mod(*rhs_dst);
collector.add_use(lhs.to_reg());
collector.add_mod(rhs_dst.to_writable_reg());
}
Inst::XmmRmiReg {
src1, src2, dst, ..
@@ -1868,14 +1914,14 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
}
Inst::XmmCmpRmR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_use(*dst);
collector.add_use(dst.to_reg());
}
Inst::Imm { dst, .. } => {
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::MovRR { src, dst, .. } => {
collector.add_use(*src);
collector.add_def(*dst);
collector.add_use(src.to_reg());
collector.add_def(dst.to_writable_reg());
}
Inst::XmmToGpr { src, dst, .. } => {
collector.add_use(src.to_reg());
@@ -1918,11 +1964,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
}
Inst::MovzxRmR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::Mov64MR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_def(*dst)
collector.add_def(dst.to_writable_reg())
}
Inst::LoadEffectiveAddress { addr: src, dst } => {
src.get_regs_as_uses(collector);
@@ -1930,41 +1976,44 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
}
Inst::MovsxRmR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::MovRM { src, dst, .. } => {
collector.add_use(*src);
collector.add_use(src.to_reg());
dst.get_regs_as_uses(collector);
}
Inst::ShiftR { num_bits, dst, .. } => {
if let Imm8Reg::Reg { reg } = num_bits {
debug_assert_eq!(*reg, regs::rcx());
if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() {
debug_assert_eq!(reg, regs::rcx());
collector.add_use(regs::rcx());
}
collector.add_mod(*dst);
collector.add_mod(dst.to_writable_reg());
}
Inst::CmpRmiR { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_use(*dst); // yes, really `add_use`
collector.add_use(dst.to_reg()); // yes, really `add_use`
}
Inst::Setcc { dst, .. } => {
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::Cmove {
consequent: src,
dst,
..
}
| Inst::XmmCmove { src, dst, .. } => {
} => {
src.get_regs_as_uses(collector);
collector.add_mod(*dst);
collector.add_mod(dst.to_writable_reg());
}
Inst::XmmCmove { src, dst, .. } => {
src.get_regs_as_uses(collector);
collector.add_mod(dst.to_writable_reg());
}
Inst::Push64 { src } => {
src.get_regs_as_uses(collector);
collector.add_mod(Writable::from_reg(regs::rsp()));
}
Inst::Pop64 { dst } => {
collector.add_def(*dst);
collector.add_def(dst.to_writable_reg());
}
Inst::CallKnown {
@@ -2149,11 +2198,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
debug_assert_eq!(*src1, dst.to_reg());
if produces_const {
src2.map_as_def(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
*src1 = dst.to_reg();
} else {
src2.map_uses(mapper);
mapper.map_mod(dst);
dst.map_mod(mapper);
*src1 = dst.to_reg();
}
}
@@ -2165,9 +2214,9 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
Inst::Div { divisor, .. } => divisor.map_uses(mapper),
Inst::MulHi { src2, .. } => src2.map_uses(mapper),
Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => {
mapper.map_mod(divisor);
divisor.map_mod(mapper);
if let Some(tmp) = tmp {
mapper.map_def(tmp)
tmp.map_def(mapper)
}
}
Inst::SignExtendData { .. } => {}
@@ -2275,8 +2324,8 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
ref mut rhs_dst,
..
} => {
mapper.map_use(lhs);
mapper.map_mod(rhs_dst);
lhs.map_use(mapper);
rhs_dst.map_mod(mapper);
}
Inst::XmmMovRM {
ref mut src,
@@ -2292,16 +2341,16 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_use(dst);
dst.map_use(mapper);
}
Inst::Imm { ref mut dst, .. } => mapper.map_def(dst),
Inst::Imm { ref mut dst, .. } => dst.map_def(mapper),
Inst::MovRR {
ref mut src,
ref mut dst,
..
} => {
mapper.map_use(src);
mapper.map_def(dst);
src.map_use(mapper);
dst.map_def(mapper);
}
Inst::XmmToGpr {
ref mut src,
@@ -2356,11 +2405,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::Mov64MR { src, dst, .. } => {
src.map_uses(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::LoadEffectiveAddress { addr: src, dst } => {
src.map_uses(mapper);
@@ -2372,14 +2421,14 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::MovRM {
ref mut src,
ref mut dst,
..
} => {
mapper.map_use(src);
src.map_use(mapper);
dst.map_uses(mapper);
}
Inst::ShiftR {
@@ -2388,7 +2437,7 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
debug_assert_eq!(*src, dst.to_reg());
mapper.map_mod(dst);
dst.map_mod(mapper);
*src = dst.to_reg();
}
Inst::CmpRmiR {
@@ -2397,9 +2446,9 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_use(dst);
dst.map_use(mapper);
}
Inst::Setcc { ref mut dst, .. } => mapper.map_def(dst),
Inst::Setcc { ref mut dst, .. } => dst.map_def(mapper),
Inst::Cmove {
consequent: ref mut src,
ref mut dst,
@@ -2407,7 +2456,7 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_mod(dst);
dst.map_mod(mapper);
*alternative = dst.to_reg();
}
Inst::XmmCmove {
@@ -2416,11 +2465,11 @@ pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
..
} => {
src.map_uses(mapper);
mapper.map_mod(dst);
dst.map_mod(mapper);
}
Inst::Push64 { ref mut src } => src.map_uses(mapper),
Inst::Pop64 { ref mut dst } => {
mapper.map_def(dst);
dst.map_def(mapper);
}
Inst::CallKnown {
@@ -2520,7 +2569,7 @@ impl MachInst for Inst {
// conceivably use `movl %reg, %reg` to zero out the top 32 bits of
// %reg.
Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => {
Some((*dst, *src))
Some((dst.to_writable_reg(), src.to_reg()))
}
// Note as well that MOVS[S|D] when used in the `XmmUnaryRmR` context are pure moves of
// scalar floating-point values (and annotate `dst` as `def`s to the register allocator)
@@ -2578,12 +2627,15 @@ impl MachInst for Inst {
size: OperandSize::Size8,
src,
dst: SyntheticAmode::NominalSPOffset { simm32 },
} => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)),
} => Some(MachInstStackOpInfo::StoreNomSPOff(
src.to_reg(),
*simm32 as i64,
)),
Self::Mov64MR {
src: SyntheticAmode::NominalSPOffset { simm32 },
dst,
} => Some(MachInstStackOpInfo::LoadNomSPOff(
dst.to_reg(),
dst.to_reg().to_reg(),
*simm32 as i64,
)),
_ => None,