T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Address review comments;

Browse Source
This commit is contained in:
Benjamin Bouvier
2020-07-16 15:19:38 +02:00
parent 5a55646fc3
commit bab337fc32
6 changed files with 213 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cranelift/codegen/src/isa/x64/abi.rs
View File

@@ -94,10 +94,10 @@ fn in_vec_reg(ty: types::Type) -> bool {
} }
fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> { fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
assert!(match call_conv { match call_conv {
CallConv::SystemV | CallConv::BaldrdashSystemV => true, CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::BaldrdashSystemV => {}
_ => false, _ => panic!("int args only supported for SysV calling convention"),
}); };
match idx { match idx {
0 => Some(regs::rdi()), 0 => Some(regs::rdi()),
1 => Some(regs::rsi()), 1 => Some(regs::rsi()),
@@ -110,10 +110,10 @@ fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
} }
fn get_fltreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> { fn get_fltreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
assert!(match call_conv { match call_conv {
CallConv::SystemV | CallConv::BaldrdashSystemV => true, CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::BaldrdashSystemV => {}
_ => false, _ => panic!("float args only supported for SysV calling convention"),
}); };
match idx { match idx {
0 => Some(regs::xmm0()), 0 => Some(regs::xmm0()),
1 => Some(regs::xmm1()), 1 => Some(regs::xmm1()),

85
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -282,32 +282,32 @@ impl fmt::Debug for AluRmiROpcode {
} }
} }
impl ToString for AluRmiROpcode { impl fmt::Display for AluRmiROpcode {
fn to_string(&self) -> String { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
format!("{:?}", self) fmt::Debug::fmt(self, f)
} }
} }
#[derive(Clone, PartialEq)] #[derive(Clone, PartialEq)]
pub enum ReadOnlyGprRmROpcode { pub enum UnaryRmROpcode {
/// Bit-scan reverse. /// Bit-scan reverse.
Bsr, Bsr,
/// Bit-scan forward. /// Bit-scan forward.
Bsf, Bsf,
} }
impl fmt::Debug for ReadOnlyGprRmROpcode { impl fmt::Debug for UnaryRmROpcode {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
match self { match self {
ReadOnlyGprRmROpcode::Bsr => write!(fmt, "bsr"), UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
ReadOnlyGprRmROpcode::Bsf => write!(fmt, "bsf"), UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
} }
} }
} }
impl ToString for ReadOnlyGprRmROpcode { impl fmt::Display for UnaryRmROpcode {
fn to_string(&self) -> String { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
format!("{:?}", self) fmt::Debug::fmt(self, f)
} }
} }
@@ -468,9 +468,9 @@ impl fmt::Debug for SseOpcode {
} }
} }
impl ToString for SseOpcode { impl fmt::Display for SseOpcode {
fn to_string(&self) -> String { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
format!("{:?}", self) fmt::Debug::fmt(self, f)
} }
} }
@@ -519,18 +519,20 @@ impl fmt::Debug for ExtMode {
} }
} }
impl ToString for ExtMode { impl fmt::Display for ExtMode {
fn to_string(&self) -> String { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
format!("{:?}", self) fmt::Debug::fmt(self, f)
} }
} }
/// These indicate the form of a scalar shift: left, signed right, unsigned right. /// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
#[derive(Clone)] #[derive(Clone)]
pub enum ShiftKind { pub enum ShiftKind {
Left, ShiftLeft,
RightZ, /// Inserts zeros in the most significant bits.
RightS, ShiftRightLogical,
/// Replicates the sign bit in the most significant bits.
ShiftRightArithmetic,
RotateLeft, RotateLeft,
RotateRight, RotateRight,
} }
@@ -538,9 +540,9 @@ pub enum ShiftKind {
impl fmt::Debug for ShiftKind { impl fmt::Debug for ShiftKind {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
let name = match self { let name = match self {
ShiftKind::Left => "shl", ShiftKind::ShiftLeft => "shl",
ShiftKind::RightZ => "shr", ShiftKind::ShiftRightLogical => "shr",
ShiftKind::RightS => "sar", ShiftKind::ShiftRightArithmetic => "sar",
ShiftKind::RotateLeft => "rol", ShiftKind::RotateLeft => "rol",
ShiftKind::RotateRight => "ror", ShiftKind::RotateRight => "ror",
}; };
@@ -548,9 +550,34 @@ impl fmt::Debug for ShiftKind {
} }
} }
impl ToString for ShiftKind { impl fmt::Display for ShiftKind {
fn to_string(&self) -> String { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
format!("{:?}", self) fmt::Debug::fmt(self, f)
}
}
/// What kind of division or remainer instruction this is?
#[derive(Clone)]
pub enum DivOrRemKind {
SignedDiv,
UnsignedDiv,
SignedRem,
UnsignedRem,
}
impl DivOrRemKind {
pub(crate) fn is_signed(&self) -> bool {
match self {
DivOrRemKind::SignedDiv | DivOrRemKind::SignedRem => true,
_ => false,
}
}
pub(crate) fn is_div(&self) -> bool {
match self {
DivOrRemKind::SignedDiv | DivOrRemKind::UnsignedDiv => true,
_ => false,
}
} }
} }
@@ -665,9 +692,9 @@ impl fmt::Debug for CC {
} }
} }
impl ToString for CC { impl fmt::Display for CC {
fn to_string(&self) -> String { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
format!("{:?}", self) fmt::Debug::fmt(self, f)
} }
} }

24
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -556,7 +556,7 @@ pub(crate) fn emit(
} }
} }
Inst::ReadOnly_Gpr_Rm_R { size, op, src, dst } => { Inst::UnaryRmR { size, op, src, dst } => {
let (prefix, rex_flags) = match size { let (prefix, rex_flags) = match size {
2 => (LegacyPrefix::_66, RexFlags::clear_w()), 2 => (LegacyPrefix::_66, RexFlags::clear_w()),
4 => (LegacyPrefix::None, RexFlags::clear_w()), 4 => (LegacyPrefix::None, RexFlags::clear_w()),
@@ -565,8 +565,8 @@ pub(crate) fn emit(
}; };
let (opcode, num_opcodes) = match op { let (opcode, num_opcodes) = match op {
ReadOnlyGprRmROpcode::Bsr => (0x0fbd, 2), UnaryRmROpcode::Bsr => (0x0fbd, 2),
ReadOnlyGprRmROpcode::Bsf => (0x0fbc, 2), UnaryRmROpcode::Bsf => (0x0fbc, 2),
}; };
match src { match src {
@@ -661,8 +661,7 @@ pub(crate) fn emit(
} }
Inst::CheckedDivOrRemSeq { Inst::CheckedDivOrRemSeq {
is_div, kind,
is_signed,
size, size,
divisor, divisor,
loc, loc,
@@ -704,7 +703,7 @@ pub(crate) fn emit(
let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero, *loc); let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero, *loc);
inst.emit(sink, flags, state); inst.emit(sink, flags, state);
let (do_op, done_label) = if *is_signed { let (do_op, done_label) = if kind.is_signed() {
// Now check if the divisor is -1. // Now check if the divisor is -1.
let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), *divisor); let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), *divisor);
inst.emit(sink, flags, state); inst.emit(sink, flags, state);
@@ -715,7 +714,7 @@ pub(crate) fn emit(
one_way_jmp(sink, CC::NZ, do_op); one_way_jmp(sink, CC::NZ, do_op);
// Here, divisor == -1. // Here, divisor == -1.
if !*is_div { if !kind.is_div() {
// x % -1 = 0; put the result into the destination, $rdx. // x % -1 = 0; put the result into the destination, $rdx.
let done_label = sink.get_label(); let done_label = sink.get_label();
@@ -756,7 +755,7 @@ pub(crate) fn emit(
} }
// Fill in the high parts: // Fill in the high parts:
if *is_signed { if kind.is_signed() {
// sign-extend the sign-bit of rax into rdx, for signed opcodes. // sign-extend the sign-bit of rax into rdx, for signed opcodes.
let inst = Inst::sign_extend_rax_to_rdx(*size); let inst = Inst::sign_extend_rax_to_rdx(*size);
inst.emit(sink, flags, state); inst.emit(sink, flags, state);
@@ -766,7 +765,7 @@ pub(crate) fn emit(
inst.emit(sink, flags, state); inst.emit(sink, flags, state);
} }
let inst = Inst::div(*size, *is_signed, RegMem::reg(*divisor), *loc); let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(*divisor), *loc);
inst.emit(sink, flags, state); inst.emit(sink, flags, state);
// Lowering takes care of moving the result back into the right register, see comment // Lowering takes care of moving the result back into the right register, see comment
@@ -1047,9 +1046,9 @@ pub(crate) fn emit(
let subopcode = match kind { let subopcode = match kind {
ShiftKind::RotateLeft => 0, ShiftKind::RotateLeft => 0,
ShiftKind::RotateRight => 1, ShiftKind::RotateRight => 1,
ShiftKind::Left => 4, ShiftKind::ShiftLeft => 4,
ShiftKind::RightZ => 5, ShiftKind::ShiftRightLogical => 5,
ShiftKind::RightS => 7, ShiftKind::ShiftRightArithmetic => 7,
}; };
let rex = if *is_64 { let rex = if *is_64 {
@@ -1550,6 +1549,7 @@ pub(crate) fn emit(
srcloc, srcloc,
} => { } => {
// The full address can be encoded in the register, with a relocation. // The full address can be encoded in the register, with a relocation.
// Generates: movabsq $name, %dst
let enc_dst = int_reg_enc(dst.to_reg()); let enc_dst = int_reg_enc(dst.to_reg());
sink.put1(0x48 | ((enc_dst >> 3) & 1)); sink.put1(0x48 | ((enc_dst >> 3) & 1));
sink.put1(0xB8 | (enc_dst & 7)); sink.put1(0xB8 | (enc_dst & 7));

48
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

@@ -1227,15 +1227,15 @@ fn test_x64_emit() {
)); ));
// ======================================================== // ========================================================
// ReadOnly_Gpr_Rm_R // UnaryRmR
insns.push(( insns.push((
Inst::read_only_gpr_rm_r(4, ReadOnlyGprRmROpcode::Bsr, RegMem::reg(rsi), w_rdi), Inst::unary_rm_r(4, UnaryRmROpcode::Bsr, RegMem::reg(rsi), w_rdi),
"0FBDFE", "0FBDFE",
"bsrl %esi, %edi", "bsrl %esi, %edi",
)); ));
insns.push(( insns.push((
Inst::read_only_gpr_rm_r(8, ReadOnlyGprRmROpcode::Bsr, RegMem::reg(r15), w_rax), Inst::unary_rm_r(8, UnaryRmROpcode::Bsr, RegMem::reg(r15), w_rax),
"490FBDC7", "490FBDC7",
"bsrq %r15, %rax", "bsrq %r15, %rax",
)); ));
@@ -2303,107 +2303,107 @@ fn test_x64_emit() {
// ======================================================== // ========================================================
// Shift_R // Shift_R
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::Left, None, w_rdi), Inst::shift_r(false, ShiftKind::ShiftLeft, None, w_rdi),
"D3E7", "D3E7",
"shll %cl, %edi", "shll %cl, %edi",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::Left, None, w_r12), Inst::shift_r(false, ShiftKind::ShiftLeft, None, w_r12),
"41D3E4", "41D3E4",
"shll %cl, %r12d", "shll %cl, %r12d",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::Left, Some(2), w_r8), Inst::shift_r(false, ShiftKind::ShiftLeft, Some(2), w_r8),
"41C1E002", "41C1E002",
"shll $2, %r8d", "shll $2, %r8d",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::Left, Some(31), w_r13), Inst::shift_r(false, ShiftKind::ShiftLeft, Some(31), w_r13),
"41C1E51F", "41C1E51F",
"shll $31, %r13d", "shll $31, %r13d",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::Left, None, w_r13), Inst::shift_r(true, ShiftKind::ShiftLeft, None, w_r13),
"49D3E5", "49D3E5",
"shlq %cl, %r13", "shlq %cl, %r13",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::Left, None, w_rdi), Inst::shift_r(true, ShiftKind::ShiftLeft, None, w_rdi),
"48D3E7", "48D3E7",
"shlq %cl, %rdi", "shlq %cl, %rdi",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::Left, Some(2), w_r8), Inst::shift_r(true, ShiftKind::ShiftLeft, Some(2), w_r8),
"49C1E002", "49C1E002",
"shlq $2, %r8", "shlq $2, %r8",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::Left, Some(3), w_rbx), Inst::shift_r(true, ShiftKind::ShiftLeft, Some(3), w_rbx),
"48C1E303", "48C1E303",
"shlq $3, %rbx", "shlq $3, %rbx",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::Left, Some(63), w_r13), Inst::shift_r(true, ShiftKind::ShiftLeft, Some(63), w_r13),
"49C1E53F", "49C1E53F",
"shlq $63, %r13", "shlq $63, %r13",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::RightZ, None, w_rdi), Inst::shift_r(false, ShiftKind::ShiftRightLogical, None, w_rdi),
"D3EF", "D3EF",
"shrl %cl, %edi", "shrl %cl, %edi",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::RightZ, Some(2), w_r8), Inst::shift_r(false, ShiftKind::ShiftRightLogical, Some(2), w_r8),
"41C1E802", "41C1E802",
"shrl $2, %r8d", "shrl $2, %r8d",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::RightZ, Some(31), w_r13), Inst::shift_r(false, ShiftKind::ShiftRightLogical, Some(31), w_r13),
"41C1ED1F", "41C1ED1F",
"shrl $31, %r13d", "shrl $31, %r13d",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::RightZ, None, w_rdi), Inst::shift_r(true, ShiftKind::ShiftRightLogical, None, w_rdi),
"48D3EF", "48D3EF",
"shrq %cl, %rdi", "shrq %cl, %rdi",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::RightZ, Some(2), w_r8), Inst::shift_r(true, ShiftKind::ShiftRightLogical, Some(2), w_r8),
"49C1E802", "49C1E802",
"shrq $2, %r8", "shrq $2, %r8",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::RightZ, Some(63), w_r13), Inst::shift_r(true, ShiftKind::ShiftRightLogical, Some(63), w_r13),
"49C1ED3F", "49C1ED3F",
"shrq $63, %r13", "shrq $63, %r13",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::RightS, None, w_rdi), Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, None, w_rdi),
"D3FF", "D3FF",
"sarl %cl, %edi", "sarl %cl, %edi",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::RightS, Some(2), w_r8), Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, Some(2), w_r8),
"41C1F802", "41C1F802",
"sarl $2, %r8d", "sarl $2, %r8d",
)); ));
insns.push(( insns.push((
Inst::shift_r(false, ShiftKind::RightS, Some(31), w_r13), Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, Some(31), w_r13),
"41C1FD1F", "41C1FD1F",
"sarl $31, %r13d", "sarl $31, %r13d",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::RightS, None, w_rdi), Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, None, w_rdi),
"48D3FF", "48D3FF",
"sarq %cl, %rdi", "sarq %cl, %rdi",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::RightS, Some(2), w_r8), Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, Some(2), w_r8),
"49C1F802", "49C1F802",
"sarq $2, %r8", "sarq $2, %r8",
)); ));
insns.push(( insns.push((
Inst::shift_r(true, ShiftKind::RightS, Some(63), w_r13), Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, Some(63), w_r13),
"49C1FD3F", "49C1FD3F",
"sarq $63, %r13", "sarq $63, %r13",
)); ));

40
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -51,9 +51,9 @@ pub enum Inst {
}, },
/// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc. /// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc.
ReadOnly_Gpr_Rm_R { UnaryRmR {
size: u8, // 2, 4 or 8 size: u8, // 2, 4 or 8
op: ReadOnlyGprRmROpcode, op: UnaryRmROpcode,
src: RegMem, src: RegMem,
dst: Writable<Reg>, dst: Writable<Reg>,
}, },
@@ -66,7 +66,7 @@ pub enum Inst {
loc: SourceLoc, loc: SourceLoc,
}, },
/// The high result of a (un)signed multiply: imul/mul using RAX:RDX. /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs.
MulHi { size: u8, signed: bool, rhs: RegMem }, MulHi { size: u8, signed: bool, rhs: RegMem },
/// A synthetic sequence to implement the right inline checks for remainder and division, /// A synthetic sequence to implement the right inline checks for remainder and division,
@@ -77,10 +77,11 @@ pub enum Inst {
/// instruction. /// instruction.
/// ///
/// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem /// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem
/// with the temporary and divisor. Make sure to zero %rdx right before this instruction! /// with the temporary and divisor registers. Make sure to zero %rdx right before this
/// instruction, or you might run into regalloc failures where %rdx is live before its first
/// def!
CheckedDivOrRemSeq { CheckedDivOrRemSeq {
is_div: bool, kind: DivOrRemKind,
is_signed: bool,
size: u8, size: u8,
divisor: Reg, divisor: Reg,
tmp: Option<Writable<Reg>>, tmp: Option<Writable<Reg>>,
@@ -283,7 +284,7 @@ pub enum Inst {
/// An instruction that will always trigger the illegal instruction exception. /// An instruction that will always trigger the illegal instruction exception.
Ud2 { trap_info: (SourceLoc, TrapCode) }, Ud2 { trap_info: (SourceLoc, TrapCode) },
/// Loads an external symbol in a register, with a relocation. /// Loads an external symbol in a register, with a relocation: movabsq $name, dst
LoadExtName { LoadExtName {
dst: Writable<Reg>, dst: Writable<Reg>,
name: Box<ExternalName>, name: Box<ExternalName>,
@@ -326,15 +327,15 @@ impl Inst {
} }
} }
pub(crate) fn read_only_gpr_rm_r( pub(crate) fn unary_rm_r(
size: u8, size: u8,
op: ReadOnlyGprRmROpcode, op: UnaryRmROpcode,
src: RegMem, src: RegMem,
dst: Writable<Reg>, dst: Writable<Reg>,
) -> Self { ) -> Self {
debug_assert!(dst.to_reg().get_class() == RegClass::I64); debug_assert!(dst.to_reg().get_class() == RegClass::I64);
debug_assert!(size == 8 || size == 4 || size == 2); debug_assert!(size == 8 || size == 4 || size == 2);
Self::ReadOnly_Gpr_Rm_R { size, op, src, dst } Self::UnaryRmR { size, op, src, dst }
} }
pub(crate) fn div(size: u8, signed: bool, divisor: RegMem, loc: SourceLoc) -> Inst { pub(crate) fn div(size: u8, signed: bool, divisor: RegMem, loc: SourceLoc) -> Inst {
@@ -667,7 +668,7 @@ impl ShowWithRRU for Inst {
show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)), show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)),
), ),
Inst::ReadOnly_Gpr_Rm_R { src, dst, op, size } => format!( Inst::UnaryRmR { src, dst, op, size } => format!(
"{} {}, {}", "{} {}, {}",
ljustify2(op.to_string(), suffixBWLQ(*size)), ljustify2(op.to_string(), suffixBWLQ(*size)),
src.show_rru_sized(mb_rru, *size), src.show_rru_sized(mb_rru, *size),
@@ -700,15 +701,18 @@ impl ShowWithRRU for Inst {
rhs.show_rru_sized(mb_rru, *size) rhs.show_rru_sized(mb_rru, *size)
), ),
Inst::CheckedDivOrRemSeq { Inst::CheckedDivOrRemSeq {
is_div, kind,
is_signed,
size, size,
divisor, divisor,
.. ..
} => format!( } => format!(
"{}{} $rax:$rdx, {}", "{} $rax:$rdx, {}",
if *is_signed { "s" } else { "u" }, match kind {
if *is_div { "div " } else { "rem " }, DivOrRemKind::SignedDiv => "sdiv",
DivOrRemKind::UnsignedDiv => "udiv",
DivOrRemKind::SignedRem => "srem",
DivOrRemKind::UnsignedRem => "urem",
},
show_ireg_sized(*divisor, mb_rru, *size), show_ireg_sized(*divisor, mb_rru, *size),
), ),
Inst::SignExtendRaxRdx { size } => match size { Inst::SignExtendRaxRdx { size } => match size {
@@ -942,7 +946,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_use(regs::rax()); collector.add_use(regs::rax());
collector.add_mod(Writable::from_reg(regs::rdx())); collector.add_mod(Writable::from_reg(regs::rdx()));
} }
Inst::ReadOnly_Gpr_Rm_R { src, dst, .. } | Inst::XMM_Mov_RM_R { src, dst, .. } => { Inst::UnaryRmR { src, dst, .. } | Inst::XMM_Mov_RM_R { src, dst, .. } => {
src.get_regs_as_uses(collector); src.get_regs_as_uses(collector);
collector.add_def(*dst); collector.add_def(*dst);
} }
@@ -1141,7 +1145,7 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
ref mut dst, ref mut dst,
.. ..
} }
| Inst::ReadOnly_Gpr_Rm_R { | Inst::UnaryRmR {
ref mut src, ref mut src,
ref mut dst, ref mut dst,
.. ..

127
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -128,16 +128,16 @@ fn input_to_reg(ctx: Ctx, spec: InsnInput) -> Reg {
} }
enum ExtSpec { enum ExtSpec {
ZeroExtend32, ZeroExtendTo32,
ZeroExtend64, ZeroExtendTo64,
SignExtend32, SignExtendTo32,
SignExtend64, SignExtendTo64,
} }
fn extend_input_to_reg(ctx: Ctx, spec: InsnInput, ext_spec: ExtSpec) -> Reg { fn extend_input_to_reg(ctx: Ctx, spec: InsnInput, ext_spec: ExtSpec) -> Reg {
let requested_size = match ext_spec { let requested_size = match ext_spec {
ExtSpec::ZeroExtend32 | ExtSpec::SignExtend32 => 32, ExtSpec::ZeroExtendTo32 | ExtSpec::SignExtendTo32 => 32,
ExtSpec::ZeroExtend64 | ExtSpec::SignExtend64 => 64, ExtSpec::ZeroExtendTo64 | ExtSpec::SignExtendTo64 => 64,
}; };
let input_size = ctx.input_ty(spec.insn, spec.input).bits(); let input_size = ctx.input_ty(spec.insn, spec.input).bits();
@@ -156,12 +156,12 @@ fn extend_input_to_reg(ctx: Ctx, spec: InsnInput, ext_spec: ExtSpec) -> Reg {
let src = input_to_reg_mem(ctx, spec); let src = input_to_reg_mem(ctx, spec);
let dst = ctx.alloc_tmp(RegClass::I64, requested_ty); let dst = ctx.alloc_tmp(RegClass::I64, requested_ty);
match ext_spec { match ext_spec {
ExtSpec::ZeroExtend32 | ExtSpec::ZeroExtend64 => { ExtSpec::ZeroExtendTo32 | ExtSpec::ZeroExtendTo64 => {
ctx.emit(Inst::movzx_rm_r( ctx.emit(Inst::movzx_rm_r(
ext_mode, src, dst, /* infallible */ None, ext_mode, src, dst, /* infallible */ None,
)) ))
} }
ExtSpec::SignExtend32 | ExtSpec::SignExtend64 => { ExtSpec::SignExtendTo32 | ExtSpec::SignExtendTo64 => {
ctx.emit(Inst::movsx_rm_r( ctx.emit(Inst::movsx_rm_r(
ext_mode, src, dst, /* infallible */ None, ext_mode, src, dst, /* infallible */ None,
)) ))
@@ -292,9 +292,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let dst = output_to_reg(ctx, outputs[0]); let dst = output_to_reg(ctx, outputs[0]);
let shift_kind = match op { let shift_kind = match op {
Opcode::Ishl => ShiftKind::Left, Opcode::Ishl => ShiftKind::ShiftLeft,
Opcode::Ushr => ShiftKind::RightZ, Opcode::Ushr => ShiftKind::ShiftRightLogical,
Opcode::Sshr => ShiftKind::RightS, Opcode::Sshr => ShiftKind::ShiftRightArithmetic,
Opcode::Rotl => ShiftKind::RotateLeft, Opcode::Rotl => ShiftKind::RotateLeft,
Opcode::Rotr => ShiftKind::RotateRight, Opcode::Rotr => ShiftKind::RotateRight,
_ => unreachable!(), _ => unreachable!(),
@@ -320,7 +320,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// sub %tmp, %dst // sub %tmp, %dst
let (ext_spec, ty) = match ctx.input_ty(insn, 0) { let (ext_spec, ty) = match ctx.input_ty(insn, 0) {
I8 | I16 => (Some(ExtSpec::ZeroExtend32), I32), I8 | I16 => (Some(ExtSpec::ZeroExtendTo32), I32),
a if a == I32 || a == I64 => (None, a), a if a == I32 || a == I64 => (None, a),
_ => unreachable!(), _ => unreachable!(),
}; };
@@ -335,9 +335,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let tmp = ctx.alloc_tmp(RegClass::I64, ty); let tmp = ctx.alloc_tmp(RegClass::I64, ty);
ctx.emit(Inst::imm_r(ty == I64, u64::max_value(), dst)); ctx.emit(Inst::imm_r(ty == I64, u64::max_value(), dst));
ctx.emit(Inst::read_only_gpr_rm_r( ctx.emit(Inst::unary_rm_r(
ty.bytes() as u8, ty.bytes() as u8,
ReadOnlyGprRmROpcode::Bsr, UnaryRmROpcode::Bsr,
src, src,
tmp, tmp,
)); ));
@@ -376,9 +376,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let tmp = ctx.alloc_tmp(RegClass::I64, ty); let tmp = ctx.alloc_tmp(RegClass::I64, ty);
ctx.emit(Inst::imm_r(false /* 64 bits */, ty.bits() as u64, tmp)); ctx.emit(Inst::imm_r(false /* 64 bits */, ty.bits() as u64, tmp));
ctx.emit(Inst::read_only_gpr_rm_r( ctx.emit(Inst::unary_rm_r(
ty.bytes() as u8, ty.bytes() as u8,
ReadOnlyGprRmROpcode::Bsf, UnaryRmROpcode::Bsf,
src, src,
dst, dst,
)); ));
@@ -395,7 +395,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// TODO when the x86 flags have use_popcnt, we can use the popcnt instruction. // TODO when the x86 flags have use_popcnt, we can use the popcnt instruction.
let (ext_spec, ty) = match ctx.input_ty(insn, 0) { let (ext_spec, ty) = match ctx.input_ty(insn, 0) {
I8 | I16 => (Some(ExtSpec::ZeroExtend32), I32), I8 | I16 => (Some(ExtSpec::ZeroExtendTo32), I32),
a if a == I32 || a == I64 => (None, a), a if a == I32 || a == I64 => (None, a),
_ => unreachable!(), _ => unreachable!(),
}; };
@@ -418,7 +418,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None)); ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None));
// shr $1, tmp1 // shr $1, tmp1
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(1),
tmp1,
));
// mov 0x7777_7777_7777_7777, cst // mov 0x7777_7777_7777_7777, cst
ctx.emit(Inst::imm_r(is_64, 0x7777777777777777, cst)); ctx.emit(Inst::imm_r(is_64, 0x7777777777777777, cst));
@@ -443,7 +448,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
// shr $1, tmp1 // shr $1, tmp1
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(1),
tmp1,
));
// and cst, tmp1 // and cst, tmp1
ctx.emit(Inst::alu_rmi_r( ctx.emit(Inst::alu_rmi_r(
@@ -462,7 +472,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
// shr $1, tmp1 // shr $1, tmp1
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(1),
tmp1,
));
// and cst, tmp1 // and cst, tmp1
ctx.emit(Inst::alu_rmi_r( ctx.emit(Inst::alu_rmi_r(
@@ -484,7 +499,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None));
// shr $4, dst // shr $4, dst
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(4), dst)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(4),
dst,
));
// add tmp2, dst // add tmp2, dst
ctx.emit(Inst::alu_rmi_r( ctx.emit(Inst::alu_rmi_r(
@@ -517,9 +537,14 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
// shr $56, dst // shr $56, dst
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(56), dst)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(56),
dst,
));
} else { } else {
debug_assert_eq!(ty, I32); assert_eq!(ty, I32);
let is_64 = false; let is_64 = false;
let tmp1 = ctx.alloc_tmp(RegClass::I64, I64); let tmp1 = ctx.alloc_tmp(RegClass::I64, I64);
@@ -529,7 +554,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None)); ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None));
// shr $1, tmp1 // shr $1, tmp1
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(1),
tmp1,
));
// andq $0x7777_7777, tmp1 // andq $0x7777_7777, tmp1
ctx.emit(Inst::alu_rmi_r( ctx.emit(Inst::alu_rmi_r(
@@ -551,7 +581,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
// shr $1, tmp1 // shr $1, tmp1
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(1),
tmp1,
));
// and 0x7777_7777, tmp1 // and 0x7777_7777, tmp1
ctx.emit(Inst::alu_rmi_r( ctx.emit(Inst::alu_rmi_r(
@@ -570,7 +605,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
// shr $1, tmp1 // shr $1, tmp1
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(1),
tmp1,
));
// and $0x7777_7777, tmp1 // and $0x7777_7777, tmp1
ctx.emit(Inst::alu_rmi_r( ctx.emit(Inst::alu_rmi_r(
@@ -592,7 +632,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None));
// shr $4, dst // shr $4, dst
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(4), dst)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(4),
dst,
));
// add tmp2, dst // add tmp2, dst
ctx.emit(Inst::alu_rmi_r( ctx.emit(Inst::alu_rmi_r(
@@ -619,7 +664,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)); ));
// shr $24, dst // shr $24, dst
ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(24), dst)); ctx.emit(Inst::shift_r(
is_64,
ShiftKind::ShiftRightLogical,
Some(24),
dst,
));
} }
} }
@@ -1085,8 +1135,14 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
Opcode::Udiv | Opcode::Urem | Opcode::Sdiv | Opcode::Srem => { Opcode::Udiv | Opcode::Urem | Opcode::Sdiv | Opcode::Srem => {
let is_div = op == Opcode::Udiv || op == Opcode::Sdiv; let kind = match op {
let is_signed = op == Opcode::Sdiv || op == Opcode::Srem; Opcode::Udiv => DivOrRemKind::UnsignedDiv,
Opcode::Sdiv => DivOrRemKind::SignedDiv,
Opcode::Urem => DivOrRemKind::UnsignedRem,
Opcode::Srem => DivOrRemKind::SignedRem,
_ => unreachable!(),
};
let is_div = kind.is_div();
let input_ty = ctx.input_ty(insn, 0); let input_ty = ctx.input_ty(insn, 0);
let size = input_ty.bytes() as u8; let size = input_ty.bytes() as u8;
@@ -1106,7 +1162,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
// pc-relative offsets that must not change, thus requiring regalloc to not // pc-relative offsets that must not change, thus requiring regalloc to not
// interfere by introducing spills and reloads. // interfere by introducing spills and reloads.
// //
// Note it keeps the result in $rax (if is_div) or $rdx (if !is_div), so that // Note it keeps the result in $rax (for divide) or $rdx (for rem), so that
// regalloc is aware of the coalescing opportunity between rax/rdx and the // regalloc is aware of the coalescing opportunity between rax/rdx and the
// destination register. // destination register.
let divisor = input_to_reg(ctx, inputs[1]); let divisor = input_to_reg(ctx, inputs[1]);
@@ -1117,8 +1173,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}; };
ctx.emit(Inst::imm_r(true, 0, Writable::from_reg(regs::rdx()))); ctx.emit(Inst::imm_r(true, 0, Writable::from_reg(regs::rdx())));
ctx.emit(Inst::CheckedDivOrRemSeq { ctx.emit(Inst::CheckedDivOrRemSeq {
is_div, kind,
is_signed,
size, size,
divisor, divisor,
tmp, tmp,
@@ -1128,7 +1183,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
let divisor = input_to_reg_mem(ctx, inputs[1]); let divisor = input_to_reg_mem(ctx, inputs[1]);
// Fill in the high parts: // Fill in the high parts:
if is_signed { if kind.is_signed() {
// sign-extend the sign-bit of rax into rdx, for signed opcodes. // sign-extend the sign-bit of rax into rdx, for signed opcodes.
ctx.emit(Inst::sign_extend_rax_to_rdx(size)); ctx.emit(Inst::sign_extend_rax_to_rdx(size));
} else { } else {
@@ -1141,7 +1196,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
} }
// Emit the actual idiv. // Emit the actual idiv.
ctx.emit(Inst::div(size, is_signed, divisor, ctx.srcloc(insn))); ctx.emit(Inst::div(size, kind.is_signed(), divisor, ctx.srcloc(insn)));
} }
// Move the result back into the destination reg. // Move the result back into the destination reg.
@@ -1346,7 +1401,7 @@ impl LowerBackend for X64Backend {
insn: branches[0], insn: branches[0],
input: 0, input: 0,
}, },
ExtSpec::ZeroExtend32, ExtSpec::ZeroExtendTo32,
); );
// Bounds-check (compute flags from idx - jt_size) and branch to default. // Bounds-check (compute flags from idx - jt_size) and branch to default.