T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x64: clean up regalloc-related semantics on several instructions. (#4811)

Browse Source 
* x64: clean up regalloc-related semantics on several instructions.

This PR removes all uses of "modify" operands on instructions in the x64
backend, and also removes all uses of "pinned vregs", or vregs that are
explicitly tied to particular physical registers. In place of both of
these mechanisms, which are legacies of the old regalloc design and
supported via compatibility code, the backend now uses operand
constraints. This is more flexible as it allows the regalloc to see the
liveranges and constraints without "reverse-engineering" move instructions.

Eventually, after removing all such uses (including in other backends
and by the ABI code), we can remove the compatibility code in regalloc2,
significantly simplifying its liverange-construction frontend and
thus allowing for higher confidence in correctness as well as possibly a
bit more compilation speed.

Curiously, there are a few extra move instructions now; they are likely
poor splitting decisions and I can try to chase these down later.

* Fix cranelift-codegen tests.

* Review feedback.
This commit is contained in:
Chris Fallin
2022-08-30 17:21:14 -07:00
committed by GitHub
parent 3ce3eeb668
commit 186c7c3b89
14 changed files with 543 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -377,11 +377,11 @@ pub(crate) fn emit(
} => {
let dividend_lo = allocs.next(dividend_lo.to_reg());
let dst_quotient = allocs.next(dst_quotient.to_reg().to_reg());
let dst_remainder = allocs.next(dst_remainder.to_reg().to_reg());
debug_assert_eq!(dividend_lo, regs::rax());
debug_assert_eq!(dst_quotient, regs::rax());
debug_assert_eq!(dst_remainder, regs::rdx());
if size.to_bits() > 8 {
let dst_remainder = allocs.next(dst_remainder.to_reg().to_reg());
debug_assert_eq!(dst_remainder, regs::rdx());
let dividend_hi = allocs.next(dividend_hi.to_reg());
debug_assert_eq!(dividend_hi, regs::rdx());
}
@@ -468,7 +468,11 @@ pub(crate) fn emit(
let src = allocs.next(src.to_reg());
let dst = allocs.next(dst.to_reg().to_reg());
debug_assert_eq!(src, regs::rax());
debug_assert_eq!(dst, regs::rdx());
if *size == OperandSize::Size8 {
debug_assert_eq!(dst, regs::rax());
} else {
debug_assert_eq!(dst, regs::rdx());
}
match size {
OperandSize::Size8 => {
sink.put1(0x66);
@@ -498,7 +502,7 @@ pub(crate) fn emit(
} => {
let dividend_lo = allocs.next(dividend_lo.to_reg());
let dividend_hi = allocs.next(dividend_hi.to_reg());
let divisor = allocs.next(divisor.to_reg().to_reg());
let divisor = allocs.next(divisor.to_reg());
let dst_quotient = allocs.next(dst_quotient.to_reg().to_reg());
let dst_remainder = allocs.next(dst_remainder.to_reg().to_reg());
let tmp = tmp.map(|tmp| allocs.next(tmp.to_reg().to_reg()));
@@ -597,18 +601,45 @@ pub(crate) fn emit(
sink.bind_label(do_op);
}
let dividend_lo = Gpr::new(regs::rax()).unwrap();
let dst_quotient = WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap());
let (dividend_hi, dst_remainder) = if *size == OperandSize::Size8 {
(
Gpr::new(regs::rax()).unwrap(),
Writable::from_reg(Gpr::new(regs::rax()).unwrap()),
)
} else {
(
Gpr::new(regs::rdx()).unwrap(),
Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
)
};
// Fill in the high parts:
if kind.is_signed() {
// sign-extend the sign-bit of rax into rdx, for signed opcodes.
let inst = Inst::sign_extend_data(*size);
let inst =
Inst::sign_extend_data(*size, dividend_lo, WritableGpr::from_reg(dividend_hi));
inst.emit(&[], sink, info, state);
} else {
} else if *size != OperandSize::Size8 {
// zero for unsigned opcodes.
let inst = Inst::imm(OperandSize::Size64, 0, Writable::from_reg(regs::rdx()));
let inst = Inst::imm(
OperandSize::Size64,
0,
Writable::from_reg(dividend_hi.to_reg()),
);
inst.emit(&[], sink, info, state);
}
let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(divisor));
let inst = Inst::div(
*size,
kind.is_signed(),
RegMem::reg(divisor),
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
);
inst.emit(&[], sink, info, state);
// Lowering takes care of moving the result back into the right register, see comment
@@ -1393,7 +1424,8 @@ pub(crate) fn emit(
// ;; generated by lowering: cmp #jmp_table_size, %idx
// jnb $default_target
// movl %idx, %tmp2
// cmovnb %tmp1, %tmp2 ;; Spectre mitigation; we require tmp1 to be zero on entry.
// mov $0, %tmp1
// cmovnb %tmp1, %tmp2 ;; Spectre mitigation.
// lea start_of_jump_table_offset(%rip), %tmp1
// movslq [%tmp1, %tmp2, 4], %tmp2 ;; shift of 2, viz. multiply index by 4
// addq %tmp2, %tmp1
@@ -1406,6 +1438,13 @@ pub(crate) fn emit(
let inst = Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(idx), tmp2);
inst.emit(&[], sink, info, state);
// Zero `tmp1` to overwrite `tmp2` with zeroes on the
// out-of-bounds case (Spectre mitigation using CMOV).
// Note that we need to do this with a move-immediate
// form, because we cannot clobber the flags.
let inst = Inst::imm(OperandSize::Size32, 0, tmp1);
inst.emit(&[], sink, info, state);
// Spectre mitigation: CMOV to zero the index if the out-of-bounds branch above misspeculated.
let inst = Inst::cmove(
OperandSize::Size64,
@@ -1768,9 +1807,21 @@ pub(crate) fn emit(
src1,
src2,
dst,
}
| Inst::XmmRmREvex3 {
op,
src1,
src2,
dst,
// `dst` reuses `src3`.
..
} => {
let dst = allocs.next(dst.to_reg().to_reg());
let src2 = allocs.next(src2.to_reg());
if let Inst::XmmRmREvex3 { src3, .. } = inst {
let src3 = allocs.next(src3.to_reg());
debug_assert_eq!(src3, dst);
}
let src1 = src1.clone().to_reg_mem().with_allocs(allocs);
let (w, opcode) = match op {
@@ -2086,7 +2137,7 @@ pub(crate) fn emit(
tmp_gpr1,
tmp_gpr2,
} => {
let src = allocs.next(src.to_reg().to_reg());
let src = allocs.next(src.to_reg());
let dst = allocs.next(dst.to_reg().to_reg());
let tmp_gpr1 = allocs.next(tmp_gpr1.to_reg().to_reg());
let tmp_gpr2 = allocs.next(tmp_gpr2.to_reg().to_reg());
@@ -2155,7 +2206,7 @@ pub(crate) fn emit(
let inst = Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightLogical,
Some(1),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 1 }).unwrap(),
Writable::from_reg(tmp_gpr1),
);
inst.emit(&[], sink, info, state);
@@ -2208,7 +2259,7 @@ pub(crate) fn emit(
tmp_gpr,
tmp_xmm,
} => {
let src = allocs.next(src.to_reg().to_reg());
let src = allocs.next(src.to_reg());
let dst = allocs.next(dst.to_reg().to_reg());
let tmp_gpr = allocs.next(tmp_gpr.to_reg().to_reg());
let tmp_xmm = allocs.next(tmp_xmm.to_reg().to_reg());
@@ -2417,7 +2468,7 @@ pub(crate) fn emit(
tmp_gpr,
tmp_xmm,
} => {
let src = allocs.next(src.to_reg().to_reg());
let src = allocs.next(src.to_reg());
let dst = allocs.next(dst.to_reg().to_reg());
let tmp_gpr = allocs.next(tmp_gpr.to_reg().to_reg());
let tmp_xmm = allocs.next(tmp_xmm.to_reg().to_reg());

216
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

@@ -1723,6 +1723,10 @@ fn test_x64_emit() {
OperandSize::Size32,
true, /*signed*/
RegMem::reg(regs::rsi()),
Gpr::new(regs::rax()).unwrap(),
Gpr::new(regs::rdx()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"F7FE",
"idiv %eax, %edx, %esi, %eax, %edx",
@@ -1732,6 +1736,10 @@ fn test_x64_emit() {
OperandSize::Size64,
true, /*signed*/
RegMem::reg(regs::r15()),
Gpr::new(regs::rax()).unwrap(),
Gpr::new(regs::rdx()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"49F7FF",
"idiv %rax, %rdx, %r15, %rax, %rdx",
@@ -1741,6 +1749,10 @@ fn test_x64_emit() {
OperandSize::Size32,
false, /*signed*/
RegMem::reg(regs::r14()),
Gpr::new(regs::rax()).unwrap(),
Gpr::new(regs::rdx()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"41F7F6",
"div %eax, %edx, %r14d, %eax, %edx",
@@ -1750,19 +1762,39 @@ fn test_x64_emit() {
OperandSize::Size64,
false, /*signed*/
RegMem::reg(regs::rdi()),
Gpr::new(regs::rax()).unwrap(),
Gpr::new(regs::rdx()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"48F7F7",
"div %rax, %rdx, %rdi, %rax, %rdx",
));
insns.push((
Inst::div(OperandSize::Size8, false, RegMem::reg(regs::rax())),
Inst::div(
OperandSize::Size8,
false,
RegMem::reg(regs::rax()),
Gpr::new(regs::rax()).unwrap(),
Gpr::new(regs::rdx()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"F6F0",
"div %al, (none), %al, %al, %dl",
"div %al, (none), %al, %al, (none)",
));
insns.push((
Inst::div(OperandSize::Size8, false, RegMem::reg(regs::rsi())),
Inst::div(
OperandSize::Size8,
false,
RegMem::reg(regs::rsi()),
Gpr::new(regs::rax()).unwrap(),
Gpr::new(regs::rdx()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"40F6F6",
"div %al, (none), %sil, %al, %dl",
"div %al, (none), %sil, %al, (none)",
));
// ========================================================
@@ -1807,25 +1839,41 @@ fn test_x64_emit() {
// ========================================================
// cbw
insns.push((
Inst::sign_extend_data(OperandSize::Size8),
Inst::sign_extend_data(
OperandSize::Size8,
Gpr::new(regs::rax()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
),
"6698",
"cbw %al, %dl",
"cbw %al, %al",
));
// ========================================================
// cdq family: SignExtendRaxRdx
insns.push((
Inst::sign_extend_data(OperandSize::Size16),
Inst::sign_extend_data(
OperandSize::Size16,
Gpr::new(regs::rax()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"6699",
"cwd %ax, %dx",
));
insns.push((
Inst::sign_extend_data(OperandSize::Size32),
Inst::sign_extend_data(
OperandSize::Size32,
Gpr::new(regs::rax()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"99",
"cdq %eax, %edx",
));
insns.push((
Inst::sign_extend_data(OperandSize::Size64),
Inst::sign_extend_data(
OperandSize::Size64,
Gpr::new(regs::rax()).unwrap(),
WritableGpr::from_reg(Gpr::new(regs::rdx()).unwrap()),
),
"4899",
"cqo %rax, %rdx",
));
@@ -2813,47 +2861,92 @@ fn test_x64_emit() {
// ========================================================
// Shift_R
insns.push((
Inst::shift_r(OperandSize::Size32, ShiftKind::ShiftLeft, None, w_rdi),
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rdi,
),
"D3E7",
"shll %cl, %edi, %edi",
));
insns.push((
Inst::shift_r(OperandSize::Size32, ShiftKind::ShiftLeft, None, w_r12),
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_r12,
),
"41D3E4",
"shll %cl, %r12d, %r12d",
));
insns.push((
Inst::shift_r(OperandSize::Size32, ShiftKind::ShiftLeft, Some(2), w_r8),
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
w_r8,
),
"41C1E002",
"shll $2, %r8d, %r8d",
));
insns.push((
Inst::shift_r(OperandSize::Size32, ShiftKind::ShiftLeft, Some(31), w_r13),
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 31 }).unwrap(),
w_r13,
),
"41C1E51F",
"shll $31, %r13d, %r13d",
));
insns.push((
Inst::shift_r(OperandSize::Size64, ShiftKind::ShiftLeft, None, w_r13),
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_r13,
),
"49D3E5",
"shlq %cl, %r13, %r13",
));
insns.push((
Inst::shift_r(OperandSize::Size64, ShiftKind::ShiftLeft, None, w_rdi),
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rdi,
),
"48D3E7",
"shlq %cl, %rdi, %rdi",
));
insns.push((
Inst::shift_r(OperandSize::Size64, ShiftKind::ShiftLeft, Some(2), w_r8),
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
w_r8,
),
"49C1E002",
"shlq $2, %r8, %r8",
));
insns.push((
Inst::shift_r(OperandSize::Size64, ShiftKind::ShiftLeft, Some(3), w_rbx),
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 3 }).unwrap(),
w_rbx,
),
"48C1E303",
"shlq $3, %rbx, %rbx",
));
insns.push((
Inst::shift_r(OperandSize::Size64, ShiftKind::ShiftLeft, Some(63), w_r13),
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 63 }).unwrap(),
w_r13,
),
"49C1E53F",
"shlq $63, %r13, %r13",
));
@@ -2861,7 +2954,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftRightLogical,
None,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rdi,
),
"D3EF",
@@ -2871,7 +2964,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftRightLogical,
Some(2),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
w_r8,
),
"41C1E802",
@@ -2881,7 +2974,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftRightLogical,
Some(31),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 31 }).unwrap(),
w_r13,
),
"41C1ED1F",
@@ -2891,7 +2984,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightLogical,
None,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rdi,
),
"48D3EF",
@@ -2901,7 +2994,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightLogical,
Some(2),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
w_r8,
),
"49C1E802",
@@ -2911,7 +3004,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightLogical,
Some(63),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 63 }).unwrap(),
w_r13,
),
"49C1ED3F",
@@ -2921,7 +3014,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftRightArithmetic,
None,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rdi,
),
"D3FF",
@@ -2931,7 +3024,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftRightArithmetic,
Some(2),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
w_r8,
),
"41C1F802",
@@ -2941,7 +3034,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size32,
ShiftKind::ShiftRightArithmetic,
Some(31),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 31 }).unwrap(),
w_r13,
),
"41C1FD1F",
@@ -2951,7 +3044,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightArithmetic,
None,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rdi,
),
"48D3FF",
@@ -2961,7 +3054,7 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightArithmetic,
Some(2),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
w_r8,
),
"49C1F802",
@@ -2971,54 +3064,99 @@ fn test_x64_emit() {
Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightArithmetic,
Some(63),
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 63 }).unwrap(),
w_r13,
),
"49C1FD3F",
"sarq $63, %r13, %r13",
));
insns.push((
Inst::shift_r(OperandSize::Size64, ShiftKind::RotateLeft, None, w_r8),
Inst::shift_r(
OperandSize::Size64,
ShiftKind::RotateLeft,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_r8,
),
"49D3C0",
"rolq %cl, %r8, %r8",
));
insns.push((
Inst::shift_r(OperandSize::Size32, ShiftKind::RotateLeft, Some(3), w_r9),
Inst::shift_r(
OperandSize::Size32,
ShiftKind::RotateLeft,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 3 }).unwrap(),
w_r9,
),
"41C1C103",
"roll $3, %r9d, %r9d",
));
insns.push((
Inst::shift_r(OperandSize::Size32, ShiftKind::RotateRight, None, w_rsi),
Inst::shift_r(
OperandSize::Size32,
ShiftKind::RotateRight,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rsi,
),
"D3CE",
"rorl %cl, %esi, %esi",
));
insns.push((
Inst::shift_r(OperandSize::Size64, ShiftKind::RotateRight, Some(5), w_r15),
Inst::shift_r(
OperandSize::Size64,
ShiftKind::RotateRight,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 5 }).unwrap(),
w_r15,
),
"49C1CF05",
"rorq $5, %r15, %r15",
));
insns.push((
Inst::shift_r(OperandSize::Size8, ShiftKind::RotateRight, None, w_rsi),
Inst::shift_r(
OperandSize::Size8,
ShiftKind::RotateRight,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rsi,
),
"40D2CE",
"rorb %cl, %sil, %sil",
));
insns.push((
Inst::shift_r(OperandSize::Size8, ShiftKind::RotateRight, None, w_rax),
Inst::shift_r(
OperandSize::Size8,
ShiftKind::RotateRight,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rax,
),
"D2C8",
"rorb %cl, %al, %al",
));
insns.push((
Inst::shift_r(OperandSize::Size8, ShiftKind::RotateRight, Some(5), w_r15),
Inst::shift_r(
OperandSize::Size8,
ShiftKind::RotateRight,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 5 }).unwrap(),
w_r15,
),
"41C0CF05",
"rorb $5, %r15b, %r15b",
));
insns.push((
Inst::shift_r(OperandSize::Size16, ShiftKind::RotateRight, None, w_rsi),
Inst::shift_r(
OperandSize::Size16,
ShiftKind::RotateRight,
Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
w_rsi,
),
"66D3CE",
"rorw %cl, %si, %si",
));
insns.push((
Inst::shift_r(OperandSize::Size16, ShiftKind::RotateRight, Some(5), w_r15),
Inst::shift_r(
OperandSize::Size16,
ShiftKind::RotateRight,
Imm8Gpr::new(Imm8Reg::Imm8 { imm: 5 }).unwrap(),
w_r15,
),
"6641C1CF05",
"rorw $5, %r15w, %r15w",
));

149
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -131,7 +131,9 @@ impl Inst {
| Inst::XmmToGpr { op, .. }
| Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()],
Inst::XmmUnaryRmREvex { op, .. } | Inst::XmmRmREvex { op, .. } => op.available_from(),
Inst::XmmUnaryRmREvex { op, .. }
| Inst::XmmRmREvex { op, .. }
| Inst::XmmRmREvex3 { op, .. } => op.available_from(),
Inst::XmmRmRVex { op, .. } => op.available_from(),
}
@@ -195,47 +197,55 @@ impl Inst {
}
}
pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst {
pub(crate) fn div(
size: OperandSize,
signed: bool,
divisor: RegMem,
dividend_lo: Gpr,
dividend_hi: Gpr,
dst_quotient: WritableGpr,
dst_remainder: WritableGpr,
) -> Inst {
divisor.assert_regclass_is(RegClass::Int);
Inst::Div {
size,
signed,
divisor: GprMem::new(divisor).unwrap(),
dividend_lo: Gpr::new(regs::rax()).unwrap(),
dividend_hi: Gpr::new(regs::rdx()).unwrap(),
dst_quotient: WritableGpr::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
}
}
pub(crate) fn checked_div_or_rem_seq(
kind: DivOrRemKind,
size: OperandSize,
divisor: Writable<Reg>,
divisor: Reg,
dividend_lo: Gpr,
dividend_hi: Gpr,
dst_quotient: WritableGpr,
dst_remainder: WritableGpr,
tmp: Option<Writable<Reg>>,
) -> Inst {
debug_assert!(divisor.to_reg().class() == RegClass::Int);
debug_assert!(divisor.class() == RegClass::Int);
debug_assert!(tmp
.map(|tmp| tmp.to_reg().class() == RegClass::Int)
.unwrap_or(true));
Inst::CheckedDivOrRemSeq {
kind,
size,
divisor: WritableGpr::from_writable_reg(divisor).unwrap(),
dividend_lo: Gpr::new(regs::rax()).unwrap(),
dividend_hi: Gpr::new(regs::rdx()).unwrap(),
dst_quotient: Writable::from_reg(Gpr::new(regs::rax()).unwrap()),
dst_remainder: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
divisor: Gpr::new(divisor).unwrap(),
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
tmp: tmp.map(|tmp| WritableGpr::from_writable_reg(tmp).unwrap()),
}
}
pub(crate) fn sign_extend_data(size: OperandSize) -> Inst {
Inst::SignExtendData {
size,
src: Gpr::new(regs::rax()).unwrap(),
dst: Writable::from_reg(Gpr::new(regs::rdx()).unwrap()),
}
pub(crate) fn sign_extend_data(size: OperandSize, src: Gpr, dst: WritableGpr) -> Inst {
Inst::SignExtendData { size, src, dst }
}
pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
@@ -415,24 +425,18 @@ impl Inst {
pub(crate) fn shift_r(
size: OperandSize,
kind: ShiftKind,
num_bits: Option<u8>,
num_bits: Imm8Gpr,
dst: Writable<Reg>,
) -> Inst {
debug_assert!(if let Some(num_bits) = num_bits {
num_bits < size.to_bits()
} else {
true
});
if let Imm8Reg::Imm8 { imm: num_bits } = num_bits.clone().to_imm8_reg() {
debug_assert!(num_bits < size.to_bits());
}
debug_assert!(dst.to_reg().class() == RegClass::Int);
Inst::ShiftR {
size,
kind,
src: Gpr::new(dst.to_reg()).unwrap(),
num_bits: Imm8Gpr::new(match num_bits {
Some(imm) => Imm8Reg::Imm8 { imm },
None => Imm8Reg::Reg { reg: regs::rcx() },
})
.unwrap(),
num_bits,
dst: WritableGpr::from_writable_reg(dst).unwrap(),
}
}
@@ -781,8 +785,11 @@ impl PrettyPrint for Inst {
let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes(), allocs);
let dst_quotient =
pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes(), allocs);
let dst_remainder =
pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes(), allocs);
let dst_remainder = if size.to_bits() > 8 {
pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes(), allocs)
} else {
"(none)".to_string()
};
let dividend_hi = if size.to_bits() > 8 {
pretty_print_reg(dividend_hi.to_reg(), size.to_bytes(), allocs)
} else {
@@ -842,7 +849,7 @@ impl PrettyPrint for Inst {
} => {
let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes(), allocs);
let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes(), allocs);
let divisor = pretty_print_reg(divisor.to_reg().to_reg(), size.to_bytes(), allocs);
let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes(), allocs);
let dst_quotient =
pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes(), allocs);
let dst_remainder =
@@ -949,12 +956,34 @@ impl PrettyPrint for Inst {
dst,
..
} => {
let src2 = pretty_print_reg(src2.to_reg(), 8, allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
let src2 = pretty_print_reg(src2.to_reg(), 8, allocs);
let src1 = src1.pretty_print(8, allocs);
format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
}
Inst::XmmRmREvex3 {
op,
src1,
src2,
src3,
dst,
..
} => {
let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
let src2 = pretty_print_reg(src2.to_reg(), 8, allocs);
let src3 = pretty_print_reg(src3.to_reg(), 8, allocs);
let src1 = src1.pretty_print(8, allocs);
format!(
"{} {}, {}, {}, {}",
ljustify(op.to_string()),
src1,
src2,
src3,
dst
)
}
Inst::XmmMinMaxSeq {
lhs,
rhs,
@@ -1084,7 +1113,7 @@ impl PrettyPrint for Inst {
tmp_gpr2,
..
} => {
let src = pretty_print_reg(src.to_reg().to_reg(), 8, allocs);
let src = pretty_print_reg(src.to_reg(), 8, allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8, allocs);
let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8, allocs);
@@ -1114,7 +1143,7 @@ impl PrettyPrint for Inst {
tmp_gpr,
is_saturating,
} => {
let src = pretty_print_reg(src.to_reg().to_reg(), src_size.to_bytes(), allocs);
let src = pretty_print_reg(src.to_reg(), src_size.to_bytes(), allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8, allocs);
let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8, allocs);
@@ -1142,7 +1171,7 @@ impl PrettyPrint for Inst {
tmp_xmm,
is_saturating,
} => {
let src = pretty_print_reg(src.to_reg().to_reg(), src_size.to_bytes(), allocs);
let src = pretty_print_reg(src.to_reg(), src_size.to_bytes(), allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8, allocs);
let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8, allocs);
@@ -1424,9 +1453,19 @@ impl PrettyPrint for Inst {
not_taken.to_string()
),
Inst::JmpTableSeq { idx, .. } => {
Inst::JmpTableSeq {
idx, tmp1, tmp2, ..
} => {
let idx = pretty_print_reg(*idx, 8, allocs);
format!("{} {}", ljustify("br_table".into()), idx)
let tmp1 = pretty_print_reg(tmp1.to_reg(), 8, allocs);
let tmp2 = pretty_print_reg(tmp2.to_reg(), 8, allocs);
format!(
"{} {}, {}, {}",
ljustify("br_table".into()),
idx,
tmp1,
tmp2
)
}
Inst::JmpUnknown { target } => {
@@ -1605,8 +1644,8 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
} => {
collector.reg_fixed_use(dividend_lo.to_reg(), regs::rax());
collector.reg_fixed_def(dst_quotient.to_writable_reg(), regs::rax());
collector.reg_fixed_def(dst_remainder.to_writable_reg(), regs::rdx());
if size.to_bits() > 8 {
collector.reg_fixed_def(dst_remainder.to_writable_reg(), regs::rdx());
collector.reg_fixed_use(dividend_hi.to_reg(), regs::rdx());
}
divisor.get_operands(collector);
@@ -1634,10 +1673,12 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
} => {
collector.reg_fixed_use(dividend_lo.to_reg(), regs::rax());
collector.reg_fixed_use(dividend_hi.to_reg(), regs::rdx());
collector.reg_mod(divisor.to_writable_reg());
collector.reg_use(divisor.to_reg());
collector.reg_fixed_def(dst_quotient.to_writable_reg(), regs::rax());
collector.reg_fixed_def(dst_remainder.to_writable_reg(), regs::rdx());
if let Some(tmp) = tmp {
// Early def so that the temporary register does not
// conflict with inputs or outputs.
collector.reg_early_def(tmp.to_writable_reg());
}
}
@@ -1718,13 +1759,25 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
dst,
..
} => {
match *op {
Avx512Opcode::Vpermi2b => collector.reg_mod(dst.to_writable_reg()),
_ => collector.reg_def(dst.to_writable_reg()),
}
assert_ne!(*op, Avx512Opcode::Vpermi2b);
collector.reg_def(dst.to_writable_reg());
collector.reg_use(src2.to_reg());
src1.get_operands(collector);
}
Inst::XmmRmREvex3 {
op,
src1,
src2,
src3,
dst,
..
} => {
assert_eq!(*op, Avx512Opcode::Vpermi2b);
collector.reg_reuse_def(dst.to_writable_reg(), 2); // Reuse `src3`.
collector.reg_use(src2.to_reg());
collector.reg_use(src3.to_reg());
src1.get_operands(collector);
}
Inst::XmmRmRImm {
op,
src1,
@@ -1795,7 +1848,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
tmp_gpr2,
..
} => {
collector.reg_mod(src.to_writable_reg());
collector.reg_use(src.to_reg());
collector.reg_def(dst.to_writable_reg());
collector.reg_early_def(tmp_gpr1.to_writable_reg());
collector.reg_early_def(tmp_gpr2.to_writable_reg());
@@ -1814,7 +1867,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
tmp_xmm,
..
} => {
collector.reg_mod(src.to_writable_reg());
collector.reg_use(src.to_reg());
collector.reg_def(dst.to_writable_reg());
collector.reg_early_def(tmp_gpr.to_writable_reg());
collector.reg_early_def(tmp_xmm.to_writable_reg());
@@ -1911,7 +1964,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
..
} => {
collector.reg_use(*idx);
collector.reg_mod(*tmp1);
collector.reg_early_def(*tmp1);
collector.reg_early_def(*tmp2);
}