T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch Cranelift over to regalloc2. (#3989)

Browse Source 
This PR switches Cranelift over to the new register allocator, regalloc2.

See [this document](https://gist.github.com/cfallin/08553421a91f150254fe878f67301801)
for a summary of the design changes. This switchover has implications for
core VCode/MachInst types and the lowering pass.

Overall, this change brings improvements to both compile time and speed of
generated code (runtime), as reported in #3942:

```
Benchmark       Compilation (wallclock)     Execution (wallclock)
blake3-scalar   25% faster                  28% faster
blake3-simd     no diff                     no diff
meshoptimizer   19% faster                  17% faster
pulldown-cmark  17% faster                  no diff
bz2             15% faster                  no diff
SpiderMonkey,   21% faster                  2% faster
  fib(30)
clang.wasm      42% faster                  N/A
```
This commit is contained in:
Chris Fallin
2022-04-14 10:28:21 -07:00
committed by GitHub
parent bfae6384aa
commit a0318f36f0
181 changed files with 16887 additions and 21587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

350
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -1,14 +1,13 @@
//! Instruction operand sub-components (aka "parts"): definitions and printing.
use super::regs::{self, show_ireg_sized};
use super::regs::{self};
use super::EmitState;
use crate::ir::condcodes::{FloatCC, IntCC};
use crate::ir::{MemFlags, Type};
use crate::isa::x64::inst::regs::pretty_print_reg;
use crate::isa::x64::inst::Inst;
use crate::machinst::*;
use regalloc::{
PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, Writable,
};
use regalloc2::VReg;
use smallvec::{smallvec, SmallVec};
use std::fmt;
use std::string::String;
@@ -23,25 +22,6 @@ pub trait FromWritableReg: Sized {
fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;
}
/// An extension trait for mapping register uses on `{Xmm,Gpr}`.
pub trait MapUseExt {
fn map_use<RM>(&mut self, mapper: &RM)
where
RM: RegMapper;
}
/// An extension trait for mapping register mods and defs on
/// `Writable{Xmm,Gpr}`.
pub trait MapDefModExt {
fn map_def<RM>(&mut self, mapper: &RM)
where
RM: RegMapper;
fn map_mod<RM>(&mut self, mapper: &RM)
where
RM: RegMapper;
}
/// A macro for defining a newtype of `Reg` that enforces some invariant about
/// the wrapped `Reg` (such as that it is of a particular register class).
macro_rules! newtype_of_reg {
@@ -55,7 +35,7 @@ macro_rules! newtype_of_reg {
|$check_reg:ident| $check:expr
) => {
/// A newtype wrapper around `Reg`.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct $newtype_reg(Reg);
impl PartialEq<Reg> for $newtype_reg {
@@ -70,12 +50,6 @@ macro_rules! newtype_of_reg {
}
}
impl PrettyPrint for $newtype_reg {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
self.0.show_rru(mb_rru)
}
}
impl $newtype_reg {
/// Create this newtype from the given register, or return `None` if the register
/// is not a valid instance of this newtype.
@@ -107,21 +81,6 @@ macro_rules! newtype_of_reg {
}
}
impl MapUseExt for $newtype_reg {
fn map_use<RM>(&mut self, mapper: &RM)
where
RM: RegMapper,
{
let mut reg = self.0;
mapper.map_use(&mut reg);
debug_assert!({
let $check_reg = reg;
$check
});
*self = $newtype_reg(reg);
}
}
pub type $newtype_writable_reg = Writable<$newtype_reg>;
#[allow(dead_code)] // Used by some newtypes and not others.
@@ -139,34 +98,6 @@ macro_rules! newtype_of_reg {
}
}
impl MapDefModExt for $newtype_writable_reg {
fn map_def<RM>(&mut self, mapper: &RM)
where
RM: RegMapper,
{
let mut reg = self.to_writable_reg();
mapper.map_def(&mut reg);
debug_assert!({
let $check_reg = reg.to_reg();
$check
});
*self = Writable::from_reg($newtype_reg(reg.to_reg()));
}
fn map_mod<RM>(&mut self, mapper: &RM)
where
RM: RegMapper,
{
let mut reg = self.to_writable_reg();
mapper.map_mod(&mut reg);
debug_assert!({
let $check_reg = reg.to_reg();
$check
});
*self = Writable::from_reg($newtype_reg(reg.to_reg()));
}
}
/// A newtype wrapper around `RegMem` for general-purpose registers.
#[derive(Clone, Debug)]
pub struct $newtype_reg_mem(RegMem);
@@ -201,44 +132,16 @@ macro_rules! newtype_of_reg {
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn map_uses<RM>(&mut self, mapper: &RM)
where
RM: RegMapper,
{
self.0.map_uses(mapper);
debug_assert!(match self.0 {
RegMem::Reg { reg: $check_reg } => $check,
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn map_as_def<RM>(&mut self, mapper: &RM)
where
RM: RegMapper,
{
self.0.map_as_def(mapper);
debug_assert!(match self.0 {
RegMem::Reg { reg: $check_reg } => $check,
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
self.0.get_regs_as_uses(collector);
pub fn get_operands<F: Fn(VReg) -> VReg>(
&self,
collector: &mut OperandCollector<'_, F>,
) {
self.0.get_operands(collector);
}
}
impl PrettyPrint for $newtype_reg_mem {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
self.0.show_rru(mb_rru)
}
}
impl PrettyPrintSized for $newtype_reg_mem {
fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String {
self.0.show_rru_sized(mb_rru, size)
fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
self.0.pretty_print(size, allocs)
}
}
@@ -278,44 +181,17 @@ macro_rules! newtype_of_reg {
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn map_uses<RM>(&mut self, mapper: &RM)
where
RM: RegMapper,
{
self.0.map_uses(mapper);
debug_assert!(match self.0 {
RegMemImm::Reg { reg: $check_reg } => $check,
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn map_as_def<RM>(&mut self, mapper: &RM)
where
RM: RegMapper,
{
self.0.map_as_def(mapper);
debug_assert!(match self.0 {
RegMemImm::Reg { reg: $check_reg } => $check,
_ => true,
});
}
#[allow(dead_code)] // Used by some newtypes and not others.
pub fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
self.0.get_regs_as_uses(collector);
pub fn get_operands<F: Fn(VReg) -> VReg>(
&self,
collector: &mut OperandCollector<'_, F>,
) {
self.0.get_operands(collector);
}
}
impl PrettyPrint for $newtype_reg_mem_imm {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
self.0.show_rru(mb_rru)
}
}
impl PrettyPrintSized for $newtype_reg_mem_imm {
fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String {
self.0.show_rru_sized(mb_rru, size)
fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
self.0.pretty_print(size, allocs)
}
}
@@ -359,7 +235,7 @@ newtype_of_reg!(
GprMem,
GprMemImm,
Imm8Gpr,
|reg| reg.get_class() == RegClass::I64
|reg| reg.class() == RegClass::Int
);
// Define a newtype of `Reg` for XMM registers.
@@ -370,7 +246,7 @@ newtype_of_reg!(
XmmMem,
XmmMemImm,
Imm8Xmm,
|reg| reg.get_class() == RegClass::V128
|reg| reg.class() == RegClass::Float
);
/// A possible addressing mode (amode) that can be used in instructions.
@@ -400,7 +276,7 @@ pub enum Amode {
impl Amode {
pub(crate) fn imm_reg(simm32: u32, base: Reg) -> Self {
debug_assert!(base.get_class() == RegClass::I64);
debug_assert!(base.class() == RegClass::Int);
Self::ImmReg {
simm32,
base,
@@ -409,8 +285,8 @@ impl Amode {
}
pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Self {
debug_assert!(base.get_class() == RegClass::I64);
debug_assert!(index.get_class() == RegClass::I64);
debug_assert!(base.class() == RegClass::Int);
debug_assert!(index.class() == RegClass::Int);
debug_assert!(shift <= 3);
Self::ImmRegRegShift {
simm32,
@@ -450,14 +326,17 @@ impl Amode {
}
/// Add the regs mentioned by `self` to `collector`.
pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
&self,
collector: &mut OperandCollector<'_, F>,
) {
match self {
Amode::ImmReg { base, .. } => {
collector.add_use(*base);
collector.reg_use(*base);
}
Amode::ImmRegRegShift { base, index, .. } => {
collector.add_use(base.to_reg());
collector.add_use(index.to_reg());
collector.reg_use(base.to_reg());
collector.reg_use(index.to_reg());
}
Amode::RipRelative { .. } => {
// RIP isn't involved in regalloc.
@@ -476,13 +355,56 @@ impl Amode {
pub(crate) fn can_trap(&self) -> bool {
!self.get_flags().notrap()
}
pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
// The order in which we consume allocs here must match the
// order in which we produce operands in get_operands() above.
match self {
&Amode::ImmReg {
simm32,
base,
flags,
} => Amode::ImmReg {
simm32,
flags,
base: allocs.next(base),
},
&Amode::ImmRegRegShift {
simm32,
base,
index,
shift,
flags,
} => Amode::ImmRegRegShift {
simm32,
shift,
flags,
base: Gpr::new(allocs.next(*base)).unwrap(),
index: Gpr::new(allocs.next(*index)).unwrap(),
},
&Amode::RipRelative { target } => Amode::RipRelative { target },
}
}
/// Offset the amode by a fixed offset.
pub(crate) fn offset(&self, offset: u32) -> Self {
let mut ret = self.clone();
match &mut ret {
&mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
&mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
_ => panic!("Cannot offset amode: {:?}", self),
}
ret
}
}
impl PrettyPrint for Amode {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self {
Amode::ImmReg { simm32, base, .. } => {
format!("{}({})", *simm32 as i32, base.show_rru(mb_rru))
// Note: size is always 8; the address is 64 bits,
// even if the addressed operand is smaller.
format!("{}({})", *simm32 as i32, pretty_print_reg(*base, 8, allocs))
}
Amode::ImmRegRegShift {
simm32,
@@ -493,8 +415,8 @@ impl PrettyPrint for Amode {
} => format!(
"{}({},{},{})",
*simm32 as i32,
base.show_rru(mb_rru),
index.show_rru(mb_rru),
pretty_print_reg(base.to_reg(), 8, allocs),
pretty_print_reg(index.to_reg(), 8, allocs),
1 << shift
),
Amode::RipRelative { ref target } => format!("label{}(%rip)", target.get()),
@@ -524,9 +446,12 @@ impl SyntheticAmode {
}
/// Add the regs mentioned by `self` to `collector`.
pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
&self,
collector: &mut OperandCollector<'_, F>,
) {
match self {
SyntheticAmode::Real(addr) => addr.get_regs_as_uses(collector),
SyntheticAmode::Real(addr) => addr.get_operands(collector),
SyntheticAmode::NominalSPOffset { .. } => {
// Nothing to do; the base is SP and isn't involved in regalloc.
}
@@ -534,16 +459,6 @@ impl SyntheticAmode {
}
}
pub(crate) fn map_uses<RM: RegMapper>(&mut self, map: &RM) {
match self {
SyntheticAmode::Real(addr) => addr.map_uses(map),
SyntheticAmode::NominalSPOffset { .. } => {
// Nothing to do.
}
SyntheticAmode::ConstantOffset(_) => {}
}
}
pub(crate) fn finalize(&self, state: &mut EmitState, buffer: &MachBuffer<Inst>) -> Amode {
match self {
SyntheticAmode::Real(addr) => addr.clone(),
@@ -561,6 +476,15 @@ impl SyntheticAmode {
}
}
}
pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
match self {
SyntheticAmode::Real(addr) => SyntheticAmode::Real(addr.with_allocs(allocs)),
&SyntheticAmode::NominalSPOffset { .. } | &SyntheticAmode::ConstantOffset { .. } => {
self.clone()
}
}
}
}
impl Into<SyntheticAmode> for Amode {
@@ -570,9 +494,10 @@ impl Into<SyntheticAmode> for Amode {
}
impl PrettyPrint for SyntheticAmode {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
fn pretty_print(&self, _size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self {
SyntheticAmode::Real(addr) => addr.show_rru(mb_rru),
// See note in `Amode` regarding constant size of `8`.
SyntheticAmode::Real(addr) => addr.pretty_print(8, allocs),
SyntheticAmode::NominalSPOffset { simm32 } => {
format!("rsp({} + virtual offset)", *simm32 as i32)
}
@@ -594,7 +519,7 @@ pub enum RegMemImm {
impl RegMemImm {
pub(crate) fn reg(reg: Reg) -> Self {
debug_assert!(reg.get_class() == RegClass::I64 || reg.get_class() == RegClass::V128);
debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
Self::Reg { reg }
}
pub(crate) fn mem(addr: impl Into<SyntheticAmode>) -> Self {
@@ -607,15 +532,18 @@ impl RegMemImm {
/// Asserts that in register mode, the reg class is the one that's expected.
pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
if let Self::Reg { reg } = self {
debug_assert_eq!(reg.get_class(), expected_reg_class);
debug_assert_eq!(reg.class(), expected_reg_class);
}
}
/// Add the regs mentioned by `self` to `collector`.
pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
&self,
collector: &mut OperandCollector<'_, F>,
) {
match self {
Self::Reg { reg } => collector.add_use(*reg),
Self::Mem { addr } => addr.get_regs_as_uses(collector),
Self::Reg { reg } => collector.reg_use(*reg),
Self::Mem { addr } => addr.get_operands(collector),
Self::Imm { .. } => {}
}
}
@@ -626,19 +554,25 @@ impl RegMemImm {
_ => None,
}
}
}
impl PrettyPrint for RegMemImm {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
self.show_rru_sized(mb_rru, 8)
pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
match self {
Self::Reg { reg } => Self::Reg {
reg: allocs.next(*reg),
},
Self::Mem { addr } => Self::Mem {
addr: addr.with_allocs(allocs),
},
Self::Imm { .. } => self.clone(),
}
}
}
impl PrettyPrintSized for RegMemImm {
fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String {
impl PrettyPrint for RegMemImm {
fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self {
Self::Reg { reg } => show_ireg_sized(*reg, mb_rru, size),
Self::Mem { addr } => addr.show_rru(mb_rru),
Self::Reg { reg } => pretty_print_reg(*reg, size, allocs),
Self::Mem { addr } => addr.pretty_print(size, allocs),
Self::Imm { simm32 } => format!("${}", *simm32 as i32),
}
}
@@ -673,7 +607,7 @@ pub enum RegMem {
impl RegMem {
pub(crate) fn reg(reg: Reg) -> Self {
debug_assert!(reg.get_class() == RegClass::I64 || reg.get_class() == RegClass::V128);
debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
Self::Reg { reg }
}
pub(crate) fn mem(addr: impl Into<SyntheticAmode>) -> Self {
@@ -682,14 +616,17 @@ impl RegMem {
/// Asserts that in register mode, the reg class is the one that's expected.
pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
if let Self::Reg { reg } = self {
debug_assert_eq!(reg.get_class(), expected_reg_class);
debug_assert_eq!(reg.class(), expected_reg_class);
}
}
/// Add the regs mentioned by `self` to `collector`.
pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) {
pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
&self,
collector: &mut OperandCollector<'_, F>,
) {
match self {
RegMem::Reg { reg } => collector.add_use(*reg),
RegMem::Mem { addr, .. } => addr.get_regs_as_uses(collector),
RegMem::Reg { reg } => collector.reg_use(*reg),
RegMem::Mem { addr, .. } => addr.get_operands(collector),
}
}
pub(crate) fn to_reg(&self) -> Option<Reg> {
@@ -698,6 +635,17 @@ impl RegMem {
_ => None,
}
}
pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
match self {
RegMem::Reg { reg } => RegMem::Reg {
reg: allocs.next(*reg),
},
RegMem::Mem { addr } => RegMem::Mem {
addr: addr.with_allocs(allocs),
},
}
}
}
impl From<Writable<Reg>> for RegMem {
@@ -707,16 +655,10 @@ impl From<Writable<Reg>> for RegMem {
}
impl PrettyPrint for RegMem {
fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String {
self.show_rru_sized(mb_rru, 8)
}
}
impl PrettyPrintSized for RegMem {
fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String {
fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self {
RegMem::Reg { reg } => show_ireg_sized(*reg, mb_rru, size),
RegMem::Mem { addr, .. } => addr.show_rru(mb_rru),
RegMem::Reg { reg } => pretty_print_reg(*reg, size, allocs),
RegMem::Mem { addr, .. } => addr.pretty_print(size, allocs),
}
}
}
@@ -1222,6 +1164,22 @@ impl SseOpcode {
_ => 8,
}
}
/// Does an XmmRmmRImm with this opcode use src1? FIXME: split
/// into separate instructions.
pub(crate) fn uses_src1(&self) -> bool {
match self {
SseOpcode::Pextrb => false,
SseOpcode::Pextrw => false,
SseOpcode::Pextrd => false,
SseOpcode::Pshufd => false,
SseOpcode::Roundss => false,
SseOpcode::Roundsd => false,
SseOpcode::Roundps => false,
SseOpcode::Roundpd => false,
_ => true,
}
}
}
impl fmt::Debug for SseOpcode {

850
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

File diff suppressed because it is too large Load Diff

414
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

File diff suppressed because it is too large Load Diff

2213
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

File diff suppressed because it is too large Load Diff

323
cranelift/codegen/src/isa/x64/inst/regs.rs
View File

@@ -1,26 +1,14 @@
//! Registers, the Universe thereof, and printing.
//! Register definitions for regalloc2.
//!
//! These are ordered by sequence number, as required in the Universe.
//! We define 16 GPRs, with indices equal to the hardware encoding,
//! and 16 XMM registers.
//!
//! The caller-saved registers are placed first in order to prefer not to clobber (requiring
//! saves/restores in prologue/epilogue code) when possible. Note that there is no other heuristic
//! in the backend that will apply such pressure; the register allocator's cost heuristics are not
//! aware of the cost of clobber-save/restore code.
//!
//! One might worry that this pessimizes code with many callsites, where using caller-saves causes
//! us to have to save them (as we are the caller) frequently. However, the register allocator
//! *should be* aware of *this* cost, because it sees that the call instruction modifies all of the
//! caller-saved (i.e., callee-clobbered) registers.
//!
//! Hence, this ordering encodes pressure in one direction (prefer not to clobber registers that we
//! ourselves have to save) and this is balanaced against the RA's pressure in the other direction
//! at callsites.
//! Note also that we make use of pinned VRegs to refer to PRegs.
use crate::machinst::{AllocationConsumer, RealReg, Reg};
use crate::settings;
use alloc::vec::Vec;
use regalloc::{
PrettyPrint, RealReg, RealRegUniverse, Reg, RegClass, RegClassInfo, NUM_REG_CLASSES,
};
use alloc::string::ToString;
use regalloc2::{MachineEnv, PReg, RegClass, VReg};
use std::string::String;
// Hardware encodings (note the special rax, rcx, rdx, rbx order).
@@ -42,53 +30,62 @@ pub const ENC_R13: u8 = 13;
pub const ENC_R14: u8 = 14;
pub const ENC_R15: u8 = 15;
fn gpr(enc: u8, index: u8) -> Reg {
Reg::new_real(RegClass::I64, enc, index)
// Constructors for Regs.
fn gpr(enc: u8) -> Reg {
let preg = PReg::new(enc as usize, RegClass::Int);
Reg::from(VReg::new(preg.index(), RegClass::Int))
}
pub(crate) fn rsi() -> Reg {
gpr(ENC_RSI, 16)
gpr(ENC_RSI)
}
pub(crate) fn rdi() -> Reg {
gpr(ENC_RDI, 17)
gpr(ENC_RDI)
}
pub(crate) fn rax() -> Reg {
gpr(ENC_RAX, 18)
gpr(ENC_RAX)
}
pub(crate) fn rcx() -> Reg {
gpr(ENC_RCX, 19)
gpr(ENC_RCX)
}
pub(crate) fn rdx() -> Reg {
gpr(ENC_RDX, 20)
gpr(ENC_RDX)
}
pub(crate) fn r8() -> Reg {
gpr(ENC_R8, 21)
gpr(ENC_R8)
}
pub(crate) fn r9() -> Reg {
gpr(ENC_R9, 22)
gpr(ENC_R9)
}
pub(crate) fn r10() -> Reg {
gpr(ENC_R10, 23)
gpr(ENC_R10)
}
pub(crate) fn r11() -> Reg {
gpr(ENC_R11, 24)
gpr(ENC_R11)
}
pub(crate) fn r12() -> Reg {
gpr(ENC_R12, 25)
gpr(ENC_R12)
}
pub(crate) fn r13() -> Reg {
gpr(ENC_R13, 26)
gpr(ENC_R13)
}
pub(crate) fn r14() -> Reg {
gpr(ENC_R14, 27)
gpr(ENC_R14)
}
pub(crate) fn rbx() -> Reg {
gpr(ENC_RBX, 28)
gpr(ENC_RBX)
}
pub(crate) fn r15() -> Reg {
// r15 is put aside since this is the pinned register.
gpr(ENC_R15, 29)
gpr(ENC_R15)
}
pub(crate) fn rsp() -> Reg {
gpr(ENC_RSP)
}
pub(crate) fn rbp() -> Reg {
gpr(ENC_RBP)
}
/// The pinned register on this architecture.
@@ -98,163 +95,177 @@ pub(crate) fn pinned_reg() -> Reg {
r15()
}
fn fpr(enc: u8, index: u8) -> Reg {
Reg::new_real(RegClass::V128, enc, index)
fn fpr(enc: u8) -> Reg {
let preg = PReg::new(enc as usize, RegClass::Float);
Reg::from(VReg::new(preg.index(), RegClass::Float))
}
pub(crate) fn xmm0() -> Reg {
fpr(0, 0)
fpr(0)
}
pub(crate) fn xmm1() -> Reg {
fpr(1, 1)
fpr(1)
}
pub(crate) fn xmm2() -> Reg {
fpr(2, 2)
fpr(2)
}
pub(crate) fn xmm3() -> Reg {
fpr(3, 3)
fpr(3)
}
pub(crate) fn xmm4() -> Reg {
fpr(4, 4)
fpr(4)
}
pub(crate) fn xmm5() -> Reg {
fpr(5, 5)
fpr(5)
}
pub(crate) fn xmm6() -> Reg {
fpr(6, 6)
fpr(6)
}
pub(crate) fn xmm7() -> Reg {
fpr(7, 7)
fpr(7)
}
pub(crate) fn xmm8() -> Reg {
fpr(8, 8)
fpr(8)
}
pub(crate) fn xmm9() -> Reg {
fpr(9, 9)
fpr(9)
}
pub(crate) fn xmm10() -> Reg {
fpr(10, 10)
fpr(10)
}
pub(crate) fn xmm11() -> Reg {
fpr(11, 11)
fpr(11)
}
pub(crate) fn xmm12() -> Reg {
fpr(12, 12)
fpr(12)
}
pub(crate) fn xmm13() -> Reg {
fpr(13, 13)
fpr(13)
}
pub(crate) fn xmm14() -> Reg {
fpr(14, 14)
fpr(14)
}
pub(crate) fn xmm15() -> Reg {
fpr(15, 15)
fpr(15)
}
pub(crate) fn rsp() -> Reg {
gpr(ENC_RSP, 30)
}
pub(crate) fn rbp() -> Reg {
gpr(ENC_RBP, 31)
}
/// Create the register universe for X64.
///
/// The ordering of registers matters, as commented in the file doc comment: assumes the
/// calling-convention is SystemV, at the moment.
pub(crate) fn create_reg_universe_systemv(flags: &settings::Flags) -> RealRegUniverse {
let mut regs = Vec::<(RealReg, String)>::new();
let mut allocable_by_class = [None; NUM_REG_CLASSES];
let use_pinned_reg = flags.enable_pinned_reg();
// XMM registers
let first_fpr = regs.len();
regs.push((xmm0().to_real_reg(), "%xmm0".into()));
regs.push((xmm1().to_real_reg(), "%xmm1".into()));
regs.push((xmm2().to_real_reg(), "%xmm2".into()));
regs.push((xmm3().to_real_reg(), "%xmm3".into()));
regs.push((xmm4().to_real_reg(), "%xmm4".into()));
regs.push((xmm5().to_real_reg(), "%xmm5".into()));
regs.push((xmm6().to_real_reg(), "%xmm6".into()));
regs.push((xmm7().to_real_reg(), "%xmm7".into()));
regs.push((xmm8().to_real_reg(), "%xmm8".into()));
regs.push((xmm9().to_real_reg(), "%xmm9".into()));
regs.push((xmm10().to_real_reg(), "%xmm10".into()));
regs.push((xmm11().to_real_reg(), "%xmm11".into()));
regs.push((xmm12().to_real_reg(), "%xmm12".into()));
regs.push((xmm13().to_real_reg(), "%xmm13".into()));
regs.push((xmm14().to_real_reg(), "%xmm14".into()));
regs.push((xmm15().to_real_reg(), "%xmm15".into()));
let last_fpr = regs.len() - 1;
// Integer regs.
let first_gpr = regs.len();
// Caller-saved, in the SystemV x86_64 ABI.
regs.push((rsi().to_real_reg(), "%rsi".into()));
regs.push((rdi().to_real_reg(), "%rdi".into()));
regs.push((rax().to_real_reg(), "%rax".into()));
regs.push((rcx().to_real_reg(), "%rcx".into()));
regs.push((rdx().to_real_reg(), "%rdx".into()));
regs.push((r8().to_real_reg(), "%r8".into()));
regs.push((r9().to_real_reg(), "%r9".into()));
regs.push((r10().to_real_reg(), "%r10".into()));
regs.push((r11().to_real_reg(), "%r11".into()));
// Callee-saved, in the SystemV x86_64 ABI.
regs.push((r12().to_real_reg(), "%r12".into()));
regs.push((r13().to_real_reg(), "%r13".into()));
regs.push((r14().to_real_reg(), "%r14".into()));
regs.push((rbx().to_real_reg(), "%rbx".into()));
// Other regs, not available to the allocator.
debug_assert_eq!(r15(), pinned_reg());
let allocable = if use_pinned_reg {
// The pinned register is not allocatable in this case, so record the length before adding
// it.
let len = regs.len();
regs.push((r15().to_real_reg(), "%r15/pinned".into()));
len
} else {
regs.push((r15().to_real_reg(), "%r15".into()));
regs.len()
};
let last_gpr = allocable - 1;
regs.push((rsp().to_real_reg(), "%rsp".into()));
regs.push((rbp().to_real_reg(), "%rbp".into()));
allocable_by_class[RegClass::I64.rc_to_usize()] = Some(RegClassInfo {
first: first_gpr,
last: last_gpr,
suggested_scratch: Some(r12().get_index()),
});
allocable_by_class[RegClass::V128.rc_to_usize()] = Some(RegClassInfo {
first: first_fpr,
last: last_fpr,
suggested_scratch: Some(xmm15().get_index()),
});
// Sanity-check: the index passed to the Reg ctor must match the order in the register list.
for (i, reg) in regs.iter().enumerate() {
assert_eq!(i, reg.0.get_index());
/// Create the register environment for x64.
pub(crate) fn create_reg_env_systemv(flags: &settings::Flags) -> MachineEnv {
fn preg(r: Reg) -> PReg {
r.to_real_reg().unwrap().into()
}
RealRegUniverse {
regs,
allocable,
allocable_by_class,
let mut env = MachineEnv {
preferred_regs_by_class: [
// Preferred GPRs: caller-saved in the SysV ABI.
vec![
preg(rsi()),
preg(rdi()),
preg(rax()),
preg(rcx()),
preg(rdx()),
preg(r8()),
preg(r9()),
// N.B.: not r10; it is our scratch reg.
preg(r11()),
],
// Preferred XMMs: all of them.
vec![
preg(xmm0()),
preg(xmm1()),
preg(xmm2()),
preg(xmm3()),
preg(xmm4()),
preg(xmm5()),
preg(xmm6()),
preg(xmm7()),
preg(xmm8()),
preg(xmm9()),
preg(xmm10()),
preg(xmm11()),
preg(xmm12()),
preg(xmm13()),
preg(xmm14()),
// N.B.: not xmm15; it is our scratch reg.
],
],
non_preferred_regs_by_class: [
// Non-preferred GPRs: callee-saved in the SysV ABI.
vec![preg(rbx()), preg(r12()), preg(r13()), preg(r14())],
// Non-preferred XMMs: none.
vec![],
],
scratch_by_class: [preg(r10()), preg(xmm15())],
fixed_stack_slots: vec![],
};
debug_assert_eq!(r15(), pinned_reg());
if !flags.enable_pinned_reg() {
env.non_preferred_regs_by_class[0].push(preg(r15()));
}
env
}
/// Give the name of a RealReg.
pub fn realreg_name(reg: RealReg) -> &'static str {
let preg = PReg::from(reg);
match preg.class() {
RegClass::Int => match preg.hw_enc() as u8 {
ENC_RAX => "%rax",
ENC_RBX => "%rbx",
ENC_RCX => "%rcx",
ENC_RDX => "%rdx",
ENC_RSI => "%rsi",
ENC_RDI => "%rdi",
ENC_RBP => "%rbp",
ENC_RSP => "%rsp",
ENC_R8 => "%r8",
ENC_R9 => "%r9",
ENC_R10 => "%r10",
ENC_R11 => "%r11",
ENC_R12 => "%r12",
ENC_R13 => "%r13",
ENC_R14 => "%r14",
ENC_R15 => "%r15",
_ => panic!("Invalid PReg: {:?}", preg),
},
RegClass::Float => match preg.hw_enc() {
0 => "%xmm0",
1 => "%xmm1",
2 => "%xmm2",
3 => "%xmm3",
4 => "%xmm4",
5 => "%xmm5",
6 => "%xmm6",
7 => "%xmm7",
8 => "%xmm8",
9 => "%xmm9",
10 => "%xmm10",
11 => "%xmm11",
12 => "%xmm12",
13 => "%xmm13",
14 => "%xmm14",
15 => "%xmm15",
_ => panic!("Invalid PReg: {:?}", preg),
},
}
}
pub fn show_reg(reg: Reg) -> String {
if let Some(rreg) = reg.to_real_reg() {
realreg_name(rreg).to_string()
} else {
format!("%{:?}", reg)
}
}
/// If `ireg` denotes an I64-classed reg, make a best-effort attempt to show its name at some
/// smaller size (4, 2 or 1 bytes).
pub fn show_ireg_sized(reg: Reg, mb_rru: Option<&RealRegUniverse>, size: u8) -> String {
let mut s = reg.show_rru(mb_rru);
pub fn show_ireg_sized(reg: Reg, size: u8) -> String {
let mut s = show_reg(reg);
if reg.get_class() != RegClass::I64 || size == 8 {
if reg.class() != RegClass::Int || size == 8 {
// We can't do any better.
return s;
}
@@ -302,3 +313,15 @@ pub fn show_ireg_sized(reg: Reg, mb_rru: Option<&RealRegUniverse>, size: u8) ->
s
}
// N.B.: this is not an `impl PrettyPrint for Reg` because it is
// specific to x64; other backends have analogous functions. The
// disambiguation happens statically by virtue of higher-level,
// x64-specific, types calling the right `pretty_print_reg`. (In other
// words, we can't pretty-print a `Reg` all by itself in a build that
// may have multiple backends; but we can pretty-print one as part of
// an x64 Inst or x64 RegMemImm.)
pub fn pretty_print_reg(reg: Reg, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
let reg = allocs.next(reg);
show_ireg_sized(reg, size)
}

11
cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs
View File

@@ -1,8 +1,8 @@
//! Unwind information for System V ABI (x86-64).
use crate::isa::unwind::systemv::RegisterMappingError;
use crate::machinst::{Reg, RegClass};
use gimli::{write::CommonInformationEntry, Encoding, Format, Register, X86_64};
use regalloc::{Reg, RegClass};
/// Creates a new x86-64 common information entry (CIE).
pub fn create_cie() -> CommonInformationEntry {
@@ -69,14 +69,13 @@ pub fn map_reg(reg: Reg) -> Result<Register, RegisterMappingError> {
X86_64::XMM15,
];
match reg.get_class() {
RegClass::I64 => {
match reg.class() {
RegClass::Int => {
// x86 GP registers have a weird mapping to DWARF registers, so we use a
// lookup table.
Ok(X86_GP_REG_MAP[reg.get_hw_encoding() as usize])
Ok(X86_GP_REG_MAP[reg.to_real_reg().unwrap().hw_enc() as usize])
}
RegClass::V128 => Ok(X86_XMM_REG_MAP[reg.get_hw_encoding() as usize]),
_ => Err(RegisterMappingError::UnsupportedRegisterBank("class?")),
RegClass::Float => Ok(X86_XMM_REG_MAP[reg.to_real_reg().unwrap().hw_enc() as usize]),
}
}

9
cranelift/codegen/src/isa/x64/inst/unwind/winx64.rs
View File

@@ -1,16 +1,15 @@
//! Unwind information for Windows x64 ABI.
use regalloc::{Reg, RegClass};
use crate::machinst::{Reg, RegClass};
pub(crate) struct RegisterMapper;
impl crate::isa::unwind::winx64::RegisterMapper<Reg> for RegisterMapper {
fn map(reg: Reg) -> crate::isa::unwind::winx64::MappedRegister {
use crate::isa::unwind::winx64::MappedRegister;
match reg.get_class() {
RegClass::I64 => MappedRegister::Int(reg.get_hw_encoding()),
RegClass::V128 => MappedRegister::Xmm(reg.get_hw_encoding()),
_ => unreachable!(),
match reg.class() {
RegClass::Int => MappedRegister::Int(reg.to_real_reg().unwrap().hw_enc()),
RegClass::Float => MappedRegister::Xmm(reg.to_real_reg().unwrap().hw_enc()),
}
}
}