T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[RFC] Dynamic Vector Support (#4200)

Browse Source 
Introduce a new concept in the IR that allows a producer to create
dynamic vector types. An IR function can now contain global value(s)
that represent a dynamic scaling factor, for a given fixed-width
vector type. A dynamic type is then created by 'multiplying' the
corresponding global value with a fixed-width type. These new types
can be used just like the existing types and the type system has a
set of hard-coded dynamic types, such as I32X4XN, which the user
defined types map onto. The dynamic types are also used explicitly
to create dynamic stack slots, which have no set size like their
existing counterparts. New IR instructions are added to access these
new stack entities.

Currently, during codegen, the dynamic scaling factor has to be
lowered to a constant so the dynamic slots do eventually have a
compile-time known size, as do spill slots.

The current lowering for aarch64 just targets Neon, using a dynamic
scale of 1.

Copyright (c) 2022, Arm Limited.
This commit is contained in:
Sam Parker
2022-07-07 20:54:39 +01:00
committed by GitHub
parent 9ae060a12a
commit 9c43749dfe
69 changed files with 2422 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
cranelift/codegen/src/ir/dfg.rs
View File

@@ -3,12 +3,13 @@
use crate::entity::{self, PrimaryMap, SecondaryMap};
use crate::ir;
use crate::ir::builder::ReplaceBuilder;
use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes};
use crate::ir::extfunc::ExtFuncData;
use crate::ir::instructions::{BranchInfo, CallInfo, InstructionData};
use crate::ir::{types, ConstantData, ConstantPool, Immediate};
use crate::ir::{
Block, FuncRef, Inst, SigRef, Signature, SourceLoc, Type, Value, ValueLabelAssignments,
ValueList, ValueListPool,
Block, DynamicType, FuncRef, Inst, SigRef, Signature, SourceLoc, Type, Value,
ValueLabelAssignments, ValueList, ValueListPool,
};
use crate::packed_option::ReservedValue;
use crate::write::write_operands;
@@ -50,6 +51,9 @@ pub struct DataFlowGraph {
/// instructions contained in each block.
blocks: PrimaryMap<Block, BlockData>,
/// Dynamic types created.
pub dynamic_types: DynamicTypes,
/// Memory pool of value lists.
///
/// The `ValueList` references into this pool appear in many places:
@@ -89,6 +93,7 @@ impl DataFlowGraph {
insts: PrimaryMap::new(),
results: SecondaryMap::new(),
blocks: PrimaryMap::new(),
dynamic_types: DynamicTypes::new(),
value_lists: ValueListPool::new(),
values: PrimaryMap::new(),
signatures: PrimaryMap::new(),
@@ -105,6 +110,7 @@ impl DataFlowGraph {
self.insts.clear();
self.results.clear();
self.blocks.clear();
self.dynamic_types.clear();
self.value_lists.clear();
self.values.clear();
self.signatures.clear();
@@ -557,6 +563,11 @@ impl DataFlowGraph {
self.insts.push(data)
}
/// Declares a dynamic vector type
pub fn make_dynamic_ty(&mut self, data: DynamicTypeData) -> DynamicType {
self.dynamic_types.push(data)
}
/// Returns an object that displays `inst`.
pub fn display_inst<'a>(&'a self, inst: Inst) -> DisplayInst<'a> {
DisplayInst(self, inst)
@@ -1104,6 +1115,20 @@ impl DataFlowGraph {
self.values[v].set_type(t);
}
/// Check that the given concrete `Type` has been defined in the function.
pub fn check_dynamic_type(&mut self, ty: Type) -> Option<Type> {
debug_assert!(ty.is_dynamic_vector());
if self
.dynamic_types
.values()
.any(|dyn_ty_data| dyn_ty_data.concrete().unwrap() == ty)
{
Some(ty)
} else {
None
}
}
/// Create result values for `inst`, reusing the provided detached values.
/// This is similar to `make_inst_results_reusing` except it's only for use
/// in the parser, which needs to reuse previously invalid values.
@@ -1130,6 +1155,10 @@ impl DataFlowGraph {
let constraints = self.insts[inst].opcode().constraints();
for res_idx in 0..constraints.num_fixed_results() {
let ty = constraints.result_type(res_idx, ctrl_typevar);
if ty.is_dynamic_vector() {
self.check_dynamic_type(ty)
.unwrap_or_else(|| panic!("Use of undeclared dynamic type: {}", ty));
}
if let Some(v) = reuse.get(res_idx) {
self.set_value_type_for_parser(*v, ty);
}

38
cranelift/codegen/src/ir/dynamic_type.rs Normal file
View File

@@ -0,0 +1,38 @@
//! Dynamic IR types
use crate::ir::entities::DynamicType;
use crate::ir::GlobalValue;
use crate::ir::PrimaryMap;
use crate::ir::Type;
#[cfg(feature = "enable-serde")]
use serde::{Deserialize, Serialize};
/// A dynamic type object which has a base vector type and a scaling factor.
#[derive(Clone)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct DynamicTypeData {
/// Base vector type, this is the minimum size of the type.
pub base_vector_ty: Type,
/// The dynamic scaling factor of the base vector type.
pub dynamic_scale: GlobalValue,
}
impl DynamicTypeData {
/// Create a new dynamic type.
pub fn new(base_vector_ty: Type, dynamic_scale: GlobalValue) -> Self {
assert!(base_vector_ty.is_vector());
Self {
base_vector_ty,
dynamic_scale,
}
}
/// Convert 'base_vector_ty' into a concrete dynamic vector type.
pub fn concrete(&self) -> Option<Type> {
self.base_vector_ty.vector_to_dynamic()
}
}
/// All allocated dynamic types.
pub type DynamicTypes = PrimaryMap<DynamicType, DynamicTypeData>;

56
cranelift/codegen/src/ir/entities.rs
View File

@@ -135,6 +135,44 @@ impl StackSlot {
}
}
/// An opaque reference to a dynamic stack slot.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct DynamicStackSlot(u32);
entity_impl!(DynamicStackSlot, "dss");
impl DynamicStackSlot {
/// Create a new stack slot reference from its number.
///
/// This method is for use by the parser.
pub fn with_number(n: u32) -> Option<Self> {
if n < u32::MAX {
Some(Self(n))
} else {
None
}
}
}
/// An opaque reference to a dynamic type.
#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct DynamicType(u32);
entity_impl!(DynamicType, "dt");
impl DynamicType {
/// Create a new dynamic type reference from its number.
///
/// This method is for use by the parser.
pub fn with_number(n: u32) -> Option<Self> {
if n < u32::MAX {
Some(Self(n))
} else {
None
}
}
}
/// An opaque reference to a global value.
///
/// A `GlobalValue` is a [`Value`](Value) that will be live across the entire
@@ -389,6 +427,10 @@ pub enum AnyEntity {
Value(Value),
/// A stack slot.
StackSlot(StackSlot),
/// A dynamic stack slot.
DynamicStackSlot(DynamicStackSlot),
/// A dynamic type
DynamicType(DynamicType),
/// A Global value.
GlobalValue(GlobalValue),
/// A jump table.
@@ -415,6 +457,8 @@ impl fmt::Display for AnyEntity {
Self::Inst(r) => r.fmt(f),
Self::Value(r) => r.fmt(f),
Self::StackSlot(r) => r.fmt(f),
Self::DynamicStackSlot(r) => r.fmt(f),
Self::DynamicType(r) => r.fmt(f),
Self::GlobalValue(r) => r.fmt(f),
Self::JumpTable(r) => r.fmt(f),
Self::Constant(r) => r.fmt(f),
@@ -457,6 +501,18 @@ impl From<StackSlot> for AnyEntity {
}
}
impl From<DynamicStackSlot> for AnyEntity {
fn from(r: DynamicStackSlot) -> Self {
Self::DynamicStackSlot(r)
}
}
impl From<DynamicType> for AnyEntity {
fn from(r: DynamicType) -> Self {
Self::DynamicType(r)
}
}
impl From<GlobalValue> for AnyEntity {
fn from(r: GlobalValue) -> Self {
Self::GlobalValue(r)

59
cranelift/codegen/src/ir/function.rs
View File

@@ -7,12 +7,12 @@ use crate::entity::{PrimaryMap, SecondaryMap};
use crate::ir;
use crate::ir::JumpTables;
use crate::ir::{
instructions::BranchInfo, Block, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap,
HeapData, Inst, InstructionData, JumpTable, JumpTableData, Opcode, SigRef, StackSlot,
StackSlotData, Table, TableData,
instructions::BranchInfo, Block, DynamicStackSlot, DynamicStackSlotData, DynamicType,
ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, InstructionData,
JumpTable, JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, Type,
};
use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature};
use crate::ir::{SourceLocs, StackSlots};
use crate::ir::{DynamicStackSlots, SourceLocs, StackSlots};
use crate::isa::CallConv;
use crate::value_label::ValueLabelsRanges;
use crate::write::write_function;
@@ -78,8 +78,11 @@ pub struct Function {
/// Signature of this function.
pub signature: Signature,
/// Stack slots allocated in this function.
pub stack_slots: StackSlots,
/// Sized stack slots allocated in this function.
pub sized_stack_slots: StackSlots,
/// Dynamic stack slots allocated in this function.
pub dynamic_stack_slots: DynamicStackSlots,
/// Global values referenced.
pub global_values: PrimaryMap<ir::GlobalValue, ir::GlobalValueData>,
@@ -120,7 +123,8 @@ impl Function {
version_marker: VersionMarker,
name,
signature: sig,
stack_slots: StackSlots::new(),
sized_stack_slots: StackSlots::new(),
dynamic_stack_slots: DynamicStackSlots::new(),
global_values: PrimaryMap::new(),
heaps: PrimaryMap::new(),
tables: PrimaryMap::new(),
@@ -135,7 +139,8 @@ impl Function {
/// Clear all data structures in this function.
pub fn clear(&mut self) {
self.signature.clear(CallConv::Fast);
self.stack_slots.clear();
self.sized_stack_slots.clear();
self.dynamic_stack_slots.clear();
self.global_values.clear();
self.heaps.clear();
self.tables.clear();
@@ -156,10 +161,16 @@ impl Function {
self.jump_tables.push(data)
}
/// Creates a stack slot in the function, to be used by `stack_load`, `stack_store` and
/// `stack_addr` instructions.
pub fn create_stack_slot(&mut self, data: StackSlotData) -> StackSlot {
self.stack_slots.push(data)
/// Creates a sized stack slot in the function, to be used by `stack_load`, `stack_store`
/// and `stack_addr` instructions.
pub fn create_sized_stack_slot(&mut self, data: StackSlotData) -> StackSlot {
self.sized_stack_slots.push(data)
}
/// Creates a dynamic stack slot in the function, to be used by `dynamic_stack_load`,
/// `dynamic_stack_store` and `dynamic_stack_addr` instructions.
pub fn create_dynamic_stack_slot(&mut self, data: DynamicStackSlotData) -> DynamicStackSlot {
self.dynamic_stack_slots.push(data)
}
/// Adds a signature which can later be used to declare an external function import.
@@ -177,6 +188,26 @@ impl Function {
self.global_values.push(data)
}
/// Find the global dyn_scale value associated with given DynamicType
pub fn get_dyn_scale(&self, ty: DynamicType) -> GlobalValue {
self.dfg.dynamic_types.get(ty).unwrap().dynamic_scale
}
/// Find the global dyn_scale for the given stack slot.
pub fn get_dynamic_slot_scale(&self, dss: DynamicStackSlot) -> GlobalValue {
let dyn_ty = self.dynamic_stack_slots.get(dss).unwrap().dyn_ty;
self.get_dyn_scale(dyn_ty)
}
/// Get a concrete `Type` from a user defined `DynamicType`.
pub fn get_concrete_dynamic_ty(&self, ty: DynamicType) -> Option<Type> {
self.dfg
.dynamic_types
.get(ty)
.unwrap_or_else(|| panic!("Undeclared dynamic vector type: {}", ty))
.concrete()
}
/// Declares a heap accessible to the function.
pub fn create_heap(&mut self, data: HeapData) -> Heap {
self.heaps.push(data)
@@ -322,8 +353,8 @@ impl Function {
/// Size occupied by all stack slots associated with this function.
///
/// Does not include any padding necessary due to offsets
pub fn stack_size(&self) -> u32 {
self.stack_slots.values().map(|ss| ss.size).sum()
pub fn fixed_stack_size(&self) -> u32 {
self.sized_stack_slots.values().map(|ss| ss.size).sum()
}
}

11
cranelift/codegen/src/ir/globalvalue.rs
View File

@@ -76,6 +76,13 @@ pub enum GlobalValueData {
/// Does this symbol refer to a thread local storage value?
tls: bool,
},
/// Value is a multiple of how many instances of `vector_type` will fit in
/// a target vector register.
DynScaleTargetConst {
/// Base vector type.
vector_type: Type,
},
}
impl GlobalValueData {
@@ -92,6 +99,7 @@ impl GlobalValueData {
match *self {
Self::VMContext { .. } | Self::Symbol { .. } => isa.pointer_type(),
Self::IAddImm { global_type, .. } | Self::Load { global_type, .. } => global_type,
Self::DynScaleTargetConst { .. } => isa.pointer_type(),
}
}
@@ -154,6 +162,9 @@ impl fmt::Display for GlobalValueData {
}
Ok(())
}
Self::DynScaleTargetConst { vector_type } => {
write!(f, "dyn_scale_target_const.{}", vector_type)
}
}
}
}

70
cranelift/codegen/src/ir/instructions.rs
View File

@@ -633,6 +633,8 @@ pub struct ValueTypeSet {
pub bools: BitSet8,
/// Allowed ref widths
pub refs: BitSet8,
/// Allowed dynamic vectors minimum lane sizes
pub dynamic_lanes: BitSet16,
}
impl ValueTypeSet {
@@ -656,8 +658,13 @@ impl ValueTypeSet {
/// Does `typ` belong to this set?
pub fn contains(self, typ: Type) -> bool {
let l2l = typ.log2_lane_count();
self.lanes.contains(l2l) && self.is_base_type(typ.lane_type())
if typ.is_dynamic_vector() {
let l2l = typ.log2_min_lane_count();
self.dynamic_lanes.contains(l2l) && self.is_base_type(typ.lane_type())
} else {
let l2l = typ.log2_lane_count();
self.lanes.contains(l2l) && self.is_base_type(typ.lane_type())
}
}
/// Get an example member of this type set.
@@ -712,6 +719,9 @@ enum OperandConstraint {
/// This operand is `ctrlType.merge_lanes()`.
MergeLanes,
/// This operands is `ctrlType.dynamic_to_vector()`.
DynamicToVector,
}
impl OperandConstraint {
@@ -738,15 +748,48 @@ impl OperandConstraint {
.expect("invalid type for half_vector"),
),
DoubleVector => Bound(ctrl_type.by(2).expect("invalid type for double_vector")),
SplitLanes => Bound(
SplitLanes => {
if ctrl_type.is_dynamic_vector() {
Bound(
ctrl_type
.dynamic_to_vector()
.expect("invalid type for dynamic_to_vector")
.split_lanes()
.expect("invalid type for split_lanes")
.vector_to_dynamic()
.expect("invalid dynamic type"),
)
} else {
Bound(
ctrl_type
.split_lanes()
.expect("invalid type for split_lanes"),
)
}
}
MergeLanes => {
if ctrl_type.is_dynamic_vector() {
Bound(
ctrl_type
.dynamic_to_vector()
.expect("invalid type for dynamic_to_vector")
.merge_lanes()
.expect("invalid type for merge_lanes")
.vector_to_dynamic()
.expect("invalid dynamic type"),
)
} else {
Bound(
ctrl_type
.merge_lanes()
.expect("invalid type for merge_lanes"),
)
}
}
DynamicToVector => Bound(
ctrl_type
.split_lanes()
.expect("invalid type for split_lanes"),
),
MergeLanes => Bound(
ctrl_type
.merge_lanes()
.expect("invalid type for merge_lanes"),
.dynamic_to_vector()
.expect("invalid type for dynamic_to_vector"),
),
}
}
@@ -860,11 +903,13 @@ mod tests {
floats: BitSet8::from_range(0, 0),
bools: BitSet8::from_range(3, 7),
refs: BitSet8::from_range(5, 7),
dynamic_lanes: BitSet16::from_range(0, 4),
};
assert!(!vts.contains(I8));
assert!(vts.contains(I32));
assert!(vts.contains(I64));
assert!(vts.contains(I32X4));
assert!(vts.contains(I32X4XN));
assert!(!vts.contains(F32));
assert!(!vts.contains(B1));
assert!(vts.contains(B8));
@@ -879,6 +924,7 @@ mod tests {
floats: BitSet8::from_range(5, 7),
bools: BitSet8::from_range(3, 7),
refs: BitSet8::from_range(0, 0),
dynamic_lanes: BitSet16::from_range(0, 8),
};
assert_eq!(vts.example().to_string(), "f32");
@@ -888,6 +934,7 @@ mod tests {
floats: BitSet8::from_range(5, 7),
bools: BitSet8::from_range(3, 7),
refs: BitSet8::from_range(0, 0),
dynamic_lanes: BitSet16::from_range(0, 8),
};
assert_eq!(vts.example().to_string(), "f32x2");
@@ -897,9 +944,11 @@ mod tests {
floats: BitSet8::from_range(0, 0),
bools: BitSet8::from_range(3, 7),
refs: BitSet8::from_range(0, 0),
dynamic_lanes: BitSet16::from_range(0, 8),
};
assert!(!vts.contains(B32X2));
assert!(vts.contains(B32X4));
assert!(vts.contains(B16X4XN));
assert_eq!(vts.example().to_string(), "b32x4");
let vts = ValueTypeSet {
@@ -909,6 +958,7 @@ mod tests {
floats: BitSet8::from_range(0, 0),
bools: BitSet8::from_range(0, 0),
refs: BitSet8::from_range(0, 0),
dynamic_lanes: BitSet16::from_range(0, 8),
};
assert!(vts.contains(I32));
assert!(vts.contains(I32X4));

10
cranelift/codegen/src/ir/mod.rs
View File

@@ -5,6 +5,7 @@ mod builder;
pub mod condcodes;
pub mod constant;
pub mod dfg;
pub mod dynamic_type;
pub mod entities;
mod extfunc;
mod extname;
@@ -33,9 +34,10 @@ pub use crate::ir::builder::{
};
pub use crate::ir::constant::{ConstantData, ConstantPool};
pub use crate::ir::dfg::{DataFlowGraph, ValueDef};
pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes};
pub use crate::ir::entities::{
Block, Constant, FuncRef, GlobalValue, Heap, Immediate, Inst, JumpTable, SigRef, StackSlot,
Table, Value,
Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, Immediate, Inst,
JumpTable, SigRef, StackSlot, Table, Value,
};
pub use crate::ir::extfunc::{
AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature,
@@ -53,7 +55,9 @@ pub use crate::ir::libcall::{get_probestack_funcref, LibCall};
pub use crate::ir::memflags::{Endianness, MemFlags};
pub use crate::ir::progpoint::{ExpandedProgramPoint, ProgramOrder, ProgramPoint};
pub use crate::ir::sourceloc::SourceLoc;
pub use crate::ir::stackslot::{StackSlotData, StackSlotKind, StackSlots};
pub use crate::ir::stackslot::{
DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKind, StackSlots,
};
pub use crate::ir::table::TableData;
pub use crate::ir::trapcode::TrapCode;
pub use crate::ir::types::Type;

113
cranelift/codegen/src/ir/stackslot.rs
View File

@@ -4,10 +4,18 @@
//!
use crate::entity::PrimaryMap;
use crate::ir::entities::{DynamicStackSlot, DynamicType};
use crate::ir::StackSlot;
use core::fmt;
use core::str::FromStr;
/// imports only needed for testing.
#[allow(unused_imports)]
use crate::ir::{DynamicTypeData, GlobalValueData};
#[allow(unused_imports)]
use crate::ir::types::*;
#[cfg(feature = "enable-serde")]
use serde::{Deserialize, Serialize};
@@ -25,6 +33,9 @@ pub enum StackSlotKind {
/// An explicit stack slot. This is a chunk of stack memory for use by the `stack_load`
/// and `stack_store` instructions.
ExplicitSlot,
/// An explicit stack slot for dynamic vector types. This is a chunk of stack memory
/// for use by the `dynamic_stack_load` and `dynamic_stack_store` instructions.
ExplicitDynamicSlot,
}
impl FromStr for StackSlotKind {
@@ -34,6 +45,7 @@ impl FromStr for StackSlotKind {
use self::StackSlotKind::*;
match s {
"explicit_slot" => Ok(ExplicitSlot),
"explicit_dynamic_slot" => Ok(ExplicitDynamicSlot),
_ => Err(()),
}
}
@@ -44,6 +56,7 @@ impl fmt::Display for StackSlotKind {
use self::StackSlotKind::*;
f.write_str(match *self {
ExplicitSlot => "explicit_slot",
ExplicitDynamicSlot => "explicit_dynamic_slot",
})
}
}
@@ -68,11 +81,15 @@ impl StackSlotData {
/// Get the alignment in bytes of this stack slot given the stack pointer alignment.
pub fn alignment(&self, max_align: StackSize) -> StackSize {
debug_assert!(max_align.is_power_of_two());
// We want to find the largest power of two that divides both `self.size` and `max_align`.
// That is the same as isolating the rightmost bit in `x`.
let x = self.size | max_align;
// C.f. Hacker's delight.
x & x.wrapping_neg()
if self.kind == StackSlotKind::ExplicitDynamicSlot {
max_align
} else {
// We want to find the largest power of two that divides both `self.size` and `max_align`.
// That is the same as isolating the rightmost bit in `x`.
let x = self.size | max_align;
// C.f. Hacker's delight.
x & x.wrapping_neg()
}
}
}
@@ -82,9 +99,43 @@ impl fmt::Display for StackSlotData {
}
}
/// Contents of a dynamic stack slot.
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct DynamicStackSlotData {
/// The kind of stack slot.
pub kind: StackSlotKind,
/// The type of this slot.
pub dyn_ty: DynamicType,
}
impl DynamicStackSlotData {
/// Create a stack slot with the specified byte size.
pub fn new(kind: StackSlotKind, dyn_ty: DynamicType) -> Self {
assert!(kind == StackSlotKind::ExplicitDynamicSlot);
Self { kind, dyn_ty }
}
/// Get the alignment in bytes of this stack slot given the stack pointer alignment.
pub fn alignment(&self, max_align: StackSize) -> StackSize {
debug_assert!(max_align.is_power_of_two());
max_align
}
}
impl fmt::Display for DynamicStackSlotData {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{} {}", self.kind, self.dyn_ty)
}
}
/// All allocated stack slots.
pub type StackSlots = PrimaryMap<StackSlot, StackSlotData>;
/// All allocated dynamic stack slots.
pub type DynamicStackSlots = PrimaryMap<DynamicStackSlot, DynamicStackSlotData>;
#[cfg(test)]
mod tests {
use super::*;
@@ -95,16 +146,56 @@ mod tests {
fn stack_slot() {
let mut func = Function::new();
let ss0 = func.create_stack_slot(StackSlotData::new(StackSlotKind::ExplicitSlot, 4));
let ss1 = func.create_stack_slot(StackSlotData::new(StackSlotKind::ExplicitSlot, 8));
let ss0 = func.create_sized_stack_slot(StackSlotData::new(StackSlotKind::ExplicitSlot, 4));
let ss1 = func.create_sized_stack_slot(StackSlotData::new(StackSlotKind::ExplicitSlot, 8));
assert_eq!(ss0.to_string(), "ss0");
assert_eq!(ss1.to_string(), "ss1");
assert_eq!(func.stack_slots[ss0].size, 4);
assert_eq!(func.stack_slots[ss1].size, 8);
assert_eq!(func.sized_stack_slots[ss0].size, 4);
assert_eq!(func.sized_stack_slots[ss1].size, 8);
assert_eq!(func.stack_slots[ss0].to_string(), "explicit_slot 4");
assert_eq!(func.stack_slots[ss1].to_string(), "explicit_slot 8");
assert_eq!(func.sized_stack_slots[ss0].to_string(), "explicit_slot 4");
assert_eq!(func.sized_stack_slots[ss1].to_string(), "explicit_slot 8");
}
#[test]
fn dynamic_stack_slot() {
let mut func = Function::new();
let int_vector_ty = I32X4;
let fp_vector_ty = F64X2;
let scale0 = GlobalValueData::DynScaleTargetConst {
vector_type: int_vector_ty,
};
let scale1 = GlobalValueData::DynScaleTargetConst {
vector_type: fp_vector_ty,
};
let gv0 = func.create_global_value(scale0);
let gv1 = func.create_global_value(scale1);
let dtd0 = DynamicTypeData::new(int_vector_ty, gv0);
let dtd1 = DynamicTypeData::new(fp_vector_ty, gv1);
let dt0 = func.dfg.make_dynamic_ty(dtd0);
let dt1 = func.dfg.make_dynamic_ty(dtd1);
let dss0 = func.create_dynamic_stack_slot(DynamicStackSlotData::new(
StackSlotKind::ExplicitDynamicSlot,
dt0,
));
let dss1 = func.create_dynamic_stack_slot(DynamicStackSlotData::new(
StackSlotKind::ExplicitDynamicSlot,
dt1,
));
assert_eq!(dss0.to_string(), "dss0");
assert_eq!(dss1.to_string(), "dss1");
assert_eq!(
func.dynamic_stack_slots[dss0].to_string(),
"explicit_dynamic_slot dt0"
);
assert_eq!(
func.dynamic_stack_slots[dss1].to_string(),
"explicit_dynamic_slot dt1"
);
}
#[test]

136
cranelift/codegen/src/ir/types.rs
View File

@@ -233,7 +233,12 @@ impl Type {
///
/// A vector type has 2 or more lanes.
pub fn is_vector(self) -> bool {
self.0 >= constants::VECTOR_BASE
self.0 >= constants::VECTOR_BASE && !self.is_dynamic_vector()
}
/// Is this a SIMD vector type with a runtime number of lanes?
pub fn is_dynamic_vector(self) -> bool {
self.0 >= constants::DYNAMIC_VECTOR_BASE
}
/// Is this a scalar boolean type?
@@ -288,19 +293,62 @@ impl Type {
///
/// A scalar type is the same as a SIMD vector type with one lane, so it returns 0.
pub fn log2_lane_count(self) -> u32 {
(self.0.saturating_sub(constants::LANE_BASE) >> 4) as u32
if self.is_dynamic_vector() {
0
} else {
(self.0.saturating_sub(constants::LANE_BASE) >> 4) as u32
}
}
/// Get log_2 of the number of lanes in this vector/dynamic type.
pub fn log2_min_lane_count(self) -> u32 {
if self.is_dynamic_vector() {
(self
.0
.saturating_sub(constants::VECTOR_BASE + constants::LANE_BASE)
>> 4) as u32
} else {
self.log2_lane_count()
}
}
/// Get the number of lanes in this SIMD vector type.
///
/// A scalar type is the same as a SIMD vector type with one lane, so it returns 1.
pub fn lane_count(self) -> u32 {
1 << self.log2_lane_count()
if self.is_dynamic_vector() {
0
} else {
1 << self.log2_lane_count()
}
}
/// Get the total number of bits used to represent this type.
pub fn bits(self) -> u32 {
self.lane_bits() * self.lane_count()
if self.is_dynamic_vector() {
0
} else {
self.lane_bits() * self.lane_count()
}
}
/// Get the minimum of lanes in this SIMD vector type, this supports both fixed and
/// dynamic types.
pub fn min_lane_count(self) -> u32 {
if self.is_dynamic_vector() {
1 << self.log2_min_lane_count()
} else {
1 << self.log2_lane_count()
}
}
/// Get the minimum number of bits used to represent this type.
pub fn min_bits(self) -> u32 {
if self.is_dynamic_vector() {
self.lane_bits() * self.min_lane_count()
} else {
self.bits()
}
}
/// Get the number of bytes used to store this type in memory.
@@ -315,23 +363,46 @@ impl Type {
/// If this is already a SIMD vector type, this produces a SIMD vector type with `n *
/// self.lane_count()` lanes.
pub fn by(self, n: u32) -> Option<Self> {
if self.is_dynamic_vector() {
return None;
}
if self.lane_bits() == 0 || !n.is_power_of_two() {
return None;
}
let log2_lanes: u32 = n.trailing_zeros();
let new_type = u32::from(self.0) + (log2_lanes << 4);
if new_type < 0x100 {
if new_type < constants::DYNAMIC_VECTOR_BASE as u32
&& (new_type as u16) < constants::DYNAMIC_VECTOR_BASE
{
Some(Self(new_type as u16))
} else {
None
}
}
/// Convert a fixed vector type to a dynamic one.
pub fn vector_to_dynamic(self) -> Option<Self> {
assert!(self.is_vector());
if self.bits() > 256 {
return None;
}
let new_ty = self.0 + constants::VECTOR_BASE;
let ty = Some(Self(new_ty));
assert!(ty.unwrap().is_dynamic_vector());
return ty;
}
/// Convert a dynamic vector type to a fixed one.
pub fn dynamic_to_vector(self) -> Option<Self> {
assert!(self.is_dynamic_vector());
Some(Self(self.0 - constants::VECTOR_BASE))
}
/// Get a SIMD vector with half the number of lanes.
///
/// There is no `double_vector()` method. Use `t.by(2)` instead.
pub fn half_vector(self) -> Option<Self> {
if self.is_vector() {
if self.is_vector() && !self.is_dynamic_vector() {
Some(Self(self.0 - 0x10))
} else {
None
@@ -418,6 +489,8 @@ impl Display for Type {
write!(f, "f{}", self.lane_bits())
} else if self.is_vector() {
write!(f, "{}x{}", self.lane_type(), self.lane_count())
} else if self.is_dynamic_vector() {
write!(f, "{:?}x{}xN", self.lane_type(), self.min_lane_count())
} else if self.is_ref() {
write!(f, "r{}", self.lane_bits())
} else {
@@ -441,6 +514,8 @@ impl Debug for Type {
write!(f, "types::F{}", self.lane_bits())
} else if self.is_vector() {
write!(f, "{:?}X{}", self.lane_type(), self.lane_count())
} else if self.is_dynamic_vector() {
write!(f, "{:?}X{}XN", self.lane_type(), self.min_lane_count())
} else if self.is_ref() {
write!(f, "types::R{}", self.lane_bits())
} else {
@@ -568,6 +643,55 @@ mod tests {
assert_eq!(F64.by(8), Some(F64X8));
}
#[test]
fn dynamic_vectors() {
// Identification.
assert_eq!(I8X16XN.is_dynamic_vector(), true);
assert_eq!(B16X4XN.is_dynamic_vector(), true);
assert_eq!(F32X8XN.is_dynamic_vector(), true);
assert_eq!(F64X4XN.is_dynamic_vector(), true);
assert_eq!(I128X2XN.is_dynamic_vector(), true);
// Lane counts.
assert_eq!(I16X8XN.lane_count(), 0);
assert_eq!(I16X8XN.min_lane_count(), 8);
// Size
assert_eq!(B32X2XN.bits(), 0);
assert_eq!(B32X2XN.min_bits(), 64);
// Change lane counts
assert_eq!(F64X4XN.half_vector(), None);
assert_eq!(I8X8XN.by(2), None);
// Conversions to and from vectors.
assert_eq!(B8.by(8).unwrap().vector_to_dynamic(), Some(B8X8XN));
assert_eq!(I8.by(16).unwrap().vector_to_dynamic(), Some(I8X16XN));
assert_eq!(I16.by(8).unwrap().vector_to_dynamic(), Some(I16X8XN));
assert_eq!(B16.by(16).unwrap().vector_to_dynamic(), Some(B16X16XN));
assert_eq!(B32.by(2).unwrap().vector_to_dynamic(), Some(B32X2XN));
assert_eq!(B32.by(8).unwrap().vector_to_dynamic(), Some(B32X8XN));
assert_eq!(I32.by(4).unwrap().vector_to_dynamic(), Some(I32X4XN));
assert_eq!(F32.by(4).unwrap().vector_to_dynamic(), Some(F32X4XN));
assert_eq!(F64.by(2).unwrap().vector_to_dynamic(), Some(F64X2XN));
assert_eq!(I128.by(2).unwrap().vector_to_dynamic(), Some(I128X2XN));
assert_eq!(I128X2XN.dynamic_to_vector(), Some(I128X2));
assert_eq!(B64X2XN.dynamic_to_vector(), Some(B64X2));
assert_eq!(F32X4XN.dynamic_to_vector(), Some(F32X4));
assert_eq!(F64X4XN.dynamic_to_vector(), Some(F64X4));
assert_eq!(I32X2XN.dynamic_to_vector(), Some(I32X2));
assert_eq!(I32X8XN.dynamic_to_vector(), Some(I32X8));
assert_eq!(I16X16XN.dynamic_to_vector(), Some(I16X16));
assert_eq!(I8X32XN.dynamic_to_vector(), Some(I8X32));
assert_eq!(I8X64.vector_to_dynamic(), None);
assert_eq!(B16X32.vector_to_dynamic(), None);
assert_eq!(F32X16.vector_to_dynamic(), None);
assert_eq!(I64X8.vector_to_dynamic(), None);
assert_eq!(I128X4.vector_to_dynamic(), None);
}
#[test]
fn format_scalars() {
assert_eq!(IFLAGS.to_string(), "iflags");