80c147d9c0
Rework br_table to use BlockCall, allowing us to avoid adding new nodes during ssa construction to hold block arguments. Additionally, many places where we previously matched on InstructionData to extract branch destinations can be replaced with a use of branch_destination or branch_destination_mut.
1855 lines
64 KiB
Rust
1855 lines
64 KiB
Rust
//! A frontend for building Cranelift IR from other languages.
|
|
use crate::ssa::{SSABuilder, SideEffects};
|
|
use crate::variable::Variable;
|
|
use core::fmt::{self, Debug};
|
|
use cranelift_codegen::cursor::{Cursor, FuncCursor};
|
|
use cranelift_codegen::entity::{EntityRef, EntitySet, SecondaryMap};
|
|
use cranelift_codegen::ir;
|
|
use cranelift_codegen::ir::condcodes::IntCC;
|
|
use cranelift_codegen::ir::{
|
|
types, AbiParam, Block, DataFlowGraph, DynamicStackSlot, DynamicStackSlotData, ExtFuncData,
|
|
ExternalName, FuncRef, Function, GlobalValue, GlobalValueData, Inst, InstBuilder,
|
|
InstBuilderBase, InstructionData, JumpTable, JumpTableData, LibCall, MemFlags, RelSourceLoc,
|
|
SigRef, Signature, StackSlot, StackSlotData, Type, Value, ValueLabel, ValueLabelAssignments,
|
|
ValueLabelStart,
|
|
};
|
|
use cranelift_codegen::isa::TargetFrontendConfig;
|
|
use cranelift_codegen::packed_option::PackedOption;
|
|
|
|
/// Structure used for translating a series of functions into Cranelift IR.
|
|
///
|
|
/// In order to reduce memory reallocations when compiling multiple functions,
|
|
/// `FunctionBuilderContext` holds various data structures which are cleared between
|
|
/// functions, rather than dropped, preserving the underlying allocations.
|
|
#[derive(Default)]
|
|
pub struct FunctionBuilderContext {
|
|
ssa: SSABuilder,
|
|
status: SecondaryMap<Block, BlockStatus>,
|
|
types: SecondaryMap<Variable, Type>,
|
|
}
|
|
|
|
/// Temporary object used to build a single Cranelift IR `Function`.
|
|
pub struct FunctionBuilder<'a> {
|
|
/// The function currently being built.
|
|
/// This field is public so the function can be re-borrowed.
|
|
pub func: &'a mut Function,
|
|
|
|
/// Source location to assign to all new instructions.
|
|
srcloc: ir::SourceLoc,
|
|
|
|
func_ctx: &'a mut FunctionBuilderContext,
|
|
position: PackedOption<Block>,
|
|
}
|
|
|
|
#[derive(Clone, Default, Eq, PartialEq)]
|
|
enum BlockStatus {
|
|
/// No instructions have been added.
|
|
#[default]
|
|
Empty,
|
|
/// Some instructions have been added, but no terminator.
|
|
Partial,
|
|
/// A terminator has been added; no further instructions may be added.
|
|
Filled,
|
|
}
|
|
|
|
impl FunctionBuilderContext {
|
|
/// Creates a FunctionBuilderContext structure. The structure is automatically cleared after
|
|
/// each [`FunctionBuilder`](struct.FunctionBuilder.html) completes translating a function.
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
fn clear(&mut self) {
|
|
self.ssa.clear();
|
|
self.status.clear();
|
|
self.types.clear();
|
|
}
|
|
|
|
fn is_empty(&self) -> bool {
|
|
self.ssa.is_empty() && self.status.is_empty() && self.types.is_empty()
|
|
}
|
|
}
|
|
|
|
/// Implementation of the [`InstBuilder`](cranelift_codegen::ir::InstBuilder) that has
|
|
/// one convenience method per Cranelift IR instruction.
|
|
pub struct FuncInstBuilder<'short, 'long: 'short> {
|
|
builder: &'short mut FunctionBuilder<'long>,
|
|
block: Block,
|
|
}
|
|
|
|
impl<'short, 'long> FuncInstBuilder<'short, 'long> {
|
|
fn new(builder: &'short mut FunctionBuilder<'long>, block: Block) -> Self {
|
|
Self { builder, block }
|
|
}
|
|
}
|
|
|
|
impl<'short, 'long> InstBuilderBase<'short> for FuncInstBuilder<'short, 'long> {
|
|
fn data_flow_graph(&self) -> &DataFlowGraph {
|
|
&self.builder.func.dfg
|
|
}
|
|
|
|
fn data_flow_graph_mut(&mut self) -> &mut DataFlowGraph {
|
|
&mut self.builder.func.dfg
|
|
}
|
|
|
|
// This implementation is richer than `InsertBuilder` because we use the data of the
|
|
// instruction being inserted to add related info to the DFG and the SSA building system,
|
|
// and perform debug sanity checks.
|
|
fn build(self, data: InstructionData, ctrl_typevar: Type) -> (Inst, &'short mut DataFlowGraph) {
|
|
// We only insert the Block in the layout when an instruction is added to it
|
|
self.builder.ensure_inserted_block();
|
|
|
|
let inst = self.builder.func.dfg.make_inst(data.clone());
|
|
self.builder.func.dfg.make_inst_results(inst, ctrl_typevar);
|
|
self.builder.func.layout.append_inst(inst, self.block);
|
|
if !self.builder.srcloc.is_default() {
|
|
self.builder.func.set_srcloc(inst, self.builder.srcloc);
|
|
}
|
|
|
|
match &self.builder.func.dfg.insts[inst] {
|
|
ir::InstructionData::Jump {
|
|
destination: dest, ..
|
|
} => {
|
|
// If the user has supplied jump arguments we must adapt the arguments of
|
|
// the destination block
|
|
let block = dest.block(&self.builder.func.dfg.value_lists);
|
|
self.builder.declare_successor(block, inst);
|
|
}
|
|
|
|
ir::InstructionData::Brif {
|
|
blocks: [branch_then, branch_else],
|
|
..
|
|
} => {
|
|
let block_then = branch_then.block(&self.builder.func.dfg.value_lists);
|
|
let block_else = branch_else.block(&self.builder.func.dfg.value_lists);
|
|
|
|
self.builder.declare_successor(block_then, inst);
|
|
if block_then != block_else {
|
|
self.builder.declare_successor(block_else, inst);
|
|
}
|
|
}
|
|
|
|
ir::InstructionData::BranchTable { table, .. } => {
|
|
let pool = &self.builder.func.dfg.value_lists;
|
|
|
|
// Unlike all other jumps/branches, jump tables are
|
|
// capable of having the same successor appear
|
|
// multiple times, so we must deduplicate.
|
|
let mut unique = EntitySet::<Block>::new();
|
|
for dest_block in self
|
|
.builder
|
|
.func
|
|
.stencil
|
|
.dfg
|
|
.jump_tables
|
|
.get(*table)
|
|
.expect("you are referencing an undeclared jump table")
|
|
.all_branches()
|
|
{
|
|
let block = dest_block.block(pool);
|
|
if !unique.insert(block) {
|
|
continue;
|
|
}
|
|
|
|
// Call `declare_block_predecessor` instead of `declare_successor` for
|
|
// avoiding the borrow checker.
|
|
self.builder
|
|
.func_ctx
|
|
.ssa
|
|
.declare_block_predecessor(block, inst);
|
|
}
|
|
}
|
|
|
|
inst => debug_assert!(!inst.opcode().is_branch()),
|
|
}
|
|
|
|
if data.opcode().is_terminator() {
|
|
self.builder.fill_current_block()
|
|
}
|
|
(inst, &mut self.builder.func.dfg)
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
|
/// An error encountered when calling [`FunctionBuilder::try_use_var`].
|
|
pub enum UseVariableError {
|
|
UsedBeforeDeclared(Variable),
|
|
}
|
|
|
|
impl fmt::Display for UseVariableError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
UseVariableError::UsedBeforeDeclared(variable) => {
|
|
write!(
|
|
f,
|
|
"variable {} was used before it was defined",
|
|
variable.index()
|
|
)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl std::error::Error for UseVariableError {}
|
|
|
|
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
|
|
/// An error encountered when calling [`FunctionBuilder::try_declare_var`].
|
|
pub enum DeclareVariableError {
|
|
DeclaredMultipleTimes(Variable),
|
|
}
|
|
|
|
impl std::error::Error for DeclareVariableError {}
|
|
|
|
impl fmt::Display for DeclareVariableError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
DeclareVariableError::DeclaredMultipleTimes(variable) => {
|
|
write!(
|
|
f,
|
|
"variable {} was declared multiple times",
|
|
variable.index()
|
|
)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
|
|
/// An error encountered when defining the initial value of a variable.
|
|
pub enum DefVariableError {
|
|
/// The variable was instantiated with a value of the wrong type.
|
|
///
|
|
/// note: to obtain the type of the value, you can call
|
|
/// [`cranelift_codegen::ir::dfg::DataFlowGraph::value_type`] (using the
|
|
/// [`FunctionBuilder.func.dfg`] field)
|
|
TypeMismatch(Variable, Value),
|
|
/// The value was defined (in a call to [`FunctionBuilder::def_var`]) before
|
|
/// it was declared (in a call to [`FunctionBuilder::declare_var`]).
|
|
DefinedBeforeDeclared(Variable),
|
|
}
|
|
|
|
impl fmt::Display for DefVariableError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
DefVariableError::TypeMismatch(variable, value) => {
|
|
write!(
|
|
f,
|
|
"the types of variable {} and value {} are not the same.
|
|
The `Value` supplied to `def_var` must be of the same type as
|
|
the variable was declared to be of in `declare_var`.",
|
|
variable.index(),
|
|
value.as_u32()
|
|
)?;
|
|
}
|
|
DefVariableError::DefinedBeforeDeclared(variable) => {
|
|
write!(
|
|
f,
|
|
"the value of variabe {} was declared before it was defined",
|
|
variable.index()
|
|
)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// This module allows you to create a function in Cranelift IR in a straightforward way, hiding
|
|
/// all the complexity of its internal representation.
|
|
///
|
|
/// The module is parametrized by one type which is the representation of variables in your
|
|
/// origin language. It offers a way to conveniently append instruction to your program flow.
|
|
/// You are responsible to split your instruction flow into extended blocks (declared with
|
|
/// `create_block`) whose properties are:
|
|
///
|
|
/// - branch and jump instructions can only point at the top of extended blocks;
|
|
/// - the last instruction of each block is a terminator instruction which has no natural successor,
|
|
/// and those instructions can only appear at the end of extended blocks.
|
|
///
|
|
/// The parameters of Cranelift IR instructions are Cranelift IR values, which can only be created
|
|
/// as results of other Cranelift IR instructions. To be able to create variables redefined multiple
|
|
/// times in your program, use the `def_var` and `use_var` command, that will maintain the
|
|
/// correspondence between your variables and Cranelift IR SSA values.
|
|
///
|
|
/// The first block for which you call `switch_to_block` will be assumed to be the beginning of
|
|
/// the function.
|
|
///
|
|
/// At creation, a `FunctionBuilder` instance borrows an already allocated `Function` which it
|
|
/// modifies with the information stored in the mutable borrowed
|
|
/// [`FunctionBuilderContext`](struct.FunctionBuilderContext.html). The function passed in
|
|
/// argument should be newly created with
|
|
/// [`Function::with_name_signature()`](Function::with_name_signature), whereas the
|
|
/// `FunctionBuilderContext` can be kept as is between two function translations.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// The functions below will panic in debug mode whenever you try to modify the Cranelift IR
|
|
/// function in a way that violate the coherence of the code. For instance: switching to a new
|
|
/// `Block` when you haven't filled the current one with a terminator instruction, inserting a
|
|
/// return instruction with arguments that don't match the function's signature.
|
|
impl<'a> FunctionBuilder<'a> {
|
|
/// Creates a new FunctionBuilder structure that will operate on a `Function` using a
|
|
/// `FunctionBuilderContext`.
|
|
pub fn new(func: &'a mut Function, func_ctx: &'a mut FunctionBuilderContext) -> Self {
|
|
debug_assert!(func_ctx.is_empty());
|
|
Self {
|
|
func,
|
|
srcloc: Default::default(),
|
|
func_ctx,
|
|
position: Default::default(),
|
|
}
|
|
}
|
|
|
|
/// Get the block that this builder is currently at.
|
|
pub fn current_block(&self) -> Option<Block> {
|
|
self.position.expand()
|
|
}
|
|
|
|
/// Set the source location that should be assigned to all new instructions.
|
|
pub fn set_srcloc(&mut self, srcloc: ir::SourceLoc) {
|
|
self.srcloc = srcloc;
|
|
}
|
|
|
|
/// Creates a new `Block` and returns its reference.
|
|
pub fn create_block(&mut self) -> Block {
|
|
let block = self.func.dfg.make_block();
|
|
self.func_ctx.ssa.declare_block(block);
|
|
block
|
|
}
|
|
|
|
/// Mark a block as "cold".
|
|
///
|
|
/// This will try to move it out of the ordinary path of execution
|
|
/// when lowered to machine code.
|
|
pub fn set_cold_block(&mut self, block: Block) {
|
|
self.func.layout.set_cold(block);
|
|
}
|
|
|
|
/// Insert `block` in the layout *after* the existing block `after`.
|
|
pub fn insert_block_after(&mut self, block: Block, after: Block) {
|
|
self.func.layout.insert_block_after(block, after);
|
|
}
|
|
|
|
/// After the call to this function, new instructions will be inserted into the designated
|
|
/// block, in the order they are declared. You must declare the types of the Block arguments
|
|
/// you will use here.
|
|
///
|
|
/// When inserting the terminator instruction (which doesn't have a fallthrough to its immediate
|
|
/// successor), the block will be declared filled and it will not be possible to append
|
|
/// instructions to it.
|
|
pub fn switch_to_block(&mut self, block: Block) {
|
|
// First we check that the previous block has been filled.
|
|
debug_assert!(
|
|
self.position.is_none()
|
|
|| self.is_unreachable()
|
|
|| self.is_pristine(self.position.unwrap())
|
|
|| self.is_filled(self.position.unwrap()),
|
|
"you have to fill your block before switching"
|
|
);
|
|
// We cannot switch to a filled block
|
|
debug_assert!(
|
|
!self.is_filled(block),
|
|
"you cannot switch to a block which is already filled"
|
|
);
|
|
|
|
// Then we change the cursor position.
|
|
self.position = PackedOption::from(block);
|
|
}
|
|
|
|
/// Declares that all the predecessors of this block are known.
|
|
///
|
|
/// Function to call with `block` as soon as the last branch instruction to `block` has been
|
|
/// created. Forgetting to call this method on every block will cause inconsistencies in the
|
|
/// produced functions.
|
|
pub fn seal_block(&mut self, block: Block) {
|
|
let side_effects = self.func_ctx.ssa.seal_block(block, self.func);
|
|
self.handle_ssa_side_effects(side_effects);
|
|
}
|
|
|
|
/// Effectively calls seal_block on all unsealed blocks in the function.
|
|
///
|
|
/// It's more efficient to seal `Block`s as soon as possible, during
|
|
/// translation, but for frontends where this is impractical to do, this
|
|
/// function can be used at the end of translating all blocks to ensure
|
|
/// that everything is sealed.
|
|
pub fn seal_all_blocks(&mut self) {
|
|
let side_effects = self.func_ctx.ssa.seal_all_blocks(self.func);
|
|
self.handle_ssa_side_effects(side_effects);
|
|
}
|
|
|
|
/// Declares the type of a variable, so that it can be used later (by calling
|
|
/// [`FunctionBuilder::use_var`]). This function will return an error if it
|
|
/// was not possible to use the variable.
|
|
pub fn try_declare_var(&mut self, var: Variable, ty: Type) -> Result<(), DeclareVariableError> {
|
|
if self.func_ctx.types[var] != types::INVALID {
|
|
return Err(DeclareVariableError::DeclaredMultipleTimes(var));
|
|
}
|
|
self.func_ctx.types[var] = ty;
|
|
Ok(())
|
|
}
|
|
|
|
/// In order to use a variable (by calling [`FunctionBuilder::use_var`]), you need
|
|
/// to first declare its type with this method.
|
|
pub fn declare_var(&mut self, var: Variable, ty: Type) {
|
|
self.try_declare_var(var, ty)
|
|
.unwrap_or_else(|_| panic!("the variable {:?} has been declared multiple times", var))
|
|
}
|
|
|
|
/// Returns the Cranelift IR necessary to use a previously defined user
|
|
/// variable, returning an error if this is not possible.
|
|
pub fn try_use_var(&mut self, var: Variable) -> Result<Value, UseVariableError> {
|
|
// Assert that we're about to add instructions to this block using the definition of the
|
|
// given variable. ssa.use_var is the only part of this crate which can add block parameters
|
|
// behind the caller's back. If we disallow calling append_block_param as soon as use_var is
|
|
// called, then we enforce a strict separation between user parameters and SSA parameters.
|
|
self.ensure_inserted_block();
|
|
|
|
let (val, side_effects) = {
|
|
let ty = *self
|
|
.func_ctx
|
|
.types
|
|
.get(var)
|
|
.ok_or(UseVariableError::UsedBeforeDeclared(var))?;
|
|
debug_assert_ne!(
|
|
ty,
|
|
types::INVALID,
|
|
"variable {:?} is used but its type has not been declared",
|
|
var
|
|
);
|
|
self.func_ctx
|
|
.ssa
|
|
.use_var(self.func, var, ty, self.position.unwrap())
|
|
};
|
|
self.handle_ssa_side_effects(side_effects);
|
|
Ok(val)
|
|
}
|
|
|
|
/// Returns the Cranelift IR value corresponding to the utilization at the current program
|
|
/// position of a previously defined user variable.
|
|
pub fn use_var(&mut self, var: Variable) -> Value {
|
|
self.try_use_var(var).unwrap_or_else(|_| {
|
|
panic!(
|
|
"variable {:?} is used but its type has not been declared",
|
|
var
|
|
)
|
|
})
|
|
}
|
|
|
|
/// Registers a new definition of a user variable. This function will return
|
|
/// an error if the value supplied does not match the type the variable was
|
|
/// declared to have.
|
|
pub fn try_def_var(&mut self, var: Variable, val: Value) -> Result<(), DefVariableError> {
|
|
let var_ty = *self
|
|
.func_ctx
|
|
.types
|
|
.get(var)
|
|
.ok_or(DefVariableError::DefinedBeforeDeclared(var))?;
|
|
if var_ty != self.func.dfg.value_type(val) {
|
|
return Err(DefVariableError::TypeMismatch(var, val));
|
|
}
|
|
|
|
self.func_ctx.ssa.def_var(var, val, self.position.unwrap());
|
|
Ok(())
|
|
}
|
|
|
|
/// Register a new definition of a user variable. The type of the value must be
|
|
/// the same as the type registered for the variable.
|
|
pub fn def_var(&mut self, var: Variable, val: Value) {
|
|
self.try_def_var(var, val)
|
|
.unwrap_or_else(|error| match error {
|
|
DefVariableError::TypeMismatch(var, val) => {
|
|
panic!(
|
|
"declared type of variable {:?} doesn't match type of value {}",
|
|
var, val
|
|
);
|
|
}
|
|
DefVariableError::DefinedBeforeDeclared(var) => {
|
|
panic!(
|
|
"variable {:?} is used but its type has not been declared",
|
|
var
|
|
);
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Set label for Value
|
|
///
|
|
/// This will not do anything unless `func.dfg.collect_debug_info` is called first.
|
|
pub fn set_val_label(&mut self, val: Value, label: ValueLabel) {
|
|
if let Some(values_labels) = self.func.stencil.dfg.values_labels.as_mut() {
|
|
use alloc::collections::btree_map::Entry;
|
|
|
|
let start = ValueLabelStart {
|
|
from: RelSourceLoc::from_base_offset(self.func.params.base_srcloc(), self.srcloc),
|
|
label,
|
|
};
|
|
|
|
match values_labels.entry(val) {
|
|
Entry::Occupied(mut e) => match e.get_mut() {
|
|
ValueLabelAssignments::Starts(starts) => starts.push(start),
|
|
_ => panic!("Unexpected ValueLabelAssignments at this stage"),
|
|
},
|
|
Entry::Vacant(e) => {
|
|
e.insert(ValueLabelAssignments::Starts(vec![start]));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Creates a jump table in the function, to be used by `br_table` instructions.
|
|
pub fn create_jump_table(&mut self, data: JumpTableData) -> JumpTable {
|
|
self.func.create_jump_table(data)
|
|
}
|
|
|
|
/// Creates a sized stack slot in the function, to be used by `stack_load`, `stack_store` and
|
|
/// `stack_addr` instructions.
|
|
pub fn create_sized_stack_slot(&mut self, data: StackSlotData) -> StackSlot {
|
|
self.func.create_sized_stack_slot(data)
|
|
}
|
|
|
|
/// Creates a dynamic stack slot in the function, to be used by `dynamic_stack_load`,
|
|
/// `dynamic_stack_store` and `dynamic_stack_addr` instructions.
|
|
pub fn create_dynamic_stack_slot(&mut self, data: DynamicStackSlotData) -> DynamicStackSlot {
|
|
self.func.create_dynamic_stack_slot(data)
|
|
}
|
|
|
|
/// Adds a signature which can later be used to declare an external function import.
|
|
pub fn import_signature(&mut self, signature: Signature) -> SigRef {
|
|
self.func.import_signature(signature)
|
|
}
|
|
|
|
/// Declare an external function import.
|
|
pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef {
|
|
self.func.import_function(data)
|
|
}
|
|
|
|
/// Declares a global value accessible to the function.
|
|
pub fn create_global_value(&mut self, data: GlobalValueData) -> GlobalValue {
|
|
self.func.create_global_value(data)
|
|
}
|
|
|
|
/// Returns an object with the [`InstBuilder`](cranelift_codegen::ir::InstBuilder)
|
|
/// trait that allows to conveniently append an instruction to the current `Block` being built.
|
|
pub fn ins<'short>(&'short mut self) -> FuncInstBuilder<'short, 'a> {
|
|
let block = self
|
|
.position
|
|
.expect("Please call switch_to_block before inserting instructions");
|
|
FuncInstBuilder::new(self, block)
|
|
}
|
|
|
|
/// Make sure that the current block is inserted in the layout.
|
|
pub fn ensure_inserted_block(&mut self) {
|
|
let block = self.position.unwrap();
|
|
if self.is_pristine(block) {
|
|
if !self.func.layout.is_block_inserted(block) {
|
|
self.func.layout.append_block(block);
|
|
}
|
|
self.func_ctx.status[block] = BlockStatus::Partial;
|
|
} else {
|
|
debug_assert!(
|
|
!self.is_filled(block),
|
|
"you cannot add an instruction to a block already filled"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Returns a `FuncCursor` pointed at the current position ready for inserting instructions.
|
|
///
|
|
/// This can be used to insert SSA code that doesn't need to access locals and that doesn't
|
|
/// need to know about `FunctionBuilder` at all.
|
|
pub fn cursor(&mut self) -> FuncCursor {
|
|
self.ensure_inserted_block();
|
|
FuncCursor::new(self.func)
|
|
.with_srcloc(self.srcloc)
|
|
.at_bottom(self.position.unwrap())
|
|
}
|
|
|
|
/// Append parameters to the given `Block` corresponding to the function
|
|
/// parameters. This can be used to set up the block parameters for the
|
|
/// entry block.
|
|
pub fn append_block_params_for_function_params(&mut self, block: Block) {
|
|
debug_assert!(
|
|
!self.func_ctx.ssa.has_any_predecessors(block),
|
|
"block parameters for function parameters should only be added to the entry block"
|
|
);
|
|
|
|
// These parameters count as "user" parameters here because they aren't
|
|
// inserted by the SSABuilder.
|
|
debug_assert!(
|
|
self.is_pristine(block),
|
|
"You can't add block parameters after adding any instruction"
|
|
);
|
|
|
|
for argtyp in &self.func.stencil.signature.params {
|
|
self.func
|
|
.stencil
|
|
.dfg
|
|
.append_block_param(block, argtyp.value_type);
|
|
}
|
|
}
|
|
|
|
/// Append parameters to the given `Block` corresponding to the function
|
|
/// return values. This can be used to set up the block parameters for a
|
|
/// function exit block.
|
|
pub fn append_block_params_for_function_returns(&mut self, block: Block) {
|
|
// These parameters count as "user" parameters here because they aren't
|
|
// inserted by the SSABuilder.
|
|
debug_assert!(
|
|
self.is_pristine(block),
|
|
"You can't add block parameters after adding any instruction"
|
|
);
|
|
|
|
for argtyp in &self.func.stencil.signature.returns {
|
|
self.func
|
|
.stencil
|
|
.dfg
|
|
.append_block_param(block, argtyp.value_type);
|
|
}
|
|
}
|
|
|
|
/// Declare that translation of the current function is complete.
|
|
///
|
|
/// This resets the state of the `FunctionBuilderContext` in preparation to
|
|
/// be used for another function.
|
|
pub fn finalize(self) {
|
|
// Check that all the `Block`s are filled and sealed.
|
|
#[cfg(debug_assertions)]
|
|
{
|
|
for block in self.func_ctx.status.keys() {
|
|
if !self.is_pristine(block) {
|
|
assert!(
|
|
self.func_ctx.ssa.is_sealed(block),
|
|
"FunctionBuilder finalized, but block {} is not sealed",
|
|
block,
|
|
);
|
|
assert!(
|
|
self.is_filled(block),
|
|
"FunctionBuilder finalized, but block {} is not filled",
|
|
block,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// In debug mode, check that all blocks are valid basic blocks.
|
|
#[cfg(debug_assertions)]
|
|
{
|
|
// Iterate manually to provide more helpful error messages.
|
|
for block in self.func_ctx.status.keys() {
|
|
if let Err((inst, msg)) = self.func.is_block_basic(block) {
|
|
let inst_str = self.func.dfg.display_inst(inst);
|
|
panic!(
|
|
"{} failed basic block invariants on {}: {}",
|
|
block, inst_str, msg
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Clear the state (but preserve the allocated buffers) in preparation
|
|
// for translation another function.
|
|
self.func_ctx.clear();
|
|
}
|
|
}
|
|
|
|
/// All the functions documented in the previous block are write-only and help you build a valid
|
|
/// Cranelift IR functions via multiple debug asserts. However, you might need to improve the
|
|
/// performance of your translation perform more complex transformations to your Cranelift IR
|
|
/// function. The functions below help you inspect the function you're creating and modify it
|
|
/// in ways that can be unsafe if used incorrectly.
|
|
impl<'a> FunctionBuilder<'a> {
|
|
/// Retrieves all the parameters for a `Block` currently inferred from the jump instructions
|
|
/// inserted that target it and the SSA construction.
|
|
pub fn block_params(&self, block: Block) -> &[Value] {
|
|
self.func.dfg.block_params(block)
|
|
}
|
|
|
|
/// Retrieves the signature with reference `sigref` previously added with `import_signature`.
|
|
pub fn signature(&self, sigref: SigRef) -> Option<&Signature> {
|
|
self.func.dfg.signatures.get(sigref)
|
|
}
|
|
|
|
/// Creates a parameter for a specific `Block` by appending it to the list of already existing
|
|
/// parameters.
|
|
///
|
|
/// **Note:** this function has to be called at the creation of the `Block` before adding
|
|
/// instructions to it, otherwise this could interfere with SSA construction.
|
|
pub fn append_block_param(&mut self, block: Block, ty: Type) -> Value {
|
|
debug_assert!(
|
|
self.is_pristine(block),
|
|
"You can't add block parameters after adding any instruction"
|
|
);
|
|
self.func.dfg.append_block_param(block, ty)
|
|
}
|
|
|
|
/// Returns the result values of an instruction.
|
|
pub fn inst_results(&self, inst: Inst) -> &[Value] {
|
|
self.func.dfg.inst_results(inst)
|
|
}
|
|
|
|
/// Changes the destination of a jump instruction after creation.
|
|
///
|
|
/// **Note:** You are responsible for maintaining the coherence with the arguments of
|
|
/// other jump instructions.
|
|
pub fn change_jump_destination(&mut self, inst: Inst, old_block: Block, new_block: Block) {
|
|
let dfg = &mut self.func.dfg;
|
|
for block in dfg.insts[inst].branch_destination_mut(&mut dfg.jump_tables) {
|
|
if block.block(&dfg.value_lists) == old_block {
|
|
self.func_ctx.ssa.remove_block_predecessor(old_block, inst);
|
|
block.set_block(new_block, &mut dfg.value_lists);
|
|
self.func_ctx.ssa.declare_block_predecessor(new_block, inst);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Returns `true` if and only if the current `Block` is sealed and has no predecessors declared.
|
|
///
|
|
/// The entry block of a function is never unreachable.
|
|
pub fn is_unreachable(&self) -> bool {
|
|
let is_entry = match self.func.layout.entry_block() {
|
|
None => false,
|
|
Some(entry) => self.position.unwrap() == entry,
|
|
};
|
|
!is_entry
|
|
&& self.func_ctx.ssa.is_sealed(self.position.unwrap())
|
|
&& !self
|
|
.func_ctx
|
|
.ssa
|
|
.has_any_predecessors(self.position.unwrap())
|
|
}
|
|
|
|
/// Returns `true` if and only if no instructions have been added since the last call to
|
|
/// `switch_to_block`.
|
|
fn is_pristine(&self, block: Block) -> bool {
|
|
self.func_ctx.status[block] == BlockStatus::Empty
|
|
}
|
|
|
|
/// Returns `true` if and only if a terminator instruction has been inserted since the
|
|
/// last call to `switch_to_block`.
|
|
fn is_filled(&self, block: Block) -> bool {
|
|
self.func_ctx.status[block] == BlockStatus::Filled
|
|
}
|
|
}
|
|
|
|
/// Helper functions
|
|
impl<'a> FunctionBuilder<'a> {
|
|
/// Calls libc.memcpy
|
|
///
|
|
/// Copies the `size` bytes from `src` to `dest`, assumes that `src + size`
|
|
/// won't overlap onto `dest`. If `dest` and `src` overlap, the behavior is
|
|
/// undefined. Applications in which `dest` and `src` might overlap should
|
|
/// use `call_memmove` instead.
|
|
pub fn call_memcpy(
|
|
&mut self,
|
|
config: TargetFrontendConfig,
|
|
dest: Value,
|
|
src: Value,
|
|
size: Value,
|
|
) {
|
|
let pointer_type = config.pointer_type();
|
|
let signature = {
|
|
let mut s = Signature::new(config.default_call_conv);
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
self.import_signature(s)
|
|
};
|
|
|
|
let libc_memcpy = self.import_function(ExtFuncData {
|
|
name: ExternalName::LibCall(LibCall::Memcpy),
|
|
signature,
|
|
colocated: false,
|
|
});
|
|
|
|
self.ins().call(libc_memcpy, &[dest, src, size]);
|
|
}
|
|
|
|
/// Optimised memcpy or memmove for small copies.
|
|
///
|
|
/// # Codegen safety
|
|
///
|
|
/// The following properties must hold to prevent UB:
|
|
///
|
|
/// * `src_align` and `dest_align` are an upper-bound on the alignment of `src` respectively `dest`.
|
|
/// * If `non_overlapping` is true, then this must be correct.
|
|
pub fn emit_small_memory_copy(
|
|
&mut self,
|
|
config: TargetFrontendConfig,
|
|
dest: Value,
|
|
src: Value,
|
|
size: u64,
|
|
dest_align: u8,
|
|
src_align: u8,
|
|
non_overlapping: bool,
|
|
mut flags: MemFlags,
|
|
) {
|
|
// Currently the result of guess work, not actual profiling.
|
|
const THRESHOLD: u64 = 4;
|
|
|
|
if size == 0 {
|
|
return;
|
|
}
|
|
|
|
let access_size = greatest_divisible_power_of_two(size);
|
|
assert!(
|
|
access_size.is_power_of_two(),
|
|
"`size` is not a power of two"
|
|
);
|
|
assert!(
|
|
access_size >= u64::from(::core::cmp::min(src_align, dest_align)),
|
|
"`size` is smaller than `dest` and `src`'s alignment value."
|
|
);
|
|
|
|
let (access_size, int_type) = if access_size <= 8 {
|
|
(access_size, Type::int((access_size * 8) as u16).unwrap())
|
|
} else {
|
|
(8, types::I64)
|
|
};
|
|
|
|
let load_and_store_amount = size / access_size;
|
|
|
|
if load_and_store_amount > THRESHOLD {
|
|
let size_value = self.ins().iconst(config.pointer_type(), size as i64);
|
|
if non_overlapping {
|
|
self.call_memcpy(config, dest, src, size_value);
|
|
} else {
|
|
self.call_memmove(config, dest, src, size_value);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if u64::from(src_align) >= access_size && u64::from(dest_align) >= access_size {
|
|
flags.set_aligned();
|
|
}
|
|
|
|
// Load all of the memory first. This is necessary in case `dest` overlaps.
|
|
// It can also improve performance a bit.
|
|
let registers: smallvec::SmallVec<[_; THRESHOLD as usize]> = (0..load_and_store_amount)
|
|
.map(|i| {
|
|
let offset = (access_size * i) as i32;
|
|
(self.ins().load(int_type, flags, src, offset), offset)
|
|
})
|
|
.collect();
|
|
|
|
for (value, offset) in registers {
|
|
self.ins().store(flags, value, dest, offset);
|
|
}
|
|
}
|
|
|
|
/// Calls libc.memset
|
|
///
|
|
/// Writes `size` bytes of i8 value `ch` to memory starting at `buffer`.
|
|
pub fn call_memset(
|
|
&mut self,
|
|
config: TargetFrontendConfig,
|
|
buffer: Value,
|
|
ch: Value,
|
|
size: Value,
|
|
) {
|
|
let pointer_type = config.pointer_type();
|
|
let signature = {
|
|
let mut s = Signature::new(config.default_call_conv);
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(types::I32));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
self.import_signature(s)
|
|
};
|
|
|
|
let libc_memset = self.import_function(ExtFuncData {
|
|
name: ExternalName::LibCall(LibCall::Memset),
|
|
signature,
|
|
colocated: false,
|
|
});
|
|
|
|
let ch = self.ins().uextend(types::I32, ch);
|
|
self.ins().call(libc_memset, &[buffer, ch, size]);
|
|
}
|
|
|
|
/// Calls libc.memset
|
|
///
|
|
/// Writes `size` bytes of value `ch` to memory starting at `buffer`.
|
|
pub fn emit_small_memset(
|
|
&mut self,
|
|
config: TargetFrontendConfig,
|
|
buffer: Value,
|
|
ch: u8,
|
|
size: u64,
|
|
buffer_align: u8,
|
|
mut flags: MemFlags,
|
|
) {
|
|
// Currently the result of guess work, not actual profiling.
|
|
const THRESHOLD: u64 = 4;
|
|
|
|
if size == 0 {
|
|
return;
|
|
}
|
|
|
|
let access_size = greatest_divisible_power_of_two(size);
|
|
assert!(
|
|
access_size.is_power_of_two(),
|
|
"`size` is not a power of two"
|
|
);
|
|
assert!(
|
|
access_size >= u64::from(buffer_align),
|
|
"`size` is smaller than `dest` and `src`'s alignment value."
|
|
);
|
|
|
|
let (access_size, int_type) = if access_size <= 8 {
|
|
(access_size, Type::int((access_size * 8) as u16).unwrap())
|
|
} else {
|
|
(8, types::I64)
|
|
};
|
|
|
|
let load_and_store_amount = size / access_size;
|
|
|
|
if load_and_store_amount > THRESHOLD {
|
|
let ch = self.ins().iconst(types::I8, i64::from(ch));
|
|
let size = self.ins().iconst(config.pointer_type(), size as i64);
|
|
self.call_memset(config, buffer, ch, size);
|
|
} else {
|
|
if u64::from(buffer_align) >= access_size {
|
|
flags.set_aligned();
|
|
}
|
|
|
|
let ch = u64::from(ch);
|
|
let raw_value = if int_type == types::I64 {
|
|
ch * 0x0101010101010101_u64
|
|
} else if int_type == types::I32 {
|
|
ch * 0x01010101_u64
|
|
} else if int_type == types::I16 {
|
|
(ch << 8) | ch
|
|
} else {
|
|
assert_eq!(int_type, types::I8);
|
|
ch
|
|
};
|
|
|
|
let value = self.ins().iconst(int_type, raw_value as i64);
|
|
for i in 0..load_and_store_amount {
|
|
let offset = (access_size * i) as i32;
|
|
self.ins().store(flags, value, buffer, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Calls libc.memmove
|
|
///
|
|
/// Copies `size` bytes from memory starting at `source` to memory starting
|
|
/// at `dest`. `source` is always read before writing to `dest`.
|
|
pub fn call_memmove(
|
|
&mut self,
|
|
config: TargetFrontendConfig,
|
|
dest: Value,
|
|
source: Value,
|
|
size: Value,
|
|
) {
|
|
let pointer_type = config.pointer_type();
|
|
let signature = {
|
|
let mut s = Signature::new(config.default_call_conv);
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
self.import_signature(s)
|
|
};
|
|
|
|
let libc_memmove = self.import_function(ExtFuncData {
|
|
name: ExternalName::LibCall(LibCall::Memmove),
|
|
signature,
|
|
colocated: false,
|
|
});
|
|
|
|
self.ins().call(libc_memmove, &[dest, source, size]);
|
|
}
|
|
|
|
/// Calls libc.memcmp
|
|
///
|
|
/// Compares `size` bytes from memory starting at `left` to memory starting
|
|
/// at `right`. Returns `0` if all `n` bytes are equal. If the first difference
|
|
/// is at offset `i`, returns a positive integer if `ugt(left[i], right[i])`
|
|
/// and a negative integer if `ult(left[i], right[i])`.
|
|
///
|
|
/// Returns a C `int`, which is currently always [`types::I32`].
|
|
pub fn call_memcmp(
|
|
&mut self,
|
|
config: TargetFrontendConfig,
|
|
left: Value,
|
|
right: Value,
|
|
size: Value,
|
|
) -> Value {
|
|
let pointer_type = config.pointer_type();
|
|
let signature = {
|
|
let mut s = Signature::new(config.default_call_conv);
|
|
s.params.reserve(3);
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.returns.push(AbiParam::new(types::I32));
|
|
self.import_signature(s)
|
|
};
|
|
|
|
let libc_memcmp = self.import_function(ExtFuncData {
|
|
name: ExternalName::LibCall(LibCall::Memcmp),
|
|
signature,
|
|
colocated: false,
|
|
});
|
|
|
|
let call = self.ins().call(libc_memcmp, &[left, right, size]);
|
|
self.func.dfg.first_result(call)
|
|
}
|
|
|
|
/// Optimised [`Self::call_memcmp`] for small copies.
|
|
///
|
|
/// This implements the byte slice comparison `int_cc(left[..size], right[..size])`.
|
|
///
|
|
/// `left_align` and `right_align` are the statically-known alignments of the
|
|
/// `left` and `right` pointers respectively. These are used to know whether
|
|
/// to mark `load`s as aligned. It's always fine to pass `1` for these, but
|
|
/// passing something higher than the true alignment may trap or otherwise
|
|
/// misbehave as described in [`MemFlags::aligned`].
|
|
///
|
|
/// Note that `memcmp` is a *big-endian* and *unsigned* comparison.
|
|
/// As such, this panics when called with `IntCC::Signed*`.
|
|
pub fn emit_small_memory_compare(
|
|
&mut self,
|
|
config: TargetFrontendConfig,
|
|
int_cc: IntCC,
|
|
left: Value,
|
|
right: Value,
|
|
size: u64,
|
|
left_align: std::num::NonZeroU8,
|
|
right_align: std::num::NonZeroU8,
|
|
flags: MemFlags,
|
|
) -> Value {
|
|
use IntCC::*;
|
|
let (zero_cc, empty_imm) = match int_cc {
|
|
//
|
|
Equal => (Equal, 1),
|
|
NotEqual => (NotEqual, 0),
|
|
|
|
UnsignedLessThan => (SignedLessThan, 0),
|
|
UnsignedGreaterThanOrEqual => (SignedGreaterThanOrEqual, 1),
|
|
UnsignedGreaterThan => (SignedGreaterThan, 0),
|
|
UnsignedLessThanOrEqual => (SignedLessThanOrEqual, 1),
|
|
|
|
SignedLessThan
|
|
| SignedGreaterThanOrEqual
|
|
| SignedGreaterThan
|
|
| SignedLessThanOrEqual => {
|
|
panic!("Signed comparison {} not supported by memcmp", int_cc)
|
|
}
|
|
};
|
|
|
|
if size == 0 {
|
|
return self.ins().iconst(types::I8, empty_imm);
|
|
}
|
|
|
|
// Future work could consider expanding this to handle more-complex scenarios.
|
|
if let Some(small_type) = size.try_into().ok().and_then(Type::int_with_byte_size) {
|
|
if let Equal | NotEqual = zero_cc {
|
|
let mut left_flags = flags;
|
|
if size == left_align.get() as u64 {
|
|
left_flags.set_aligned();
|
|
}
|
|
let mut right_flags = flags;
|
|
if size == right_align.get() as u64 {
|
|
right_flags.set_aligned();
|
|
}
|
|
let left_val = self.ins().load(small_type, left_flags, left, 0);
|
|
let right_val = self.ins().load(small_type, right_flags, right, 0);
|
|
return self.ins().icmp(int_cc, left_val, right_val);
|
|
} else if small_type == types::I8 {
|
|
// Once the big-endian loads from wasmtime#2492 are implemented in
|
|
// the backends, we could easily handle comparisons for more sizes here.
|
|
// But for now, just handle single bytes where we don't need to worry.
|
|
|
|
let mut aligned_flags = flags;
|
|
aligned_flags.set_aligned();
|
|
let left_val = self.ins().load(small_type, aligned_flags, left, 0);
|
|
let right_val = self.ins().load(small_type, aligned_flags, right, 0);
|
|
return self.ins().icmp(int_cc, left_val, right_val);
|
|
}
|
|
}
|
|
|
|
let pointer_type = config.pointer_type();
|
|
let size = self.ins().iconst(pointer_type, size as i64);
|
|
let cmp = self.call_memcmp(config, left, right, size);
|
|
self.ins().icmp_imm(zero_cc, cmp, 0)
|
|
}
|
|
}
|
|
|
|
fn greatest_divisible_power_of_two(size: u64) -> u64 {
|
|
(size as i64 & -(size as i64)) as u64
|
|
}
|
|
|
|
// Helper functions
|
|
impl<'a> FunctionBuilder<'a> {
|
|
/// A Block is 'filled' when a terminator instruction is present.
|
|
fn fill_current_block(&mut self) {
|
|
self.func_ctx.status[self.position.unwrap()] = BlockStatus::Filled;
|
|
}
|
|
|
|
fn declare_successor(&mut self, dest_block: Block, jump_inst: Inst) {
|
|
self.func_ctx
|
|
.ssa
|
|
.declare_block_predecessor(dest_block, jump_inst);
|
|
}
|
|
|
|
fn handle_ssa_side_effects(&mut self, side_effects: SideEffects) {
|
|
for split_block in side_effects.split_blocks_created {
|
|
self.func_ctx.status[split_block] = BlockStatus::Filled;
|
|
}
|
|
for modified_block in side_effects.instructions_added_to_blocks {
|
|
if self.is_pristine(modified_block) {
|
|
self.func_ctx.status[modified_block] = BlockStatus::Partial;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::greatest_divisible_power_of_two;
|
|
use crate::frontend::{
|
|
DeclareVariableError, DefVariableError, FunctionBuilder, FunctionBuilderContext,
|
|
UseVariableError,
|
|
};
|
|
use crate::Variable;
|
|
use alloc::string::ToString;
|
|
use cranelift_codegen::entity::EntityRef;
|
|
use cranelift_codegen::ir::condcodes::IntCC;
|
|
use cranelift_codegen::ir::{types::*, UserFuncName};
|
|
use cranelift_codegen::ir::{AbiParam, Function, InstBuilder, MemFlags, Signature, Value};
|
|
use cranelift_codegen::isa::{CallConv, TargetFrontendConfig, TargetIsa};
|
|
use cranelift_codegen::settings;
|
|
use cranelift_codegen::verifier::verify_function;
|
|
use target_lexicon::PointerWidth;
|
|
|
|
fn sample_function(lazy_seal: bool) {
|
|
let mut sig = Signature::new(CallConv::SystemV);
|
|
sig.returns.push(AbiParam::new(I32));
|
|
sig.params.push(AbiParam::new(I32));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let block1 = builder.create_block();
|
|
let block2 = builder.create_block();
|
|
let block3 = builder.create_block();
|
|
let x = Variable::new(0);
|
|
let y = Variable::new(1);
|
|
let z = Variable::new(2);
|
|
builder.declare_var(x, I32);
|
|
builder.declare_var(y, I32);
|
|
builder.declare_var(z, I32);
|
|
builder.append_block_params_for_function_params(block0);
|
|
|
|
builder.switch_to_block(block0);
|
|
if !lazy_seal {
|
|
builder.seal_block(block0);
|
|
}
|
|
{
|
|
let tmp = builder.block_params(block0)[0]; // the first function parameter
|
|
builder.def_var(x, tmp);
|
|
}
|
|
{
|
|
let tmp = builder.ins().iconst(I32, 2);
|
|
builder.def_var(y, tmp);
|
|
}
|
|
{
|
|
let arg1 = builder.use_var(x);
|
|
let arg2 = builder.use_var(y);
|
|
let tmp = builder.ins().iadd(arg1, arg2);
|
|
builder.def_var(z, tmp);
|
|
}
|
|
builder.ins().jump(block1, &[]);
|
|
|
|
builder.switch_to_block(block1);
|
|
{
|
|
let arg1 = builder.use_var(y);
|
|
let arg2 = builder.use_var(z);
|
|
let tmp = builder.ins().iadd(arg1, arg2);
|
|
builder.def_var(z, tmp);
|
|
}
|
|
{
|
|
let arg = builder.use_var(y);
|
|
builder.ins().brif(arg, block3, &[], block2, &[]);
|
|
}
|
|
|
|
builder.switch_to_block(block2);
|
|
if !lazy_seal {
|
|
builder.seal_block(block2);
|
|
}
|
|
{
|
|
let arg1 = builder.use_var(z);
|
|
let arg2 = builder.use_var(x);
|
|
let tmp = builder.ins().isub(arg1, arg2);
|
|
builder.def_var(z, tmp);
|
|
}
|
|
{
|
|
let arg = builder.use_var(y);
|
|
builder.ins().return_(&[arg]);
|
|
}
|
|
|
|
builder.switch_to_block(block3);
|
|
if !lazy_seal {
|
|
builder.seal_block(block3);
|
|
}
|
|
|
|
{
|
|
let arg1 = builder.use_var(y);
|
|
let arg2 = builder.use_var(x);
|
|
let tmp = builder.ins().isub(arg1, arg2);
|
|
builder.def_var(y, tmp);
|
|
}
|
|
builder.ins().jump(block1, &[]);
|
|
if !lazy_seal {
|
|
builder.seal_block(block1);
|
|
}
|
|
|
|
if lazy_seal {
|
|
builder.seal_all_blocks();
|
|
}
|
|
|
|
builder.finalize();
|
|
}
|
|
|
|
let flags = settings::Flags::new(settings::builder());
|
|
// println!("{}", func.display(None));
|
|
if let Err(errors) = verify_function(&func, &flags) {
|
|
panic!("{}\n{}", func.display(), errors)
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn sample() {
|
|
sample_function(false)
|
|
}
|
|
|
|
#[test]
|
|
fn sample_with_lazy_seal() {
|
|
sample_function(true)
|
|
}
|
|
|
|
#[track_caller]
|
|
fn check(func: &Function, expected_ir: &str) {
|
|
let actual_ir = func.display().to_string();
|
|
assert!(
|
|
expected_ir == actual_ir,
|
|
"Expected:\n{}\nGot:\n{}",
|
|
expected_ir,
|
|
actual_ir
|
|
);
|
|
}
|
|
|
|
/// Helper function to construct a fixed frontend configuration.
|
|
fn systemv_frontend_config() -> TargetFrontendConfig {
|
|
TargetFrontendConfig {
|
|
default_call_conv: CallConv::SystemV,
|
|
pointer_width: PointerWidth::U64,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn memcpy() {
|
|
let frontend_config = systemv_frontend_config();
|
|
let mut sig = Signature::new(frontend_config.default_call_conv);
|
|
sig.returns.push(AbiParam::new(I32));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let x = Variable::new(0);
|
|
let y = Variable::new(1);
|
|
let z = Variable::new(2);
|
|
builder.declare_var(x, frontend_config.pointer_type());
|
|
builder.declare_var(y, frontend_config.pointer_type());
|
|
builder.declare_var(z, I32);
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
let src = builder.use_var(x);
|
|
let dest = builder.use_var(y);
|
|
let size = builder.use_var(y);
|
|
builder.call_memcpy(frontend_config, dest, src, size);
|
|
builder.ins().return_(&[size]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
"function %sample() -> i32 system_v {
|
|
sig0 = (i64, i64, i64) system_v
|
|
fn0 = %Memcpy sig0
|
|
|
|
block0:
|
|
v3 = iconst.i64 0
|
|
v1 -> v3
|
|
v2 = iconst.i64 0
|
|
v0 -> v2
|
|
call fn0(v1, v0, v1) ; v1 = 0, v0 = 0, v1 = 0
|
|
return v1 ; v1 = 0
|
|
}
|
|
",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn small_memcpy() {
|
|
let frontend_config = systemv_frontend_config();
|
|
let mut sig = Signature::new(frontend_config.default_call_conv);
|
|
sig.returns.push(AbiParam::new(I32));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let x = Variable::new(0);
|
|
let y = Variable::new(16);
|
|
builder.declare_var(x, frontend_config.pointer_type());
|
|
builder.declare_var(y, frontend_config.pointer_type());
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
let src = builder.use_var(x);
|
|
let dest = builder.use_var(y);
|
|
let size = 8;
|
|
builder.emit_small_memory_copy(
|
|
frontend_config,
|
|
dest,
|
|
src,
|
|
size,
|
|
8,
|
|
8,
|
|
true,
|
|
MemFlags::new(),
|
|
);
|
|
builder.ins().return_(&[dest]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
"function %sample() -> i32 system_v {
|
|
block0:
|
|
v4 = iconst.i64 0
|
|
v1 -> v4
|
|
v3 = iconst.i64 0
|
|
v0 -> v3
|
|
v2 = load.i64 aligned v0 ; v0 = 0
|
|
store aligned v2, v1 ; v1 = 0
|
|
return v1 ; v1 = 0
|
|
}
|
|
",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn not_so_small_memcpy() {
|
|
let frontend_config = systemv_frontend_config();
|
|
let mut sig = Signature::new(frontend_config.default_call_conv);
|
|
sig.returns.push(AbiParam::new(I32));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let x = Variable::new(0);
|
|
let y = Variable::new(16);
|
|
builder.declare_var(x, frontend_config.pointer_type());
|
|
builder.declare_var(y, frontend_config.pointer_type());
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
let src = builder.use_var(x);
|
|
let dest = builder.use_var(y);
|
|
let size = 8192;
|
|
builder.emit_small_memory_copy(
|
|
frontend_config,
|
|
dest,
|
|
src,
|
|
size,
|
|
8,
|
|
8,
|
|
true,
|
|
MemFlags::new(),
|
|
);
|
|
builder.ins().return_(&[dest]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
"function %sample() -> i32 system_v {
|
|
sig0 = (i64, i64, i64) system_v
|
|
fn0 = %Memcpy sig0
|
|
|
|
block0:
|
|
v4 = iconst.i64 0
|
|
v1 -> v4
|
|
v3 = iconst.i64 0
|
|
v0 -> v3
|
|
v2 = iconst.i64 8192
|
|
call fn0(v1, v0, v2) ; v1 = 0, v0 = 0, v2 = 8192
|
|
return v1 ; v1 = 0
|
|
}
|
|
",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn small_memset() {
|
|
let frontend_config = systemv_frontend_config();
|
|
let mut sig = Signature::new(frontend_config.default_call_conv);
|
|
sig.returns.push(AbiParam::new(I32));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let y = Variable::new(16);
|
|
builder.declare_var(y, frontend_config.pointer_type());
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
let dest = builder.use_var(y);
|
|
let size = 8;
|
|
builder.emit_small_memset(frontend_config, dest, 1, size, 8, MemFlags::new());
|
|
builder.ins().return_(&[dest]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
"function %sample() -> i32 system_v {
|
|
block0:
|
|
v2 = iconst.i64 0
|
|
v0 -> v2
|
|
v1 = iconst.i64 0x0101_0101_0101_0101
|
|
store aligned v1, v0 ; v1 = 0x0101_0101_0101_0101, v0 = 0
|
|
return v0 ; v0 = 0
|
|
}
|
|
",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn not_so_small_memset() {
|
|
let frontend_config = systemv_frontend_config();
|
|
let mut sig = Signature::new(frontend_config.default_call_conv);
|
|
sig.returns.push(AbiParam::new(I32));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let y = Variable::new(16);
|
|
builder.declare_var(y, frontend_config.pointer_type());
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
let dest = builder.use_var(y);
|
|
let size = 8192;
|
|
builder.emit_small_memset(frontend_config, dest, 1, size, 8, MemFlags::new());
|
|
builder.ins().return_(&[dest]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
"function %sample() -> i32 system_v {
|
|
sig0 = (i64, i32, i64) system_v
|
|
fn0 = %Memset sig0
|
|
|
|
block0:
|
|
v4 = iconst.i64 0
|
|
v0 -> v4
|
|
v1 = iconst.i8 1
|
|
v2 = iconst.i64 8192
|
|
v3 = uextend.i32 v1 ; v1 = 1
|
|
call fn0(v0, v3, v2) ; v0 = 0, v2 = 8192
|
|
return v0 ; v0 = 0
|
|
}
|
|
",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn memcmp() {
|
|
use core::str::FromStr;
|
|
use cranelift_codegen::isa;
|
|
|
|
let shared_builder = settings::builder();
|
|
let shared_flags = settings::Flags::new(shared_builder);
|
|
|
|
let triple =
|
|
::target_lexicon::Triple::from_str("x86_64").expect("Couldn't create x86_64 triple");
|
|
|
|
let target = isa::lookup(triple)
|
|
.ok()
|
|
.map(|b| b.finish(shared_flags))
|
|
.expect("This test requires x86_64 support.")
|
|
.expect("Should be able to create backend with default flags");
|
|
|
|
let mut sig = Signature::new(target.default_call_conv());
|
|
sig.returns.push(AbiParam::new(I32));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let x = Variable::new(0);
|
|
let y = Variable::new(1);
|
|
let z = Variable::new(2);
|
|
builder.declare_var(x, target.pointer_type());
|
|
builder.declare_var(y, target.pointer_type());
|
|
builder.declare_var(z, target.pointer_type());
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
let left = builder.use_var(x);
|
|
let right = builder.use_var(y);
|
|
let size = builder.use_var(z);
|
|
let cmp = builder.call_memcmp(target.frontend_config(), left, right, size);
|
|
builder.ins().return_(&[cmp]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
"function %sample() -> i32 system_v {
|
|
sig0 = (i64, i64, i64) -> i32 system_v
|
|
fn0 = %Memcmp sig0
|
|
|
|
block0:
|
|
v6 = iconst.i64 0
|
|
v2 -> v6
|
|
v5 = iconst.i64 0
|
|
v1 -> v5
|
|
v4 = iconst.i64 0
|
|
v0 -> v4
|
|
v3 = call fn0(v0, v1, v2) ; v0 = 0, v1 = 0, v2 = 0
|
|
return v3
|
|
}
|
|
",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn small_memcmp_zero_size() {
|
|
let align_eight = std::num::NonZeroU8::new(8).unwrap();
|
|
small_memcmp_helper(
|
|
"
|
|
block0:
|
|
v4 = iconst.i64 0
|
|
v1 -> v4
|
|
v3 = iconst.i64 0
|
|
v0 -> v3
|
|
v2 = iconst.i8 1
|
|
return v2 ; v2 = 1",
|
|
|builder, target, x, y| {
|
|
builder.emit_small_memory_compare(
|
|
target.frontend_config(),
|
|
IntCC::UnsignedGreaterThanOrEqual,
|
|
x,
|
|
y,
|
|
0,
|
|
align_eight,
|
|
align_eight,
|
|
MemFlags::new(),
|
|
)
|
|
},
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn small_memcmp_byte_ugt() {
|
|
let align_one = std::num::NonZeroU8::new(1).unwrap();
|
|
small_memcmp_helper(
|
|
"
|
|
block0:
|
|
v6 = iconst.i64 0
|
|
v1 -> v6
|
|
v5 = iconst.i64 0
|
|
v0 -> v5
|
|
v2 = load.i8 aligned v0 ; v0 = 0
|
|
v3 = load.i8 aligned v1 ; v1 = 0
|
|
v4 = icmp ugt v2, v3
|
|
return v4",
|
|
|builder, target, x, y| {
|
|
builder.emit_small_memory_compare(
|
|
target.frontend_config(),
|
|
IntCC::UnsignedGreaterThan,
|
|
x,
|
|
y,
|
|
1,
|
|
align_one,
|
|
align_one,
|
|
MemFlags::new(),
|
|
)
|
|
},
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn small_memcmp_aligned_eq() {
|
|
let align_four = std::num::NonZeroU8::new(4).unwrap();
|
|
small_memcmp_helper(
|
|
"
|
|
block0:
|
|
v6 = iconst.i64 0
|
|
v1 -> v6
|
|
v5 = iconst.i64 0
|
|
v0 -> v5
|
|
v2 = load.i32 aligned v0 ; v0 = 0
|
|
v3 = load.i32 aligned v1 ; v1 = 0
|
|
v4 = icmp eq v2, v3
|
|
return v4",
|
|
|builder, target, x, y| {
|
|
builder.emit_small_memory_compare(
|
|
target.frontend_config(),
|
|
IntCC::Equal,
|
|
x,
|
|
y,
|
|
4,
|
|
align_four,
|
|
align_four,
|
|
MemFlags::new(),
|
|
)
|
|
},
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn small_memcmp_ipv6_ne() {
|
|
let align_two = std::num::NonZeroU8::new(2).unwrap();
|
|
small_memcmp_helper(
|
|
"
|
|
block0:
|
|
v6 = iconst.i64 0
|
|
v1 -> v6
|
|
v5 = iconst.i64 0
|
|
v0 -> v5
|
|
v2 = load.i128 v0 ; v0 = 0
|
|
v3 = load.i128 v1 ; v1 = 0
|
|
v4 = icmp ne v2, v3
|
|
return v4",
|
|
|builder, target, x, y| {
|
|
builder.emit_small_memory_compare(
|
|
target.frontend_config(),
|
|
IntCC::NotEqual,
|
|
x,
|
|
y,
|
|
16,
|
|
align_two,
|
|
align_two,
|
|
MemFlags::new(),
|
|
)
|
|
},
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn small_memcmp_odd_size_uge() {
|
|
let one = std::num::NonZeroU8::new(1).unwrap();
|
|
small_memcmp_helper(
|
|
"
|
|
sig0 = (i64, i64, i64) -> i32 system_v
|
|
fn0 = %Memcmp sig0
|
|
|
|
block0:
|
|
v6 = iconst.i64 0
|
|
v1 -> v6
|
|
v5 = iconst.i64 0
|
|
v0 -> v5
|
|
v2 = iconst.i64 3
|
|
v3 = call fn0(v0, v1, v2) ; v0 = 0, v1 = 0, v2 = 3
|
|
v4 = icmp_imm sge v3, 0
|
|
return v4",
|
|
|builder, target, x, y| {
|
|
builder.emit_small_memory_compare(
|
|
target.frontend_config(),
|
|
IntCC::UnsignedGreaterThanOrEqual,
|
|
x,
|
|
y,
|
|
3,
|
|
one,
|
|
one,
|
|
MemFlags::new(),
|
|
)
|
|
},
|
|
);
|
|
}
|
|
|
|
fn small_memcmp_helper(
|
|
expected: &str,
|
|
f: impl FnOnce(&mut FunctionBuilder, &dyn TargetIsa, Value, Value) -> Value,
|
|
) {
|
|
use core::str::FromStr;
|
|
use cranelift_codegen::isa;
|
|
|
|
let shared_builder = settings::builder();
|
|
let shared_flags = settings::Flags::new(shared_builder);
|
|
|
|
let triple =
|
|
::target_lexicon::Triple::from_str("x86_64").expect("Couldn't create x86_64 triple");
|
|
|
|
let target = isa::lookup(triple)
|
|
.ok()
|
|
.map(|b| b.finish(shared_flags))
|
|
.expect("This test requires x86_64 support.")
|
|
.expect("Should be able to create backend with default flags");
|
|
|
|
let mut sig = Signature::new(target.default_call_conv());
|
|
sig.returns.push(AbiParam::new(I8));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let x = Variable::new(0);
|
|
let y = Variable::new(1);
|
|
builder.declare_var(x, target.pointer_type());
|
|
builder.declare_var(y, target.pointer_type());
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
let left = builder.use_var(x);
|
|
let right = builder.use_var(y);
|
|
let ret = f(&mut builder, &*target, left, right);
|
|
builder.ins().return_(&[ret]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
&format!("function %sample() -> i8 system_v {{{}\n}}\n", expected),
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn undef_vector_vars() {
|
|
let mut sig = Signature::new(CallConv::SystemV);
|
|
sig.returns.push(AbiParam::new(I8X16));
|
|
sig.returns.push(AbiParam::new(I8X16));
|
|
sig.returns.push(AbiParam::new(F32X4));
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
let a = Variable::new(0);
|
|
let b = Variable::new(1);
|
|
let c = Variable::new(2);
|
|
builder.declare_var(a, I8X16);
|
|
builder.declare_var(b, I8X16);
|
|
builder.declare_var(c, F32X4);
|
|
builder.switch_to_block(block0);
|
|
|
|
let a = builder.use_var(a);
|
|
let b = builder.use_var(b);
|
|
let c = builder.use_var(c);
|
|
builder.ins().return_(&[a, b, c]);
|
|
|
|
builder.seal_all_blocks();
|
|
builder.finalize();
|
|
}
|
|
|
|
check(
|
|
&func,
|
|
"function %sample() -> i8x16, i8x16, f32x4 system_v {
|
|
const0 = 0x00000000000000000000000000000000
|
|
|
|
block0:
|
|
v5 = f32const 0.0
|
|
v6 = splat.f32x4 v5 ; v5 = 0.0
|
|
v2 -> v6
|
|
v4 = vconst.i8x16 const0
|
|
v1 -> v4
|
|
v3 = vconst.i8x16 const0
|
|
v0 -> v3
|
|
return v0, v1, v2 ; v0 = const0, v1 = const0
|
|
}
|
|
",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_greatest_divisible_power_of_two() {
|
|
assert_eq!(64, greatest_divisible_power_of_two(64));
|
|
assert_eq!(16, greatest_divisible_power_of_two(48));
|
|
assert_eq!(8, greatest_divisible_power_of_two(24));
|
|
assert_eq!(1, greatest_divisible_power_of_two(25));
|
|
}
|
|
|
|
#[test]
|
|
fn try_use_var() {
|
|
let sig = Signature::new(CallConv::SystemV);
|
|
|
|
let mut fn_ctx = FunctionBuilderContext::new();
|
|
let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
|
|
{
|
|
let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
|
|
|
|
let block0 = builder.create_block();
|
|
builder.append_block_params_for_function_params(block0);
|
|
builder.switch_to_block(block0);
|
|
|
|
assert_eq!(
|
|
builder.try_use_var(Variable::from_u32(0)),
|
|
Err(UseVariableError::UsedBeforeDeclared(Variable::from_u32(0)))
|
|
);
|
|
|
|
let value = builder.ins().iconst(cranelift_codegen::ir::types::I32, 0);
|
|
|
|
assert_eq!(
|
|
builder.try_def_var(Variable::from_u32(0), value),
|
|
Err(DefVariableError::DefinedBeforeDeclared(Variable::from_u32(
|
|
0
|
|
)))
|
|
);
|
|
|
|
builder.declare_var(Variable::from_u32(0), cranelift_codegen::ir::types::I32);
|
|
assert_eq!(
|
|
builder.try_declare_var(Variable::from_u32(0), cranelift_codegen::ir::types::I32),
|
|
Err(DeclareVariableError::DeclaredMultipleTimes(
|
|
Variable::from_u32(0)
|
|
))
|
|
);
|
|
}
|
|
}
|
|
}
|