T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

moved crates in lib/ to src/, renamed crates, modified some files' text (#660)

Browse Source 
moved crates in lib/ to src/, renamed crates, modified some files' text (#660)
This commit is contained in:
lazypassion
2019-01-28 18:56:54 -05:00
committed by Dan Gohman
parent 54959cf5bb
commit 747ad3c4c5
508 changed files with 94 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

716
cranelift/codegen/src/legalizer/boundary.rs Normal file
View File

@@ -0,0 +1,716 @@
//! Legalize ABI boundaries.
//!
//! This legalizer sub-module contains code for dealing with ABI boundaries:
//!
//! - Function arguments passed to the entry block.
//! - Function arguments passed to call instructions.
//! - Return values from call instructions.
//! - Return values passed to return instructions.
//!
//! The ABI boundary legalization happens in two phases:
//!
//! 1. The `legalize_signatures` function rewrites all the preamble signatures with ABI information
//! and possibly new argument types. It also rewrites the entry block arguments to match.
//! 2. The `handle_call_abi` and `handle_return_abi` functions rewrite call and return instructions
//! to match the new ABI signatures.
//!
//! Between the two phases, preamble signatures and call/return arguments don't match. This
//! intermediate state doesn't type check.
use crate::abi::{legalize_abi_value, ValueConversion};
use crate::cursor::{Cursor, FuncCursor};
use crate::flowgraph::ControlFlowGraph;
use crate::ir::instructions::CallInfo;
use crate::ir::{
AbiParam, ArgumentLoc, ArgumentPurpose, DataFlowGraph, Ebb, Function, Inst, InstBuilder,
SigRef, Signature, Type, Value, ValueLoc,
};
use crate::isa::TargetIsa;
use crate::legalizer::split::{isplit, vsplit};
use log::debug;
use std::vec::Vec;
/// Legalize all the function signatures in `func`.
///
/// This changes all signatures to be ABI-compliant with full `ArgumentLoc` annotations. It doesn't
/// change the entry block arguments, calls, or return instructions, so this can leave the function
/// in a state with type discrepancies.
pub fn legalize_signatures(func: &mut Function, isa: &TargetIsa) {
legalize_signature(&mut func.signature, true, isa);
for sig_data in func.dfg.signatures.values_mut() {
legalize_signature(sig_data, false, isa);
}
if let Some(entry) = func.layout.entry_block() {
legalize_entry_params(func, entry);
spill_entry_params(func, entry);
}
}
/// Legalize the libcall signature, which we may generate on the fly after
/// `legalize_signatures` has been called.
pub fn legalize_libcall_signature(signature: &mut Signature, isa: &TargetIsa) {
legalize_signature(signature, false, isa);
}
/// Legalize the given signature.
///
/// `current` is true if this is the signature for the current function.
fn legalize_signature(signature: &mut Signature, current: bool, isa: &TargetIsa) {
isa.legalize_signature(signature, current);
}
/// Legalize the entry block parameters after `func`'s signature has been legalized.
///
/// The legalized signature may contain more parameters than the original signature, and the
/// parameter types have been changed. This function goes through the parameters of the entry EBB
/// and replaces them with parameters of the right type for the ABI.
///
/// The original entry EBB parameters are computed from the new ABI parameters by code inserted at
/// the top of the entry block.
fn legalize_entry_params(func: &mut Function, entry: Ebb) {
let mut has_sret = false;
let mut has_link = false;
let mut has_vmctx = false;
let mut has_sigid = false;
let mut has_stack_limit = false;
// Insert position for argument conversion code.
// We want to insert instructions before the first instruction in the entry block.
// If the entry block is empty, append instructions to it instead.
let mut pos = FuncCursor::new(func).at_first_inst(entry);
// Keep track of the argument types in the ABI-legalized signature.
let mut abi_arg = 0;
// Process the EBB parameters one at a time, possibly replacing one argument with multiple new
// ones. We do this by detaching the entry EBB parameters first.
let ebb_params = pos.func.dfg.detach_ebb_params(entry);
let mut old_arg = 0;
while let Some(arg) = ebb_params.get(old_arg, &pos.func.dfg.value_lists) {
old_arg += 1;
let abi_type = pos.func.signature.params[abi_arg];
let arg_type = pos.func.dfg.value_type(arg);
if arg_type == abi_type.value_type {
// No value translation is necessary, this argument matches the ABI type.
// Just use the original EBB argument value. This is the most common case.
pos.func.dfg.attach_ebb_param(entry, arg);
match abi_type.purpose {
ArgumentPurpose::Normal => {}
ArgumentPurpose::FramePointer => {}
ArgumentPurpose::CalleeSaved => {}
ArgumentPurpose::StructReturn => {
debug_assert!(!has_sret, "Multiple sret arguments found");
has_sret = true;
}
ArgumentPurpose::VMContext => {
debug_assert!(!has_vmctx, "Multiple vmctx arguments found");
has_vmctx = true;
}
ArgumentPurpose::SignatureId => {
debug_assert!(!has_sigid, "Multiple sigid arguments found");
has_sigid = true;
}
ArgumentPurpose::StackLimit => {
debug_assert!(!has_stack_limit, "Multiple stack_limit arguments found");
has_stack_limit = true;
}
_ => panic!("Unexpected special-purpose arg {}", abi_type),
}
abi_arg += 1;
} else {
// Compute the value we want for `arg` from the legalized ABI parameters.
let mut get_arg = |func: &mut Function, ty| {
let abi_type = func.signature.params[abi_arg];
debug_assert_eq!(
abi_type.purpose,
ArgumentPurpose::Normal,
"Can't legalize special-purpose argument"
);
if ty == abi_type.value_type {
abi_arg += 1;
Ok(func.dfg.append_ebb_param(entry, ty))
} else {
Err(abi_type)
}
};
let converted = convert_from_abi(&mut pos, arg_type, Some(arg), &mut get_arg);
// The old `arg` is no longer an attached EBB argument, but there are probably still
// uses of the value.
debug_assert_eq!(pos.func.dfg.resolve_aliases(arg), converted);
}
}
// The legalized signature may contain additional parameters representing special-purpose
// registers.
for &arg in &pos.func.signature.params[abi_arg..] {
match arg.purpose {
// Any normal parameters should have been processed above.
ArgumentPurpose::Normal => {
panic!("Leftover arg: {}", arg);
}
// The callee-save parameters should not appear until after register allocation is
// done.
ArgumentPurpose::FramePointer | ArgumentPurpose::CalleeSaved => {
panic!("Premature callee-saved arg {}", arg);
}
// These can be meaningfully added by `legalize_signature()`.
ArgumentPurpose::Link => {
debug_assert!(!has_link, "Multiple link parameters found");
has_link = true;
}
ArgumentPurpose::StructReturn => {
debug_assert!(!has_sret, "Multiple sret parameters found");
has_sret = true;
}
ArgumentPurpose::VMContext => {
debug_assert!(!has_vmctx, "Multiple vmctx parameters found");
has_vmctx = true;
}
ArgumentPurpose::SignatureId => {
debug_assert!(!has_sigid, "Multiple sigid parameters found");
has_sigid = true;
}
ArgumentPurpose::StackLimit => {
debug_assert!(!has_stack_limit, "Multiple stack_limit parameters found");
has_stack_limit = true;
}
}
// Just create entry block values to match here. We will use them in `handle_return_abi()`
// below.
pos.func.dfg.append_ebb_param(entry, arg.value_type);
}
}
/// Legalize the results returned from a call instruction to match the ABI signature.
///
/// The cursor `pos` points to a call instruction with at least one return value. The cursor will
/// be left pointing after the instructions inserted to convert the return values.
///
/// This function is very similar to the `legalize_entry_params` function above.
///
/// Returns the possibly new instruction representing the call.
fn legalize_inst_results<ResType>(pos: &mut FuncCursor, mut get_abi_type: ResType) -> Inst
where
ResType: FnMut(&Function, usize) -> AbiParam,
{
let call = pos
.current_inst()
.expect("Cursor must point to a call instruction");
// We theoretically allow for call instructions that return a number of fixed results before
// the call return values. In practice, it doesn't happen.
debug_assert_eq!(
pos.func.dfg[call]
.opcode()
.constraints()
.num_fixed_results(),
0,
"Fixed results on calls not supported"
);
let results = pos.func.dfg.detach_results(call);
let mut next_res = 0;
let mut abi_res = 0;
// Point immediately after the call.
pos.next_inst();
while let Some(res) = results.get(next_res, &pos.func.dfg.value_lists) {
next_res += 1;
let res_type = pos.func.dfg.value_type(res);
if res_type == get_abi_type(pos.func, abi_res).value_type {
// No value translation is necessary, this result matches the ABI type.
pos.func.dfg.attach_result(call, res);
abi_res += 1;
} else {
let mut get_res = |func: &mut Function, ty| {
let abi_type = get_abi_type(func, abi_res);
if ty == abi_type.value_type {
let last_res = func.dfg.append_result(call, ty);
abi_res += 1;
Ok(last_res)
} else {
Err(abi_type)
}
};
let v = convert_from_abi(pos, res_type, Some(res), &mut get_res);
debug_assert_eq!(pos.func.dfg.resolve_aliases(res), v);
}
}
call
}
/// Compute original value of type `ty` from the legalized ABI arguments.
///
/// The conversion is recursive, controlled by the `get_arg` closure which is called to retrieve an
/// ABI argument. It returns:
///
/// - `Ok(arg)` if the requested type matches the next ABI argument.
/// - `Err(arg_type)` if further conversions are needed from the ABI argument `arg_type`.
///
/// If the `into_result` value is provided, the converted result will be written into that value.
fn convert_from_abi<GetArg>(
pos: &mut FuncCursor,
ty: Type,
into_result: Option<Value>,
get_arg: &mut GetArg,
) -> Value
where
GetArg: FnMut(&mut Function, Type) -> Result<Value, AbiParam>,
{
// Terminate the recursion when we get the desired type.
let arg_type = match get_arg(pos.func, ty) {
Ok(v) => {
debug_assert_eq!(pos.func.dfg.value_type(v), ty);
debug_assert_eq!(into_result, None);
return v;
}
Err(t) => t,
};
// Reconstruct how `ty` was legalized into the `arg_type` argument.
let conversion = legalize_abi_value(ty, &arg_type);
debug!("convert_from_abi({}): {:?}", ty, conversion);
// The conversion describes value to ABI argument. We implement the reverse conversion here.
match conversion {
// Construct a `ty` by concatenating two ABI integers.
ValueConversion::IntSplit => {
let abi_ty = ty.half_width().expect("Invalid type for conversion");
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
debug!(
"intsplit {}: {}, {}: {}",
lo,
pos.func.dfg.value_type(lo),
hi,
pos.func.dfg.value_type(hi)
);
pos.ins().with_results([into_result]).iconcat(lo, hi)
}
// Construct a `ty` by concatenating two halves of a vector.
ValueConversion::VectorSplit => {
let abi_ty = ty.half_vector().expect("Invalid type for conversion");
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).vconcat(lo, hi)
}
// Construct a `ty` by bit-casting from an integer type.
ValueConversion::IntBits => {
debug_assert!(!ty.is_int());
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).bitcast(ty, arg)
}
// ABI argument is a sign-extended version of the value we want.
ValueConversion::Sext(abi_ty) => {
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
// TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
// We could insert an `assert_sreduce` which would fold with a following `sextend` of
// this value.
pos.ins().with_results([into_result]).ireduce(ty, arg)
}
ValueConversion::Uext(abi_ty) => {
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
// TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
// We could insert an `assert_ureduce` which would fold with a following `uextend` of
// this value.
pos.ins().with_results([into_result]).ireduce(ty, arg)
}
}
}
/// Convert `value` to match an ABI signature by inserting instructions at `pos`.
///
/// This may require expanding the value to multiple ABI arguments. The conversion process is
/// recursive and controlled by the `put_arg` closure. When a candidate argument value is presented
/// to the closure, it will perform one of two actions:
///
/// 1. If the suggested argument has an acceptable value type, consume it by adding it to the list
/// of arguments and return `Ok(())`.
/// 2. If the suggested argument doesn't have the right value type, don't change anything, but
/// return the `Err(AbiParam)` that is needed.
///
fn convert_to_abi<PutArg>(
pos: &mut FuncCursor,
cfg: &ControlFlowGraph,
value: Value,
put_arg: &mut PutArg,
) where
PutArg: FnMut(&mut Function, Value) -> Result<(), AbiParam>,
{
// Start by invoking the closure to either terminate the recursion or get the argument type
// we're trying to match.
let arg_type = match put_arg(pos.func, value) {
Ok(_) => return,
Err(t) => t,
};
let ty = pos.func.dfg.value_type(value);
match legalize_abi_value(ty, &arg_type) {
ValueConversion::IntSplit => {
let curpos = pos.position();
let srcloc = pos.srcloc();
let (lo, hi) = isplit(&mut pos.func, cfg, curpos, srcloc, value);
convert_to_abi(pos, cfg, lo, put_arg);
convert_to_abi(pos, cfg, hi, put_arg);
}
ValueConversion::VectorSplit => {
let curpos = pos.position();
let srcloc = pos.srcloc();
let (lo, hi) = vsplit(&mut pos.func, cfg, curpos, srcloc, value);
convert_to_abi(pos, cfg, lo, put_arg);
convert_to_abi(pos, cfg, hi, put_arg);
}
ValueConversion::IntBits => {
debug_assert!(!ty.is_int());
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
let arg = pos.ins().bitcast(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
ValueConversion::Sext(abi_ty) => {
let arg = pos.ins().sextend(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
ValueConversion::Uext(abi_ty) => {
let arg = pos.ins().uextend(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
}
}
/// Check if a sequence of arguments match a desired sequence of argument types.
fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool {
let arg_types = args.iter().map(|&v| dfg.value_type(v));
let sig_types = types.iter().map(|&at| at.value_type);
arg_types.eq(sig_types)
}
/// Check if the arguments of the call `inst` match the signature.
///
/// Returns `Ok(())` if the signature matches and no changes are needed, or `Err(sig_ref)` if the
/// signature doesn't match.
fn check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Result<(), SigRef> {
// Extract the signature and argument values.
let (sig_ref, args) = match dfg[inst].analyze_call(&dfg.value_lists) {
CallInfo::Direct(func, args) => (dfg.ext_funcs[func].signature, args),
CallInfo::Indirect(sig_ref, args) => (sig_ref, args),
CallInfo::NotACall => panic!("Expected call, got {:?}", dfg[inst]),
};
let sig = &dfg.signatures[sig_ref];
if check_arg_types(dfg, args, &sig.params[..])
&& check_arg_types(dfg, dfg.inst_results(inst), &sig.returns[..])
{
// All types check out.
Ok(())
} else {
// Call types need fixing.
Err(sig_ref)
}
}
/// Check if the arguments of the return `inst` match the signature.
fn check_return_signature(dfg: &DataFlowGraph, inst: Inst, sig: &Signature) -> bool {
check_arg_types(dfg, dfg.inst_variable_args(inst), &sig.returns)
}
/// Insert ABI conversion code for the arguments to the call or return instruction at `pos`.
///
/// - `abi_args` is the number of arguments that the ABI signature requires.
/// - `get_abi_type` is a closure that can provide the desired `AbiParam` for a given ABI
/// argument number in `0..abi_args`.
///
fn legalize_inst_arguments<ArgType>(
pos: &mut FuncCursor,
cfg: &ControlFlowGraph,
abi_args: usize,
mut get_abi_type: ArgType,
) where
ArgType: FnMut(&Function, usize) -> AbiParam,
{
let inst = pos
.current_inst()
.expect("Cursor must point to a call instruction");
// Lift the value list out of the call instruction so we modify it.
let mut vlist = pos.func.dfg[inst]
.take_value_list()
.expect("Call must have a value list");
// The value list contains all arguments to the instruction, including the callee on an
// indirect call which isn't part of the call arguments that must match the ABI signature.
// Figure out how many fixed values are at the front of the list. We won't touch those.
let num_fixed_values = pos.func.dfg[inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
let have_args = vlist.len(&pos.func.dfg.value_lists) - num_fixed_values;
// Grow the value list to the right size and shift all the existing arguments to the right.
// This lets us write the new argument values into the list without overwriting the old
// arguments.
//
// Before:
//
// <--> fixed_values
// <-----------> have_args
// [FFFFOOOOOOOOOOOOO]
//
// After grow_at():
//
// <--> fixed_values
// <-----------> have_args
// <------------------> abi_args
// [FFFF-------OOOOOOOOOOOOO]
// ^
// old_arg_offset
//
// After writing the new arguments:
//
// <--> fixed_values
// <------------------> abi_args
// [FFFFNNNNNNNNNNNNNNNNNNNN]
//
vlist.grow_at(
num_fixed_values,
abi_args - have_args,
&mut pos.func.dfg.value_lists,
);
let old_arg_offset = num_fixed_values + abi_args - have_args;
let mut abi_arg = 0;
for old_arg in 0..have_args {
let old_value = vlist
.get(old_arg_offset + old_arg, &pos.func.dfg.value_lists)
.unwrap();
let mut put_arg = |func: &mut Function, arg| {
let abi_type = get_abi_type(func, abi_arg);
if func.dfg.value_type(arg) == abi_type.value_type {
// This is the argument type we need.
vlist.as_mut_slice(&mut func.dfg.value_lists)[num_fixed_values + abi_arg] = arg;
abi_arg += 1;
Ok(())
} else {
Err(abi_type)
}
};
convert_to_abi(pos, cfg, old_value, &mut put_arg);
}
// Put the modified value list back.
pos.func.dfg[inst].put_value_list(vlist);
}
/// Insert ABI conversion code before and after the call instruction at `pos`.
///
/// Instructions inserted before the call will compute the appropriate ABI values for the
/// callee's new ABI-legalized signature. The function call arguments are rewritten in place to
/// match the new signature.
///
/// Instructions will be inserted after the call to convert returned ABI values back to the
/// original return values. The call's result values will be adapted to match the new signature.
///
/// Returns `true` if any instructions were inserted.
pub fn handle_call_abi(mut inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
let pos = &mut FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Start by checking if the argument types already match the signature.
let sig_ref = match check_call_signature(&pos.func.dfg, inst) {
Ok(_) => return spill_call_arguments(pos),
Err(s) => s,
};
// OK, we need to fix the call arguments to match the ABI signature.
let abi_args = pos.func.dfg.signatures[sig_ref].params.len();
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
func.dfg.signatures[sig_ref].params[abi_arg]
});
if !pos.func.dfg.signatures[sig_ref].returns.is_empty() {
inst = legalize_inst_results(pos, |func, abi_res| {
func.dfg.signatures[sig_ref].returns[abi_res]
});
}
debug_assert!(
check_call_signature(&pos.func.dfg, inst).is_ok(),
"Signature still wrong: {}, {}{}",
pos.func.dfg.display_inst(inst, None),
sig_ref,
pos.func.dfg.signatures[sig_ref]
);
// Go back and insert spills for any stack arguments.
pos.goto_inst(inst);
spill_call_arguments(pos);
// Yes, we changed stuff.
true
}
/// Insert ABI conversion code before and after the return instruction at `inst`.
///
/// Return `true` if any instructions were inserted.
pub fn handle_return_abi(inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
// Check if the returned types already match the signature.
if check_return_signature(&func.dfg, inst, &func.signature) {
return false;
}
// Count the special-purpose return values (`link`, `sret`, and `vmctx`) that were appended to
// the legalized signature.
let special_args = func
.signature
.returns
.iter()
.rev()
.take_while(|&rt| {
rt.purpose == ArgumentPurpose::Link
|| rt.purpose == ArgumentPurpose::StructReturn
|| rt.purpose == ArgumentPurpose::VMContext
})
.count();
let abi_args = func.signature.returns.len() - special_args;
let pos = &mut FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
func.signature.returns[abi_arg]
});
debug_assert_eq!(pos.func.dfg.inst_variable_args(inst).len(), abi_args);
// Append special return arguments for any `sret`, `link`, and `vmctx` return values added to
// the legalized signature. These values should simply be propagated from the entry block
// arguments.
if special_args > 0 {
debug!(
"Adding {} special-purpose arguments to {}",
special_args,
pos.func.dfg.display_inst(inst, None)
);
let mut vlist = pos.func.dfg[inst].take_value_list().unwrap();
for arg in &pos.func.signature.returns[abi_args..] {
match arg.purpose {
ArgumentPurpose::Link
| ArgumentPurpose::StructReturn
| ArgumentPurpose::VMContext => {}
ArgumentPurpose::Normal => panic!("unexpected return value {}", arg),
_ => panic!("Unsupported special purpose return value {}", arg),
}
// A `link`/`sret`/`vmctx` return value can only appear in a signature that has a
// unique matching argument. They are appended at the end, so search the signature from
// the end.
let idx = pos
.func
.signature
.params
.iter()
.rposition(|t| t.purpose == arg.purpose)
.expect("No matching special purpose argument.");
// Get the corresponding entry block value and add it to the return instruction's
// arguments.
let val = pos
.func
.dfg
.ebb_params(pos.func.layout.entry_block().unwrap())[idx];
debug_assert_eq!(pos.func.dfg.value_type(val), arg.value_type);
vlist.push(val, &mut pos.func.dfg.value_lists);
}
pos.func.dfg[inst].put_value_list(vlist);
}
debug_assert!(
check_return_signature(&pos.func.dfg, inst, &pos.func.signature),
"Signature still wrong: {} / signature {}",
pos.func.dfg.display_inst(inst, None),
pos.func.signature
);
// Yes, we changed stuff.
true
}
/// Assign stack slots to incoming function parameters on the stack.
///
/// Values that are passed into the function on the stack must be assigned to an `IncomingArg`
/// stack slot already during legalization.
fn spill_entry_params(func: &mut Function, entry: Ebb) {
for (abi, &arg) in func.signature.params.iter().zip(func.dfg.ebb_params(entry)) {
if let ArgumentLoc::Stack(offset) = abi.location {
let ss = func.stack_slots.make_incoming_arg(abi.value_type, offset);
func.locations[arg] = ValueLoc::Stack(ss);
}
}
}
/// Assign stack slots to outgoing function arguments on the stack.
///
/// Values that are passed to a called function on the stack must be assigned to a matching
/// `OutgoingArg` stack slot. The assignment must happen immediately before the call.
///
/// TODO: The outgoing stack slots can be written a bit earlier, as long as there are no branches
/// or calls between writing the stack slots and the call instruction. Writing the slots earlier
/// could help reduce register pressure before the call.
fn spill_call_arguments(pos: &mut FuncCursor) -> bool {
let inst = pos
.current_inst()
.expect("Cursor must point to a call instruction");
let sig_ref = pos
.func
.dfg
.call_signature(inst)
.expect("Call instruction expected.");
// Start by building a list of stack slots and arguments to be replaced.
// This requires borrowing `pos.func.dfg`, so we can't change anything.
let arglist = {
let locations = &pos.func.locations;
let stack_slots = &mut pos.func.stack_slots;
pos.func
.dfg
.inst_variable_args(inst)
.iter()
.zip(&pos.func.dfg.signatures[sig_ref].params)
.enumerate()
.filter_map(|(idx, (&arg, abi))| {
match abi.location {
ArgumentLoc::Stack(offset) => {
// Assign `arg` to a new stack slot, unless it's already in the correct
// slot. The legalization needs to be idempotent, so we should see a
// correct outgoing slot on the second pass.
let ss = stack_slots.get_outgoing_arg(abi.value_type, offset);
if locations[arg] != ValueLoc::Stack(ss) {
Some((idx, arg, ss))
} else {
None
}
}
_ => None,
}
})
.collect::<Vec<_>>()
};
if arglist.is_empty() {
return false;
}
// Insert the spill instructions and rewrite call arguments.
for (idx, arg, ss) in arglist {
let stack_val = pos.ins().spill(arg);
pos.func.locations[stack_val] = ValueLoc::Stack(ss);
pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val;
}
// We changed stuff.
true
}

54
cranelift/codegen/src/legalizer/call.rs Normal file
View File

@@ -0,0 +1,54 @@
//! Legalization of calls.
//!
//! This module exports the `expand_call` function which transforms a `call`
//! instruction into `func_addr` and `call_indirect` instructions.
use crate::cursor::{Cursor, FuncCursor};
use crate::flowgraph::ControlFlowGraph;
use crate::ir::{self, InstBuilder};
use crate::isa::TargetIsa;
/// Expand a `call` instruction. This lowers it to a `call_indirect`, which
/// is only done if the ABI doesn't support direct calls.
pub fn expand_call(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
// Unpack the instruction.
let (func_ref, old_args) = match func.dfg[inst] {
ir::InstructionData::Call {
opcode,
ref args,
func_ref,
} => {
debug_assert_eq!(opcode, ir::Opcode::Call);
(func_ref, args.clone())
}
_ => panic!("Wanted call: {}", func.dfg.display_inst(inst, None)),
};
let ptr_ty = isa.pointer_type();
let sig = func.dfg.ext_funcs[func_ref].signature;
let callee = {
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
pos.ins().func_addr(ptr_ty, func_ref)
};
let mut new_args = ir::ValueList::default();
new_args.push(callee, &mut func.dfg.value_lists);
for i in 0..old_args.len(&func.dfg.value_lists) {
new_args.push(
old_args.as_slice(&func.dfg.value_lists)[i],
&mut func.dfg.value_lists,
);
}
func.dfg
.replace(inst)
.CallIndirect(ir::Opcode::CallIndirect, ptr_ty, sig, new_args);
}

129
cranelift/codegen/src/legalizer/globalvalue.rs Normal file
View File

@@ -0,0 +1,129 @@
//! Legalization of global values.
//!
//! This module exports the `expand_global_value` function which transforms a `global_value`
//! instruction into code that depends on the kind of global value referenced.
use crate::cursor::{Cursor, FuncCursor};
use crate::flowgraph::ControlFlowGraph;
use crate::ir::{self, InstBuilder};
use crate::isa::TargetIsa;
/// Expand a `global_value` instruction according to the definition of the global value.
pub fn expand_global_value(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
// Unpack the instruction.
let gv = match func.dfg[inst] {
ir::InstructionData::UnaryGlobalValue {
opcode,
global_value,
} => {
debug_assert_eq!(opcode, ir::Opcode::GlobalValue);
global_value
}
_ => panic!("Wanted global_value: {}", func.dfg.display_inst(inst, None)),
};
match func.global_values[gv] {
ir::GlobalValueData::VMContext => vmctx_addr(inst, func),
ir::GlobalValueData::IAddImm {
base,
offset,
global_type,
} => iadd_imm_addr(inst, func, base, offset.into(), global_type),
ir::GlobalValueData::Load {
base,
offset,
global_type,
readonly,
} => load_addr(inst, func, base, offset, global_type, readonly, isa),
ir::GlobalValueData::Symbol { .. } => symbol(inst, func, gv, isa),
}
}
/// Expand a `global_value` instruction for a vmctx global.
fn vmctx_addr(inst: ir::Inst, func: &mut ir::Function) {
// Get the value representing the `vmctx` argument.
let vmctx = func
.special_param(ir::ArgumentPurpose::VMContext)
.expect("Missing vmctx parameter");
// Replace the `global_value` instruction's value with an alias to the vmctx arg.
let result = func.dfg.first_result(inst);
func.dfg.clear_results(inst);
func.dfg.change_to_alias(result, vmctx);
func.layout.remove_inst(inst);
}
/// Expand a `global_value` instruction for an iadd_imm global.
fn iadd_imm_addr(
inst: ir::Inst,
func: &mut ir::Function,
base: ir::GlobalValue,
offset: i64,
global_type: ir::Type,
) {
let mut pos = FuncCursor::new(func).at_inst(inst);
// Get the value for the lhs. For tidiness, expand VMContext here so that we avoid
// `vmctx_addr` which creates an otherwise unneeded value alias.
let lhs = if let ir::GlobalValueData::VMContext = pos.func.global_values[base] {
pos.func
.special_param(ir::ArgumentPurpose::VMContext)
.expect("Missing vmctx parameter")
} else {
pos.ins().global_value(global_type, base)
};
// Simply replace the `global_value` instruction with an `iadd_imm`, reusing the result value.
pos.func.dfg.replace(inst).iadd_imm(lhs, offset);
}
/// Expand a `global_value` instruction for a load global.
fn load_addr(
inst: ir::Inst,
func: &mut ir::Function,
base: ir::GlobalValue,
offset: ir::immediates::Offset32,
global_type: ir::Type,
readonly: bool,
isa: &TargetIsa,
) {
// We need to load a pointer from the `base` global value, so insert a new `global_value`
// instruction. This depends on the iterative legalization loop. Note that the IR verifier
// detects any cycles in the `load` globals.
let ptr_ty = isa.pointer_type();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Get the value for the base. For tidiness, expand VMContext here so that we avoid
// `vmctx_addr` which creates an otherwise unneeded value alias.
let base_addr = if let ir::GlobalValueData::VMContext = pos.func.global_values[base] {
pos.func
.special_param(ir::ArgumentPurpose::VMContext)
.expect("Missing vmctx parameter")
} else {
pos.ins().global_value(ptr_ty, base)
};
// Global-value loads are always notrap and aligned. They may be readonly.
let mut mflags = ir::MemFlags::trusted();
if readonly {
mflags.set_readonly();
}
// Perform the load.
pos.func
.dfg
.replace(inst)
.load(global_type, mflags, base_addr, offset);
}
/// Expand a `global_value` instruction for a symbolic name global.
fn symbol(inst: ir::Inst, func: &mut ir::Function, gv: ir::GlobalValue, isa: &TargetIsa) {
let ptr_ty = isa.pointer_type();
func.dfg.replace(inst).symbol_value(ptr_ty, gv);
}

161
cranelift/codegen/src/legalizer/heap.rs Normal file
View File

@@ -0,0 +1,161 @@
//! Legalization of heaps.
//!
//! This module exports the `expand_heap_addr` function which transforms a `heap_addr`
//! instruction into code that depends on the kind of heap referenced.
use crate::cursor::{Cursor, FuncCursor};
use crate::flowgraph::ControlFlowGraph;
use crate::ir::condcodes::IntCC;
use crate::ir::{self, InstBuilder};
use crate::isa::TargetIsa;
/// Expand a `heap_addr` instruction according to the definition of the heap.
pub fn expand_heap_addr(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
// Unpack the instruction.
let (heap, offset, access_size) = match func.dfg[inst] {
ir::InstructionData::HeapAddr {
opcode,
heap,
arg,
imm,
} => {
debug_assert_eq!(opcode, ir::Opcode::HeapAddr);
(heap, arg, imm.into())
}
_ => panic!("Wanted heap_addr: {}", func.dfg.display_inst(inst, None)),
};
match func.heaps[heap].style {
ir::HeapStyle::Dynamic { bound_gv } => {
dynamic_addr(inst, heap, offset, access_size, bound_gv, func)
}
ir::HeapStyle::Static { bound } => {
static_addr(inst, heap, offset, access_size, bound.into(), func, cfg)
}
}
}
/// Expand a `heap_addr` for a dynamic heap.
fn dynamic_addr(
inst: ir::Inst,
heap: ir::Heap,
offset: ir::Value,
access_size: u32,
bound_gv: ir::GlobalValue,
func: &mut ir::Function,
) {
let access_size = u64::from(access_size);
let offset_ty = func.dfg.value_type(offset);
let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
let min_size = func.heaps[heap].min_size.into();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Start with the bounds check. Trap if `offset + access_size > bound`.
let bound = pos.ins().global_value(offset_ty, bound_gv);
let oob;
if access_size == 1 {
// `offset > bound - 1` is the same as `offset >= bound`.
oob = pos
.ins()
.icmp(IntCC::UnsignedGreaterThanOrEqual, offset, bound);
} else if access_size <= min_size {
// We know that bound >= min_size, so here we can compare `offset > bound - access_size`
// without wrapping.
let adj_bound = pos.ins().iadd_imm(bound, -(access_size as i64));
oob = pos
.ins()
.icmp(IntCC::UnsignedGreaterThan, offset, adj_bound);
} else {
// We need an overflow check for the adjusted offset.
let access_size_val = pos.ins().iconst(offset_ty, access_size as i64);
let (adj_offset, overflow) = pos.ins().iadd_cout(offset, access_size_val);
pos.ins().trapnz(overflow, ir::TrapCode::HeapOutOfBounds);
oob = pos
.ins()
.icmp(IntCC::UnsignedGreaterThan, adj_offset, bound);
}
pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
compute_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}
/// Expand a `heap_addr` for a static heap.
fn static_addr(
inst: ir::Inst,
heap: ir::Heap,
offset: ir::Value,
access_size: u32,
bound: u64,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
) {
let access_size = u64::from(access_size);
let offset_ty = func.dfg.value_type(offset);
let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Start with the bounds check. Trap if `offset + access_size > bound`.
if access_size > bound {
// This will simply always trap since `offset >= 0`.
pos.ins().trap(ir::TrapCode::HeapOutOfBounds);
pos.func.dfg.replace(inst).iconst(addr_ty, 0);
// Split Ebb, as the trap is a terminator instruction.
let curr_ebb = pos.current_ebb().expect("Cursor is not in an ebb");
let new_ebb = pos.func.dfg.make_ebb();
pos.insert_ebb(new_ebb);
cfg.recompute_ebb(pos.func, curr_ebb);
cfg.recompute_ebb(pos.func, new_ebb);
return;
}
// Check `offset > limit` which is now known non-negative.
let limit = bound - access_size;
// We may be able to omit the check entirely for 32-bit offsets if the heap bound is 4 GB or
// more.
if offset_ty != ir::types::I32 || limit < 0xffff_ffff {
let oob = if limit & 1 == 1 {
// Prefer testing `offset >= limit - 1` when limit is odd because an even number is
// likely to be a convenient constant on ARM and other RISC architectures.
pos.ins()
.icmp_imm(IntCC::UnsignedGreaterThanOrEqual, offset, limit as i64 - 1)
} else {
pos.ins()
.icmp_imm(IntCC::UnsignedGreaterThan, offset, limit as i64)
};
pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
}
compute_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}
/// Emit code for the base address computation of a `heap_addr` instruction.
fn compute_addr(
inst: ir::Inst,
heap: ir::Heap,
addr_ty: ir::Type,
mut offset: ir::Value,
offset_ty: ir::Type,
func: &mut ir::Function,
) {
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Convert `offset` to `addr_ty`.
if offset_ty != addr_ty {
offset = pos.ins().uextend(addr_ty, offset);
}
// Add the heap base address base
let base_gv = pos.func.heaps[heap].base;
let base = pos.ins().global_value(addr_ty, base_gv);
pos.func.dfg.replace(inst).iadd(base, offset);
}

31
cranelift/codegen/src/legalizer/libcall.rs Normal file
View File

@@ -0,0 +1,31 @@
//! Expanding instructions as runtime library calls.
use crate::ir;
use crate::ir::{get_libcall_funcref, InstBuilder};
use crate::isa::TargetIsa;
use crate::legalizer::boundary::legalize_libcall_signature;
use std::vec::Vec;
/// Try to expand `inst` as a library call, returning true is successful.
pub fn expand_as_libcall(inst: ir::Inst, func: &mut ir::Function, isa: &TargetIsa) -> bool {
// Does the opcode/ctrl_type combo even have a well-known runtime library name.
let libcall = match ir::LibCall::for_inst(func.dfg[inst].opcode(), func.dfg.ctrl_typevar(inst))
{
Some(lc) => lc,
None => return false,
};
// Now we convert `inst` to a call. First save the arguments.
let mut args = Vec::new();
args.extend_from_slice(func.dfg.inst_args(inst));
// The replace builder will preserve the instruction result values.
let funcref = get_libcall_funcref(libcall, func, inst, isa);
func.dfg.replace(inst).call(funcref, &args);
// Ask the ISA to legalize the signature.
let fn_data = &func.dfg.ext_funcs[funcref];
let sig_data = &mut func.dfg.signatures[fn_data.signature];
legalize_libcall_signature(sig_data, isa);
true
}

440
cranelift/codegen/src/legalizer/mod.rs Normal file
View File

@@ -0,0 +1,440 @@
//! Legalize instructions.
//!
//! A legal instruction is one that can be mapped directly to a machine code instruction for the
//! target ISA. The `legalize_function()` function takes as input any function and transforms it
//! into an equivalent function using only legal instructions.
//!
//! The characteristics of legal instructions depend on the target ISA, so any given instruction
//! can be legal for one ISA and illegal for another.
//!
//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map
//! which provides a legal encoding recipe for every instruction.
//!
//! The legalizer does not deal with register allocation constraints. These constraints are derived
//! from the encoding recipes, and solved later by the register allocator.
use crate::bitset::BitSet;
use crate::cursor::{Cursor, FuncCursor};
use crate::flowgraph::ControlFlowGraph;
use crate::ir::types::I32;
use crate::ir::{self, InstBuilder, MemFlags};
use crate::isa::TargetIsa;
use crate::timing;
mod boundary;
mod call;
mod globalvalue;
mod heap;
mod libcall;
mod split;
mod table;
use self::call::expand_call;
use self::globalvalue::expand_global_value;
use self::heap::expand_heap_addr;
use self::libcall::expand_as_libcall;
use self::table::expand_table_addr;
/// Legalize `inst` for `isa`. Return true if any changes to the code were
/// made; return false if the instruction was successfully encoded as is.
fn legalize_inst(
inst: ir::Inst,
pos: &mut FuncCursor,
cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) -> bool {
let opcode = pos.func.dfg[inst].opcode();
// Check for ABI boundaries that need to be converted to the legalized signature.
if opcode.is_call() {
if boundary::handle_call_abi(inst, pos.func, cfg) {
return true;
}
} else if opcode.is_return() {
if boundary::handle_return_abi(inst, pos.func, cfg) {
return true;
}
} else if opcode.is_branch() {
split::simplify_branch_arguments(&mut pos.func.dfg, inst);
}
match pos.func.update_encoding(inst, isa) {
Ok(()) => false,
Err(action) => {
// We should transform the instruction into legal equivalents.
// If the current instruction was replaced, we need to double back and revisit
// the expanded sequence. This is both to assign encodings and possible to
// expand further.
// There's a risk of infinite looping here if the legalization patterns are
// unsound. Should we attempt to detect that?
if action(inst, pos.func, cfg, isa) {
return true;
}
// We don't have any pattern expansion for this instruction either.
// Try converting it to a library call as a last resort.
expand_as_libcall(inst, pos.func, isa)
}
}
}
/// Legalize `func` for `isa`.
///
/// - Transform any instructions that don't have a legal representation in `isa`.
/// - Fill out `func.encodings`.
///
pub fn legalize_function(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa: &TargetIsa) {
let _tt = timing::legalize();
debug_assert!(cfg.is_valid());
boundary::legalize_signatures(func, isa);
func.encodings.resize(func.dfg.num_insts());
let mut pos = FuncCursor::new(func);
// Process EBBs in layout order. Some legalization actions may split the current EBB or append
// new ones to the end. We need to make sure we visit those new EBBs too.
while let Some(_ebb) = pos.next_ebb() {
// Keep track of the cursor position before the instruction being processed, so we can
// double back when replacing instructions.
let mut prev_pos = pos.position();
while let Some(inst) = pos.next_inst() {
if legalize_inst(inst, &mut pos, cfg, isa) {
// Go back and legalize the inserted return value conversion instructions.
pos.set_position(prev_pos);
} else {
// Remember this position in case we need to double back.
prev_pos = pos.position();
}
}
}
// Now that we've lowered all br_tables, we don't need the jump tables anymore.
if !isa.flags().jump_tables_enabled() {
pos.func.jump_tables.clear();
}
}
// Include legalization patterns that were generated by `gen_legalizer.py` from the `XForms` in
// `cranelift-codegen/meta-python/base/legalize.py`.
//
// Concretely, this defines private functions `narrow()`, and `expand()`.
include!(concat!(env!("OUT_DIR"), "/legalizer.rs"));
/// Custom expansion for conditional trap instructions.
/// TODO: Add CFG support to the Python patterns so we won't have to do this.
fn expand_cond_trap(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
// Parse the instruction.
let trapz;
let (arg, code) = match func.dfg[inst] {
ir::InstructionData::CondTrap { opcode, arg, code } => {
// We want to branch *over* an unconditional trap.
trapz = match opcode {
ir::Opcode::Trapz => true,
ir::Opcode::Trapnz => false,
_ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst, None)),
};
(arg, code)
}
_ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst, None)),
};
// Split the EBB after `inst`:
//
// trapnz arg
//
// Becomes:
//
// brz arg, new_ebb
// trap
// new_ebb:
//
let old_ebb = func.layout.pp_ebb(inst);
let new_ebb = func.dfg.make_ebb();
if trapz {
func.dfg.replace(inst).brnz(arg, new_ebb, &[]);
} else {
func.dfg.replace(inst).brz(arg, new_ebb, &[]);
}
let mut pos = FuncCursor::new(func).after_inst(inst);
pos.use_srcloc(inst);
pos.ins().trap(code);
pos.insert_ebb(new_ebb);
// Finally update the CFG.
cfg.recompute_ebb(pos.func, old_ebb);
cfg.recompute_ebb(pos.func, new_ebb);
}
/// Jump tables.
fn expand_br_table(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
if isa.flags().jump_tables_enabled() {
expand_br_table_jt(inst, func, cfg, isa);
} else {
expand_br_table_conds(inst, func, cfg, isa);
}
}
/// Expand br_table to jump table.
fn expand_br_table_jt(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
use crate::ir::condcodes::IntCC;
let (arg, default_ebb, table) = match func.dfg[inst] {
ir::InstructionData::BranchTable {
opcode: ir::Opcode::BrTable,
arg,
destination,
table,
} => (arg, destination, table),
_ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
};
let table_size = func.jump_tables[table].len();
let addr_ty = isa.pointer_type();
let entry_ty = I32;
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Bounds check
let oob = pos
.ins()
.icmp_imm(IntCC::UnsignedGreaterThanOrEqual, arg, table_size as i64);
pos.ins().brnz(oob, default_ebb, &[]);
let base_addr = pos.ins().jump_table_base(addr_ty, table);
let entry = pos
.ins()
.jump_table_entry(addr_ty, arg, base_addr, entry_ty.bytes() as u8, table);
let addr = pos.ins().iadd(base_addr, entry);
pos.ins().indirect_jump_table_br(addr, table);
let ebb = pos.current_ebb().unwrap();
pos.remove_inst();
cfg.recompute_ebb(pos.func, ebb);
}
/// Expand br_table to series of conditionals.
fn expand_br_table_conds(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
use crate::ir::condcodes::IntCC;
let (arg, default_ebb, table) = match func.dfg[inst] {
ir::InstructionData::BranchTable {
opcode: ir::Opcode::BrTable,
arg,
destination,
table,
} => (arg, destination, table),
_ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
};
// This is a poor man's jump table using just a sequence of conditional branches.
let table_size = func.jump_tables[table].len();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
for i in 0..table_size {
let dest = pos.func.jump_tables[table].as_slice()[i];
let t = pos.ins().icmp_imm(IntCC::Equal, arg, i as i64);
pos.ins().brnz(t, dest, &[]);
}
// `br_table` jumps to the default destination if nothing matches
pos.ins().jump(default_ebb, &[]);
let ebb = pos.current_ebb().unwrap();
pos.remove_inst();
cfg.recompute_ebb(pos.func, ebb);
}
/// Expand the select instruction.
///
/// Conditional moves are available in some ISAs for some register classes. The remaining selects
/// are handled by a branch.
fn expand_select(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
let (ctrl, tval, fval) = match func.dfg[inst] {
ir::InstructionData::Ternary {
opcode: ir::Opcode::Select,
args,
} => (args[0], args[1], args[2]),
_ => panic!("Expected select: {}", func.dfg.display_inst(inst, None)),
};
// Replace `result = select ctrl, tval, fval` with:
//
// brnz ctrl, new_ebb(tval)
// jump new_ebb(fval)
// new_ebb(result):
let old_ebb = func.layout.pp_ebb(inst);
let result = func.dfg.first_result(inst);
func.dfg.clear_results(inst);
let new_ebb = func.dfg.make_ebb();
func.dfg.attach_ebb_param(new_ebb, result);
func.dfg.replace(inst).brnz(ctrl, new_ebb, &[tval]);
let mut pos = FuncCursor::new(func).after_inst(inst);
pos.use_srcloc(inst);
pos.ins().jump(new_ebb, &[fval]);
pos.insert_ebb(new_ebb);
cfg.recompute_ebb(pos.func, new_ebb);
cfg.recompute_ebb(pos.func, old_ebb);
}
fn expand_br_icmp(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
let (cond, a, b, destination, ebb_args) = match func.dfg[inst] {
ir::InstructionData::BranchIcmp {
cond,
destination,
ref args,
..
} => (
cond,
args.get(0, &func.dfg.value_lists).unwrap(),
args.get(1, &func.dfg.value_lists).unwrap(),
destination,
args.as_slice(&func.dfg.value_lists)[2..].to_vec(),
),
_ => panic!("Expected br_icmp {}", func.dfg.display_inst(inst, None)),
};
let old_ebb = func.layout.pp_ebb(inst);
func.dfg.clear_results(inst);
let icmp_res = func.dfg.replace(inst).icmp(cond, a, b);
let mut pos = FuncCursor::new(func).after_inst(inst);
pos.use_srcloc(inst);
pos.ins().brnz(icmp_res, destination, &ebb_args);
cfg.recompute_ebb(pos.func, destination);
cfg.recompute_ebb(pos.func, old_ebb);
}
/// Expand illegal `f32const` and `f64const` instructions.
fn expand_fconst(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
let ty = func.dfg.value_type(func.dfg.first_result(inst));
debug_assert!(!ty.is_vector(), "Only scalar fconst supported: {}", ty);
// In the future, we may want to generate constant pool entries for these constants, but for
// now use an `iconst` and a bit cast.
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
let ival = match pos.func.dfg[inst] {
ir::InstructionData::UnaryIeee32 {
opcode: ir::Opcode::F32const,
imm,
} => pos.ins().iconst(ir::types::I32, i64::from(imm.bits())),
ir::InstructionData::UnaryIeee64 {
opcode: ir::Opcode::F64const,
imm,
} => pos.ins().iconst(ir::types::I64, imm.bits() as i64),
_ => panic!("Expected fconst: {}", pos.func.dfg.display_inst(inst, None)),
};
pos.func.dfg.replace(inst).bitcast(ty, ival);
}
/// Expand illegal `stack_load` instructions.
fn expand_stack_load(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
let ty = func.dfg.value_type(func.dfg.first_result(inst));
let addr_ty = isa.pointer_type();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
let (stack_slot, offset) = match pos.func.dfg[inst] {
ir::InstructionData::StackLoad {
opcode: _opcode,
stack_slot,
offset,
} => (stack_slot, offset),
_ => panic!(
"Expected stack_load: {}",
pos.func.dfg.display_inst(inst, None)
),
};
let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
// Stack slots are required to be accessible and aligned.
let mflags = MemFlags::trusted();
pos.func.dfg.replace(inst).load(ty, mflags, addr, 0);
}
/// Expand illegal `stack_store` instructions.
fn expand_stack_store(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
let addr_ty = isa.pointer_type();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
let (val, stack_slot, offset) = match pos.func.dfg[inst] {
ir::InstructionData::StackStore {
opcode: _opcode,
arg,
stack_slot,
offset,
} => (arg, stack_slot, offset),
_ => panic!(
"Expected stack_store: {}",
pos.func.dfg.display_inst(inst, None)
),
};
let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
let mut mflags = MemFlags::new();
// Stack slots are required to be accessible and aligned.
mflags.set_notrap();
mflags.set_aligned();
pos.func.dfg.replace(inst).store(mflags, val, addr, 0);
}

345
cranelift/codegen/src/legalizer/split.rs Normal file
View File

@@ -0,0 +1,345 @@
//! Value splitting.
//!
//! Some value types are too large to fit in registers, so they need to be split into smaller parts
//! that the ISA can operate on. There's two dimensions of splitting, represented by two
//! complementary instruction pairs:
//!
//! - `isplit` and `iconcat` for splitting integer types into smaller integers.
//! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same
//! lane types.
//!
//! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably
//! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers.
//! This breakdown is handled by the ABI lowering.
//!
//! When legalizing a single instruction, it is wrapped in splits and concatenations:
//!
//!```clif
//! v1 = bxor.i64 v2, v3
//! ```
//!
//! becomes:
//!
//!```clif
//! v20, v21 = isplit v2
//! v30, v31 = isplit v3
//! v10 = bxor.i32 v20, v30
//! v11 = bxor.i32 v21, v31
//! v1 = iconcat v10, v11
//! ```
//!
//! This local expansion approach still leaves the original `i64` values in the code as operands on
//! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as
//! values are constantly split and concatenated.
//!
//! # Optimized splitting
//!
//! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value,
//! first check if the value is defined by the corresponding concatenation. If so, then just use
//! the two concatenation inputs directly:
//!
//! ```clif
//! v4 = iadd_imm.i64 v1, 1
//! ```
//!
//! becomes, using the expanded code from above:
//!
//! ```clif
//! v40, v5 = iadd_imm_cout.i32 v10, 1
//! v6 = bint.i32
//! v41 = iadd.i32 v11, v6
//! v4 = iconcat v40, v41
//! ```
//!
//! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they
//! can be trivially deleted by a dead code elimination pass.
//!
//! # EBB arguments
//!
//! If all instructions that produce an `i64` value are legalized as above, we will eventually end
//! up with no `i64` values anywhere, except for EBB arguments. We can work around this by
//! iteratively splitting EBB arguments too. That should leave us with no illegal value types
//! anywhere.
//!
//! It is possible to have circular dependencies of EBB arguments that are never used by any real
//! instructions. These loops will remain in the program.
use crate::cursor::{Cursor, CursorPosition, FuncCursor};
use crate::flowgraph::{BasicBlock, ControlFlowGraph};
use crate::ir::{self, Ebb, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef};
use core::iter;
use std::vec::Vec;
/// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values
/// if possible.
pub fn isplit(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
) -> (Value, Value) {
split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat)
}
/// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if
/// possible.
pub fn vsplit(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
) -> (Value, Value) {
split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat)
}
/// After splitting an EBB argument, we need to go back and fix up all of the predecessor
/// instructions. This is potentially a recursive operation, but we don't implement it recursively
/// since that could use up too muck stack.
///
/// Instead, the repairs are deferred and placed on a work list in stack form.
struct Repair {
concat: Opcode,
// The argument type after splitting.
split_type: Type,
// The destination EBB whose arguments have been split.
ebb: Ebb,
// Number of the original EBB argument which has been replaced by the low part.
num: usize,
// Number of the new EBB argument which represents the high part after the split.
hi_num: usize,
}
/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode.
fn split_any(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
concat: Opcode,
) -> (Value, Value) {
let mut repairs = Vec::new();
let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc);
let result = split_value(pos, value, concat, &mut repairs);
// We have split the value requested, and now we may need to fix some EBB predecessors.
while let Some(repair) = repairs.pop() {
for BasicBlock { inst, .. } in cfg.pred_iter(repair.ebb) {
let branch_opc = pos.func.dfg[inst].opcode();
debug_assert!(
branch_opc.is_branch(),
"Predecessor not a branch: {}",
pos.func.dfg.display_inst(inst, None)
);
let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments();
let mut args = pos.func.dfg[inst]
.take_value_list()
.expect("Branches must have value lists.");
let num_args = args.len(&pos.func.dfg.value_lists);
// Get the old value passed to the EBB argument we're repairing.
let old_arg = args
.get(num_fixed_args + repair.num, &pos.func.dfg.value_lists)
.expect("Too few branch arguments");
// It's possible that the CFG's predecessor list has duplicates. Detect them here.
if pos.func.dfg.value_type(old_arg) == repair.split_type {
pos.func.dfg[inst].put_value_list(args);
continue;
}
// Split the old argument, possibly causing more repairs to be scheduled.
pos.goto_inst(inst);
let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs);
// The `lo` part replaces the original argument.
*args
.get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists)
.unwrap() = lo;
// The `hi` part goes at the end. Since multiple repairs may have been scheduled to the
// same EBB, there could be multiple arguments missing.
if num_args > num_fixed_args + repair.hi_num {
*args
.get_mut(
num_fixed_args + repair.hi_num,
&mut pos.func.dfg.value_lists,
)
.unwrap() = hi;
} else {
// We need to append one or more arguments. If we're adding more than one argument,
// there must be pending repairs on the stack that will fill in the correct values
// instead of `hi`.
args.extend(
iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args),
&mut pos.func.dfg.value_lists,
);
}
// Put the value list back after manipulating it.
pos.func.dfg[inst].put_value_list(args);
}
}
result
}
/// Split a single value using the integer or vector semantics given by the `concat` opcode.
///
/// If the value is defined by a `concat` instruction, just reuse the operand values of that
/// instruction.
///
/// Return the two new values representing the parts of `value`.
fn split_value(
pos: &mut FuncCursor,
value: Value,
concat: Opcode,
repairs: &mut Vec<Repair>,
) -> (Value, Value) {
let value = pos.func.dfg.resolve_aliases(value);
let mut reuse = None;
match pos.func.dfg.value_def(value) {
ValueDef::Result(inst, num) => {
// This is an instruction result. See if the value was created by a `concat`
// instruction.
if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] {
debug_assert_eq!(num, 0);
if opcode == concat {
reuse = Some((args[0], args[1]));
}
}
}
ValueDef::Param(ebb, num) => {
// This is an EBB parameter. We can split the parameter value unless this is the entry
// block.
if pos.func.layout.entry_block() != Some(ebb) {
// We are going to replace the parameter at `num` with two new arguments.
// Determine the new value types.
let ty = pos.func.dfg.value_type(value);
let split_type = match concat {
Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"),
Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"),
_ => panic!("Unhandled concat opcode: {}", concat),
};
// Since the `repairs` stack potentially contains other parameter numbers for
// `ebb`, avoid shifting and renumbering EBB parameters. It could invalidate other
// `repairs` entries.
//
// Replace the original `value` with the low part, and append the high part at the
// end of the argument list.
let lo = pos.func.dfg.replace_ebb_param(value, split_type);
let hi_num = pos.func.dfg.num_ebb_params(ebb);
let hi = pos.func.dfg.append_ebb_param(ebb, split_type);
reuse = Some((lo, hi));
// Now the original value is dangling. Insert a concatenation instruction that can
// compute it from the two new parameters. This also serves as a record of what we
// did so a future call to this function doesn't have to redo the work.
//
// Note that it is safe to move `pos` here since `reuse` was set above, so we don't
// need to insert a split instruction before returning.
pos.goto_first_inst(ebb);
pos.ins()
.with_result(value)
.Binary(concat, split_type, lo, hi);
// Finally, splitting the EBB parameter is not enough. We also have to repair all
// of the predecessor instructions that branch here.
add_repair(concat, split_type, ebb, num, hi_num, repairs);
}
}
}
// Did the code above succeed in finding values we can reuse?
if let Some(pair) = reuse {
pair
} else {
// No, we'll just have to insert the requested split instruction at `pos`. Note that `pos`
// has not been moved by the EBB argument code above when `reuse` is `None`.
match concat {
Opcode::Iconcat => pos.ins().isplit(value),
Opcode::Vconcat => pos.ins().vsplit(value),
_ => panic!("Unhandled concat opcode: {}", concat),
}
}
}
// Add a repair entry to the work list.
fn add_repair(
concat: Opcode,
split_type: Type,
ebb: Ebb,
num: usize,
hi_num: usize,
repairs: &mut Vec<Repair>,
) {
repairs.push(Repair {
concat,
split_type,
ebb,
num,
hi_num,
});
}
/// Strip concat-split chains. Return a simpler way of computing the same value.
///
/// Given this input:
///
/// ```clif
/// v10 = iconcat v1, v2
/// v11, v12 = isplit v10
/// ```
///
/// This function resolves `v11` to `v1` and `v12` to `v2`.
fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value {
let value = dfg.resolve_aliases(value);
// Deconstruct a split instruction.
let split_res;
let concat_opc;
let split_arg;
if let ValueDef::Result(inst, num) = dfg.value_def(value) {
split_res = num;
concat_opc = match dfg[inst].opcode() {
Opcode::Isplit => Opcode::Iconcat,
Opcode::Vsplit => Opcode::Vconcat,
_ => return value,
};
split_arg = dfg.inst_args(inst)[0];
} else {
return value;
}
// See if split_arg is defined by a concatenation instruction.
if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) {
if dfg[inst].opcode() == concat_opc {
return dfg.inst_args(inst)[split_res];
}
}
value
}
/// Simplify the arguments to a branch *after* the instructions leading up to the branch have been
/// legalized.
///
/// The branch argument repairs performed by `split_any()` above may be performed on branches that
/// have not yet been legalized. The repaired arguments can be defined by actual split
/// instructions in that case.
///
/// After legalizing the instructions computing the value that was split, it is likely that we can
/// avoid depending on the split instruction. Its input probably comes from a concatenation.
pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) {
let mut new_args = Vec::new();
for &arg in dfg.inst_args(branch) {
let new_arg = resolve_splits(dfg, arg);
new_args.push(new_arg);
}
dfg.inst_args_mut(branch).copy_from_slice(&new_args);
}

113
cranelift/codegen/src/legalizer/table.rs Normal file
View File

@@ -0,0 +1,113 @@
//! Legalization of tables.
//!
//! This module exports the `expand_table_addr` function which transforms a `table_addr`
//! instruction into code that depends on the kind of table referenced.
use crate::cursor::{Cursor, FuncCursor};
use crate::flowgraph::ControlFlowGraph;
use crate::ir::condcodes::IntCC;
use crate::ir::immediates::Offset32;
use crate::ir::{self, InstBuilder};
use crate::isa::TargetIsa;
/// Expand a `table_addr` instruction according to the definition of the table.
pub fn expand_table_addr(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
// Unpack the instruction.
let (table, index, element_offset) = match func.dfg[inst] {
ir::InstructionData::TableAddr {
opcode,
table,
arg,
offset,
} => {
debug_assert_eq!(opcode, ir::Opcode::TableAddr);
(table, arg, offset)
}
_ => panic!("Wanted table_addr: {}", func.dfg.display_inst(inst, None)),
};
dynamic_addr(inst, table, index, element_offset, func);
}
/// Expand a `table_addr` for a dynamic table.
fn dynamic_addr(
inst: ir::Inst,
table: ir::Table,
index: ir::Value,
element_offset: Offset32,
func: &mut ir::Function,
) {
let bound_gv = func.tables[table].bound_gv;
let index_ty = func.dfg.value_type(index);
let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Start with the bounds check. Trap if `index + 1 > bound`.
let bound = pos.ins().global_value(index_ty, bound_gv);
// `index > bound - 1` is the same as `index >= bound`.
let oob = pos
.ins()
.icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound);
pos.ins().trapnz(oob, ir::TrapCode::TableOutOfBounds);
compute_addr(
inst,
table,
addr_ty,
index,
index_ty,
element_offset,
pos.func,
);
}
/// Emit code for the base address computation of a `table_addr` instruction.
fn compute_addr(
inst: ir::Inst,
table: ir::Table,
addr_ty: ir::Type,
mut index: ir::Value,
index_ty: ir::Type,
element_offset: Offset32,
func: &mut ir::Function,
) {
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Convert `index` to `addr_ty`.
if index_ty != addr_ty {
index = pos.ins().uextend(addr_ty, index);
}
// Add the table base address base
let base_gv = pos.func.tables[table].base_gv;
let base = pos.ins().global_value(addr_ty, base_gv);
let element_size = pos.func.tables[table].element_size;
let mut offset;
let element_size: u64 = element_size.into();
if element_size == 1 {
offset = index;
} else if element_size.is_power_of_two() {
offset = pos
.ins()
.ishl_imm(index, i64::from(element_size.trailing_zeros()));
} else {
offset = pos.ins().imul_imm(index, element_size as i64);
}
if element_offset == Offset32::new(0) {
pos.func.dfg.replace(inst).iadd(base, offset);
} else {
let imm: i64 = element_offset.into();
offset = pos.ins().iadd(base, offset);
pos.func.dfg.replace(inst).iadd_imm(offset, imm);
}
}