T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into no_std

Browse Source
This commit is contained in:
morenzg
2018-04-17 16:41:27 -04:00
parent 07693048f0 58380f38e8
commit a10a6a0df0
364 changed files with 3412 additions and 1024 deletions
Download Patch File Download Diff File Expand all files Collapse all files

683
lib/codegen/src/legalizer/boundary.rs Normal file
View File

@@ -0,0 +1,683 @@
//! Legalize ABI boundaries.
//!
//! This legalizer sub-module contains code for dealing with ABI boundaries:
//!
//! - Function arguments passed to the entry block.
//! - Function arguments passed to call instructions.
//! - Return values from call instructions.
//! - Return values passed to return instructions.
//!
//! The ABI boundary legalization happens in two phases:
//!
//! 1. The `legalize_signatures` function rewrites all the preamble signatures with ABI information
//! and possibly new argument types. It also rewrites the entry block arguments to match.
//! 2. The `handle_call_abi` and `handle_return_abi` functions rewrite call and return instructions
//! to match the new ABI signatures.
//!
//! Between the two phases, preamble signatures and call/return arguments don't match. This
//! intermediate state doesn't type check.
use abi::{legalize_abi_value, ValueConversion};
use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::instructions::CallInfo;
use ir::{AbiParam, ArgumentLoc, ArgumentPurpose, DataFlowGraph, Ebb, Function, Inst, InstBuilder,
SigRef, Signature, Type, Value, ValueLoc};
use isa::TargetIsa;
use legalizer::split::{isplit, vsplit};
use std::vec::Vec;
/// Legalize all the function signatures in `func`.
///
/// This changes all signatures to be ABI-compliant with full `ArgumentLoc` annotations. It doesn't
/// change the entry block arguments, calls, or return instructions, so this can leave the function
/// in a state with type discrepancies.
pub fn legalize_signatures(func: &mut Function, isa: &TargetIsa) {
isa.legalize_signature(&mut func.signature, true);
func.signature.compute_argument_bytes();
for sig_data in func.dfg.signatures.values_mut() {
isa.legalize_signature(sig_data, false);
sig_data.compute_argument_bytes();
}
if let Some(entry) = func.layout.entry_block() {
legalize_entry_params(func, entry);
spill_entry_params(func, entry);
}
}
/// Legalize the entry block parameters after `func`'s signature has been legalized.
///
/// The legalized signature may contain more parameters than the original signature, and the
/// parameter types have been changed. This function goes through the parameters of the entry EBB
/// and replaces them with parameters of the right type for the ABI.
///
/// The original entry EBB parameters are computed from the new ABI parameters by code inserted at
/// the top of the entry block.
fn legalize_entry_params(func: &mut Function, entry: Ebb) {
let mut has_sret = false;
let mut has_link = false;
let mut has_vmctx = false;
let mut has_sigid = false;
// Insert position for argument conversion code.
// We want to insert instructions before the first instruction in the entry block.
// If the entry block is empty, append instructions to it instead.
let mut pos = FuncCursor::new(func).at_first_inst(entry);
// Keep track of the argument types in the ABI-legalized signature.
let mut abi_arg = 0;
// Process the EBB parameters one at a time, possibly replacing one argument with multiple new
// ones. We do this by detaching the entry EBB parameters first.
let ebb_params = pos.func.dfg.detach_ebb_params(entry);
let mut old_arg = 0;
while let Some(arg) = ebb_params.get(old_arg, &pos.func.dfg.value_lists) {
old_arg += 1;
let abi_type = pos.func.signature.params[abi_arg];
let arg_type = pos.func.dfg.value_type(arg);
if arg_type == abi_type.value_type {
// No value translation is necessary, this argument matches the ABI type.
// Just use the original EBB argument value. This is the most common case.
pos.func.dfg.attach_ebb_param(entry, arg);
match abi_type.purpose {
ArgumentPurpose::Normal => {}
ArgumentPurpose::FramePointer => {}
ArgumentPurpose::CalleeSaved => {}
ArgumentPurpose::StructReturn => {
debug_assert!(!has_sret, "Multiple sret arguments found");
has_sret = true;
}
ArgumentPurpose::VMContext => {
debug_assert!(!has_vmctx, "Multiple vmctx arguments found");
has_vmctx = true;
}
ArgumentPurpose::SignatureId => {
debug_assert!(!has_sigid, "Multiple sigid arguments found");
has_sigid = true;
}
_ => panic!("Unexpected special-purpose arg {}", abi_type),
}
abi_arg += 1;
} else {
// Compute the value we want for `arg` from the legalized ABI parameters.
let mut get_arg = |func: &mut Function, ty| {
let abi_type = func.signature.params[abi_arg];
debug_assert_eq!(
abi_type.purpose,
ArgumentPurpose::Normal,
"Can't legalize special-purpose argument"
);
if ty == abi_type.value_type {
abi_arg += 1;
Ok(func.dfg.append_ebb_param(entry, ty))
} else {
Err(abi_type)
}
};
let converted = convert_from_abi(&mut pos, arg_type, Some(arg), &mut get_arg);
// The old `arg` is no longer an attached EBB argument, but there are probably still
// uses of the value.
debug_assert_eq!(pos.func.dfg.resolve_aliases(arg), converted);
}
}
// The legalized signature may contain additional parameters representing special-purpose
// registers.
for &arg in &pos.func.signature.params[abi_arg..] {
match arg.purpose {
// Any normal parameters should have been processed above.
ArgumentPurpose::Normal => {
panic!("Leftover arg: {}", arg);
}
// The callee-save parameters should not appear until after register allocation is
// done.
ArgumentPurpose::FramePointer |
ArgumentPurpose::CalleeSaved => {
panic!("Premature callee-saved arg {}", arg);
}
// These can be meaningfully added by `legalize_signature()`.
ArgumentPurpose::Link => {
debug_assert!(!has_link, "Multiple link parameters found");
has_link = true;
}
ArgumentPurpose::StructReturn => {
debug_assert!(!has_sret, "Multiple sret parameters found");
has_sret = true;
}
ArgumentPurpose::VMContext => {
debug_assert!(!has_vmctx, "Multiple vmctx parameters found");
has_vmctx = true;
}
ArgumentPurpose::SignatureId => {
debug_assert!(!has_sigid, "Multiple sigid parameters found");
has_sigid = true;
}
}
// Just create entry block values to match here. We will use them in `handle_return_abi()`
// below.
pos.func.dfg.append_ebb_param(entry, arg.value_type);
}
}
/// Legalize the results returned from a call instruction to match the ABI signature.
///
/// The cursor `pos` points to a call instruction with at least one return value. The cursor will
/// be left pointing after the instructions inserted to convert the return values.
///
/// This function is very similar to the `legalize_entry_params` function above.
///
/// Returns the possibly new instruction representing the call.
fn legalize_inst_results<ResType>(pos: &mut FuncCursor, mut get_abi_type: ResType) -> Inst
where
ResType: FnMut(&Function, usize) -> AbiParam,
{
let call = pos.current_inst().expect(
"Cursor must point to a call instruction",
);
// We theoretically allow for call instructions that return a number of fixed results before
// the call return values. In practice, it doesn't happen.
let fixed_results = pos.func.dfg[call].opcode().constraints().fixed_results();
debug_assert_eq!(fixed_results, 0, "Fixed results on calls not supported");
let results = pos.func.dfg.detach_results(call);
let mut next_res = 0;
let mut abi_res = 0;
// Point immediately after the call.
pos.next_inst();
while let Some(res) = results.get(next_res, &pos.func.dfg.value_lists) {
next_res += 1;
let res_type = pos.func.dfg.value_type(res);
if res_type == get_abi_type(pos.func, abi_res).value_type {
// No value translation is necessary, this result matches the ABI type.
pos.func.dfg.attach_result(call, res);
abi_res += 1;
} else {
let mut get_res = |func: &mut Function, ty| {
let abi_type = get_abi_type(func, abi_res);
if ty == abi_type.value_type {
let last_res = func.dfg.append_result(call, ty);
abi_res += 1;
Ok(last_res)
} else {
Err(abi_type)
}
};
let v = convert_from_abi(pos, res_type, Some(res), &mut get_res);
debug_assert_eq!(pos.func.dfg.resolve_aliases(res), v);
}
}
call
}
/// Compute original value of type `ty` from the legalized ABI arguments.
///
/// The conversion is recursive, controlled by the `get_arg` closure which is called to retrieve an
/// ABI argument. It returns:
///
/// - `Ok(arg)` if the requested type matches the next ABI argument.
/// - `Err(arg_type)` if further conversions are needed from the ABI argument `arg_type`.
///
/// If the `into_result` value is provided, the converted result will be written into that value.
fn convert_from_abi<GetArg>(
pos: &mut FuncCursor,
ty: Type,
into_result: Option<Value>,
get_arg: &mut GetArg,
) -> Value
where
GetArg: FnMut(&mut Function, Type) -> Result<Value, AbiParam>,
{
// Terminate the recursion when we get the desired type.
let arg_type = match get_arg(pos.func, ty) {
Ok(v) => {
debug_assert_eq!(pos.func.dfg.value_type(v), ty);
debug_assert_eq!(into_result, None);
return v;
}
Err(t) => t,
};
// Reconstruct how `ty` was legalized into the `arg_type` argument.
let conversion = legalize_abi_value(ty, &arg_type);
dbg!("convert_from_abi({}): {:?}", ty, conversion);
// The conversion describes value to ABI argument. We implement the reverse conversion here.
match conversion {
// Construct a `ty` by concatenating two ABI integers.
ValueConversion::IntSplit => {
let abi_ty = ty.half_width().expect("Invalid type for conversion");
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
dbg!(
"intsplit {}: {}, {}: {}",
lo,
pos.func.dfg.value_type(lo),
hi,
pos.func.dfg.value_type(hi)
);
pos.ins().with_results([into_result]).iconcat(lo, hi)
}
// Construct a `ty` by concatenating two halves of a vector.
ValueConversion::VectorSplit => {
let abi_ty = ty.half_vector().expect("Invalid type for conversion");
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).vconcat(lo, hi)
}
// Construct a `ty` by bit-casting from an integer type.
ValueConversion::IntBits => {
debug_assert!(!ty.is_int());
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).bitcast(ty, arg)
}
// ABI argument is a sign-extended version of the value we want.
ValueConversion::Sext(abi_ty) => {
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
// TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
// We could insert an `assert_sreduce` which would fold with a following `sextend` of
// this value.
pos.ins().with_results([into_result]).ireduce(ty, arg)
}
ValueConversion::Uext(abi_ty) => {
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
// TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
// We could insert an `assert_ureduce` which would fold with a following `uextend` of
// this value.
pos.ins().with_results([into_result]).ireduce(ty, arg)
}
}
}
/// Convert `value` to match an ABI signature by inserting instructions at `pos`.
///
/// This may require expanding the value to multiple ABI arguments. The conversion process is
/// recursive and controlled by the `put_arg` closure. When a candidate argument value is presented
/// to the closure, it will perform one of two actions:
///
/// 1. If the suggested argument has an acceptable value type, consume it by adding it to the list
/// of arguments and return `Ok(())`.
/// 2. If the suggested argument doesn't have the right value type, don't change anything, but
/// return the `Err(AbiParam)` that is needed.
///
fn convert_to_abi<PutArg>(
pos: &mut FuncCursor,
cfg: &ControlFlowGraph,
value: Value,
put_arg: &mut PutArg,
) where
PutArg: FnMut(&mut Function, Value) -> Result<(), AbiParam>,
{
// Start by invoking the closure to either terminate the recursion or get the argument type
// we're trying to match.
let arg_type = match put_arg(pos.func, value) {
Ok(_) => return,
Err(t) => t,
};
let ty = pos.func.dfg.value_type(value);
match legalize_abi_value(ty, &arg_type) {
ValueConversion::IntSplit => {
let curpos = pos.position();
let srcloc = pos.srcloc();
let (lo, hi) = isplit(&mut pos.func, cfg, curpos, srcloc, value);
convert_to_abi(pos, cfg, lo, put_arg);
convert_to_abi(pos, cfg, hi, put_arg);
}
ValueConversion::VectorSplit => {
let curpos = pos.position();
let srcloc = pos.srcloc();
let (lo, hi) = vsplit(&mut pos.func, cfg, curpos, srcloc, value);
convert_to_abi(pos, cfg, lo, put_arg);
convert_to_abi(pos, cfg, hi, put_arg);
}
ValueConversion::IntBits => {
debug_assert!(!ty.is_int());
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
let arg = pos.ins().bitcast(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
ValueConversion::Sext(abi_ty) => {
let arg = pos.ins().sextend(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
ValueConversion::Uext(abi_ty) => {
let arg = pos.ins().uextend(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
}
}
/// Check if a sequence of arguments match a desired sequence of argument types.
fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool {
let arg_types = args.iter().map(|&v| dfg.value_type(v));
let sig_types = types.iter().map(|&at| at.value_type);
arg_types.eq(sig_types)
}
/// Check if the arguments of the call `inst` match the signature.
///
/// Returns `Ok(())` if the signature matches and no changes are needed, or `Err(sig_ref)` if the
/// signature doesn't match.
fn check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Result<(), SigRef> {
// Extract the signature and argument values.
let (sig_ref, args) = match dfg[inst].analyze_call(&dfg.value_lists) {
CallInfo::Direct(func, args) => (dfg.ext_funcs[func].signature, args),
CallInfo::Indirect(sig_ref, args) => (sig_ref, args),
CallInfo::NotACall => panic!("Expected call, got {:?}", dfg[inst]),
};
let sig = &dfg.signatures[sig_ref];
if check_arg_types(dfg, args, &sig.params[..]) &&
check_arg_types(dfg, dfg.inst_results(inst), &sig.returns[..])
{
// All types check out.
Ok(())
} else {
// Call types need fixing.
Err(sig_ref)
}
}
/// Check if the arguments of the return `inst` match the signature.
fn check_return_signature(dfg: &DataFlowGraph, inst: Inst, sig: &Signature) -> bool {
check_arg_types(dfg, dfg.inst_variable_args(inst), &sig.returns)
}
/// Insert ABI conversion code for the arguments to the call or return instruction at `pos`.
///
/// - `abi_args` is the number of arguments that the ABI signature requires.
/// - `get_abi_type` is a closure that can provide the desired `AbiParam` for a given ABI
/// argument number in `0..abi_args`.
///
fn legalize_inst_arguments<ArgType>(
pos: &mut FuncCursor,
cfg: &ControlFlowGraph,
abi_args: usize,
mut get_abi_type: ArgType,
) where
ArgType: FnMut(&Function, usize) -> AbiParam,
{
let inst = pos.current_inst().expect(
"Cursor must point to a call instruction",
);
// Lift the value list out of the call instruction so we modify it.
let mut vlist = pos.func.dfg[inst].take_value_list().expect(
"Call must have a value list",
);
// The value list contains all arguments to the instruction, including the callee on an
// indirect call which isn't part of the call arguments that must match the ABI signature.
// Figure out how many fixed values are at the front of the list. We won't touch those.
let fixed_values = pos.func.dfg[inst]
.opcode()
.constraints()
.fixed_value_arguments();
let have_args = vlist.len(&pos.func.dfg.value_lists) - fixed_values;
// Grow the value list to the right size and shift all the existing arguments to the right.
// This lets us write the new argument values into the list without overwriting the old
// arguments.
//
// Before:
//
// <--> fixed_values
// <-----------> have_args
// [FFFFOOOOOOOOOOOOO]
//
// After grow_at():
//
// <--> fixed_values
// <-----------> have_args
// <------------------> abi_args
// [FFFF-------OOOOOOOOOOOOO]
// ^
// old_arg_offset
//
// After writing the new arguments:
//
// <--> fixed_values
// <------------------> abi_args
// [FFFFNNNNNNNNNNNNNNNNNNNN]
//
vlist.grow_at(
fixed_values,
abi_args - have_args,
&mut pos.func.dfg.value_lists,
);
let old_arg_offset = fixed_values + abi_args - have_args;
let mut abi_arg = 0;
for old_arg in 0..have_args {
let old_value = vlist
.get(old_arg_offset + old_arg, &pos.func.dfg.value_lists)
.unwrap();
let mut put_arg = |func: &mut Function, arg| {
let abi_type = get_abi_type(func, abi_arg);
if func.dfg.value_type(arg) == abi_type.value_type {
// This is the argument type we need.
vlist.as_mut_slice(&mut func.dfg.value_lists)[fixed_values + abi_arg] = arg;
abi_arg += 1;
Ok(())
} else {
Err(abi_type)
}
};
convert_to_abi(pos, cfg, old_value, &mut put_arg);
}
// Put the modified value list back.
pos.func.dfg[inst].put_value_list(vlist);
}
/// Insert ABI conversion code before and after the call instruction at `pos`.
///
/// Instructions inserted before the call will compute the appropriate ABI values for the
/// callee's new ABI-legalized signature. The function call arguments are rewritten in place to
/// match the new signature.
///
/// Instructions will be inserted after the call to convert returned ABI values back to the
/// original return values. The call's result values will be adapted to match the new signature.
///
/// Returns `true` if any instructions were inserted.
pub fn handle_call_abi(mut inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
let pos = &mut FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Start by checking if the argument types already match the signature.
let sig_ref = match check_call_signature(&pos.func.dfg, inst) {
Ok(_) => return spill_call_arguments(pos),
Err(s) => s,
};
// OK, we need to fix the call arguments to match the ABI signature.
let abi_args = pos.func.dfg.signatures[sig_ref].params.len();
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
func.dfg.signatures[sig_ref].params[abi_arg]
});
if !pos.func.dfg.signatures[sig_ref].returns.is_empty() {
inst = legalize_inst_results(pos, |func, abi_res| {
func.dfg.signatures[sig_ref].returns[abi_res]
});
}
debug_assert!(
check_call_signature(&pos.func.dfg, inst).is_ok(),
"Signature still wrong: {}, {}{}",
pos.func.dfg.display_inst(inst, None),
sig_ref,
pos.func.dfg.signatures[sig_ref]
);
// Go back and insert spills for any stack arguments.
pos.goto_inst(inst);
spill_call_arguments(pos);
// Yes, we changed stuff.
true
}
/// Insert ABI conversion code before and after the return instruction at `inst`.
///
/// Return `true` if any instructions were inserted.
pub fn handle_return_abi(inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
// Check if the returned types already match the signature.
if check_return_signature(&func.dfg, inst, &func.signature) {
return false;
}
// Count the special-purpose return values (`link`, `sret`, and `vmctx`) that were appended to
// the legalized signature.
let special_args = func.signature
.returns
.iter()
.rev()
.take_while(|&rt| {
rt.purpose == ArgumentPurpose::Link || rt.purpose == ArgumentPurpose::StructReturn ||
rt.purpose == ArgumentPurpose::VMContext
})
.count();
let abi_args = func.signature.returns.len() - special_args;
let pos = &mut FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
func.signature.returns[abi_arg]
});
debug_assert_eq!(pos.func.dfg.inst_variable_args(inst).len(), abi_args);
// Append special return arguments for any `sret`, `link`, and `vmctx` return values added to
// the legalized signature. These values should simply be propagated from the entry block
// arguments.
if special_args > 0 {
dbg!(
"Adding {} special-purpose arguments to {}",
special_args,
pos.func.dfg.display_inst(inst, None)
);
let mut vlist = pos.func.dfg[inst].take_value_list().unwrap();
for arg in &pos.func.signature.returns[abi_args..] {
match arg.purpose {
ArgumentPurpose::Link |
ArgumentPurpose::StructReturn |
ArgumentPurpose::VMContext => {}
ArgumentPurpose::Normal => panic!("unexpected return value {}", arg),
_ => panic!("Unsupported special purpose return value {}", arg),
}
// A `link`/`sret`/`vmctx` return value can only appear in a signature that has a
// unique matching argument. They are appended at the end, so search the signature from
// the end.
let idx = pos.func
.signature
.params
.iter()
.rposition(|t| t.purpose == arg.purpose)
.expect("No matching special purpose argument.");
// Get the corresponding entry block value and add it to the return instruction's
// arguments.
let val = pos.func.dfg.ebb_params(
pos.func.layout.entry_block().unwrap(),
)
[idx];
debug_assert_eq!(pos.func.dfg.value_type(val), arg.value_type);
vlist.push(val, &mut pos.func.dfg.value_lists);
}
pos.func.dfg[inst].put_value_list(vlist);
}
debug_assert!(
check_return_signature(&pos.func.dfg, inst, &pos.func.signature),
"Signature still wrong: {} / signature {}",
pos.func.dfg.display_inst(inst, None),
pos.func.signature
);
// Yes, we changed stuff.
true
}
/// Assign stack slots to incoming function parameters on the stack.
///
/// Values that are passed into the function on the stack must be assigned to an `IncomingArg`
/// stack slot already during legalization.
fn spill_entry_params(func: &mut Function, entry: Ebb) {
for (abi, &arg) in func.signature.params.iter().zip(func.dfg.ebb_params(entry)) {
if let ArgumentLoc::Stack(offset) = abi.location {
let ss = func.stack_slots.make_incoming_arg(abi.value_type, offset);
func.locations[arg] = ValueLoc::Stack(ss);
}
}
}
/// Assign stack slots to outgoing function arguments on the stack.
///
/// Values that are passed to a called function on the stack must be assigned to a matching
/// `OutgoingArg` stack slot. The assignment must happen immediately before the call.
///
/// TODO: The outgoing stack slots can be written a bit earlier, as long as there are no branches
/// or calls between writing the stack slots and the call instruction. Writing the slots earlier
/// could help reduce register pressure before the call.
fn spill_call_arguments(pos: &mut FuncCursor) -> bool {
let inst = pos.current_inst().expect(
"Cursor must point to a call instruction",
);
let sig_ref = pos.func.dfg.call_signature(inst).expect(
"Call instruction expected.",
);
// Start by building a list of stack slots and arguments to be replaced.
// This requires borrowing `pos.func.dfg`, so we can't change anything.
let arglist = {
let locations = &pos.func.locations;
let stack_slots = &mut pos.func.stack_slots;
pos.func
.dfg
.inst_variable_args(inst)
.iter()
.zip(&pos.func.dfg.signatures[sig_ref].params)
.enumerate()
.filter_map(|(idx, (&arg, abi))| {
match abi.location {
ArgumentLoc::Stack(offset) => {
// Assign `arg` to a new stack slot, unless it's already in the correct
// slot. The legalization needs to be idempotent, so we should see a
// correct outgoing slot on the second pass.
let ss = stack_slots.get_outgoing_arg(abi.value_type, offset);
if locations[arg] != ValueLoc::Stack(ss) {
Some((idx, arg, ss))
} else {
None
}
}
_ => None,
}
})
.collect::<Vec<_>>()
};
if arglist.is_empty() {
return false;
}
// Insert the spill instructions and rewrite call arguments.
for (idx, arg, ss) in arglist {
let stack_val = pos.ins().spill(arg);
pos.func.locations[stack_val] = ValueLoc::Stack(ss);
pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val;
}
// We changed stuff.
true
}

60
lib/codegen/src/legalizer/call.rs Normal file
View File

@@ -0,0 +1,60 @@
//! Legalization of calls.
//!
//! This module exports the `expand_call` function which transforms a `call`
//! instruction into `func_addr` and `call_indirect` instructions.
use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::{self, InstBuilder};
use isa::TargetIsa;
/// Expand a `call` instruction.
pub fn expand_call(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
// Unpack the instruction.
let (func_ref, old_args) = match func.dfg[inst] {
ir::InstructionData::Call {
opcode,
ref args,
func_ref,
} => {
debug_assert_eq!(opcode, ir::Opcode::Call);
(func_ref, args.clone())
}
_ => panic!("Wanted call: {}", func.dfg.display_inst(inst, None)),
};
let ptr_ty = if isa.flags().is_64bit() {
ir::types::I64
} else {
ir::types::I32
};
let sig = func.dfg.ext_funcs[func_ref].signature;
let callee = {
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
pos.ins().func_addr(ptr_ty, func_ref)
};
let mut new_args = ir::ValueList::default();
new_args.push(callee, &mut func.dfg.value_lists);
for i in 0..old_args.len(&func.dfg.value_lists) {
new_args.push(
old_args.as_slice(&func.dfg.value_lists)[i],
&mut func.dfg.value_lists,
);
}
func.dfg.replace(inst).CallIndirect(
ir::Opcode::CallIndirect,
ptr_ty,
sig,
new_args,
);
}

67
lib/codegen/src/legalizer/globalvar.rs Normal file
View File

@@ -0,0 +1,67 @@
//! Legalization of global variables.
//!
//! This module exports the `expand_global_addr` function which transforms a `global_addr`
//! instruction into code that depends on the kind of global variable referenced.
use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::{self, InstBuilder};
use isa::TargetIsa;
/// Expand a `global_addr` instruction according to the definition of the global variable.
pub fn expand_global_addr(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
// Unpack the instruction.
let gv = match func.dfg[inst] {
ir::InstructionData::UnaryGlobalVar { opcode, global_var } => {
debug_assert_eq!(opcode, ir::Opcode::GlobalAddr);
global_var
}
_ => panic!("Wanted global_addr: {}", func.dfg.display_inst(inst, None)),
};
match func.global_vars[gv] {
ir::GlobalVarData::VMContext { offset } => vmctx_addr(inst, func, offset.into()),
ir::GlobalVarData::Deref { base, offset } => deref_addr(inst, func, base, offset.into()),
ir::GlobalVarData::Sym { .. } => globalsym(inst, func, gv),
}
}
/// Expand a `global_addr` instruction for a vmctx global.
fn vmctx_addr(inst: ir::Inst, func: &mut ir::Function, offset: i64) {
// Get the value representing the `vmctx` argument.
let vmctx = func.special_param(ir::ArgumentPurpose::VMContext).expect(
"Missing vmctx parameter",
);
// Simply replace the `global_addr` instruction with an `iadd_imm`, reusing the result value.
func.dfg.replace(inst).iadd_imm(vmctx, offset);
}
/// Expand a `global_addr` instruction for a deref global.
fn deref_addr(inst: ir::Inst, func: &mut ir::Function, base: ir::GlobalVar, offset: i64) {
// We need to load a pointer from the `base` global variable, so insert a new `global_addr`
// instruction. This depends on the iterative legalization loop. Note that the IR verifier
// detects any cycles in the `deref` globals.
let ptr_ty = func.dfg.value_type(func.dfg.first_result(inst));
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
let base_addr = pos.ins().global_addr(ptr_ty, base);
let mut mflags = ir::MemFlags::new();
// Deref globals are required to be accessible and aligned.
mflags.set_notrap();
mflags.set_aligned();
let base_ptr = pos.ins().load(ptr_ty, mflags, base_addr, 0);
pos.func.dfg.replace(inst).iadd_imm(base_ptr, offset);
}
/// Expand a `global_addr` instruction for a symbolic name global.
fn globalsym(inst: ir::Inst, func: &mut ir::Function, gv: ir::GlobalVar) {
let ptr_ty = func.dfg.value_type(func.dfg.first_result(inst));
func.dfg.replace(inst).globalsym_addr(ptr_ty, gv);
}

190
lib/codegen/src/legalizer/heap.rs Normal file
View File

@@ -0,0 +1,190 @@
//! Legalization of heaps.
//!
//! This module exports the `expand_heap_addr` function which transforms a `heap_addr`
//! instruction into code that depends on the kind of heap referenced.
use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::condcodes::IntCC;
use ir::{self, InstBuilder, MemFlags};
use isa::TargetIsa;
/// Expand a `heap_addr` instruction according to the definition of the heap.
pub fn expand_heap_addr(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
// Unpack the instruction.
let (heap, offset, size) = match func.dfg[inst] {
ir::InstructionData::HeapAddr {
opcode,
heap,
arg,
imm,
} => {
debug_assert_eq!(opcode, ir::Opcode::HeapAddr);
(heap, arg, imm.into())
}
_ => panic!("Wanted heap_addr: {}", func.dfg.display_inst(inst, None)),
};
match func.heaps[heap].style {
ir::HeapStyle::Dynamic { bound_gv } => {
dynamic_addr(inst, heap, offset, size, bound_gv, func)
}
ir::HeapStyle::Static { bound } => {
static_addr(inst, heap, offset, size, bound.into(), func, cfg)
}
}
}
/// Expand a `heap_addr` for a dynamic heap.
fn dynamic_addr(
inst: ir::Inst,
heap: ir::Heap,
offset: ir::Value,
size: u32,
bound_gv: ir::GlobalVar,
func: &mut ir::Function,
) {
let size = i64::from(size);
let offset_ty = func.dfg.value_type(offset);
let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
let min_size = func.heaps[heap].min_size.into();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Start with the bounds check. Trap if `offset + size > bound`.
let bound_addr = pos.ins().global_addr(addr_ty, bound_gv);
let mut mflags = MemFlags::new();
// The bound variable is requied to be accessible and aligned.
mflags.set_notrap();
mflags.set_aligned();
let bound = pos.ins().load(offset_ty, mflags, bound_addr, 0);
let oob;
if size == 1 {
// `offset > bound - 1` is the same as `offset >= bound`.
oob = pos.ins().icmp(
IntCC::UnsignedGreaterThanOrEqual,
offset,
bound,
);
} else if size <= min_size {
// We know that bound >= min_size, so here we can compare `offset > bound - size` without
// wrapping.
let adj_bound = pos.ins().iadd_imm(bound, -size);
oob = pos.ins().icmp(
IntCC::UnsignedGreaterThan,
offset,
adj_bound,
);
} else {
// We need an overflow check for the adjusted offset.
let size_val = pos.ins().iconst(offset_ty, size);
let (adj_offset, overflow) = pos.ins().iadd_cout(offset, size_val);
pos.ins().trapnz(overflow, ir::TrapCode::HeapOutOfBounds);
oob = pos.ins().icmp(
IntCC::UnsignedGreaterThan,
adj_offset,
bound,
);
}
pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
offset_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}
/// Expand a `heap_addr` for a static heap.
fn static_addr(
inst: ir::Inst,
heap: ir::Heap,
offset: ir::Value,
size: u32,
bound: i64,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
) {
let size = i64::from(size);
let offset_ty = func.dfg.value_type(offset);
let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Start with the bounds check. Trap if `offset + size > bound`.
if size > bound {
// This will simply always trap since `offset >= 0`.
pos.ins().trap(ir::TrapCode::HeapOutOfBounds);
pos.func.dfg.replace(inst).iconst(addr_ty, 0);
// Split Ebb, as the trap is a terminator instruction.
let curr_ebb = pos.current_ebb().expect("Cursor is not in an ebb");
let new_ebb = pos.func.dfg.make_ebb();
pos.insert_ebb(new_ebb);
cfg.recompute_ebb(pos.func, curr_ebb);
cfg.recompute_ebb(pos.func, new_ebb);
return;
}
// Check `offset > limit` which is now known non-negative.
let limit = bound - size;
// We may be able to omit the check entirely for 32-bit offsets if the heap bound is 4 GB or
// more.
if offset_ty != ir::types::I32 || limit < 0xffff_ffff {
let oob = if limit & 1 == 1 {
// Prefer testing `offset >= limit - 1` when limit is odd because an even number is
// likely to be a convenient constant on ARM and other RISC architectures.
pos.ins().icmp_imm(
IntCC::UnsignedGreaterThanOrEqual,
offset,
limit - 1,
)
} else {
pos.ins().icmp_imm(
IntCC::UnsignedGreaterThan,
offset,
limit,
)
};
pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
}
offset_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}
/// Emit code for the base address computation of a `heap_addr` instruction.
///
///
fn offset_addr(
inst: ir::Inst,
heap: ir::Heap,
addr_ty: ir::Type,
mut offset: ir::Value,
offset_ty: ir::Type,
func: &mut ir::Function,
) {
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Convert `offset` to `addr_ty`.
if offset_ty != addr_ty {
offset = pos.ins().uextend(addr_ty, offset);
}
// Add the heap base address base
match pos.func.heaps[heap].base {
ir::HeapBase::ReservedReg => unimplemented!(),
ir::HeapBase::GlobalVar(base_gv) => {
let base_addr = pos.ins().global_addr(addr_ty, base_gv);
let mut mflags = MemFlags::new();
// The base address variable is requied to be accessible and aligned.
mflags.set_notrap();
mflags.set_aligned();
let base = pos.ins().load(addr_ty, mflags, base_addr, 0);
pos.func.dfg.replace(inst).iadd(base, offset);
}
}
}

63
lib/codegen/src/legalizer/libcall.rs Normal file
View File

@@ -0,0 +1,63 @@
//! Expanding instructions as runtime library calls.
use ir;
use ir::InstBuilder;
/// Try to expand `inst` as a library call, returning true is successful.
pub fn expand_as_libcall(inst: ir::Inst, func: &mut ir::Function) -> bool {
// Does the opcode/ctrl_type combo even have a well-known runtime library name.
let libcall =
match ir::LibCall::for_inst(func.dfg[inst].opcode(), func.dfg.ctrl_typevar(inst)) {
Some(lc) => lc,
None => return false,
};
let funcref = find_funcref(libcall, func).unwrap_or_else(|| make_funcref(libcall, inst, func));
// Now we convert `inst` to a call. First save the arguments.
let mut args = vec![];
args.extend_from_slice(func.dfg.inst_args(inst));
// The replace builder will preserve the instruction result values.
func.dfg.replace(inst).call(funcref, &args);
// TODO: ask the ISA to legalize the signature.
true
}
/// Get the existing function reference for `libcall` in `func` if it exists.
fn find_funcref(libcall: ir::LibCall, func: &ir::Function) -> Option<ir::FuncRef> {
// We're assuming that all libcall function decls are at the end.
// If we get this wrong, worst case we'll have duplicate libcall decls which is harmless.
for (fref, func_data) in func.dfg.ext_funcs.iter().rev() {
match func_data.name {
ir::ExternalName::LibCall(lc) => {
if lc == libcall {
return Some(fref);
}
}
_ => break,
}
}
None
}
/// Create a funcref for `libcall` with a signature matching `inst`.
fn make_funcref(libcall: ir::LibCall, inst: ir::Inst, func: &mut ir::Function) -> ir::FuncRef {
// Start with a system_v calling convention. We'll give the ISA a chance to change it.
let mut sig = ir::Signature::new(ir::CallConv::SystemV);
for &v in func.dfg.inst_args(inst) {
sig.params.push(ir::AbiParam::new(func.dfg.value_type(v)));
}
for &v in func.dfg.inst_results(inst) {
sig.returns.push(ir::AbiParam::new(func.dfg.value_type(v)));
}
let sigref = func.import_signature(sig);
// TODO: Can libcalls be colocated in some circumstances?
func.import_function(ir::ExtFuncData {
name: ir::ExternalName::LibCall(libcall),
signature: sigref,
colocated: false,
})
}

302
lib/codegen/src/legalizer/mod.rs Normal file
View File

@@ -0,0 +1,302 @@
//! Legalize instructions.
//!
//! A legal instruction is one that can be mapped directly to a machine code instruction for the
//! target ISA. The `legalize_function()` function takes as input any function and transforms it
//! into an equivalent function using only legal instructions.
//!
//! The characteristics of legal instructions depend on the target ISA, so any given instruction
//! can be legal for one ISA and illegal for another.
//!
//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map
//! which provides a legal encoding recipe for every instruction.
//!
//! The legalizer does not deal with register allocation constraints. These constraints are derived
//! from the encoding recipes, and solved later by the register allocator.
use bitset::BitSet;
use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::{self, InstBuilder};
use isa::TargetIsa;
use timing;
mod boundary;
mod call;
mod globalvar;
mod heap;
mod libcall;
mod split;
use self::globalvar::expand_global_addr;
use self::heap::expand_heap_addr;
use self::call::expand_call;
use self::libcall::expand_as_libcall;
/// Legalize `func` for `isa`.
///
/// - Transform any instructions that don't have a legal representation in `isa`.
/// - Fill out `func.encodings`.
///
pub fn legalize_function(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa: &TargetIsa) {
let _tt = timing::legalize();
debug_assert!(cfg.is_valid());
boundary::legalize_signatures(func, isa);
func.encodings.resize(func.dfg.num_insts());
let mut pos = FuncCursor::new(func);
// Process EBBs in layout order. Some legalization actions may split the current EBB or append
// new ones to the end. We need to make sure we visit those new EBBs too.
while let Some(_ebb) = pos.next_ebb() {
// Keep track of the cursor position before the instruction being processed, so we can
// double back when replacing instructions.
let mut prev_pos = pos.position();
while let Some(inst) = pos.next_inst() {
let opcode = pos.func.dfg[inst].opcode();
// Check for ABI boundaries that need to be converted to the legalized signature.
if opcode.is_call() {
if boundary::handle_call_abi(inst, pos.func, cfg) {
// Go back and legalize the inserted argument conversion instructions.
pos.set_position(prev_pos);
continue;
}
} else if opcode.is_return() {
if boundary::handle_return_abi(inst, pos.func, cfg) {
// Go back and legalize the inserted return value conversion instructions.
pos.set_position(prev_pos);
continue;
}
} else if opcode.is_branch() {
split::simplify_branch_arguments(&mut pos.func.dfg, inst);
}
match pos.func.update_encoding(inst, isa) {
Ok(()) => {}
Err(action) => {
// We should transform the instruction into legal equivalents.
let changed = action(inst, pos.func, cfg, isa);
// If the current instruction was replaced, we need to double back and revisit
// the expanded sequence. This is both to assign encodings and possible to
// expand further.
// There's a risk of infinite looping here if the legalization patterns are
// unsound. Should we attempt to detect that?
if changed {
pos.set_position(prev_pos);
continue;
}
// We don't have any pattern expansion for this instruction either.
// Try converting it to a library call as a last resort.
if expand_as_libcall(inst, pos.func) {
pos.set_position(prev_pos);
continue;
}
}
}
// Remember this position in case we need to double back.
prev_pos = pos.position();
}
}
}
// Include legalization patterns that were generated by `gen_legalizer.py` from the `XForms` in
// `lib/codegen/meta/base/legalize.py`.
//
// Concretely, this defines private functions `narrow()`, and `expand()`.
include!(concat!(env!("OUT_DIR"), "/legalizer.rs"));
/// Custom expansion for conditional trap instructions.
/// TODO: Add CFG support to the Python patterns so we won't have to do this.
fn expand_cond_trap(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
// Parse the instruction.
let trapz;
let (arg, code) = match func.dfg[inst] {
ir::InstructionData::CondTrap { opcode, arg, code } => {
// We want to branch *over* an unconditional trap.
trapz = match opcode {
ir::Opcode::Trapz => true,
ir::Opcode::Trapnz => false,
_ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst, None)),
};
(arg, code)
}
_ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst, None)),
};
// Split the EBB after `inst`:
//
// trapnz arg
//
// Becomes:
//
// brz arg, new_ebb
// trap
// new_ebb:
//
let old_ebb = func.layout.pp_ebb(inst);
let new_ebb = func.dfg.make_ebb();
if trapz {
func.dfg.replace(inst).brnz(arg, new_ebb, &[]);
} else {
func.dfg.replace(inst).brz(arg, new_ebb, &[]);
}
let mut pos = FuncCursor::new(func).after_inst(inst);
pos.use_srcloc(inst);
pos.ins().trap(code);
pos.insert_ebb(new_ebb);
// Finally update the CFG.
cfg.recompute_ebb(pos.func, old_ebb);
cfg.recompute_ebb(pos.func, new_ebb);
}
/// Jump tables.
fn expand_br_table(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
use ir::condcodes::IntCC;
let (arg, table) = match func.dfg[inst] {
ir::InstructionData::BranchTable {
opcode: ir::Opcode::BrTable,
arg,
table,
} => (arg, table),
_ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
};
// This is a poor man's jump table using just a sequence of conditional branches.
// TODO: Lower into a jump table load and indirect branch.
let table_size = func.jump_tables[table].len();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
for i in 0..table_size {
if let Some(dest) = pos.func.jump_tables[table].get_entry(i) {
let t = pos.ins().icmp_imm(IntCC::Equal, arg, i as i64);
pos.ins().brnz(t, dest, &[]);
}
}
// `br_table` falls through when nothing matches.
let ebb = pos.current_ebb().unwrap();
pos.remove_inst();
cfg.recompute_ebb(pos.func, ebb);
}
/// Expand the select instruction.
///
/// Conditional moves are available in some ISAs for some register classes. The remaining selects
/// are handled by a branch.
fn expand_select(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
let (ctrl, tval, fval) = match func.dfg[inst] {
ir::InstructionData::Ternary {
opcode: ir::Opcode::Select,
args,
} => (args[0], args[1], args[2]),
_ => panic!("Expected select: {}", func.dfg.display_inst(inst, None)),
};
// Replace `result = select ctrl, tval, fval` with:
//
// brnz ctrl, new_ebb(tval)
// jump new_ebb(fval)
// new_ebb(result):
let old_ebb = func.layout.pp_ebb(inst);
let result = func.dfg.first_result(inst);
func.dfg.clear_results(inst);
let new_ebb = func.dfg.make_ebb();
func.dfg.attach_ebb_param(new_ebb, result);
func.dfg.replace(inst).brnz(ctrl, new_ebb, &[tval]);
let mut pos = FuncCursor::new(func).after_inst(inst);
pos.use_srcloc(inst);
pos.ins().jump(new_ebb, &[fval]);
pos.insert_ebb(new_ebb);
cfg.recompute_ebb(pos.func, new_ebb);
cfg.recompute_ebb(pos.func, old_ebb);
}
/// Expand illegal `f32const` and `f64const` instructions.
fn expand_fconst(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
_isa: &TargetIsa,
) {
let ty = func.dfg.value_type(func.dfg.first_result(inst));
debug_assert!(!ty.is_vector(), "Only scalar fconst supported: {}", ty);
// In the future, we may want to generate constant pool entries for these constants, but for
// now use an `iconst` and a bit cast.
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
let ival = match pos.func.dfg[inst] {
ir::InstructionData::UnaryIeee32 {
opcode: ir::Opcode::F32const,
imm,
} => pos.ins().iconst(ir::types::I32, i64::from(imm.bits())),
ir::InstructionData::UnaryIeee64 {
opcode: ir::Opcode::F64const,
imm,
} => pos.ins().iconst(ir::types::I64, imm.bits() as i64),
_ => panic!("Expected fconst: {}", pos.func.dfg.display_inst(inst, None)),
};
pos.func.dfg.replace(inst).bitcast(ty, ival);
}
/// Expand the stack check instruction.
pub fn expand_stack_check(
inst: ir::Inst,
func: &mut ir::Function,
_cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
use ir::condcodes::IntCC;
let gv = match func.dfg[inst] {
ir::InstructionData::UnaryGlobalVar { global_var, .. } => global_var,
_ => panic!("Want stack_check: {}", func.dfg.display_inst(inst, isa)),
};
let ptr_ty = if isa.flags().is_64bit() {
ir::types::I64
} else {
ir::types::I32
};
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
let limit_addr = pos.ins().global_addr(ptr_ty, gv);
let mut mflags = ir::MemFlags::new();
mflags.set_aligned();
mflags.set_notrap();
let limit = pos.ins().load(ptr_ty, mflags, limit_addr, 0);
let cflags = pos.ins().ifcmp_sp(limit);
pos.func.dfg.replace(inst).trapif(
IntCC::UnsignedGreaterThanOrEqual,
cflags,
ir::TrapCode::StackOverflow,
);
}

342
lib/codegen/src/legalizer/split.rs Normal file
View File

@@ -0,0 +1,342 @@
//! Value splitting.
//!
//! Some value types are too large to fit in registers, so they need to be split into smaller parts
//! that the ISA can operate on. There's two dimensions of splitting, represented by two
//! complementary instruction pairs:
//!
//! - `isplit` and `iconcat` for splitting integer types into smaller integers.
//! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same
//! lane types.
//!
//! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably
//! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers.
//! This breakdown is handled by the ABI lowering.
//!
//! When legalizing a single instruction, it is wrapped in splits and concatenations:
//!
//!```cton
//! v1 = bxor.i64 v2, v3
//! ```
//!
//! becomes:
//!
//!```cton
//! v20, v21 = isplit v2
//! v30, v31 = isplit v3
//! v10 = bxor.i32 v20, v30
//! v11 = bxor.i32 v21, v31
//! v1 = iconcat v10, v11
//! ```
//!
//! This local expansion approach still leaves the original `i64` values in the code as operands on
//! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as
//! values are constantly split and concatenated.
//!
//! # Optimized splitting
//!
//! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value,
//! first check if the value is defined by the corresponding concatenation. If so, then just use
//! the two concatenation inputs directly:
//!
//! ```cton
//! v4 = iadd_imm.i64 v1, 1
//! ```
//!
//! becomes, using the expanded code from above:
//!
//! ```cton
//! v40, v5 = iadd_imm_cout.i32 v10, 1
//! v6 = bint.i32
//! v41 = iadd.i32 v11, v6
//! v4 = iconcat v40, v41
//! ```
//!
//! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they
//! can be trivially deleted by a dead code elimination pass.
//!
//! # EBB arguments
//!
//! If all instructions that produce an `i64` value are legalized as above, we will eventually end
//! up with no `i64` values anywhere, except for EBB arguments. We can work around this by
//! iteratively splitting EBB arguments too. That should leave us with no illegal value types
//! anywhere.
//!
//! It is possible to have circular dependencies of EBB arguments that are never used by any real
//! instructions. These loops will remain in the program.
use cursor::{Cursor, CursorPosition, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::{self, Ebb, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef};
use std::iter;
use std::vec::Vec;
/// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values
/// if possible.
pub fn isplit(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
) -> (Value, Value) {
split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat)
}
/// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if
/// possible.
pub fn vsplit(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
) -> (Value, Value) {
split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat)
}
/// After splitting an EBB argument, we need to go back and fix up all of the predecessor
/// instructions. This is potentially a recursive operation, but we don't implement it recursively
/// since that could use up too muck stack.
///
/// Instead, the repairs are deferred and placed on a work list in stack form.
struct Repair {
concat: Opcode,
// The argument type after splitting.
split_type: Type,
// The destination EBB whose arguments have been split.
ebb: Ebb,
// Number of the original EBB argument which has been replaced by the low part.
num: usize,
// Number of the new EBB argument which represents the high part after the split.
hi_num: usize,
}
/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode.
fn split_any(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
concat: Opcode,
) -> (Value, Value) {
let mut repairs = Vec::new();
let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc);
let result = split_value(pos, value, concat, &mut repairs);
// We have split the value requested, and now we may need to fix some EBB predecessors.
while let Some(repair) = repairs.pop() {
for (_, inst) in cfg.pred_iter(repair.ebb) {
let branch_opc = pos.func.dfg[inst].opcode();
debug_assert!(
branch_opc.is_branch(),
"Predecessor not a branch: {}",
pos.func.dfg.display_inst(inst, None)
);
let fixed_args = branch_opc.constraints().fixed_value_arguments();
let mut args = pos.func.dfg[inst].take_value_list().expect(
"Branches must have value lists.",
);
let num_args = args.len(&pos.func.dfg.value_lists);
// Get the old value passed to the EBB argument we're repairing.
let old_arg = args.get(fixed_args + repair.num, &pos.func.dfg.value_lists)
.expect("Too few branch arguments");
// It's possible that the CFG's predecessor list has duplicates. Detect them here.
if pos.func.dfg.value_type(old_arg) == repair.split_type {
pos.func.dfg[inst].put_value_list(args);
continue;
}
// Split the old argument, possibly causing more repairs to be scheduled.
pos.goto_inst(inst);
let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs);
// The `lo` part replaces the original argument.
*args.get_mut(fixed_args + repair.num, &mut pos.func.dfg.value_lists)
.unwrap() = lo;
// The `hi` part goes at the end. Since multiple repairs may have been scheduled to the
// same EBB, there could be multiple arguments missing.
if num_args > fixed_args + repair.hi_num {
*args.get_mut(fixed_args + repair.hi_num, &mut pos.func.dfg.value_lists)
.unwrap() = hi;
} else {
// We need to append one or more arguments. If we're adding more than one argument,
// there must be pending repairs on the stack that will fill in the correct values
// instead of `hi`.
args.extend(
iter::repeat(hi).take(1 + fixed_args + repair.hi_num - num_args),
&mut pos.func.dfg.value_lists,
);
}
// Put the value list back after manipulating it.
pos.func.dfg[inst].put_value_list(args);
}
}
result
}
/// Split a single value using the integer or vector semantics given by the `concat` opcode.
///
/// If the value is defined by a `concat` instruction, just reuse the operand values of that
/// instruction.
///
/// Return the two new values representing the parts of `value`.
fn split_value(
pos: &mut FuncCursor,
value: Value,
concat: Opcode,
repairs: &mut Vec<Repair>,
) -> (Value, Value) {
let value = pos.func.dfg.resolve_aliases(value);
let mut reuse = None;
match pos.func.dfg.value_def(value) {
ValueDef::Result(inst, num) => {
// This is an instruction result. See if the value was created by a `concat`
// instruction.
if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] {
debug_assert_eq!(num, 0);
if opcode == concat {
reuse = Some((args[0], args[1]));
}
}
}
ValueDef::Param(ebb, num) => {
// This is an EBB parameter. We can split the parameter value unless this is the entry
// block.
if pos.func.layout.entry_block() != Some(ebb) {
// We are going to replace the parameter at `num` with two new arguments.
// Determine the new value types.
let ty = pos.func.dfg.value_type(value);
let split_type = match concat {
Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"),
Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"),
_ => panic!("Unhandled concat opcode: {}", concat),
};
// Since the `repairs` stack potentially contains other parameter numbers for
// `ebb`, avoid shifting and renumbering EBB parameters. It could invalidate other
// `repairs` entries.
//
// Replace the original `value` with the low part, and append the high part at the
// end of the argument list.
let lo = pos.func.dfg.replace_ebb_param(value, split_type);
let hi_num = pos.func.dfg.num_ebb_params(ebb);
let hi = pos.func.dfg.append_ebb_param(ebb, split_type);
reuse = Some((lo, hi));
// Now the original value is dangling. Insert a concatenation instruction that can
// compute it from the two new parameters. This also serves as a record of what we
// did so a future call to this function doesn't have to redo the work.
//
// Note that it is safe to move `pos` here since `reuse` was set above, so we don't
// need to insert a split instruction before returning.
pos.goto_first_inst(ebb);
pos.ins().with_result(value).Binary(
concat,
split_type,
lo,
hi,
);
// Finally, splitting the EBB parameter is not enough. We also have to repair all
// of the predecessor instructions that branch here.
add_repair(concat, split_type, ebb, num, hi_num, repairs);
}
}
}
// Did the code above succeed in finding values we can reuse?
if let Some(pair) = reuse {
pair
} else {
// No, we'll just have to insert the requested split instruction at `pos`. Note that `pos`
// has not been moved by the EBB argument code above when `reuse` is `None`.
match concat {
Opcode::Iconcat => pos.ins().isplit(value),
Opcode::Vconcat => pos.ins().vsplit(value),
_ => panic!("Unhandled concat opcode: {}", concat),
}
}
}
// Add a repair entry to the work list.
fn add_repair(
concat: Opcode,
split_type: Type,
ebb: Ebb,
num: usize,
hi_num: usize,
repairs: &mut Vec<Repair>,
) {
repairs.push(Repair {
concat,
split_type,
ebb,
num,
hi_num,
});
}
/// Strip concat-split chains. Return a simpler way of computing the same value.
///
/// Given this input:
///
/// ```cton
/// v10 = iconcat v1, v2
/// v11, v12 = isplit v10
/// ```
///
/// This function resolves `v11` to `v1` and `v12` to `v2`.
fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value {
let value = dfg.resolve_aliases(value);
// Deconstruct a split instruction.
let split_res;
let concat_opc;
let split_arg;
if let ValueDef::Result(inst, num) = dfg.value_def(value) {
split_res = num;
concat_opc = match dfg[inst].opcode() {
Opcode::Isplit => Opcode::Iconcat,
Opcode::Vsplit => Opcode::Vconcat,
_ => return value,
};
split_arg = dfg.inst_args(inst)[0];
} else {
return value;
}
// See if split_arg is defined by a concatenation instruction.
if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) {
if dfg[inst].opcode() == concat_opc {
return dfg.inst_args(inst)[split_res];
}
}
value
}
/// Simplify the arguments to a branch *after* the instructions leading up to the branch have been
/// legalized.
///
/// The branch argument repairs performed by `split_any()` above may be performed on branches that
/// have not yet been legalized. The repaired arguments can be defined by actual split
/// instructions in that case.
///
/// After legalizing the instructions computing the value that was split, it is likely that we can
/// avoid depending on the split instruction. Its input probably comes from a concatenation.
pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) {
let mut new_args = Vec::new();
for &arg in dfg.inst_args(branch) {
let new_arg = resolve_splits(dfg, arg);
new_args.push(new_arg);
}
dfg.inst_args_mut(branch).copy_from_slice(&new_args);
}