1173 lines
44 KiB
Rust
1173 lines
44 KiB
Rust
//! Legalize ABI boundaries.
|
|
//!
|
|
//! This legalizer sub-module contains code for dealing with ABI boundaries:
|
|
//!
|
|
//! - Function arguments passed to the entry block.
|
|
//! - Function arguments passed to call instructions.
|
|
//! - Return values from call instructions.
|
|
//! - Return values passed to return instructions.
|
|
//!
|
|
//! The ABI boundary legalization happens in two phases:
|
|
//!
|
|
//! 1. The `legalize_signatures` function rewrites all the preamble signatures with ABI information
|
|
//! and possibly new argument types. It also rewrites the entry block arguments to match.
|
|
//! 2. The `handle_call_abi` and `handle_return_abi` functions rewrite call and return instructions
|
|
//! to match the new ABI signatures.
|
|
//!
|
|
//! Between the two phases, preamble signatures and call/return arguments don't match. This
|
|
//! intermediate state doesn't type check.
|
|
|
|
use crate::abi::{legalize_abi_value, ValueConversion};
|
|
use crate::cursor::{Cursor, FuncCursor};
|
|
use crate::flowgraph::ControlFlowGraph;
|
|
use crate::ir::instructions::CallInfo;
|
|
use crate::ir::{
|
|
AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, ExtFuncData, ExternalName,
|
|
Function, Inst, InstBuilder, LibCall, MemFlags, SigRef, Signature, StackSlotData,
|
|
StackSlotKind, Type, Value, ValueLoc,
|
|
};
|
|
use crate::isa::TargetIsa;
|
|
use crate::legalizer::split::{isplit, vsplit};
|
|
use alloc::borrow::Cow;
|
|
use alloc::vec::Vec;
|
|
use core::mem;
|
|
use cranelift_entity::EntityList;
|
|
use log::debug;
|
|
|
|
/// Legalize all the function signatures in `func`.
|
|
///
|
|
/// This changes all signatures to be ABI-compliant with full `ArgumentLoc` annotations. It doesn't
|
|
/// change the entry block arguments, calls, or return instructions, so this can leave the function
|
|
/// in a state with type discrepancies.
|
|
pub fn legalize_signatures(func: &mut Function, isa: &dyn TargetIsa) {
|
|
if let Some(new) = legalize_signature(&func.signature, true, isa) {
|
|
let old = mem::replace(&mut func.signature, new);
|
|
func.old_signature = Some(old);
|
|
}
|
|
|
|
for (sig_ref, sig_data) in func.dfg.signatures.iter_mut() {
|
|
if let Some(new) = legalize_signature(sig_data, false, isa) {
|
|
let old = mem::replace(sig_data, new);
|
|
func.dfg.old_signatures[sig_ref] = Some(old);
|
|
}
|
|
}
|
|
|
|
if let Some(entry) = func.layout.entry_block() {
|
|
legalize_entry_params(func, entry);
|
|
spill_entry_params(func, entry);
|
|
}
|
|
}
|
|
|
|
/// Legalize the libcall signature, which we may generate on the fly after
|
|
/// `legalize_signatures` has been called.
|
|
pub fn legalize_libcall_signature(signature: &mut Signature, isa: &dyn TargetIsa) {
|
|
if let Some(s) = legalize_signature(signature, false, isa) {
|
|
*signature = s;
|
|
}
|
|
}
|
|
|
|
/// Legalize the given signature.
|
|
///
|
|
/// `current` is true if this is the signature for the current function.
|
|
fn legalize_signature(
|
|
signature: &Signature,
|
|
current: bool,
|
|
isa: &dyn TargetIsa,
|
|
) -> Option<Signature> {
|
|
let mut cow = Cow::Borrowed(signature);
|
|
isa.legalize_signature(&mut cow, current);
|
|
match cow {
|
|
Cow::Borrowed(_) => None,
|
|
Cow::Owned(s) => Some(s),
|
|
}
|
|
}
|
|
|
|
/// Legalize the entry block parameters after `func`'s signature has been legalized.
|
|
///
|
|
/// The legalized signature may contain more parameters than the original signature, and the
|
|
/// parameter types have been changed. This function goes through the parameters of the entry block
|
|
/// and replaces them with parameters of the right type for the ABI.
|
|
///
|
|
/// The original entry block parameters are computed from the new ABI parameters by code inserted at
|
|
/// the top of the entry block.
|
|
fn legalize_entry_params(func: &mut Function, entry: Block) {
|
|
let mut has_sret = false;
|
|
let mut has_link = false;
|
|
let mut has_vmctx = false;
|
|
let mut has_sigid = false;
|
|
let mut has_stack_limit = false;
|
|
|
|
// Insert position for argument conversion code.
|
|
// We want to insert instructions before the first instruction in the entry block.
|
|
// If the entry block is empty, append instructions to it instead.
|
|
let mut pos = FuncCursor::new(func).at_first_inst(entry);
|
|
|
|
// Keep track of the argument types in the ABI-legalized signature.
|
|
let mut abi_arg = 0;
|
|
|
|
// Process the block parameters one at a time, possibly replacing one argument with multiple new
|
|
// ones. We do this by detaching the entry block parameters first.
|
|
let block_params = pos.func.dfg.detach_block_params(entry);
|
|
let mut old_arg = 0;
|
|
while let Some(arg) = block_params.get(old_arg, &pos.func.dfg.value_lists) {
|
|
old_arg += 1;
|
|
|
|
let abi_type = pos.func.signature.params[abi_arg];
|
|
let arg_type = pos.func.dfg.value_type(arg);
|
|
match abi_type.purpose {
|
|
ArgumentPurpose::StructArgument(size) => {
|
|
let offset = if let ArgumentLoc::Stack(offset) = abi_type.location {
|
|
offset
|
|
} else {
|
|
unreachable!("StructArgument must already have a Stack ArgumentLoc assigned");
|
|
};
|
|
let ss = pos.func.stack_slots.make_incoming_arg(size, offset);
|
|
let struct_arg = pos.ins().stack_addr(arg_type, ss, 0);
|
|
pos.func.dfg.change_to_alias(arg, struct_arg);
|
|
let dummy = pos
|
|
.func
|
|
.dfg
|
|
.append_block_param(entry, crate::ir::types::SARG_T);
|
|
pos.func.locations[dummy] = ValueLoc::Stack(ss);
|
|
abi_arg += 1;
|
|
continue;
|
|
}
|
|
_ => {}
|
|
}
|
|
if arg_type == abi_type.value_type {
|
|
// No value translation is necessary, this argument matches the ABI type.
|
|
// Just use the original block argument value. This is the most common case.
|
|
pos.func.dfg.attach_block_param(entry, arg);
|
|
match abi_type.purpose {
|
|
ArgumentPurpose::Normal => {}
|
|
ArgumentPurpose::StructArgument(_) => unreachable!("Handled above"),
|
|
ArgumentPurpose::FramePointer => {}
|
|
ArgumentPurpose::CalleeSaved => {}
|
|
ArgumentPurpose::StructReturn => {
|
|
debug_assert!(!has_sret, "Multiple sret arguments found");
|
|
has_sret = true;
|
|
}
|
|
ArgumentPurpose::VMContext => {
|
|
debug_assert!(!has_vmctx, "Multiple vmctx arguments found");
|
|
has_vmctx = true;
|
|
}
|
|
ArgumentPurpose::SignatureId => {
|
|
debug_assert!(!has_sigid, "Multiple sigid arguments found");
|
|
has_sigid = true;
|
|
}
|
|
ArgumentPurpose::StackLimit => {
|
|
debug_assert!(!has_stack_limit, "Multiple stack_limit arguments found");
|
|
has_stack_limit = true;
|
|
}
|
|
ArgumentPurpose::Link => panic!("Unexpected link arg {}", abi_type),
|
|
}
|
|
abi_arg += 1;
|
|
} else {
|
|
// Compute the value we want for `arg` from the legalized ABI parameters.
|
|
let mut get_arg = |func: &mut Function, ty| {
|
|
let abi_type = func.signature.params[abi_arg];
|
|
debug_assert_eq!(
|
|
abi_type.purpose,
|
|
ArgumentPurpose::Normal,
|
|
"Can't legalize special-purpose argument"
|
|
);
|
|
if ty == abi_type.value_type {
|
|
abi_arg += 1;
|
|
Ok(func.dfg.append_block_param(entry, ty))
|
|
} else {
|
|
Err(abi_type)
|
|
}
|
|
};
|
|
let converted = convert_from_abi(&mut pos, arg_type, Some(arg), &mut get_arg);
|
|
// The old `arg` is no longer an attached block argument, but there are probably still
|
|
// uses of the value.
|
|
debug_assert_eq!(pos.func.dfg.resolve_aliases(arg), converted);
|
|
}
|
|
}
|
|
|
|
// The legalized signature may contain additional parameters representing special-purpose
|
|
// registers.
|
|
for &arg in &pos.func.signature.params[abi_arg..] {
|
|
match arg.purpose {
|
|
// Any normal parameters should have been processed above.
|
|
ArgumentPurpose::Normal | ArgumentPurpose::StructArgument(_) => {
|
|
panic!("Leftover arg: {}", arg);
|
|
}
|
|
// The callee-save parameters should not appear until after register allocation is
|
|
// done.
|
|
ArgumentPurpose::FramePointer | ArgumentPurpose::CalleeSaved => {
|
|
panic!("Premature callee-saved arg {}", arg);
|
|
}
|
|
// These can be meaningfully added by `legalize_signature()`.
|
|
ArgumentPurpose::Link => {
|
|
debug_assert!(!has_link, "Multiple link parameters found");
|
|
has_link = true;
|
|
}
|
|
ArgumentPurpose::StructReturn => {
|
|
debug_assert!(!has_sret, "Multiple sret parameters found");
|
|
has_sret = true;
|
|
}
|
|
ArgumentPurpose::VMContext => {
|
|
debug_assert!(!has_vmctx, "Multiple vmctx parameters found");
|
|
has_vmctx = true;
|
|
}
|
|
ArgumentPurpose::SignatureId => {
|
|
debug_assert!(!has_sigid, "Multiple sigid parameters found");
|
|
has_sigid = true;
|
|
}
|
|
ArgumentPurpose::StackLimit => {
|
|
debug_assert!(!has_stack_limit, "Multiple stack_limit parameters found");
|
|
has_stack_limit = true;
|
|
}
|
|
}
|
|
|
|
// Just create entry block values to match here. We will use them in `handle_return_abi()`
|
|
// below.
|
|
pos.func.dfg.append_block_param(entry, arg.value_type);
|
|
}
|
|
}
|
|
|
|
/// Legalize the results returned from a call instruction to match the ABI signature.
|
|
///
|
|
/// The cursor `pos` points to a call instruction with at least one return value. The cursor will
|
|
/// be left pointing after the instructions inserted to convert the return values.
|
|
///
|
|
/// This function is very similar to the `legalize_entry_params` function above.
|
|
///
|
|
/// Returns the possibly new instruction representing the call.
|
|
fn legalize_inst_results<ResType>(pos: &mut FuncCursor, mut get_abi_type: ResType) -> Inst
|
|
where
|
|
ResType: FnMut(&Function, usize) -> AbiParam,
|
|
{
|
|
let call = pos
|
|
.current_inst()
|
|
.expect("Cursor must point to a call instruction");
|
|
|
|
// We theoretically allow for call instructions that return a number of fixed results before
|
|
// the call return values. In practice, it doesn't happen.
|
|
debug_assert_eq!(
|
|
pos.func.dfg[call]
|
|
.opcode()
|
|
.constraints()
|
|
.num_fixed_results(),
|
|
0,
|
|
"Fixed results on calls not supported"
|
|
);
|
|
|
|
let results = pos.func.dfg.detach_results(call);
|
|
let mut next_res = 0;
|
|
let mut abi_res = 0;
|
|
|
|
// Point immediately after the call.
|
|
pos.next_inst();
|
|
|
|
while let Some(res) = results.get(next_res, &pos.func.dfg.value_lists) {
|
|
next_res += 1;
|
|
|
|
let res_type = pos.func.dfg.value_type(res);
|
|
if res_type == get_abi_type(pos.func, abi_res).value_type {
|
|
// No value translation is necessary, this result matches the ABI type.
|
|
pos.func.dfg.attach_result(call, res);
|
|
abi_res += 1;
|
|
} else {
|
|
let mut get_res = |func: &mut Function, ty| {
|
|
let abi_type = get_abi_type(func, abi_res);
|
|
if ty == abi_type.value_type {
|
|
let last_res = func.dfg.append_result(call, ty);
|
|
abi_res += 1;
|
|
Ok(last_res)
|
|
} else {
|
|
Err(abi_type)
|
|
}
|
|
};
|
|
let v = convert_from_abi(pos, res_type, Some(res), &mut get_res);
|
|
debug_assert_eq!(pos.func.dfg.resolve_aliases(res), v);
|
|
}
|
|
}
|
|
|
|
call
|
|
}
|
|
|
|
fn assert_is_valid_sret_legalization(
|
|
old_ret_list: &EntityList<Value>,
|
|
old_sig: &Signature,
|
|
new_sig: &Signature,
|
|
pos: &FuncCursor,
|
|
) {
|
|
debug_assert_eq!(
|
|
old_sig.returns.len(),
|
|
old_ret_list.len(&pos.func.dfg.value_lists)
|
|
);
|
|
|
|
// Assert that the only difference in special parameters is that there
|
|
// is an appended struct return pointer parameter.
|
|
let old_special_params: Vec<_> = old_sig
|
|
.params
|
|
.iter()
|
|
.filter(|r| r.purpose != ArgumentPurpose::Normal)
|
|
.collect();
|
|
let new_special_params: Vec<_> = new_sig
|
|
.params
|
|
.iter()
|
|
.filter(|r| r.purpose != ArgumentPurpose::Normal)
|
|
.collect();
|
|
debug_assert_eq!(old_special_params.len() + 1, new_special_params.len());
|
|
debug_assert!(old_special_params
|
|
.iter()
|
|
.zip(&new_special_params)
|
|
.all(|(old, new)| old.purpose == new.purpose));
|
|
debug_assert_eq!(
|
|
new_special_params.last().unwrap().purpose,
|
|
ArgumentPurpose::StructReturn
|
|
);
|
|
|
|
// If the special returns have changed at all, then the only change
|
|
// should be that the struct return pointer is returned back out of the
|
|
// function, so that callers don't have to load its stack address again.
|
|
let old_special_returns: Vec<_> = old_sig
|
|
.returns
|
|
.iter()
|
|
.filter(|r| r.purpose != ArgumentPurpose::Normal)
|
|
.collect();
|
|
let new_special_returns: Vec<_> = new_sig
|
|
.returns
|
|
.iter()
|
|
.filter(|r| r.purpose != ArgumentPurpose::Normal)
|
|
.collect();
|
|
debug_assert!(old_special_returns
|
|
.iter()
|
|
.zip(&new_special_returns)
|
|
.all(|(old, new)| old.purpose == new.purpose));
|
|
debug_assert!(
|
|
old_special_returns.len() == new_special_returns.len()
|
|
|| (old_special_returns.len() + 1 == new_special_returns.len()
|
|
&& new_special_returns.last().unwrap().purpose == ArgumentPurpose::StructReturn)
|
|
);
|
|
}
|
|
|
|
fn legalize_sret_call(isa: &dyn TargetIsa, pos: &mut FuncCursor, sig_ref: SigRef, call: Inst) {
|
|
let old_ret_list = pos.func.dfg.detach_results(call);
|
|
let old_sig = pos.func.dfg.old_signatures[sig_ref]
|
|
.take()
|
|
.expect("must have an old signature when using an `sret` parameter");
|
|
|
|
// We make a bunch of assumptions about the shape of the old, multi-return
|
|
// signature and the new, sret-using signature in this legalization
|
|
// function. Assert that these assumptions hold true in debug mode.
|
|
if cfg!(debug_assertions) {
|
|
assert_is_valid_sret_legalization(
|
|
&old_ret_list,
|
|
&old_sig,
|
|
&pos.func.dfg.signatures[sig_ref],
|
|
&pos,
|
|
);
|
|
}
|
|
|
|
// Go through and remove all normal return values from the `call`
|
|
// instruction's returns list. These will be stored into the stack slot that
|
|
// the sret points to. At the same time, calculate the size of the sret
|
|
// stack slot.
|
|
let mut sret_slot_size = 0;
|
|
for (i, ret) in old_sig.returns.iter().enumerate() {
|
|
let v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
|
|
let ty = pos.func.dfg.value_type(v);
|
|
if ret.purpose == ArgumentPurpose::Normal {
|
|
debug_assert_eq!(ret.location, ArgumentLoc::Unassigned);
|
|
let ty = legalized_type_for_sret(ty);
|
|
let size = ty.bytes();
|
|
sret_slot_size = round_up_to_multiple_of_type_align(sret_slot_size, ty) + size;
|
|
} else {
|
|
let new_v = pos.func.dfg.append_result(call, ty);
|
|
pos.func.dfg.change_to_alias(v, new_v);
|
|
}
|
|
}
|
|
|
|
let stack_slot = pos.func.stack_slots.push(StackSlotData {
|
|
kind: StackSlotKind::StructReturnSlot,
|
|
size: sret_slot_size,
|
|
offset: None,
|
|
});
|
|
|
|
// Append the sret pointer to the `call` instruction's arguments.
|
|
let ptr_type = Type::triple_pointer_type(isa.triple());
|
|
let sret_arg = pos.ins().stack_addr(ptr_type, stack_slot, 0);
|
|
pos.func.dfg.append_inst_arg(call, sret_arg);
|
|
|
|
// The sret pointer might be returned by the signature as well. If so, we
|
|
// need to add it to the `call` instruction's results list.
|
|
//
|
|
// Additionally, when the sret is explicitly returned in this calling
|
|
// convention, then use it when loading the sret returns back into ssa
|
|
// values to avoid keeping the original `sret_arg` live and potentially
|
|
// having to do spills and fills.
|
|
let sret =
|
|
if pos.func.dfg.signatures[sig_ref].uses_special_return(ArgumentPurpose::StructReturn) {
|
|
pos.func.dfg.append_result(call, ptr_type)
|
|
} else {
|
|
sret_arg
|
|
};
|
|
|
|
// Finally, load each of the call's return values out of the sret stack
|
|
// slot.
|
|
pos.goto_after_inst(call);
|
|
let mut offset = 0;
|
|
for i in 0..old_ret_list.len(&pos.func.dfg.value_lists) {
|
|
if old_sig.returns[i].purpose != ArgumentPurpose::Normal {
|
|
continue;
|
|
}
|
|
|
|
let old_v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
|
|
let ty = pos.func.dfg.value_type(old_v);
|
|
let mut legalized_ty = legalized_type_for_sret(ty);
|
|
|
|
offset = round_up_to_multiple_of_type_align(offset, legalized_ty);
|
|
|
|
let new_legalized_v =
|
|
pos.ins()
|
|
.load(legalized_ty, MemFlags::trusted(), sret, offset as i32);
|
|
|
|
// "Illegalize" the loaded value from the legalized type back to its
|
|
// original `ty`. This is basically the opposite of
|
|
// `legalize_type_for_sret_store`.
|
|
let mut new_v = new_legalized_v;
|
|
if ty.is_bool() {
|
|
legalized_ty = legalized_ty.as_bool_pedantic();
|
|
new_v = pos.ins().raw_bitcast(legalized_ty, new_v);
|
|
|
|
if ty.bits() < legalized_ty.bits() {
|
|
legalized_ty = ty;
|
|
new_v = pos.ins().breduce(legalized_ty, new_v);
|
|
}
|
|
}
|
|
|
|
pos.func.dfg.change_to_alias(old_v, new_v);
|
|
|
|
offset += legalized_ty.bytes();
|
|
}
|
|
|
|
pos.func.dfg.old_signatures[sig_ref] = Some(old_sig);
|
|
}
|
|
|
|
/// Compute original value of type `ty` from the legalized ABI arguments.
|
|
///
|
|
/// The conversion is recursive, controlled by the `get_arg` closure which is called to retrieve an
|
|
/// ABI argument. It returns:
|
|
///
|
|
/// - `Ok(arg)` if the requested type matches the next ABI argument.
|
|
/// - `Err(arg_type)` if further conversions are needed from the ABI argument `arg_type`.
|
|
///
|
|
/// If the `into_result` value is provided, the converted result will be written into that value.
|
|
fn convert_from_abi<GetArg>(
|
|
pos: &mut FuncCursor,
|
|
ty: Type,
|
|
into_result: Option<Value>,
|
|
get_arg: &mut GetArg,
|
|
) -> Value
|
|
where
|
|
GetArg: FnMut(&mut Function, Type) -> Result<Value, AbiParam>,
|
|
{
|
|
// Terminate the recursion when we get the desired type.
|
|
let arg_type = match get_arg(pos.func, ty) {
|
|
Ok(v) => {
|
|
debug_assert_eq!(pos.func.dfg.value_type(v), ty);
|
|
debug_assert_eq!(into_result, None);
|
|
return v;
|
|
}
|
|
Err(t) => t,
|
|
};
|
|
|
|
// Reconstruct how `ty` was legalized into the `arg_type` argument.
|
|
let conversion = legalize_abi_value(ty, &arg_type);
|
|
|
|
debug!("convert_from_abi({}): {:?}", ty, conversion);
|
|
|
|
// The conversion describes value to ABI argument. We implement the reverse conversion here.
|
|
match conversion {
|
|
// Construct a `ty` by concatenating two ABI integers.
|
|
ValueConversion::IntSplit => {
|
|
let abi_ty = ty.half_width().expect("Invalid type for conversion");
|
|
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
debug!(
|
|
"intsplit {}: {}, {}: {}",
|
|
lo,
|
|
pos.func.dfg.value_type(lo),
|
|
hi,
|
|
pos.func.dfg.value_type(hi)
|
|
);
|
|
pos.ins().with_results([into_result]).iconcat(lo, hi)
|
|
}
|
|
// Construct a `ty` by concatenating two halves of a vector.
|
|
ValueConversion::VectorSplit => {
|
|
let abi_ty = ty.half_vector().expect("Invalid type for conversion");
|
|
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
pos.ins().with_results([into_result]).vconcat(lo, hi)
|
|
}
|
|
// Construct a `ty` by bit-casting from an integer type.
|
|
ValueConversion::IntBits => {
|
|
debug_assert!(!ty.is_int());
|
|
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
|
|
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
pos.ins().with_results([into_result]).bitcast(ty, arg)
|
|
}
|
|
// ABI argument is a sign-extended version of the value we want.
|
|
ValueConversion::Sext(abi_ty) => {
|
|
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
// TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
|
|
// We could insert an `assert_sreduce` which would fold with a following `sextend` of
|
|
// this value.
|
|
pos.ins().with_results([into_result]).ireduce(ty, arg)
|
|
}
|
|
ValueConversion::Uext(abi_ty) => {
|
|
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
// TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
|
|
// We could insert an `assert_ureduce` which would fold with a following `uextend` of
|
|
// this value.
|
|
pos.ins().with_results([into_result]).ireduce(ty, arg)
|
|
}
|
|
// ABI argument is a pointer to the value we want.
|
|
ValueConversion::Pointer(abi_ty) => {
|
|
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
|
|
pos.ins()
|
|
.with_results([into_result])
|
|
.load(ty, MemFlags::new(), arg, 0)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Convert `value` to match an ABI signature by inserting instructions at `pos`.
|
|
///
|
|
/// This may require expanding the value to multiple ABI arguments. The conversion process is
|
|
/// recursive and controlled by the `put_arg` closure. When a candidate argument value is presented
|
|
/// to the closure, it will perform one of two actions:
|
|
///
|
|
/// 1. If the suggested argument has an acceptable value type, consume it by adding it to the list
|
|
/// of arguments and return `Ok(())`.
|
|
/// 2. If the suggested argument doesn't have the right value type, don't change anything, but
|
|
/// return the `Err(AbiParam)` that is needed.
|
|
///
|
|
fn convert_to_abi<PutArg>(
|
|
pos: &mut FuncCursor,
|
|
cfg: &ControlFlowGraph,
|
|
value: Value,
|
|
put_arg: &mut PutArg,
|
|
) where
|
|
PutArg: FnMut(&mut Function, Value) -> Result<(), AbiParam>,
|
|
{
|
|
// Start by invoking the closure to either terminate the recursion or get the argument type
|
|
// we're trying to match.
|
|
let arg_type = match put_arg(pos.func, value) {
|
|
Ok(_) => return,
|
|
Err(t) => t,
|
|
};
|
|
|
|
let ty = pos.func.dfg.value_type(value);
|
|
match legalize_abi_value(ty, &arg_type) {
|
|
ValueConversion::IntSplit => {
|
|
let curpos = pos.position();
|
|
let srcloc = pos.srcloc();
|
|
let (lo, hi) = isplit(&mut pos.func, cfg, curpos, srcloc, value);
|
|
convert_to_abi(pos, cfg, lo, put_arg);
|
|
convert_to_abi(pos, cfg, hi, put_arg);
|
|
}
|
|
ValueConversion::VectorSplit => {
|
|
let curpos = pos.position();
|
|
let srcloc = pos.srcloc();
|
|
let (lo, hi) = vsplit(&mut pos.func, cfg, curpos, srcloc, value);
|
|
convert_to_abi(pos, cfg, lo, put_arg);
|
|
convert_to_abi(pos, cfg, hi, put_arg);
|
|
}
|
|
ValueConversion::IntBits => {
|
|
debug_assert!(!ty.is_int());
|
|
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
|
|
let arg = pos.ins().bitcast(abi_ty, value);
|
|
convert_to_abi(pos, cfg, arg, put_arg);
|
|
}
|
|
ValueConversion::Sext(abi_ty) => {
|
|
let arg = pos.ins().sextend(abi_ty, value);
|
|
convert_to_abi(pos, cfg, arg, put_arg);
|
|
}
|
|
ValueConversion::Uext(abi_ty) => {
|
|
let arg = pos.ins().uextend(abi_ty, value);
|
|
convert_to_abi(pos, cfg, arg, put_arg);
|
|
}
|
|
ValueConversion::Pointer(abi_ty) => {
|
|
// Note: This conversion can only happen for call arguments,
|
|
// so we can allocate the value on stack safely.
|
|
let stack_slot = pos.func.create_stack_slot(StackSlotData {
|
|
kind: StackSlotKind::ExplicitSlot,
|
|
size: ty.bytes(),
|
|
offset: None,
|
|
});
|
|
let arg = pos.ins().stack_addr(abi_ty, stack_slot, 0);
|
|
pos.ins().store(MemFlags::new(), value, arg, 0);
|
|
convert_to_abi(pos, cfg, arg, put_arg);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Check if a sequence of arguments match a desired sequence of argument types.
|
|
fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool {
|
|
args.len() == types.len()
|
|
&& args.iter().zip(types.iter()).all(|(v, at)| {
|
|
if let ArgumentPurpose::StructArgument(_) = at.purpose {
|
|
true
|
|
} else {
|
|
dfg.value_type(*v) == at.value_type
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Check if the arguments of the call `inst` match the signature.
|
|
///
|
|
/// Returns `Ok(())` if the signature matches and no changes are needed, or `Err(sig_ref)` if the
|
|
/// signature doesn't match.
|
|
fn check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Result<(), SigRef> {
|
|
// Extract the signature and argument values.
|
|
let (sig_ref, args) = match dfg[inst].analyze_call(&dfg.value_lists) {
|
|
CallInfo::Direct(func, args) => (dfg.ext_funcs[func].signature, args),
|
|
CallInfo::Indirect(sig_ref, args) => (sig_ref, args),
|
|
CallInfo::NotACall => panic!("Expected call, got {:?}", dfg[inst]),
|
|
};
|
|
let sig = &dfg.signatures[sig_ref];
|
|
|
|
if check_arg_types(dfg, args, &sig.params[..])
|
|
&& check_arg_types(dfg, dfg.inst_results(inst), &sig.returns[..])
|
|
{
|
|
// All types check out.
|
|
Ok(())
|
|
} else {
|
|
// Call types need fixing.
|
|
Err(sig_ref)
|
|
}
|
|
}
|
|
|
|
/// Check if the arguments of the return `inst` match the signature.
|
|
fn check_return_signature(dfg: &DataFlowGraph, inst: Inst, sig: &Signature) -> bool {
|
|
check_arg_types(dfg, dfg.inst_variable_args(inst), &sig.returns)
|
|
}
|
|
|
|
/// Insert ABI conversion code for the arguments to the call or return instruction at `pos`.
|
|
///
|
|
/// - `abi_args` is the number of arguments that the ABI signature requires.
|
|
/// - `get_abi_type` is a closure that can provide the desired `AbiParam` for a given ABI
|
|
/// argument number in `0..abi_args`.
|
|
///
|
|
fn legalize_inst_arguments<ArgType>(
|
|
pos: &mut FuncCursor,
|
|
cfg: &ControlFlowGraph,
|
|
abi_args: usize,
|
|
mut get_abi_type: ArgType,
|
|
) where
|
|
ArgType: FnMut(&Function, usize) -> AbiParam,
|
|
{
|
|
let inst = pos
|
|
.current_inst()
|
|
.expect("Cursor must point to a call instruction");
|
|
|
|
// Lift the value list out of the call instruction so we modify it.
|
|
let mut vlist = pos.func.dfg[inst]
|
|
.take_value_list()
|
|
.expect("Call must have a value list");
|
|
|
|
// The value list contains all arguments to the instruction, including the callee on an
|
|
// indirect call which isn't part of the call arguments that must match the ABI signature.
|
|
// Figure out how many fixed values are at the front of the list. We won't touch those.
|
|
let num_fixed_values = pos.func.dfg[inst]
|
|
.opcode()
|
|
.constraints()
|
|
.num_fixed_value_arguments();
|
|
let have_args = vlist.len(&pos.func.dfg.value_lists) - num_fixed_values;
|
|
if abi_args < have_args {
|
|
// This happens with multiple return values after we've legalized the
|
|
// signature but haven't legalized the return instruction yet. This
|
|
// legalization is handled in `handle_return_abi`.
|
|
pos.func.dfg[inst].put_value_list(vlist);
|
|
return;
|
|
}
|
|
|
|
// Grow the value list to the right size and shift all the existing arguments to the right.
|
|
// This lets us write the new argument values into the list without overwriting the old
|
|
// arguments.
|
|
//
|
|
// Before:
|
|
//
|
|
// <--> fixed_values
|
|
// <-----------> have_args
|
|
// [FFFFOOOOOOOOOOOOO]
|
|
//
|
|
// After grow_at():
|
|
//
|
|
// <--> fixed_values
|
|
// <-----------> have_args
|
|
// <------------------> abi_args
|
|
// [FFFF-------OOOOOOOOOOOOO]
|
|
// ^
|
|
// old_arg_offset
|
|
//
|
|
// After writing the new arguments:
|
|
//
|
|
// <--> fixed_values
|
|
// <------------------> abi_args
|
|
// [FFFFNNNNNNNNNNNNNNNNNNNN]
|
|
//
|
|
vlist.grow_at(
|
|
num_fixed_values,
|
|
abi_args - have_args,
|
|
&mut pos.func.dfg.value_lists,
|
|
);
|
|
let old_arg_offset = num_fixed_values + abi_args - have_args;
|
|
|
|
let mut abi_arg = 0;
|
|
for old_arg in 0..have_args {
|
|
let old_value = vlist
|
|
.get(old_arg_offset + old_arg, &pos.func.dfg.value_lists)
|
|
.unwrap();
|
|
let mut put_arg = |func: &mut Function, arg| {
|
|
let abi_type = get_abi_type(func, abi_arg);
|
|
let struct_argument = if let ArgumentPurpose::StructArgument(_) = abi_type.purpose {
|
|
true
|
|
} else {
|
|
false
|
|
};
|
|
if func.dfg.value_type(arg) == abi_type.value_type || struct_argument {
|
|
// This is the argument type we need.
|
|
vlist.as_mut_slice(&mut func.dfg.value_lists)[num_fixed_values + abi_arg] = arg;
|
|
abi_arg += 1;
|
|
Ok(())
|
|
} else {
|
|
Err(abi_type)
|
|
}
|
|
};
|
|
convert_to_abi(pos, cfg, old_value, &mut put_arg);
|
|
}
|
|
|
|
// Put the modified value list back.
|
|
pos.func.dfg[inst].put_value_list(vlist);
|
|
}
|
|
|
|
/// Ensure that the `ty` being returned is a type that can be loaded and stored
|
|
/// (potentially after another narrowing legalization) from memory, since it
|
|
/// will go into the `sret` space.
|
|
fn legalized_type_for_sret(ty: Type) -> Type {
|
|
if ty.is_bool() {
|
|
let bits = std::cmp::max(8, ty.bits());
|
|
Type::int(bits).unwrap()
|
|
} else {
|
|
ty
|
|
}
|
|
}
|
|
|
|
/// Insert any legalization code required to ensure that `val` can be stored
|
|
/// into the `sret` memory. Returns the (potentially new, potentially
|
|
/// unmodified) legalized value and its type.
|
|
fn legalize_type_for_sret_store(pos: &mut FuncCursor, val: Value, ty: Type) -> (Value, Type) {
|
|
if ty.is_bool() {
|
|
let bits = std::cmp::max(8, ty.bits());
|
|
let ty = Type::int(bits).unwrap();
|
|
let val = pos.ins().bint(ty, val);
|
|
(val, ty)
|
|
} else {
|
|
(val, ty)
|
|
}
|
|
}
|
|
|
|
/// Insert ABI conversion code before and after the call instruction at `pos`.
|
|
///
|
|
/// Instructions inserted before the call will compute the appropriate ABI values for the
|
|
/// callee's new ABI-legalized signature. The function call arguments are rewritten in place to
|
|
/// match the new signature.
|
|
///
|
|
/// Instructions will be inserted after the call to convert returned ABI values back to the
|
|
/// original return values. The call's result values will be adapted to match the new signature.
|
|
///
|
|
/// Returns `true` if any instructions were inserted.
|
|
pub fn handle_call_abi(
|
|
isa: &dyn TargetIsa,
|
|
mut inst: Inst,
|
|
func: &mut Function,
|
|
cfg: &ControlFlowGraph,
|
|
) -> bool {
|
|
let pos = &mut FuncCursor::new(func).at_inst(inst);
|
|
pos.use_srcloc(inst);
|
|
|
|
// Start by checking if the argument types already match the signature.
|
|
let sig_ref = match check_call_signature(&pos.func.dfg, inst) {
|
|
Ok(_) => return spill_call_arguments(pos, isa),
|
|
Err(s) => s,
|
|
};
|
|
|
|
let sig = &pos.func.dfg.signatures[sig_ref];
|
|
let old_sig = &pos.func.dfg.old_signatures[sig_ref];
|
|
|
|
if sig.uses_struct_return_param()
|
|
&& old_sig
|
|
.as_ref()
|
|
.map_or(false, |s| !s.uses_struct_return_param())
|
|
{
|
|
legalize_sret_call(isa, pos, sig_ref, inst);
|
|
} else {
|
|
if !pos.func.dfg.signatures[sig_ref].returns.is_empty() {
|
|
inst = legalize_inst_results(pos, |func, abi_res| {
|
|
func.dfg.signatures[sig_ref].returns[abi_res]
|
|
});
|
|
}
|
|
}
|
|
|
|
// Go back and fix the call arguments to match the ABI signature.
|
|
pos.goto_inst(inst);
|
|
let abi_args = pos.func.dfg.signatures[sig_ref].params.len();
|
|
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
|
|
func.dfg.signatures[sig_ref].params[abi_arg]
|
|
});
|
|
|
|
debug_assert!(
|
|
check_call_signature(&pos.func.dfg, inst).is_ok(),
|
|
"Signature still wrong: {}, {}{}",
|
|
pos.func.dfg.display_inst(inst, None),
|
|
sig_ref,
|
|
pos.func.dfg.signatures[sig_ref]
|
|
);
|
|
|
|
// Go back and insert spills for any stack arguments.
|
|
pos.goto_inst(inst);
|
|
spill_call_arguments(pos, isa);
|
|
|
|
// Yes, we changed stuff.
|
|
true
|
|
}
|
|
|
|
/// Insert ABI conversion code before and after the return instruction at `inst`.
|
|
///
|
|
/// Return `true` if any instructions were inserted.
|
|
pub fn handle_return_abi(inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
|
|
// Check if the returned types already match the signature.
|
|
if check_return_signature(&func.dfg, inst, &func.signature) {
|
|
return false;
|
|
}
|
|
|
|
// Count the special-purpose return values (`link`, `sret`, and `vmctx`) that were appended to
|
|
// the legalized signature.
|
|
let special_args = func
|
|
.signature
|
|
.returns
|
|
.iter()
|
|
.rev()
|
|
.take_while(|&rt| {
|
|
rt.purpose == ArgumentPurpose::Link
|
|
|| rt.purpose == ArgumentPurpose::StructReturn
|
|
|| rt.purpose == ArgumentPurpose::VMContext
|
|
})
|
|
.count();
|
|
let abi_args = func.signature.returns.len() - special_args;
|
|
|
|
let pos = &mut FuncCursor::new(func).at_inst(inst);
|
|
pos.use_srcloc(inst);
|
|
|
|
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
|
|
let arg = func.signature.returns[abi_arg];
|
|
debug_assert!(
|
|
!arg.legalized_to_pointer,
|
|
"Return value cannot be legalized to pointer"
|
|
);
|
|
arg
|
|
});
|
|
// Append special return arguments for any `sret`, `link`, and `vmctx` return values added to
|
|
// the legalized signature. These values should simply be propagated from the entry block
|
|
// arguments.
|
|
if special_args > 0 {
|
|
debug!(
|
|
"Adding {} special-purpose arguments to {}",
|
|
special_args,
|
|
pos.func.dfg.display_inst(inst, None)
|
|
);
|
|
let mut vlist = pos.func.dfg[inst].take_value_list().unwrap();
|
|
let mut sret = None;
|
|
|
|
for arg in &pos.func.signature.returns[abi_args..] {
|
|
match arg.purpose {
|
|
ArgumentPurpose::Link
|
|
| ArgumentPurpose::StructReturn
|
|
| ArgumentPurpose::VMContext => {}
|
|
ArgumentPurpose::Normal => panic!("unexpected return value {}", arg),
|
|
_ => panic!("Unsupported special purpose return value {}", arg),
|
|
}
|
|
// A `link`/`sret`/`vmctx` return value can only appear in a signature that has a
|
|
// unique matching argument. They are appended at the end, so search the signature from
|
|
// the end.
|
|
let idx = pos
|
|
.func
|
|
.signature
|
|
.params
|
|
.iter()
|
|
.rposition(|t| t.purpose == arg.purpose)
|
|
.expect("No matching special purpose argument.");
|
|
// Get the corresponding entry block value and add it to the return instruction's
|
|
// arguments.
|
|
let val = pos
|
|
.func
|
|
.dfg
|
|
.block_params(pos.func.layout.entry_block().unwrap())[idx];
|
|
debug_assert_eq!(pos.func.dfg.value_type(val), arg.value_type);
|
|
vlist.push(val, &mut pos.func.dfg.value_lists);
|
|
|
|
if let ArgumentPurpose::StructReturn = arg.purpose {
|
|
sret = Some(val);
|
|
}
|
|
}
|
|
|
|
// Store all the regular returns into the retptr space and remove them
|
|
// from the `return` instruction's value list.
|
|
if let Some(sret) = sret {
|
|
let mut offset = 0;
|
|
let num_regular_rets = vlist.len(&pos.func.dfg.value_lists) - special_args;
|
|
for i in 0..num_regular_rets {
|
|
debug_assert_eq!(
|
|
pos.func.old_signature.as_ref().unwrap().returns[i].purpose,
|
|
ArgumentPurpose::Normal,
|
|
);
|
|
|
|
// The next return value to process is always at `0`, since the
|
|
// list is emptied as we iterate.
|
|
let v = vlist.get(0, &pos.func.dfg.value_lists).unwrap();
|
|
let ty = pos.func.dfg.value_type(v);
|
|
let (v, ty) = legalize_type_for_sret_store(pos, v, ty);
|
|
|
|
let size = ty.bytes();
|
|
offset = round_up_to_multiple_of_type_align(offset, ty);
|
|
|
|
pos.ins().store(MemFlags::trusted(), v, sret, offset as i32);
|
|
vlist.remove(0, &mut pos.func.dfg.value_lists);
|
|
|
|
offset += size;
|
|
}
|
|
}
|
|
pos.func.dfg[inst].put_value_list(vlist);
|
|
}
|
|
|
|
debug_assert_eq!(
|
|
pos.func.dfg.inst_variable_args(inst).len(),
|
|
abi_args + special_args
|
|
);
|
|
debug_assert!(
|
|
check_return_signature(&pos.func.dfg, inst, &pos.func.signature),
|
|
"Signature still wrong: {} / signature {}",
|
|
pos.func.dfg.display_inst(inst, None),
|
|
pos.func.signature
|
|
);
|
|
|
|
// Yes, we changed stuff.
|
|
true
|
|
}
|
|
|
|
fn round_up_to_multiple_of_type_align(bytes: u32, ty: Type) -> u32 {
|
|
// We don't have a dedicated alignment for types, so assume they are
|
|
// size-aligned.
|
|
let align = ty.bytes();
|
|
round_up_to_multiple_of_pow2(bytes, align)
|
|
}
|
|
|
|
/// Round `n` up to the next multiple of `to` that is greater than or equal to
|
|
/// `n`.
|
|
///
|
|
/// `to` must be a power of two and greater than zero.
|
|
///
|
|
/// This is useful for rounding an offset or pointer up to some type's required
|
|
/// alignment.
|
|
fn round_up_to_multiple_of_pow2(n: u32, to: u32) -> u32 {
|
|
debug_assert!(to > 0);
|
|
debug_assert!(to.is_power_of_two());
|
|
|
|
// The simple version of this function is
|
|
//
|
|
// (n + to - 1) / to * to
|
|
//
|
|
// Consider the numerator: `n + to - 1`. This is ensuring that if there is
|
|
// any remainder for `n / to`, then the result of the division is one
|
|
// greater than `n / to`, and that otherwise we get exactly the same result
|
|
// as `n / to` due to integer division rounding off the remainder. In other
|
|
// words, we only round up if `n` is not aligned to `to`.
|
|
//
|
|
// However, we know `to` is a power of two, and therefore `anything / to` is
|
|
// equivalent to `anything >> log2(to)` and `anything * to` is equivalent to
|
|
// `anything << log2(to)`. We can therefore rewrite our simplified function
|
|
// into the following:
|
|
//
|
|
// (n + to - 1) >> log2(to) << log2(to)
|
|
//
|
|
// But shifting a value right by some number of bits `b` and then shifting
|
|
// it left by that same number of bits `b` is equivalent to clearing the
|
|
// bottom `b` bits of the number. We can clear the bottom `b` bits of a
|
|
// number by bit-wise and'ing the number with the bit-wise not of `2^b - 1`.
|
|
// Plugging this into our function and simplifying, we get:
|
|
//
|
|
// (n + to - 1) >> log2(to) << log2(to)
|
|
// = (n + to - 1) & !(2^log2(to) - 1)
|
|
// = (n + to - 1) & !(to - 1)
|
|
//
|
|
// And now we have the final version of this function!
|
|
|
|
(n + to - 1) & !(to - 1)
|
|
}
|
|
|
|
/// Assign stack slots to incoming function parameters on the stack.
|
|
///
|
|
/// Values that are passed into the function on the stack must be assigned to an `IncomingArg`
|
|
/// stack slot already during legalization.
|
|
fn spill_entry_params(func: &mut Function, entry: Block) {
|
|
for (abi, &arg) in func
|
|
.signature
|
|
.params
|
|
.iter()
|
|
.zip(func.dfg.block_params(entry))
|
|
{
|
|
if let ArgumentPurpose::StructArgument(_) = abi.purpose {
|
|
} else if let ArgumentLoc::Stack(offset) = abi.location {
|
|
let ss = func
|
|
.stack_slots
|
|
.make_incoming_arg(abi.value_type.bytes(), offset);
|
|
func.locations[arg] = ValueLoc::Stack(ss);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Assign stack slots to outgoing function arguments on the stack.
|
|
///
|
|
/// Values that are passed to a called function on the stack must be assigned to a matching
|
|
/// `OutgoingArg` stack slot. The assignment must happen immediately before the call.
|
|
///
|
|
/// TODO: The outgoing stack slots can be written a bit earlier, as long as there are no branches
|
|
/// or calls between writing the stack slots and the call instruction. Writing the slots earlier
|
|
/// could help reduce register pressure before the call.
|
|
fn spill_call_arguments(pos: &mut FuncCursor, isa: &dyn TargetIsa) -> bool {
|
|
let inst = pos
|
|
.current_inst()
|
|
.expect("Cursor must point to a call instruction");
|
|
let sig_ref = pos
|
|
.func
|
|
.dfg
|
|
.call_signature(inst)
|
|
.expect("Call instruction expected.");
|
|
|
|
// Start by building a list of stack slots and arguments to be replaced.
|
|
// This requires borrowing `pos.func.dfg`, so we can't change anything.
|
|
let arglist = {
|
|
let locations = &pos.func.locations;
|
|
let stack_slots = &mut pos.func.stack_slots;
|
|
pos.func
|
|
.dfg
|
|
.inst_variable_args(inst)
|
|
.iter()
|
|
.zip(&pos.func.dfg.signatures[sig_ref].params)
|
|
.enumerate()
|
|
.filter_map(|(idx, (&arg, abi))| {
|
|
match abi.location {
|
|
ArgumentLoc::Stack(offset) => {
|
|
// Assign `arg` to a new stack slot, unless it's already in the correct
|
|
// slot. The legalization needs to be idempotent, so we should see a
|
|
// correct outgoing slot on the second pass.
|
|
let (ss, size) = match abi.purpose {
|
|
ArgumentPurpose::StructArgument(size) => {
|
|
(stack_slots.get_outgoing_arg(size, offset), Some(size))
|
|
}
|
|
_ => (
|
|
stack_slots.get_outgoing_arg(abi.value_type.bytes(), offset),
|
|
None,
|
|
),
|
|
};
|
|
if locations[arg] != ValueLoc::Stack(ss) {
|
|
Some((idx, arg, ss, size))
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
_ => None,
|
|
}
|
|
})
|
|
.collect::<Vec<_>>()
|
|
};
|
|
|
|
if arglist.is_empty() {
|
|
return false;
|
|
}
|
|
|
|
let mut libc_memcpy = None;
|
|
let mut import_memcpy = |func: &mut Function, pointer_type| {
|
|
if let Some(libc_memcpy) = libc_memcpy {
|
|
return libc_memcpy;
|
|
}
|
|
|
|
let signature = {
|
|
let mut s = Signature::new(isa.default_call_conv());
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
s.params.push(AbiParam::new(pointer_type));
|
|
legalize_libcall_signature(&mut s, isa);
|
|
func.import_signature(s)
|
|
};
|
|
|
|
let func = func.import_function(ExtFuncData {
|
|
name: ExternalName::LibCall(LibCall::Memcpy),
|
|
signature,
|
|
colocated: false,
|
|
});
|
|
libc_memcpy = Some(func);
|
|
func
|
|
};
|
|
|
|
// Insert the spill instructions and rewrite call arguments.
|
|
for (idx, arg, ss, size) in arglist {
|
|
let stack_val = if let Some(size) = size {
|
|
// Struct argument
|
|
let pointer_type = pos.func.dfg.value_type(arg);
|
|
let src = arg;
|
|
let dest = pos.ins().stack_addr(pointer_type, ss, 0);
|
|
let size = pos.ins().iconst(pointer_type, i64::from(size));
|
|
|
|
let libc_memcpy = import_memcpy(pos.func, pointer_type);
|
|
pos.ins().call(libc_memcpy, &[dest, src, size]);
|
|
pos.ins().dummy_sarg_t()
|
|
} else {
|
|
// Non struct argument
|
|
pos.ins().spill(arg)
|
|
};
|
|
pos.func.locations[stack_val] = ValueLoc::Stack(ss);
|
|
pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val;
|
|
}
|
|
|
|
// We changed stuff.
|
|
true
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::round_up_to_multiple_of_pow2;
|
|
|
|
#[test]
|
|
fn round_up_to_multiple_of_pow2_works() {
|
|
for (n, to, expected) in vec![
|
|
(0, 1, 0),
|
|
(1, 1, 1),
|
|
(2, 1, 2),
|
|
(0, 2, 0),
|
|
(1, 2, 2),
|
|
(2, 2, 2),
|
|
(3, 2, 4),
|
|
(0, 4, 0),
|
|
(1, 4, 4),
|
|
(2, 4, 4),
|
|
(3, 4, 4),
|
|
(4, 4, 4),
|
|
(5, 4, 8),
|
|
] {
|
|
let actual = round_up_to_multiple_of_pow2(n, to);
|
|
assert_eq!(
|
|
actual, expected,
|
|
"round_up_to_multiple_of_pow2(n = {}, to = {}) = {} (expected {})",
|
|
n, to, actual, expected
|
|
);
|
|
}
|
|
}
|
|
}
|