Use value lists for call arguments.

Add a new kind of instruction format that keeps all of its value
arguments in a value list. These value lists are all allocated out of
the dfg.value_lists memory pool.

Instruction formats with the value_list property set store *all* of
their value arguments in a single value list. There is no distinction
between fixed arguments and variable arguments.

Change the Call instruction format to use the value list representation
for its arguments.

This change is only the beginning. The intent is to eliminate the
boxed_storage instruction formats completely. Value lists use less
memory, and when the transition is complete, InstructionData will have a
trivial Drop implementation.

lib/cretonne/meta/base/formats.py

@@ -47,7 +47,7 @@ Branch = InstructionFormat(VALUE, ebb, VARIABLE_ARGS, boxed_storage=True)
 BranchTable = InstructionFormat(VALUE, jump_table)
 
 Call = InstructionFormat(
-        func_ref, VARIABLE_ARGS, multiple_results=True, boxed_storage=True)
+        func_ref, VARIABLE_ARGS, multiple_results=True, value_list=True)
 IndirectCall = InstructionFormat(
         sig_ref, VALUE, VARIABLE_ARGS,
         multiple_results=True, boxed_storage=True)

lib/cretonne/meta/cdsl/formats.py

@@ -34,6 +34,8 @@ class InstructionFormat(object):
         enums.
     :param multiple_results: Set to `True` if this instruction format allows
         more than one result to be produced.
+    :param value_list: Set to `True` if this instruction format uses a
+        `ValueList` member to store its value operands.
     :param boxed_storage: Set to `True` is this instruction format requires a
         `data: Box<...>` pointer to additional storage in its `InstructionData`
         variant.
@@ -52,6 +54,7 @@ class InstructionFormat(object):
         # type: (*Union[OperandKind, Tuple[str, OperandKind]], **Any) -> None # noqa
         self.name = kwargs.get('name', None)  # type: str
         self.multiple_results = kwargs.get('multiple_results', False)
+        self.has_value_list = kwargs.get('value_list', False)
         self.boxed_storage = kwargs.get('boxed_storage', False)
         self.members = list()  # type: List[str]
         self.kinds = tuple(self._process_member_names(kinds))

lib/cretonne/meta/gen_instr.py

@@ -58,17 +58,32 @@ def gen_arguments_method(fmt, is_mut):
     method = 'arguments'
     mut = ''
     rslice = 'ref_slice'
+    as_slice = 'as_slice'
     if is_mut:
         method += '_mut'
         mut = 'mut '
         rslice += '_mut'
+        as_slice = 'as_mut_slice'
 
     with fmt.indented(
-            'pub fn {f}(&{m}self) -> [&{m}[Value]; 2] {{'
+            'pub fn {f}<\'a>(&\'a {m}self, pool: &\'a {m}ValueListPool) -> '
+            '[&{m}[Value]; 2] {{'
             .format(f=method, m=mut), '}'):
         with fmt.indented('match *self {', '}'):
             for f in InstructionFormat.all_formats:
                 n = 'InstructionData::' + f.name
+
+                # Formats with a value list put all of their arguments in the
+                # list. We don't split them up, just return it all as variable
+                # arguments. (I expect the distinction to go away).
+                if f.has_value_list:
+                    arg = ''.format(mut)
+                    fmt.line(
+                        '{} {{ ref {}args, .. }} => '
+                        '[ &{}[], args.{}(pool) ],'
+                        .format(n, mut, mut, as_slice))
+                    continue
+
                 has_varargs = cdsl.operands.VARIABLE_ARGS in f.kinds
                 # Formats with both fixed and variable arguments delegate to
                 # the data struct. We need to work around borrow checker quirks
@@ -472,7 +487,11 @@ def gen_format_constructor(iform, fmt):
     """
 
     # Construct method arguments.
-    args = ['self', 'opcode: Opcode']
+    if iform.has_value_list:
+        args = ['mut self']
+    else:
+        args = ['self']
+    args.append('opcode: Opcode')
 
     if iform.multiple_results:
         args.append('ctrl_typevar: Type')
@@ -484,6 +503,9 @@ def gen_format_constructor(iform, fmt):
 
     # Normal operand arguments.
     for idx, kind in enumerate(iform.kinds):
+        if kind is cdsl.operands.VARIABLE_ARGS and iform.has_value_list:
+            args.append('op{}: &[Value]'.format(idx, kind.rust_type))
+        else:
             args.append('op{}: {}'.format(idx, kind.rust_type))
 
     proto = '{}({})'.format(iform.name, ', '.join(args))
@@ -492,6 +514,21 @@ def gen_format_constructor(iform, fmt):
     fmt.doc_comment(str(iform))
     fmt.line('#[allow(non_snake_case)]')
     with fmt.indented('fn {} {{'.format(proto), '}'):
+        # Start by constructing a value list with *all* the arguments.
+        if iform.has_value_list:
+            fmt.line('let mut vlist = ValueList::default();')
+            with fmt.indented('{', '}'):
+                fmt.line(
+                        'let pool = '
+                        '&mut self.data_flow_graph_mut().value_lists;')
+                for idx, kind in enumerate(iform.kinds):
+                    if kind is cdsl.operands.VALUE:
+                        fmt.line('vlist.push(op{}, pool);'.format(idx))
+                    elif kind is cdsl.operands.VARIABLE_ARGS:
+                        fmt.line(
+                                'vlist.extend(op{}.iter().cloned(), pool);'
+                                .format(idx))
+
         # Generate the instruction data.
         with fmt.indented(
                 'let data = InstructionData::{} {{'.format(iform.name), '};'):
@@ -520,7 +557,10 @@ def gen_member_inits(iform, fmt):
     """
 
     # Values first.
-    if len(iform.value_operands) == 1:
+    if iform.has_value_list:
+        # Value-list formats put *all* arguments in the list.
+        fmt.line('args: vlist,')
+    elif len(iform.value_operands) == 1:
         fmt.line('arg: op{},'.format(iform.value_operands[0]))
     elif len(iform.value_operands) > 1:
         fmt.line('args: [{}],'.format(
@@ -528,6 +568,8 @@ def gen_member_inits(iform, fmt):
 
     # Immediates and entity references.
     for idx, member in enumerate(iform.members):
+        if iform.has_value_list and member == 'varargs':
+            continue
         if member:
             fmt.line('{}: op{},'.format(member, idx))
 
@@ -557,6 +599,9 @@ def gen_inst_builder(inst, fmt):
             t = 'T{}{}'.format(1 + len(tmpl_types), op.kind.name)
             tmpl_types.append('{}: Into<{}>'.format(t, op.kind.rust_type))
             into_args.append(op.name)
+        elif (inst.format.has_value_list and
+                op.kind is cdsl.operands.VARIABLE_ARGS):
+            t = '&[Value]'
         else:
             t = op.kind.rust_type
         args.append('{}: {}'.format(op.name, t))

lib/cretonne/src/entity_list.rs

@@ -54,6 +54,11 @@ use entity_map::EntityRef;
 ///
 /// All of the list methods that take a pool reference must be given the same pool reference every
 /// time they are called. Otherwise data structures will be corrupted.
+///
+/// Entity lists can be cloned, but that operation should only be used as part of cloning the whole
+/// function they belong to. *Cloning an entity list does not allocate new memory for the clone*.
+/// It creates an alias of the same memory.
+#[derive(Clone, Debug)]
 pub struct EntityList<T: EntityRef> {
     index: u32,
     unused: PhantomData<T>,
@@ -70,6 +75,7 @@ impl<T: EntityRef> Default for EntityList<T> {
 }
 
 /// A memory pool for storing lists of `T`.
+#[derive(Clone)]
 pub struct ListPool<T: EntityRef> {
     // The main array containing the lists.
     data: Vec<T>,

lib/cretonne/src/ir/builder.rs

@@ -5,7 +5,7 @@
 
 use ir::{types, instructions};
 use ir::{InstructionData, DataFlowGraph, Cursor};
-use ir::{Opcode, Type, Inst, Value, Ebb, JumpTable, VariableArgs, SigRef, FuncRef};
+use ir::{Opcode, Type, Inst, Value, Ebb, JumpTable, VariableArgs, SigRef, FuncRef, ValueList};
 use ir::immediates::{Imm64, Uimm8, Ieee32, Ieee64, ImmVector};
 use ir::condcodes::{IntCC, FloatCC};
 
@@ -21,6 +21,7 @@ pub trait InstBuilderBase<'f>: Sized {
     /// Get an immutable reference to the data flow graph that will hold the constructed
     /// instructions.
     fn data_flow_graph(&self) -> &DataFlowGraph;
+    fn data_flow_graph_mut(&mut self) -> &mut DataFlowGraph;
 
     /// Insert a simple instruction and return a reference to it.
     ///
@@ -76,6 +77,10 @@ impl<'c, 'fc, 'fd> InstBuilderBase<'fd> for InsertBuilder<'c, 'fc, 'fd> {
         self.dfg
     }
 
+    fn data_flow_graph_mut(&mut self) -> &mut DataFlowGraph {
+        self.dfg
+    }
+
     fn simple_instruction(self, data: InstructionData) -> (Inst, &'fd mut DataFlowGraph) {
         let inst = self.dfg.make_inst(data);
         self.pos.insert_inst(inst);
@@ -129,6 +134,10 @@ impl<'f> InstBuilderBase<'f> for ReplaceBuilder<'f> {
         self.dfg
     }
 
+    fn data_flow_graph_mut(&mut self) -> &mut DataFlowGraph {
+        self.dfg
+    }
+
     fn simple_instruction(self, data: InstructionData) -> (Inst, &'f mut DataFlowGraph) {
         // The replacement instruction cannot generate multiple results, so verify that the old
         // instruction's secondary results have been detached.

lib/cretonne/src/ir/dfg.rs

@@ -1,6 +1,6 @@
 //! Data flow graph tracking Instructions, Values, and EBBs.
 
-use ir::{Ebb, Inst, Value, Type, SigRef, Signature, FuncRef};
+use ir::{Ebb, Inst, Value, Type, SigRef, Signature, FuncRef, ValueListPool};
 use ir::entities::ExpandedValue;
 use ir::instructions::{Opcode, InstructionData, CallInfo};
 use ir::extfunc::ExtFuncData;
@@ -24,7 +24,10 @@ pub struct DataFlowGraph {
     /// Data about all of the instructions in the function, including opcodes and operands.
     /// The instructions in this map are not in program order. That is tracked by `Layout`, along
     /// with the EBB containing each instruction.
-    insts: EntityMap<Inst, InstructionData>,
+    pub insts: EntityMap<Inst, InstructionData>,
+
+    /// Memory pool of value lists referenced by instructions in `insts`.
+    pub value_lists: ValueListPool,
 
     /// Extended basic blocks in the function and their arguments.
     /// This map is not in program order. That is handled by `Layout`, and so is the sequence of
@@ -56,6 +59,7 @@ impl DataFlowGraph {
     pub fn new() -> DataFlowGraph {
         DataFlowGraph {
             insts: EntityMap::new(),
+            value_lists: ValueListPool::new(),
             ebbs: EntityMap::new(),
             extended_values: Vec::new(),
             signatures: EntityMap::new(),
@@ -429,7 +433,7 @@ impl DataFlowGraph {
     /// Get the call signature of a direct or indirect call instruction.
     /// Returns `None` if `inst` is not a call instruction.
     pub fn call_signature(&self, inst: Inst) -> Option<SigRef> {
-        match self.insts[inst].analyze_call() {
+        match self.insts[inst].analyze_call(&self.value_lists) {
             CallInfo::NotACall => None,
             CallInfo::Direct(f, _) => Some(self.ext_funcs[f].signature),
             CallInfo::Indirect(s, _) => Some(s),

lib/cretonne/src/ir/instructions.rs

@@ -16,8 +16,17 @@ use ir::condcodes::*;
 use ir::types;
 use ir::DataFlowGraph;
 
-use ref_slice::*;
+use entity_list;
 use packed_option::PackedOption;
+use ref_slice::{ref_slice, ref_slice_mut};
+
+/// Some instructions use an external list of argument values because there is not enough space in
+/// the 16-byte `InstructionData` struct. These value lists are stored in a memory pool in
+/// `dfg.value_lists`.
+pub type ValueList = entity_list::EntityList<Value>;
+
+/// Memory pool for holding value lists. See `ValueList`.
+pub type ValueListPool = entity_list::ListPool<Value>;
 
 // Include code generated by `lib/cretonne/meta/gen_instr.py`. This file contains:
 //
@@ -204,7 +213,8 @@ pub enum InstructionData {
         opcode: Opcode,
         ty: Type,
         second_result: PackedOption<Value>,
-        data: Box<CallData>,
+        func_ref: FuncRef,
+        args: ValueList,
     },
     IndirectCall {
         opcode: Opcode,
@@ -244,6 +254,13 @@ impl VariableArgs {
     pub fn is_empty(&self) -> bool {
         self.0.is_empty()
     }
+
+    /// Convert this to a value list in `pool`.
+    pub fn into_value_list(self, pool: &mut ValueListPool) -> ValueList {
+        let mut vlist = ValueList::default();
+        vlist.extend(self.0, pool);
+        vlist
+    }
 }
 
 // Coerce `VariableArgs` into a `&[Value]` slice.
@@ -364,16 +381,6 @@ impl Display for BranchData {
     }
 }
 
-/// Payload of a call instruction.
-#[derive(Clone, Debug)]
-pub struct CallData {
-    /// Callee function.
-    pub func_ref: FuncRef,
-
-    /// Dynamically sized array containing call argument values.
-    pub varargs: VariableArgs,
-}
-
 /// Payload of an indirect call instruction.
 #[derive(Clone, Debug)]
 pub struct IndirectCallData {
@@ -434,10 +441,10 @@ impl ReturnRegData {
 impl InstructionData {
     /// Execute a closure once for each argument to this instruction.
     /// See also the `arguments()` method.
-    pub fn each_arg<F>(&self, mut func: F)
+    pub fn each_arg<F>(&self, pool: &ValueListPool, mut func: F)
         where F: FnMut(Value)
     {
-        for part in &self.arguments() {
+        for part in &self.arguments(pool) {
             for &arg in part.iter() {
                 func(arg);
             }
@@ -446,10 +453,10 @@ impl InstructionData {
 
     /// Execute a closure with a mutable reference to each argument to this instruction.
     /// See also the `arguments_mut()` method.
-    pub fn each_arg_mut<F>(&mut self, mut func: F)
+    pub fn each_arg_mut<F>(&mut self, pool: &mut ValueListPool, mut func: F)
         where F: FnMut(&mut Value)
     {
-        for part in &mut self.arguments_mut() {
+        for part in &mut self.arguments_mut(pool) {
             for arg in part.iter_mut() {
                 func(arg);
             }
@@ -476,10 +483,10 @@ impl InstructionData {
     /// Return information about a call instruction.
     ///
     /// Any instruction that can call another function reveals its call signature here.
-    pub fn analyze_call<'a>(&'a self) -> CallInfo<'a> {
+    pub fn analyze_call<'a>(&'a self, pool: &'a ValueListPool) -> CallInfo<'a> {
         match self {
-            &InstructionData::Call { ref data, .. } => {
+            &InstructionData::Call { func_ref, ref args, .. } => {
-                CallInfo::Direct(data.func_ref, &data.varargs)
+                CallInfo::Direct(func_ref, &args.as_slice(pool))
             }
             &InstructionData::IndirectCall { ref data, .. } => {
                 CallInfo::Indirect(data.sig_ref, &data.varargs)

lib/cretonne/src/ir/mod.rs

@@ -20,7 +20,7 @@ pub use ir::funcname::FunctionName;
 pub use ir::extfunc::{Signature, ArgumentType, ArgumentExtension, ExtFuncData};
 pub use ir::types::Type;
 pub use ir::entities::{Ebb, Inst, Value, StackSlot, JumpTable, FuncRef, SigRef};
-pub use ir::instructions::{Opcode, InstructionData, VariableArgs};
+pub use ir::instructions::{Opcode, InstructionData, VariableArgs, ValueList, ValueListPool};
 pub use ir::stackslot::StackSlotData;
 pub use ir::jumptable::JumpTableData;
 pub use ir::valueloc::{ValueLoc, ArgumentLoc};

lib/cretonne/src/regalloc/coloring.rs

@@ -299,7 +299,8 @@ impl<'a> Context<'a> {
                         }
                         ConstraintKind::Tied(arg_index) => {
                             // This def must use the same register as a fixed instruction argument.
-                            let loc = locations[dfg[inst].arguments()[0][arg_index as usize]];
+                            let arg = dfg[inst].arguments(&dfg.value_lists)[0][arg_index as usize];
+                            let loc = locations[arg];
                             *locations.ensure(lv.value) = loc;
                             // Mark the reused register. It's not really clear if we support tied
                             // stack operands. We could do that for some Intel read-modify-write

lib/cretonne/src/regalloc/liveness.rs

@@ -318,7 +318,7 @@ impl Liveness {
                 let mut operand_constraints =
                     recipe_constraints.get(recipe).map(|c| c.ins).unwrap_or(&[]).iter();
 
-                func.dfg[inst].each_arg(|arg| {
+                func.dfg[inst].each_arg(&func.dfg.value_lists, |arg| {
                     // Get the live range, create it as a dead range if necessary.
                     let lr = get_or_create(&mut self.ranges, arg, func, recipe_constraints);
 

lib/cretonne/src/write.rs

@@ -6,7 +6,7 @@
 
 use ir::{Function, Ebb, Inst, Value, Type};
 use isa::{TargetIsa, RegInfo};
-use std::fmt::{Result, Error, Write};
+use std::fmt::{self, Result, Error, Write};
 use std::result;
 
 /// Write `func` to `w` as equivalent text.
@@ -157,7 +157,7 @@ fn type_suffix(func: &Function, inst: Inst) -> Option<Type> {
 
 // Write out any value aliases appearing in `inst`.
 fn write_value_aliases(w: &mut Write, func: &Function, inst: Inst, indent: usize) -> Result {
-    for &arg in func.dfg[inst].arguments().iter().flat_map(|x| x.iter()) {
+    for &arg in func.dfg[inst].arguments(&func.dfg.value_lists).iter().flat_map(|x| x.iter()) {
         let resolved = func.dfg.resolve_aliases(arg);
         if resolved != arg {
             writeln!(w, "{1:0$}{2} -> {3}", indent, "", arg, resolved)?;
@@ -247,7 +247,12 @@ fn write_instruction(w: &mut Write,
         Jump { ref data, .. } => writeln!(w, " {}", data),
         Branch { ref data, .. } => writeln!(w, " {}", data),
         BranchTable { arg, table, .. } => writeln!(w, " {}, {}", arg, table),
-        Call { ref data, .. } => writeln!(w, " {}({})", data.func_ref, data.varargs),
+        Call { func_ref, ref args, .. } => {
+            writeln!(w,
+                     " {}({})",
+                     func_ref,
+                     DisplayValues(args.as_slice(&func.dfg.value_lists)))
+        }
         IndirectCall { ref data, .. } => {
             writeln!(w, " {}, {}({})", data.sig_ref, data.arg, data.varargs)
         }
@@ -268,6 +273,22 @@ fn write_instruction(w: &mut Write,
     }
 }
 
+/// Displayable slice of values.
+struct DisplayValues<'a>(&'a [Value]);
+
+impl<'a> fmt::Display for DisplayValues<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result {
+        for (i, val) in self.0.iter().enumerate() {
+            if i == 0 {
+                write!(f, "{}", val)?;
+            } else {
+                write!(f, ", {}", val)?;
+            }
+        }
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use ir::{Function, FunctionName, StackSlotData};

lib/reader/src/parser.rs

@@ -15,8 +15,8 @@ use cretonne::ir::types::VOID;
 use cretonne::ir::immediates::{Imm64, Ieee32, Ieee64};
 use cretonne::ir::entities::AnyEntity;
 use cretonne::ir::instructions::{InstructionFormat, InstructionData, VariableArgs,
-                                 TernaryOverflowData, JumpData, BranchData, CallData,
+                                 TernaryOverflowData, JumpData, BranchData, IndirectCallData,
-                                 IndirectCallData, ReturnData, ReturnRegData};
+                                 ReturnData, ReturnRegData};
 use cretonne::isa::{self, TargetIsa, Encoding};
 use cretonne::settings;
 use testfile::{TestFile, Details, Comment};
@@ -167,7 +167,8 @@ impl<'a> Context<'a> {
         for ebb in self.function.layout.ebbs() {
             for inst in self.function.layout.ebb_insts(ebb) {
                 let loc = inst.into();
-                match self.function.dfg[inst] {
+                let value_lists = &mut self.function.dfg.value_lists;
+                match self.function.dfg.insts[inst] {
                     InstructionData::Nullary { .. } |
                     InstructionData::UnaryImm { .. } |
                     InstructionData::UnaryIeee32 { .. } |
@@ -210,8 +211,8 @@ impl<'a> Context<'a> {
                         self.map.rewrite_values(&mut data.varargs, loc)?;
                     }
 
-                    InstructionData::Call { ref mut data, .. } => {
+                    InstructionData::Call { ref mut args, .. } => {
-                        self.map.rewrite_values(&mut data.varargs, loc)?;
+                        self.map.rewrite_values(args.as_mut_slice(value_lists), loc)?;
                     }
 
                     InstructionData::IndirectCall { ref mut data, .. } => {
@@ -1300,7 +1301,10 @@ impl<'a> Parser<'a> {
 
     // Parse the operands following the instruction opcode.
     // This depends on the format of the opcode.
-    fn parse_inst_operands(&mut self, ctx: &Context, opcode: Opcode) -> Result<InstructionData> {
+    fn parse_inst_operands(&mut self,
+                           ctx: &mut Context,
+                           opcode: Opcode)
+                           -> Result<InstructionData> {
         Ok(match opcode.format() {
             InstructionFormat::Nullary => {
                 InstructionData::Nullary {
@@ -1506,10 +1510,8 @@ impl<'a> Parser<'a> {
                     opcode: opcode,
                     ty: VOID,
                     second_result: None.into(),
-                    data: Box::new(CallData {
                     func_ref: func_ref,
-                        varargs: args,
+                    args: args.into_value_list(&mut ctx.function.dfg.value_lists),
-                    }),
                 }
             }
             InstructionFormat::IndirectCall => {