T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move default blocks into jump tables (#5756)

Browse Source 
Move the default block off of the br_table instrution, and into the JumpTable that it references.
This commit is contained in:
Trevor Elliott
2023-02-10 08:53:30 -08:00
committed by GitHub
parent 49613be393
commit d99783fc91
21 changed files with 118 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cranelift/codegen/meta/src/shared/entities.rs
View File

@@ -15,10 +15,6 @@ pub(crate) struct EntityRefs {
/// This is primarily used in control flow instructions.
pub(crate) block_call: OperandKind,
/// A reference to a basic block in the same function.
/// This is primarily used in control flow instructions.
pub(crate) label: OperandKind,
/// A reference to a basic block in the same function, with its arguments provided.
/// This is primarily used in control flow instructions.
pub(crate) block_then: OperandKind,
@@ -63,12 +59,6 @@ impl EntityRefs {
"a basic block in the same function, with its arguments provided.",
),
label: new(
"destination",
"ir::Block",
"a basic block in the same function.",
),
block_then: new(
"block_then",
"ir::BlockCall",

1
cranelift/codegen/meta/src/shared/formats.rs
View File

@@ -117,7 +117,6 @@ impl Formats {
branch_table: Builder::new("BranchTable")
.value()
.imm(&entities.label)
.imm(&entities.jump_table)
.build(),

3
cranelift/codegen/meta/src/shared/instructions.rs
View File

@@ -19,7 +19,6 @@ fn define_control_flow(
) {
let block_call = &Operand::new("block_call", &entities.block_call)
.with_doc("Destination basic block, with its arguments provided");
let label = &Operand::new("label", &entities.label).with_doc("Destination basic block");
ig.push(
Inst::new(
@@ -93,7 +92,7 @@ fn define_control_flow(
"#,
&formats.branch_table,
)
.operands_in(vec![x, label, JT])
.operands_in(vec![x, JT])
.branches(),
);
}

18
cranelift/codegen/src/inst_predicates.rs
View File

@@ -190,16 +190,16 @@ pub(crate) fn visit_block_succs<F: FnMut(Inst, Block, bool)>(
visit(inst, block_else.block(&f.dfg.value_lists), false);
}
ir::InstructionData::BranchTable {
table,
destination: dest,
..
} => {
// The default block is reached via a direct conditional branch,
// so it is not part of the table.
visit(inst, *dest, false);
ir::InstructionData::BranchTable { table, .. } => {
let table = &f.stencil.dfg.jump_tables[*table];
for &dest in f.stencil.dfg.jump_tables[*table].as_slice() {
// The default block is reached via a direct conditional branch,
// so it is not part of the table. We visit the default block first
// explicitly, as some callers of visit_block_succs depend on that
// ordering.
visit(inst, table.default_block(), false);
for &dest in table.as_slice() {
visit(inst, dest, true);
}
}

12
cranelift/codegen/src/ir/function.rs
View File

@@ -295,19 +295,11 @@ impl FunctionStencil {
}
}
InstructionData::BranchTable {
table,
destination: default_dest,
..
} => {
self.dfg.jump_tables[*table].iter_mut().for_each(|entry| {
InstructionData::BranchTable { table, .. } => {
for entry in self.dfg.jump_tables[*table].all_branches_mut() {
if *entry == old_dest {
*entry = new_dest;
}
});
if *default_dest == old_dest {
*default_dest = new_dest;
}
}

111
cranelift/codegen/src/ir/jumptable.rs
View File

@@ -14,6 +14,12 @@ use serde::{Deserialize, Serialize};
/// Contents of a jump table.
///
/// All jump tables use 0-based indexing and are densely populated.
///
/// The default block for the jump table is stored as the last element of the underlying vector,
/// and is not included in the length of the jump table. It can be accessed through the
/// `default_block` and `default_block_mut` functions. All blocks may be iterated using the
/// `all_branches` and `all_branches_mut` functions, which will both iterate over the default block
/// last.
#[derive(Clone, PartialEq, Hash)]
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
pub struct JumpTableData {
@@ -22,72 +28,71 @@ pub struct JumpTableData {
}
impl JumpTableData {
/// Create a new empty jump table.
pub fn new() -> Self {
Self { table: Vec::new() }
}
/// Create a new empty jump table with the specified capacity.
pub fn with_capacity(capacity: usize) -> Self {
Self {
table: Vec::with_capacity(capacity),
}
}
/// Create a new jump table with the provided blocks
pub fn with_blocks(table: Vec<Block>) -> Self {
pub fn new(def: Block, mut table: Vec<Block>) -> Self {
table.push(def);
Self { table }
}
/// Get the number of table entries.
pub fn len(&self) -> usize {
self.table.len()
/// Fetch the default block for this jump table.
pub fn default_block(&self) -> Block {
*self.table.last().unwrap()
}
/// Append a table entry.
pub fn push_entry(&mut self, dest: Block) {
self.table.push(dest)
/// Mutable access to the default block of this jump table.
pub fn default_block_mut(&mut self) -> &mut Block {
self.table.last_mut().unwrap()
}
/// Checks if any of the entries branch to `block`.
pub fn branches_to(&self, block: Block) -> bool {
self.table.iter().any(|target_block| *target_block == block)
}
/// Access the whole table as a slice.
pub fn as_slice(&self) -> &[Block] {
/// The jump table and default block as a single slice. The default block will always be last.
pub fn all_branches(&self) -> &[Block] {
self.table.as_slice()
}
/// Access the whole table as a mutable slice.
pub fn as_mut_slice(&mut self) -> &mut [Block] {
/// The jump table and default block as a single mutable slice. The default block will always
/// be last.
pub fn all_branches_mut(&mut self) -> &mut [Block] {
self.table.as_mut_slice()
}
/// Returns an iterator over the table.
/// Access the jump table as a slice. This excludes the default block.
pub fn as_slice(&self) -> &[Block] {
let last = self.table.len() - 1;
&self.table.as_slice()[0..last]
}
/// Access the jump table as a mutable slice. This excludes the default block.
pub fn as_mut_slice(&mut self) -> &mut [Block] {
let last = self.table.len() - 1;
&mut self.table.as_mut_slice()[0..last]
}
/// Returns an iterator to the jump table, excluding the default block.
#[deprecated(since = "7.0.0", note = "please use `.as_slice()` instead")]
pub fn iter(&self) -> Iter<Block> {
self.table.iter()
self.as_slice().iter()
}
/// Returns an iterator that allows modifying each value.
/// Returns an iterator that allows modifying each value, excluding the default block.
#[deprecated(since = "7.0.0", note = "please use `.as_mut_slice()` instead")]
pub fn iter_mut(&mut self) -> IterMut<Block> {
self.table.iter_mut()
self.as_mut_slice().iter_mut()
}
/// Clears all entries in this jump table.
/// Clears all entries in this jump table, except for the default block.
pub fn clear(&mut self) {
self.table.clear();
self.table.drain(0..self.table.len() - 1);
}
}
impl Display for JumpTableData {
fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
write!(fmt, "[")?;
match self.table.first() {
None => (),
Some(first) => write!(fmt, "{}", first)?,
}
for block in self.table.iter().skip(1) {
write!(fmt, ", {}", block)?;
write!(fmt, "{}, [", self.default_block())?;
if let Some((first, rest)) = self.as_slice().split_first() {
write!(fmt, "{}", first)?;
for block in rest {
write!(fmt, ", {}", block)?;
}
}
write!(fmt, "]")
}
@@ -102,31 +107,33 @@ mod tests {
#[test]
fn empty() {
let jt = JumpTableData::new();
let def = Block::new(0);
let jt = JumpTableData::new(def, vec![]);
assert_eq!(jt.all_branches().get(0), Some(&def));
assert_eq!(jt.as_slice().get(0), None);
assert_eq!(jt.as_slice().get(10), None);
assert_eq!(jt.to_string(), "[]");
assert_eq!(jt.to_string(), "block0, []");
let v = jt.as_slice();
assert_eq!(v, []);
assert_eq!(jt.all_branches(), [def]);
assert_eq!(jt.as_slice(), []);
}
#[test]
fn insert() {
let def = Block::new(0);
let e1 = Block::new(1);
let e2 = Block::new(2);
let mut jt = JumpTableData::new();
let jt = JumpTableData::new(def, vec![e1, e2, e1]);
jt.push_entry(e1);
jt.push_entry(e2);
jt.push_entry(e1);
assert_eq!(jt.default_block(), def);
assert_eq!(jt.to_string(), "block0, [block1, block2, block1]");
assert_eq!(jt.to_string(), "[block1, block2, block1]");
let v = jt.as_slice();
assert_eq!(v, [e1, e2, e1]);
assert_eq!(jt.all_branches(), [e1, e2, e1, def]);
assert_eq!(jt.as_slice(), [e1, e2, e1]);
}
}

2
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -2515,7 +2515,7 @@
;; `targets` contains the default target with the list of branch targets
;; concatenated.
(rule (lower_branch (br_table idx _ _) targets)
(rule (lower_branch (br_table idx _) targets)
(let ((jt_size u32 (targets_jt_size targets))
(_ InstOutput (side_effect
(emit_island (targets_jt_space targets))))

9
cranelift/codegen/src/isa/aarch64/mod.rs
View File

@@ -361,11 +361,10 @@ mod test {
let mut pos = FuncCursor::new(&mut func);
pos.insert_block(bb0);
let mut jt_data = JumpTableData::new();
jt_data.push_entry(bb1);
jt_data.push_entry(bb2);
let jt = pos.func.create_jump_table(jt_data);
pos.ins().br_table(arg0, bb3, jt);
let jt = pos
.func
.create_jump_table(JumpTableData::new(bb3, vec![bb1, bb2]));
pos.ins().br_table(arg0, jt);
pos.insert_block(bb1);
let v1 = pos.ins().iconst(I32, 1);

2
cranelift/codegen/src/isa/riscv64/inst.isle
View File

@@ -1942,7 +1942,7 @@
(extern constructor lower_br_table lower_br_table)
(rule
(lower_branch (br_table index _ _) targets)
(lower_branch (br_table index _) targets)
(lower_br_table index targets))
(decl load_ra () Reg)

2
cranelift/codegen/src/isa/s390x/lower.isle
View File

@@ -3725,7 +3725,7 @@
;; Jump table. `targets` contains the default target followed by the
;; list of branch targets per index value.
(rule (lower_branch (br_table val_idx _ _) targets)
(rule (lower_branch (br_table val_idx _) targets)
(let ((idx Reg (put_in_reg_zext64 val_idx))
;; Bounds-check the index and branch to default.
;; This is an internal branch that is not a terminator insn.

2
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -2942,7 +2942,7 @@
;; Rules for `br_table` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower_branch (br_table idx @ (value_type ty) _ _) (jump_table_targets default_target jt_targets))
(rule (lower_branch (br_table idx @ (value_type ty) _) (jump_table_targets default_target jt_targets))
(emit_side_effect (jmp_table_seq ty idx default_target jt_targets)))
;; Rules for `select_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

9
cranelift/codegen/src/isa/x64/mod.rs
View File

@@ -408,11 +408,10 @@ mod test {
let mut pos = FuncCursor::new(&mut func);
pos.insert_block(bb0);
let mut jt_data = JumpTableData::new();
jt_data.push_entry(bb1);
jt_data.push_entry(bb2);
let jt = pos.func.create_jump_table(jt_data);
pos.ins().br_table(arg0, bb3, jt);
let jt = pos
.func
.create_jump_table(JumpTableData::new(bb3, vec![bb1, bb2]));
pos.ins().br_table(arg0, jt);
pos.insert_block(bb1);
let v1 = pos.ins().iconst(I32, 1);

26
cranelift/codegen/src/verifier/mod.rs
View File

@@ -579,10 +579,7 @@ impl<'a> Verifier<'a> {
self.verify_block(inst, block_then.block(&self.func.dfg.value_lists), errors)?;
self.verify_block(inst, block_else.block(&self.func.dfg.value_lists), errors)?;
}
BranchTable {
table, destination, ..
} => {
self.verify_block(inst, destination, errors)?;
BranchTable { table, .. } => {
self.verify_jump_table(inst, table, errors)?;
}
Call {
@@ -852,7 +849,7 @@ impl<'a> Verifier<'a> {
format!("invalid jump table reference {}", j),
))
} else {
for &block in self.func.stencil.dfg.jump_tables[j].as_slice() {
for &block in self.func.stencil.dfg.jump_tables[j].all_branches() {
self.verify_block(inst, block, errors)?;
}
Ok(())
@@ -1322,23 +1319,8 @@ impl<'a> Verifier<'a> {
let args_else = block_else.args_slice(&self.func.dfg.value_lists);
self.typecheck_variable_args_iterator(inst, iter, args_else, errors)?;
}
ir::InstructionData::BranchTable {
table,
destination: block,
..
} => {
let arg_count = self.func.dfg.num_block_params(*block);
if arg_count != 0 {
return errors.nonfatal((
inst,
self.context(inst),
format!(
"takes no arguments, but had target {} with {} arguments",
block, arg_count,
),
));
}
for block in self.func.stencil.dfg.jump_tables[*table].iter() {
ir::InstructionData::BranchTable { table, .. } => {
for block in self.func.stencil.dfg.jump_tables[*table].all_branches() {
let arg_count = self.func.dfg.num_block_params(*block);
if arg_count != 0 {
return errors.nonfatal((

7
cranelift/codegen/src/write.rs
View File

@@ -421,12 +421,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
write!(w, " {}, {}", arg, block_then.display(pool))?;
write!(w, ", {}", block_else.display(pool))
}
BranchTable {
arg,
destination,
table,
..
} => write!(w, " {}, {}, {}", arg, destination, jump_tables[table]),
BranchTable { arg, table, .. } => write!(w, " {}, {}", arg, jump_tables[table]),
Call {
func_ref, ref args, ..
} => write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool))),

12
cranelift/frontend/src/frontend.rs
View File

@@ -129,9 +129,7 @@ impl<'short, 'long> InstBuilderBase<'short> for FuncInstBuilder<'short, 'long> {
}
}
ir::InstructionData::BranchTable {
table, destination, ..
} => {
ir::InstructionData::BranchTable { table, .. } => {
// Unlike all other jumps/branches, jump tables are
// capable of having the same successor appear
// multiple times, so we must deduplicate.
@@ -144,9 +142,12 @@ impl<'short, 'long> InstBuilderBase<'short> for FuncInstBuilder<'short, 'long> {
.jump_tables
.get(*table)
.expect("you are referencing an undeclared jump table")
.iter()
.filter(|&dest_block| unique.insert(*dest_block))
.all_branches()
{
if !unique.insert(*dest_block) {
continue;
}
// Call `declare_block_predecessor` instead of `declare_successor` for
// avoiding the borrow checker.
self.builder
@@ -154,7 +155,6 @@ impl<'short, 'long> InstBuilderBase<'short> for FuncInstBuilder<'short, 'long> {
.ssa
.declare_block_predecessor(*dest_block, inst);
}
self.builder.declare_successor(*destination, inst);
}
inst => debug_assert!(!inst.opcode().is_branch()),

19
cranelift/frontend/src/ssa.rs
View File

@@ -591,27 +591,18 @@ impl SSABuilder {
}
None
}
InstructionData::BranchTable {
table: jt,
destination,
..
} => {
InstructionData::BranchTable { table: jt, .. } => {
// In the case of a jump table, the situation is tricky because br_table doesn't
// support arguments. We have to split the critical edge.
let middle_block = dfg.blocks.add();
func.stencil.layout.append_block(middle_block);
let table = &mut dfg.jump_tables[*jt];
for block in table.iter_mut() {
for block in dfg.jump_tables[*jt].all_branches_mut() {
if *block == dest_block {
*block = middle_block;
}
}
if *destination == dest_block {
*destination = middle_block;
}
let mut cur = FuncCursor::new(func).at_bottom(middle_block);
let middle_jump_inst = cur.ins().jump(dest_block, &[val]);
Some((middle_block, middle_jump_inst))
@@ -1001,9 +992,6 @@ mod tests {
let block0 = func.dfg.make_block();
let block1 = func.dfg.make_block();
let block2 = func.dfg.make_block();
let mut jump_table = JumpTableData::new();
jump_table.push_entry(block2);
jump_table.push_entry(block1);
{
let mut cur = FuncCursor::new(&mut func);
cur.insert_block(block0);
@@ -1022,9 +1010,10 @@ mod tests {
ssa.def_var(x_var, x1, block0);
ssa.use_var(&mut func, x_var, I32, block0).0;
let br_table = {
let jump_table = JumpTableData::new(block2, vec![block2, block1]);
let jt = func.create_jump_table(jump_table);
let mut cur = FuncCursor::new(&mut func).at_bottom(block0);
cur.ins().br_table(x1, block2, jt)
cur.ins().br_table(x1, jt)
};
// block1

4
cranelift/frontend/src/switch.rs
View File

@@ -220,7 +220,7 @@ impl Switch {
"Jump tables bigger than 2^32-1 are not yet supported"
);
let jt_data = JumpTableData::with_blocks(Vec::from(blocks));
let jt_data = JumpTableData::new(otherwise, Vec::from(blocks));
let jump_table = bx.create_jump_table(jt_data);
let discr = if first_index == 0 {
@@ -256,7 +256,7 @@ impl Switch {
_ => discr,
};
bx.ins().br_table(discr, otherwise, jump_table);
bx.ins().br_table(discr, jump_table);
}
/// Build the switch

4
cranelift/fuzzgen/src/function_generator.rs
View File

@@ -1614,12 +1614,12 @@ where
}
BlockTerminator::BrTable(default, targets) => {
// Create jump tables on demand
let jt = builder.create_jump_table(JumpTableData::with_blocks(targets));
let jt = builder.create_jump_table(JumpTableData::new(default, targets));
// br_table only supports I32
let val = builder.use_var(self.get_variable_of_type(I32)?);
builder.ins().br_table(val, default, jt);
builder.ins().br_table(val, jt);
}
BlockTerminator::Switch(_type, default, entries) => {
let mut switch = Switch::new();

7
cranelift/interpreter/src/step.rs
View File

@@ -373,10 +373,7 @@ where
}
}
Opcode::BrTable => {
if let InstructionData::BranchTable {
table, destination, ..
} = inst
{
if let InstructionData::BranchTable { table, .. } = inst {
let jt_data = &state.get_current_function().stencil.dfg.jump_tables[table];
// Convert to usize to remove negative indexes from the following operations
@@ -384,7 +381,7 @@ where
.ok()
.and_then(|i| jt_data.as_slice().get(i))
.copied()
.unwrap_or(destination);
.unwrap_or(jt_data.default_block());
ControlFlow::ContinueAt(jump_target, SmallVec::new())
} else {

19
cranelift/reader/src/parser.rs
View File

@@ -1767,15 +1767,15 @@ impl<'a> Parser<'a> {
//
// jump-table-lit ::= "[" block {"," block } "]"
// | "[]"
fn parse_jump_table(&mut self) -> ParseResult<JumpTableData> {
fn parse_jump_table(&mut self, def: Block) -> ParseResult<JumpTableData> {
self.match_token(Token::LBracket, "expected '[' before jump table contents")?;
let mut data = JumpTableData::new();
let mut data = Vec::new();
match self.token() {
Some(Token::Block(dest)) => {
self.consume();
data.push_entry(dest);
data.push(dest);
loop {
match self.token() {
@@ -1783,7 +1783,7 @@ impl<'a> Parser<'a> {
self.consume();
if let Some(Token::Block(dest)) = self.token() {
self.consume();
data.push_entry(dest);
data.push(dest);
} else {
return err!(self.loc, "expected jump_table entry");
}
@@ -1799,7 +1799,7 @@ impl<'a> Parser<'a> {
self.consume();
Ok(data)
Ok(JumpTableData::new(def, data))
}
// Parse a constant decl.
@@ -2571,14 +2571,9 @@ impl<'a> Parser<'a> {
self.match_token(Token::Comma, "expected ',' between operands")?;
let block_num = self.match_block("expected branch destination block")?;
self.match_token(Token::Comma, "expected ',' between operands")?;
let table_data = self.parse_jump_table()?;
let table_data = self.parse_jump_table(block_num)?;
let table = ctx.function.dfg.jump_tables.push(table_data);
InstructionData::BranchTable {
opcode,
arg,
destination: block_num,
table,
}
InstructionData::BranchTable { opcode, arg, table }
}
InstructionFormat::TernaryImm8 => {
let lhs = self.match_value("expected SSA value first operand")?;

14
cranelift/wasm/src/code_translator.rs
View File

@@ -515,7 +515,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
}
};
let val = state.pop1();
let mut data = JumpTableData::with_capacity(targets.len() as usize);
let mut data = Vec::with_capacity(targets.len() as usize);
if jump_args_count == 0 {
// No jump arguments
for depth in targets.targets() {
@@ -526,16 +526,16 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
frame.set_branched_to_exit();
frame.br_destination()
};
data.push_entry(block);
data.push(block);
}
let jt = builder.create_jump_table(data);
let block = {
let i = state.control_stack.len() - 1 - (default as usize);
let frame = &mut state.control_stack[i];
frame.set_branched_to_exit();
frame.br_destination()
};
builder.ins().br_table(val, block, jt);
let jt = builder.create_jump_table(JumpTableData::new(block, data));
builder.ins().br_table(val, jt);
} else {
// Here we have jump arguments, but Cranelift's br_table doesn't support them
// We then proceed to split the edges going out of the br_table
@@ -552,7 +552,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
*entry.insert(block)
}
};
data.push_entry(branch_block);
data.push(branch_block);
}
let default_branch_block = match dest_block_map.entry(default as usize) {
hash_map::Entry::Occupied(entry) => *entry.get(),
@@ -562,8 +562,8 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
*entry.insert(block)
}
};
let jt = builder.create_jump_table(data);
builder.ins().br_table(val, default_branch_block, jt);
let jt = builder.create_jump_table(JumpTableData::new(default_branch_block, data));
builder.ins().br_table(val, jt);
for (depth, dest_block) in dest_block_sequence {
builder.switch_to_block(dest_block);
builder.seal_block(dest_block);