T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement jump tables (#453)

Browse Source 
* Add 'jump_table_entry' and 'indirect_jump' instructions.

* Update CodeSink to keep track of code size. Pretty up clif-util's disassembly output.

* Only disassemble the machine portion of output. Pretty print the read-only data after it.

* Update switch frontend code to use new br_table instruction w/ default.
This commit is contained in:
Tyler McMullen
2018-10-03 11:04:21 -06:00
committed by Dan Gohman
parent de1d82b4ba
commit 79cea5e18b
39 changed files with 627 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
lib/codegen/src/binemit/memorysink.rs
View File

@@ -32,6 +32,8 @@ use std::ptr::write_unaligned;
pub struct MemoryCodeSink<'a> {
data: *mut u8,
offset: isize,
/// Size of the machine code portion of output
pub code_size: isize,
relocs: &'a mut RelocSink,
traps: &'a mut TrapSink,
}
@@ -49,6 +51,7 @@ impl<'a> MemoryCodeSink<'a> {
MemoryCodeSink {
data,
offset: 0,
code_size: 0,
relocs,
traps,
}
@@ -131,6 +134,10 @@ impl<'a> CodeSink for MemoryCodeSink<'a> {
let ofs = self.offset();
self.traps.trap(ofs, srcloc, code);
}
fn begin_rodata(&mut self) {
self.code_size = self.offset;
}
}
/// A `TrapSink` implementation that does nothing, which is convenient when

19
lib/codegen/src/binemit/mod.rs
View File

@@ -94,6 +94,9 @@ pub trait CodeSink {
/// Add trap information for the current offset.
fn trap(&mut self, TrapCode, SourceLoc);
/// Code output is complete, read-only data may follow.
fn begin_rodata(&mut self);
}
/// Report a bad encoding error.
@@ -123,4 +126,20 @@ where
emit_inst(func, inst, &mut divert, sink);
}
}
sink.begin_rodata();
// output jump tables
for (jt, jt_data) in func.jump_tables.iter() {
let jt_offset = func.jt_offsets[jt];
for idx in 0..jt_data.len() {
match jt_data.get_entry(idx) {
Some(ebb) => {
let rel_offset: i32 = func.offsets[ebb] as i32 - jt_offset as i32;
sink.put4(rel_offset as u32)
}
None => sink.put4(0),
}
}
}
}

7
lib/codegen/src/binemit/relaxation.rs
View File

@@ -95,6 +95,13 @@ pub fn relax_branches(func: &mut Function, isa: &TargetIsa) -> CodegenResult<Cod
}
}
for (jt, jt_data) in func.jump_tables.iter() {
func.jt_offsets[jt] = offset;
// TODO: this should be computed based on the min size needed to hold
// the furthest branch.
offset += jt_data.len() as u32 * 4;
}
Ok(offset)
}

7
lib/codegen/src/cfg_printer.rs
View File

@@ -48,8 +48,11 @@ impl<'a> CFGPrinter<'a> {
BranchInfo::SingleDest(dest, _) => {
write!(w, " | <{}>{} {}", inst, idata.opcode(), dest)?
}
BranchInfo::Table(table) => {
write!(w, " | <{}>{} {}", inst, idata.opcode(), table)?
BranchInfo::Table(table, dest) => {
write!(w, " | <{}>{} {}", inst, idata.opcode(), table)?;
if let Some(dest) = dest {
write!(w, " {}", dest)?
}
}
BranchInfo::NotABranch => {}
}

25
lib/codegen/src/dominator_tree.rs
View File

@@ -345,18 +345,13 @@ impl DominatorTree {
fn push_successors(&mut self, func: &Function, ebb: Ebb) {
for inst in func.layout.ebb_insts(ebb) {
match func.dfg.analyze_branch(inst) {
BranchInfo::SingleDest(succ, _) => {
if self.nodes[succ].rpo_number == 0 {
self.nodes[succ].rpo_number = SEEN;
self.stack.push(succ);
}
}
BranchInfo::Table(jt) => {
BranchInfo::SingleDest(succ, _) => self.push_if_unseen(succ),
BranchInfo::Table(jt, dest) => {
for (_, succ) in func.jump_tables[jt].entries() {
if self.nodes[succ].rpo_number == 0 {
self.nodes[succ].rpo_number = SEEN;
self.stack.push(succ);
}
self.push_if_unseen(succ);
}
if let Some(dest) = dest {
self.push_if_unseen(dest);
}
}
BranchInfo::NotABranch => {}
@@ -364,6 +359,14 @@ impl DominatorTree {
}
}
/// Push `ebb` onto `self.stack` if it has not already been seen.
fn push_if_unseen(&mut self, ebb: Ebb) {
if self.nodes[ebb].rpo_number == 0 {
self.nodes[ebb].rpo_number = SEEN;
self.stack.push(ebb);
}
}
/// Build a dominator tree from a control flow graph using Keith D. Cooper's
/// "Simple, Fast Dominator Algorithm."
fn compute_domtree(&mut self, func: &Function, cfg: &ControlFlowGraph) {

5
lib/codegen/src/flowgraph.rs
View File

@@ -125,7 +125,10 @@ impl ControlFlowGraph {
BranchInfo::SingleDest(dest, _) => {
self.add_edge(ebb, inst, dest);
}
BranchInfo::Table(jt) => {
BranchInfo::Table(jt, dest) => {
if let Some(dest) = dest {
self.add_edge(ebb, inst, dest);
}
for (_, dest) in func.jump_tables[jt].entries() {
self.add_edge(ebb, inst, dest);
}

7
lib/codegen/src/ir/function.rs
View File

@@ -11,7 +11,8 @@ use ir::{
Ebb, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, JumpTable,
JumpTableData, SigRef, StackSlot, StackSlotData, Table, TableData,
};
use ir::{EbbOffsets, InstEncodings, JumpTables, SourceLocs, StackSlots, ValueLocations};
use ir::{EbbOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations};
use ir::{JumpTableOffsets, JumpTables};
use isa::{EncInfo, Encoding, Legalize, TargetIsa};
use settings::CallConv;
use std::fmt;
@@ -64,6 +65,9 @@ pub struct Function {
/// in the textual IR format.
pub offsets: EbbOffsets,
/// Code offsets of Jump Table headers.
pub jt_offsets: JumpTableOffsets,
/// Source locations.
///
/// Track the original source location for each instruction. The source locations are not
@@ -87,6 +91,7 @@ impl Function {
encodings: SecondaryMap::new(),
locations: SecondaryMap::new(),
offsets: SecondaryMap::new(),
jt_offsets: SecondaryMap::new(),
srclocs: SecondaryMap::new(),
}
}

11
lib/codegen/src/ir/instructions.rs
View File

@@ -194,7 +194,10 @@ impl InstructionData {
ref args,
..
} => BranchInfo::SingleDest(destination, &args.as_slice(pool)[2..]),
InstructionData::BranchTable { table, .. } => BranchInfo::Table(table),
InstructionData::BranchTable {
table, destination, ..
} => BranchInfo::Table(table, Some(destination)),
InstructionData::IndirectJump { table, .. } => BranchInfo::Table(table, None),
_ => {
debug_assert!(!self.opcode().is_branch());
BranchInfo::NotABranch
@@ -213,7 +216,7 @@ impl InstructionData {
| InstructionData::BranchInt { destination, .. }
| InstructionData::BranchFloat { destination, .. }
| InstructionData::BranchIcmp { destination, .. } => Some(destination),
InstructionData::BranchTable { .. } => None,
InstructionData::BranchTable { .. } | InstructionData::IndirectJump { .. } => None,
_ => {
debug_assert!(!self.opcode().is_branch());
None
@@ -284,8 +287,8 @@ pub enum BranchInfo<'a> {
/// This is a branch or jump to a single destination EBB, possibly taking value arguments.
SingleDest(Ebb, &'a [Value]),
/// This is a jump table branch which can have many destination EBBs.
Table(JumpTable),
/// This is a jump table branch which can have many destination EBBs and maybe one default EBB.
Table(JumpTable, Option<Ebb>),
}
/// Information about call instructions.

5
lib/codegen/src/ir/jumptable.rs
View File

@@ -45,6 +45,11 @@ impl JumpTableData {
self.table.len()
}
/// Boolean that is false if the table has missing entries.
pub fn fully_dense(&self) -> bool {
self.holes == 0
}
/// Set a table entry.
///
/// The table will grow as needed to fit `idx`.

3
lib/codegen/src/ir/mod.rs
View File

@@ -62,5 +62,8 @@ pub type InstEncodings = SecondaryMap<Inst, isa::Encoding>;
/// Code offsets for EBBs.
pub type EbbOffsets = SecondaryMap<Ebb, binemit::CodeOffset>;
/// Code offsets for Jump Tables.
pub type JumpTableOffsets = SecondaryMap<JumpTable, binemit::CodeOffset>;
/// Source locations for instructions.
pub type SourceLocs = SecondaryMap<Inst, SourceLoc>;

8
lib/codegen/src/isa/x86/binemit.rs
View File

@@ -3,7 +3,7 @@
use super::registers::RU;
use binemit::{bad_encoding, CodeSink, Reloc};
use ir::condcodes::{CondCode, FloatCC, IntCC};
use ir::{Ebb, Function, Inst, InstructionData, Opcode, TrapCode};
use ir::{Ebb, Function, Inst, InstructionData, JumpTable, Opcode, TrapCode};
use isa::{RegUnit, StackBase, StackBaseMask, StackRef};
use regalloc::RegDiversions;
@@ -249,6 +249,7 @@ fn sib_noindex<CS: CodeSink + ?Sized>(base: RegUnit, sink: &mut CS) {
sink.put1(b);
}
/// Emit a SIB byte with a scale, base, and index.
fn sib<CS: CodeSink + ?Sized>(scale: u8, index: RegUnit, base: RegUnit, sink: &mut CS) {
// SIB SS_III_BBB.
debug_assert_eq!(scale & !0x03, 0, "Scale out of range");
@@ -332,3 +333,8 @@ fn disp4<CS: CodeSink + ?Sized>(destination: Ebb, func: &Function, sink: &mut CS
let delta = func.offsets[destination].wrapping_sub(sink.offset() + 4);
sink.put4(delta);
}
fn jt_disp4<CS: CodeSink + ?Sized>(jt: JumpTable, func: &Function, sink: &mut CS) {
let delta = func.jt_offsets[jt].wrapping_sub(sink.offset() + 4);
sink.put4(delta);
}

77
lib/codegen/src/legalizer/mod.rs
View File

@@ -16,6 +16,7 @@
use bitset::BitSet;
use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::types::I32;
use ir::{self, InstBuilder, MemFlags};
use isa::TargetIsa;
use timing;
@@ -170,6 +171,72 @@ fn expand_cond_trap(
/// Jump tables.
fn expand_br_table(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
if isa.flags().jump_tables_enabled() {
expand_br_table_jt(inst, func, cfg, isa);
} else {
expand_br_table_conds(inst, func, cfg, isa);
}
}
/// Expand br_table to jump table.
fn expand_br_table_jt(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
use ir::condcodes::IntCC;
let (arg, default_ebb, table) = match func.dfg[inst] {
ir::InstructionData::BranchTable {
opcode: ir::Opcode::BrTable,
arg,
destination,
table,
} => (arg, destination, table),
_ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
};
let table_size = func.jump_tables[table].len();
let table_is_fully_dense = func.jump_tables[table].fully_dense();
let addr_ty = isa.pointer_type();
let entry_ty = I32;
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Bounds check
let oob = pos
.ins()
.icmp_imm(IntCC::UnsignedGreaterThanOrEqual, arg, table_size as i64);
pos.ins().brnz(oob, default_ebb, &[]);
let base_addr = pos.ins().jump_table_base(addr_ty, table);
let entry = pos
.ins()
.jump_table_entry(addr_ty, arg, base_addr, entry_ty.bytes() as u8, table);
// If the table isn't fully dense, zero-check the entry.
if !table_is_fully_dense {
pos.ins().brz(entry, default_ebb, &[]);
}
let addr = pos.ins().iadd(base_addr, entry);
pos.ins().indirect_jump_table_br(addr, table);
let ebb = pos.current_ebb().unwrap();
pos.remove_inst();
cfg.recompute_ebb(pos.func, ebb);
}
/// Expand br_table to series of conditionals.
fn expand_br_table_conds(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
@@ -177,17 +244,17 @@ fn expand_br_table(
) {
use ir::condcodes::IntCC;
let (arg, table) = match func.dfg[inst] {
let (arg, default_ebb, table) = match func.dfg[inst] {
ir::InstructionData::BranchTable {
opcode: ir::Opcode::BrTable,
arg,
destination,
table,
} => (arg, table),
} => (arg, destination, table),
_ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
};
// This is a poor man's jump table using just a sequence of conditional branches.
// TODO: Lower into a jump table load and indirect branch.
let table_size = func.jump_tables[table].len();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
@@ -199,7 +266,9 @@ fn expand_br_table(
}
}
// `br_table` falls through when nothing matches.
// `br_table` jumps to the default destination if nothing matches
pos.ins().jump(default_ebb, &[]);
let ebb = pos.current_ebb().unwrap();
pos.remove_inst();
cfg.recompute_ebb(pos.func, ebb);

12
lib/codegen/src/regalloc/coloring.rs
View File

@@ -900,11 +900,15 @@ impl<'a> Context<'a> {
let lr = &self.liveness[value];
lr.is_livein(ebb, ctx)
}
Table(jt) => {
Table(jt, ebb) => {
let lr = &self.liveness[value];
!lr.is_local() && self.cur.func.jump_tables[jt]
.entries()
.any(|(_, ebb)| lr.is_livein(ebb, ctx))
!lr.is_local()
&& (ebb.map_or(false, |ebb| lr.is_livein(ebb, ctx)) || self
.cur
.func
.jump_tables[jt]
.entries()
.any(|(_, ebb)| lr.is_livein(ebb, ctx)))
}
}
}

3
lib/codegen/src/settings.rs
View File

@@ -380,7 +380,8 @@ mod tests {
allones_funcaddrs = false\n\
probestack_enabled = true\n\
probestack_func_adjusts_sp = false\n\
probestack_size_log2 = 12\n"
probestack_size_log2 = 12\n\
jump_tables_enabled = true\n"
);
assert_eq!(f.opt_level(), super::OptLevel::Default);
assert_eq!(f.enable_simd(), true);

7
lib/codegen/src/verifier/flags.rs
View File

@@ -135,7 +135,12 @@ impl<'a> FlagsVerifier<'a> {
merge(&mut live_val, val, inst, errors)?;
}
}
BranchInfo::Table(jt) => {
BranchInfo::Table(jt, dest) => {
if let Some(dest) = dest {
if let Some(val) = self.livein[dest].expand() {
merge(&mut live_val, val, inst, errors)?;
}
}
for (_, dest) in self.func.jump_tables[jt].entries() {
if let Some(val) = self.livein[dest].expand() {
merge(&mut live_val, val, inst, errors)?;

14
lib/codegen/src/verifier/locations.rs
View File

@@ -332,9 +332,21 @@ impl<'a> LocationVerifier<'a> {
}
}
}
Table(jt) => {
Table(jt, ebb) => {
for d in divert.all() {
let lr = &liveness[d.value];
if let Some(ebb) = ebb {
if lr.is_livein(ebb, liveness.context(&self.func.layout)) {
return fatal!(
errors,
inst,
"{} is diverted to {} and live in to {}",
d.value,
d.to.display(&self.reginfo),
ebb
);
}
}
for (_, ebb) in self.func.jump_tables[jt].entries() {
if lr.is_livein(ebb, liveness.context(&self.func.layout)) {
return fatal!(

19
lib/codegen/src/verifier/mod.rs
View File

@@ -657,7 +657,10 @@ impl<'a> Verifier<'a> {
self.verify_ebb(inst, destination, errors)?;
self.verify_value_list(inst, args, errors)?;
}
BranchTable { table, .. } => {
BranchTable { table, .. }
| BranchTableBase { table, .. }
| BranchTableEntry { table, .. }
| IndirectJump { table, .. } => {
self.verify_jump_table(inst, table, errors)?;
}
Call {
@@ -1213,7 +1216,19 @@ impl<'a> Verifier<'a> {
.map(|&v| self.func.dfg.value_type(v));
self.typecheck_variable_args_iterator(inst, iter, errors)?;
}
BranchInfo::Table(table) => {
BranchInfo::Table(table, ebb) => {
if let Some(ebb) = ebb {
let arg_count = self.func.dfg.num_ebb_params(ebb);
if arg_count != 0 {
return nonfatal!(
errors,
inst,
"takes no arguments, but had target {} with {} arguments",
ebb,
arg_count
);
}
}
for (_, ebb) in self.func.jump_tables[table].entries() {
let arg_count = self.func.dfg.num_ebb_params(ebb);
if arg_count != 0 {

12
lib/codegen/src/write.rs
View File

@@ -443,7 +443,17 @@ pub fn write_operands(
write!(w, " {} {}, {}, {}", cond, args[0], args[1], destination)?;
write_ebb_args(w, &args[2..])
}
BranchTable { arg, table, .. } => write!(w, " {}, {}", arg, table),
BranchTable {
arg,
destination,
table,
..
} => write!(w, " {}, {}, {}", arg, destination, table),
BranchTableBase { table, .. } => write!(w, " {}", table),
BranchTableEntry {
args, imm, table, ..
} => write!(w, " {}, {}, {}, {}", args[0], args[1], imm, table),
IndirectJump { arg, table, .. } => write!(w, " {}, {}", arg, table),
Call {
func_ref, ref args, ..
} => write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool))),