T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement jump tables (#453)

Browse Source 
* Add 'jump_table_entry' and 'indirect_jump' instructions.

* Update CodeSink to keep track of code size. Pretty up clif-util's disassembly output.

* Only disassemble the machine portion of output. Pretty print the read-only data after it.

* Update switch frontend code to use new br_table instruction w/ default.
This commit is contained in:
Tyler McMullen
2018-10-03 11:04:21 -06:00
committed by Dan Gohman
parent de1d82b4ba
commit 79cea5e18b
39 changed files with 627 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
cranelift/filetests/isa/x86/binary64.clif
View File

@@ -1402,3 +1402,41 @@ ebb0:
trap user0 ; bin: user0 0f 0b
}
; Tests for i64 jump table instructions.
function %I64_JT(i64 [%rdi]) {
jt0 = jump_table ebb1, ebb2, ebb3
ebb0(v0: i64 [%rdi]):
; Note: The next two lines will need to change whenever instructions are
; added or removed from this test.
[-, %rax] v1 = jump_table_base.i64 jt0 ; bin: 48 8d 05 00000039
[-, %r10] v2 = jump_table_base.i64 jt0 ; bin: 4c 8d 15 00000032
[-, %rbx] v10 = iconst.i64 1
[-, %r13] v11 = iconst.i64 2
[-, %rax] v20 = jump_table_entry.i64 v10, v1, 4, jt0 ; bin: 48 63 04 98
[-, %rax] v21 = jump_table_entry.i64 v10, v2, 4, jt0 ; bin: 49 63 04 9a
[-, %rax] v22 = jump_table_entry.i64 v11, v1, 4, jt0 ; bin: 4a 63 04 a8
[-, %rax] v23 = jump_table_entry.i64 v11, v2, 4, jt0 ; bin: 4b 63 04 aa
[-, %r10] v30 = jump_table_entry.i64 v10, v1, 4, jt0 ; bin: 4c 63 14 98
[-, %r10] v31 = jump_table_entry.i64 v10, v2, 4, jt0 ; bin: 4d 63 14 9a
[-, %r10] v32 = jump_table_entry.i64 v11, v1, 4, jt0 ; bin: 4e 63 14 a8
[-, %r10] v33 = jump_table_entry.i64 v11, v2, 4, jt0 ; bin: 4f 63 14 aa
fallthrough ebb10
ebb10:
indirect_jump_table_br v10, jt0 ; bin: ff e3
ebb11:
indirect_jump_table_br v11, jt0 ; bin: 41 ff e5
ebb1:
fallthrough ebb2
ebb2:
fallthrough ebb3
ebb3:
trap user0
}

4
cranelift/filetests/isa/x86/legalize-br-table.clif
View File

@@ -9,7 +9,9 @@ function u0:0(i64) system_v {
ebb0(v0: i64):
v1 = stack_addr.i64 ss0
v2 = load.i8 v1
br_table v2, jt0
br_table v2, ebb2, jt0
ebb2:
jump ebb1
ebb1:

12
cranelift/filetests/parser/branch.clif
View File

@@ -85,21 +85,22 @@ function %jumptable(i32) {
jt2 = jump_table 0, 0, ebb10, ebb40, ebb20, ebb30
ebb10(v3: i32):
br_table v3, jt2
trap user1
br_table v3, ebb50, jt2
ebb20:
trap user2
ebb30:
trap user3
ebb40:
trap user4
ebb50:
trap user1
}
; sameln: function %jumptable(i32) fast {
; check: jt2 = jump_table 0, 0, ebb10, ebb40, ebb20, ebb30
; check: jt200 = jump_table 0
; check: ebb10(v3: i32):
; nextln: br_table v3, jt2
; nextln: trap user1
; nextln: br_table v3, ebb50, jt2
; nextln:
; nextln: ebb20:
; nextln: trap user2
@@ -109,4 +110,7 @@ ebb40:
; nextln:
; nextln: ebb40:
; nextln: trap user4
; nextln:
; nextln: ebb50:
; nextln: trap user1
; nextln: }

6
cranelift/filetests/verifier/type_check.clif
View File

@@ -71,10 +71,12 @@ function %jump_table_args() {
jt1 = jump_table ebb1
ebb0:
v0 = iconst.i32 0
br_table v0, jt1 ; error: takes no arguments, but had target ebb1 with 1 arguments
return
br_table v0, ebb2, jt1 ; error: takes no arguments, but had target ebb1 with 1 arguments
ebb1(v5: i32):
return
ebb2:
return
}
function %jump_args() {

4
cranelift/filetests/wasm/control.clif
View File

@@ -51,7 +51,9 @@ function %br_table(i32) {
jt0 = jump_table ebb3, ebb1, 0, ebb2
ebb0(v0: i32):
br_table v0, jt0
br_table v0, ebb4, jt0
ebb4:
trap oob
ebb1:

62
cranelift/src/compile.rs
View File

@@ -97,12 +97,20 @@ fn handle_module(
context.func = func;
// Compile and encode the result to machine code.
let total_size = context
.compile(isa)
.map_err(|err| pretty_error(&context.func, Some(isa), err))?;
let mut mem = Vec::new();
mem.resize(total_size as usize, 0);
let mut relocs = PrintRelocs { flag_print };
let mut traps = PrintTraps { flag_print };
context
.compile_and_emit(isa, &mut mem, &mut relocs, &mut traps)
.map_err(|err| pretty_error(&context.func, Some(isa), err))?;
let mut code_sink: binemit::MemoryCodeSink;
unsafe {
code_sink = binemit::MemoryCodeSink::new(mem.as_mut_ptr(), &mut relocs, &mut traps);
}
isa.emit_function_to_memory(&context.func, &mut code_sink);
if flag_print {
println!("{}", context.func.display(isa));
@@ -121,17 +129,41 @@ fn handle_module(
}
println!();
print_disassembly(isa, &mem)?;
print_disassembly(isa, &mem[0..code_sink.code_size as usize])?;
print_readonly_data(&mem[code_sink.code_size as usize..total_size as usize]);
}
}
Ok(())
}
fn print_readonly_data(mem: &[u8]) {
if mem.len() == 0 {
return;
}
println!("\nFollowed by {} bytes of read-only data:", mem.len());
for (i, byte) in mem.iter().enumerate() {
if i % 16 == 0 {
if i != 0 {
println!();
}
print!("{:4}: ", i);
}
if i % 4 == 0 {
print!(" ");
}
print!("{:02x} ", byte);
}
println!();
}
cfg_if! {
if #[cfg(feature = "disas")] {
use capstone::prelude::*;
use target_lexicon::Architecture;
use std::fmt::Write;
fn get_disassembler(isa: &TargetIsa) -> Result<Capstone, String> {
let cs = match isa.triple().architecture {
@@ -168,10 +200,28 @@ cfg_if! {
fn print_disassembly(isa: &TargetIsa, mem: &[u8]) -> Result<(), String> {
let mut cs = get_disassembler(isa)?;
println!("\nDisassembly:");
println!("\nDisassembly of {} bytes:", mem.len());
let insns = cs.disasm_all(&mem, 0x0).unwrap();
for i in insns.iter() {
println!("{}", i);
let mut line = String::new();
write!(&mut line, "{:4x}:\t", i.address()).unwrap();
let mut bytes_str = String::new();
for b in i.bytes() {
write!(&mut bytes_str, "{:02x} ", b).unwrap();
}
write!(&mut line, "{:21}\t", bytes_str).unwrap();
if let Some(s) = i.mnemonic() {
write!(&mut line, "{}\t", s).unwrap();
}
if let Some(s) = i.op_str() {
write!(&mut line, "{}", s).unwrap();
}
println!("{}", line);
}
Ok(())
}

5
lib/codegen/meta-python/base/formats.py
View File

@@ -50,7 +50,10 @@ Branch = InstructionFormat(VALUE, ebb, VARIABLE_ARGS)
BranchInt = InstructionFormat(intcc, VALUE, ebb, VARIABLE_ARGS)
BranchFloat = InstructionFormat(floatcc, VALUE, ebb, VARIABLE_ARGS)
BranchIcmp = InstructionFormat(intcc, VALUE, VALUE, ebb, VARIABLE_ARGS)
BranchTable = InstructionFormat(VALUE, entities.jump_table)
BranchTable = InstructionFormat(VALUE, ebb, entities.jump_table)
BranchTableEntry = InstructionFormat(VALUE, VALUE, uimm8, entities.jump_table)
BranchTableBase = InstructionFormat(entities.jump_table)
IndirectJump = InstructionFormat(VALUE, entities.jump_table)
Call = InstructionFormat(func_ref, VARIABLE_ARGS)
CallIndirect = InstructionFormat(sig_ref, VALUE, VARIABLE_ARGS)

41
lib/codegen/meta-python/base/instructions.py
View File

@@ -130,6 +130,8 @@ brff = Instruction(
ins=(Cond, f, EBB, args), is_branch=True)
x = Operand('x', iB, doc='index into jump table')
Entry = TypeVar('Entry', 'A scalar integer type', ints=True)
entry = Operand('entry', Entry, doc='entry of jump table')
JT = Operand('JT', entities.jump_table)
br_table = Instruction(
'br_table', r"""
@@ -137,12 +139,47 @@ br_table = Instruction(
Use ``x`` as an unsigned index into the jump table ``JT``. If a jump
table entry is found, branch to the corresponding EBB. If no entry was
found fall through to the next instruction.
found or the index is out-of-bounds, branch to the given default EBB.
Note that this branch instruction can't pass arguments to the targeted
blocks. Split critical edges as needed to work around this.
""",
ins=(x, JT), is_branch=True)
ins=(x, EBB, JT), is_branch=True, is_terminator=True)
Size = Operand('Size', uimm8, 'Size in bytes')
jump_table_entry = Instruction(
'jump_table_entry', r"""
Get an entry from a jump table.
Load a serialized ``entry`` from a jump table ``JT`` at a given index
``addr`` with a specific ``Size``. The retrieved entry may need to be
decoded after loading, depending upon the jump table type used.
Currently, the only type supported is entries which are relative to the
base of the jump table.
""",
ins=(x, addr, Size, JT), outs=entry)
jump_table_base = Instruction(
'jump_table_base', r"""
Get the absolute base address of a jump table.
This is used for jump tables wherein the entries are stored relative to
the base of jump table. In order to use these, generated code should first
load an entry using ``jump_table_entry``, then use this instruction to add
the relative base back to it.
""",
ins=JT, outs=addr)
indirect_jump_table_br = Instruction(
'indirect_jump_table_br', r"""
Branch indirectly via a jump table entry.
Unconditionally jump via a jump table entry that was previously loaded
with the ``jump_table_entry`` instruction.
""",
ins=(addr, JT),
is_branch=True, is_indirect_branch=True, is_terminator=True)
code = Operand('code', trapcode)
trap = Instruction(

4
lib/codegen/meta-python/base/legalize.py
View File

@@ -306,10 +306,10 @@ widen.legalize(
for int_ty in [types.i8, types.i16]:
widen.legalize(
br_table.bind(int_ty)(x, y),
br_table.bind(int_ty)(x, y, z),
Rtl(
b << uextend.i32(x),
br_table(b, y),
br_table(b, y, z),
)
)

9
lib/codegen/meta-python/base/settings.py
View File

@@ -166,4 +166,13 @@ probestack_size_log2 = NumSetting(
""",
default=12)
#
# Jump table options.
#
jump_tables_enabled = BoolSetting(
"""
Enable the use of jump tables in generated machine code.
""",
default=True)
group.close(globals())

3
lib/codegen/meta-python/cdsl/instructions.py
View File

@@ -89,6 +89,7 @@ class Instruction(object):
:param constraints: Tuple of instruction-specific TypeConstraints.
:param is_terminator: This is a terminator instruction.
:param is_branch: This is a branch instruction.
:param is_indirect_branch: This is an indirect branch instruction.
:param is_call: This is a call instruction.
:param is_return: This is a return instruction.
:param can_trap: This instruction can trap.
@@ -103,6 +104,8 @@ class Instruction(object):
ATTRIBS = {
'is_terminator': 'True for instructions that terminate the EBB.',
'is_branch': 'True for all branch or jump instructions.',
'is_indirect_branch':
'True for all indirect branch or jump instructions.',
'is_call': 'Is this a call instruction?',
'is_return': 'Is this a return instruction?',
'can_load': 'Can this instruction read from memory?',

2
lib/codegen/meta-python/cdsl/isa.py
View File

@@ -469,7 +469,7 @@ class Encoding(object):
"Format {} must match recipe: {}".format(
self.inst.format, recipe.format))
if self.inst.is_branch:
if self.inst.is_branch and not self.inst.is_indirect_branch:
assert recipe.branch_range, (
'Recipe {} for {} must have a branch_range'
.format(recipe, self.inst.name))

11
lib/codegen/meta-python/isa/x86/encodings.py
View File

@@ -507,6 +507,17 @@ enc_both(base.brz.b1, r.t8jccd_abcd, 0x84)
enc_both(base.brnz.b1, r.t8jccb_abcd, 0x75)
enc_both(base.brnz.b1, r.t8jccd_abcd, 0x85)
#
# Jump tables
#
X86_64.enc(base.jump_table_entry.i64.any.any, *r.jt_entry.rex(0x63, w=1))
X86_32.enc(base.jump_table_entry.i32.any.any, *r.jt_entry(0x8b))
X86_64.enc(base.jump_table_base.i64, *r.jt_base.rex(0x8d, w=1))
X86_32.enc(base.jump_table_base.i32, *r.jt_base(0x8d))
enc_x86_64(base.indirect_jump_table_br.i64, r.indirect_jmp, 0xff, rrr=4)
X86_32.enc(base.indirect_jump_table_br.i32, *r.indirect_jmp(0xff, rrr=4))
#
# Trap as ud2
#

45
lib/codegen/meta-python/isa/x86/recipes.py
View File

@@ -14,6 +14,7 @@ from base.formats import IntCompare, IntCompareImm, FloatCompare
from base.formats import IntCond, FloatCond
from base.formats import IntSelect, IntCondTrap, FloatCondTrap
from base.formats import Jump, Branch, BranchInt, BranchFloat
from base.formats import BranchTableEntry, BranchTableBase, IndirectJump
from base.formats import Ternary, FuncAddr, UnaryGlobalValue
from base.formats import RegMove, RegSpill, RegFill, CopySpecial
from base.formats import LoadComplex, StoreComplex
@@ -276,6 +277,18 @@ def floatccs(iform):
return Or(*(IsEqual(iform.cond, cc) for cc in supported_floatccs))
def valid_scale(iform):
# type: (InstructionFormat) -> PredNode
"""
Return an instruction predicate that checks if `iform.imm` is a valid
`scale` for a SIB byte.
"""
return Or(IsEqual(iform.imm, 1),
IsEqual(iform.imm, 2),
IsEqual(iform.imm, 4),
IsEqual(iform.imm, 8))
# A null unary instruction that takes a GPR register. Can be used for identity
# copies and no-op conversions.
null = EncRecipe('null', Unary, size=0, ins=GPR, outs=0, emit='')
@@ -1473,6 +1486,38 @@ brfd = TailRecipe(
disp4(destination, func, sink);
''')
indirect_jmp = TailRecipe(
'indirect_jmp', IndirectJump, size=1, ins=GPR, outs=(),
clobbers_flags=False,
emit='''
PUT_OP(bits, rex1(in_reg0), sink);
modrm_r_bits(in_reg0, bits, sink);
''')
jt_entry = TailRecipe(
'jt_entry', BranchTableEntry, size=2,
ins=(GPR_DEREF_SAFE, GPR_ZERO_DEREF_SAFE),
outs=(GPR),
clobbers_flags=False,
instp=valid_scale(BranchTableEntry),
emit='''
PUT_OP(bits, rex3(in_reg1, out_reg0, in_reg0), sink);
modrm_sib(out_reg0, sink);
sib(imm.trailing_zeros() as u8, in_reg0, in_reg1, sink);
''')
jt_base = TailRecipe(
'jt_base', BranchTableBase, size=5, ins=(), outs=(GPR),
clobbers_flags=False,
emit='''
// No reloc is needed here as the jump table is emitted directly after
// the function body.
PUT_OP(bits, rex2(0, out_reg0), sink);
modrm_riprel(out_reg0, sink);
jt_disp4(table, func, sink);
''')
#
# Test flags and set a register.
#

7
lib/codegen/src/binemit/memorysink.rs
View File

@@ -32,6 +32,8 @@ use std::ptr::write_unaligned;
pub struct MemoryCodeSink<'a> {
data: *mut u8,
offset: isize,
/// Size of the machine code portion of output
pub code_size: isize,
relocs: &'a mut RelocSink,
traps: &'a mut TrapSink,
}
@@ -49,6 +51,7 @@ impl<'a> MemoryCodeSink<'a> {
MemoryCodeSink {
data,
offset: 0,
code_size: 0,
relocs,
traps,
}
@@ -131,6 +134,10 @@ impl<'a> CodeSink for MemoryCodeSink<'a> {
let ofs = self.offset();
self.traps.trap(ofs, srcloc, code);
}
fn begin_rodata(&mut self) {
self.code_size = self.offset;
}
}
/// A `TrapSink` implementation that does nothing, which is convenient when

19
lib/codegen/src/binemit/mod.rs
View File

@@ -94,6 +94,9 @@ pub trait CodeSink {
/// Add trap information for the current offset.
fn trap(&mut self, TrapCode, SourceLoc);
/// Code output is complete, read-only data may follow.
fn begin_rodata(&mut self);
}
/// Report a bad encoding error.
@@ -123,4 +126,20 @@ where
emit_inst(func, inst, &mut divert, sink);
}
}
sink.begin_rodata();
// output jump tables
for (jt, jt_data) in func.jump_tables.iter() {
let jt_offset = func.jt_offsets[jt];
for idx in 0..jt_data.len() {
match jt_data.get_entry(idx) {
Some(ebb) => {
let rel_offset: i32 = func.offsets[ebb] as i32 - jt_offset as i32;
sink.put4(rel_offset as u32)
}
None => sink.put4(0),
}
}
}
}

7
lib/codegen/src/binemit/relaxation.rs
View File

@@ -95,6 +95,13 @@ pub fn relax_branches(func: &mut Function, isa: &TargetIsa) -> CodegenResult<Cod
}
}
for (jt, jt_data) in func.jump_tables.iter() {
func.jt_offsets[jt] = offset;
// TODO: this should be computed based on the min size needed to hold
// the furthest branch.
offset += jt_data.len() as u32 * 4;
}
Ok(offset)
}

7
lib/codegen/src/cfg_printer.rs
View File

@@ -48,8 +48,11 @@ impl<'a> CFGPrinter<'a> {
BranchInfo::SingleDest(dest, _) => {
write!(w, " | <{}>{} {}", inst, idata.opcode(), dest)?
}
BranchInfo::Table(table) => {
write!(w, " | <{}>{} {}", inst, idata.opcode(), table)?
BranchInfo::Table(table, dest) => {
write!(w, " | <{}>{} {}", inst, idata.opcode(), table)?;
if let Some(dest) = dest {
write!(w, " {}", dest)?
}
}
BranchInfo::NotABranch => {}
}

23
lib/codegen/src/dominator_tree.rs
View File

@@ -345,18 +345,13 @@ impl DominatorTree {
fn push_successors(&mut self, func: &Function, ebb: Ebb) {
for inst in func.layout.ebb_insts(ebb) {
match func.dfg.analyze_branch(inst) {
BranchInfo::SingleDest(succ, _) => {
if self.nodes[succ].rpo_number == 0 {
self.nodes[succ].rpo_number = SEEN;
self.stack.push(succ);
}
}
BranchInfo::Table(jt) => {
BranchInfo::SingleDest(succ, _) => self.push_if_unseen(succ),
BranchInfo::Table(jt, dest) => {
for (_, succ) in func.jump_tables[jt].entries() {
if self.nodes[succ].rpo_number == 0 {
self.nodes[succ].rpo_number = SEEN;
self.stack.push(succ);
self.push_if_unseen(succ);
}
if let Some(dest) = dest {
self.push_if_unseen(dest);
}
}
BranchInfo::NotABranch => {}
@@ -364,6 +359,14 @@ impl DominatorTree {
}
}
/// Push `ebb` onto `self.stack` if it has not already been seen.
fn push_if_unseen(&mut self, ebb: Ebb) {
if self.nodes[ebb].rpo_number == 0 {
self.nodes[ebb].rpo_number = SEEN;
self.stack.push(ebb);
}
}
/// Build a dominator tree from a control flow graph using Keith D. Cooper's
/// "Simple, Fast Dominator Algorithm."
fn compute_domtree(&mut self, func: &Function, cfg: &ControlFlowGraph) {

5
lib/codegen/src/flowgraph.rs
View File

@@ -125,7 +125,10 @@ impl ControlFlowGraph {
BranchInfo::SingleDest(dest, _) => {
self.add_edge(ebb, inst, dest);
}
BranchInfo::Table(jt) => {
BranchInfo::Table(jt, dest) => {
if let Some(dest) = dest {
self.add_edge(ebb, inst, dest);
}
for (_, dest) in func.jump_tables[jt].entries() {
self.add_edge(ebb, inst, dest);
}

7
lib/codegen/src/ir/function.rs
View File

@@ -11,7 +11,8 @@ use ir::{
Ebb, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, JumpTable,
JumpTableData, SigRef, StackSlot, StackSlotData, Table, TableData,
};
use ir::{EbbOffsets, InstEncodings, JumpTables, SourceLocs, StackSlots, ValueLocations};
use ir::{EbbOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations};
use ir::{JumpTableOffsets, JumpTables};
use isa::{EncInfo, Encoding, Legalize, TargetIsa};
use settings::CallConv;
use std::fmt;
@@ -64,6 +65,9 @@ pub struct Function {
/// in the textual IR format.
pub offsets: EbbOffsets,
/// Code offsets of Jump Table headers.
pub jt_offsets: JumpTableOffsets,
/// Source locations.
///
/// Track the original source location for each instruction. The source locations are not
@@ -87,6 +91,7 @@ impl Function {
encodings: SecondaryMap::new(),
locations: SecondaryMap::new(),
offsets: SecondaryMap::new(),
jt_offsets: SecondaryMap::new(),
srclocs: SecondaryMap::new(),
}
}

11
lib/codegen/src/ir/instructions.rs
View File

@@ -194,7 +194,10 @@ impl InstructionData {
ref args,
..
} => BranchInfo::SingleDest(destination, &args.as_slice(pool)[2..]),
InstructionData::BranchTable { table, .. } => BranchInfo::Table(table),
InstructionData::BranchTable {
table, destination, ..
} => BranchInfo::Table(table, Some(destination)),
InstructionData::IndirectJump { table, .. } => BranchInfo::Table(table, None),
_ => {
debug_assert!(!self.opcode().is_branch());
BranchInfo::NotABranch
@@ -213,7 +216,7 @@ impl InstructionData {
| InstructionData::BranchInt { destination, .. }
| InstructionData::BranchFloat { destination, .. }
| InstructionData::BranchIcmp { destination, .. } => Some(destination),
InstructionData::BranchTable { .. } => None,
InstructionData::BranchTable { .. } | InstructionData::IndirectJump { .. } => None,
_ => {
debug_assert!(!self.opcode().is_branch());
None
@@ -284,8 +287,8 @@ pub enum BranchInfo<'a> {
/// This is a branch or jump to a single destination EBB, possibly taking value arguments.
SingleDest(Ebb, &'a [Value]),
/// This is a jump table branch which can have many destination EBBs.
Table(JumpTable),
/// This is a jump table branch which can have many destination EBBs and maybe one default EBB.
Table(JumpTable, Option<Ebb>),
}
/// Information about call instructions.

5
lib/codegen/src/ir/jumptable.rs
View File

@@ -45,6 +45,11 @@ impl JumpTableData {
self.table.len()
}
/// Boolean that is false if the table has missing entries.
pub fn fully_dense(&self) -> bool {
self.holes == 0
}
/// Set a table entry.
///
/// The table will grow as needed to fit `idx`.

3
lib/codegen/src/ir/mod.rs
View File

@@ -62,5 +62,8 @@ pub type InstEncodings = SecondaryMap<Inst, isa::Encoding>;
/// Code offsets for EBBs.
pub type EbbOffsets = SecondaryMap<Ebb, binemit::CodeOffset>;
/// Code offsets for Jump Tables.
pub type JumpTableOffsets = SecondaryMap<JumpTable, binemit::CodeOffset>;
/// Source locations for instructions.
pub type SourceLocs = SecondaryMap<Inst, SourceLoc>;

8
lib/codegen/src/isa/x86/binemit.rs
View File

@@ -3,7 +3,7 @@
use super::registers::RU;
use binemit::{bad_encoding, CodeSink, Reloc};
use ir::condcodes::{CondCode, FloatCC, IntCC};
use ir::{Ebb, Function, Inst, InstructionData, Opcode, TrapCode};
use ir::{Ebb, Function, Inst, InstructionData, JumpTable, Opcode, TrapCode};
use isa::{RegUnit, StackBase, StackBaseMask, StackRef};
use regalloc::RegDiversions;
@@ -249,6 +249,7 @@ fn sib_noindex<CS: CodeSink + ?Sized>(base: RegUnit, sink: &mut CS) {
sink.put1(b);
}
/// Emit a SIB byte with a scale, base, and index.
fn sib<CS: CodeSink + ?Sized>(scale: u8, index: RegUnit, base: RegUnit, sink: &mut CS) {
// SIB SS_III_BBB.
debug_assert_eq!(scale & !0x03, 0, "Scale out of range");
@@ -332,3 +333,8 @@ fn disp4<CS: CodeSink + ?Sized>(destination: Ebb, func: &Function, sink: &mut CS
let delta = func.offsets[destination].wrapping_sub(sink.offset() + 4);
sink.put4(delta);
}
fn jt_disp4<CS: CodeSink + ?Sized>(jt: JumpTable, func: &Function, sink: &mut CS) {
let delta = func.jt_offsets[jt].wrapping_sub(sink.offset() + 4);
sink.put4(delta);
}

77
lib/codegen/src/legalizer/mod.rs
View File

@@ -16,6 +16,7 @@
use bitset::BitSet;
use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::types::I32;
use ir::{self, InstBuilder, MemFlags};
use isa::TargetIsa;
use timing;
@@ -170,6 +171,72 @@ fn expand_cond_trap(
/// Jump tables.
fn expand_br_table(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
if isa.flags().jump_tables_enabled() {
expand_br_table_jt(inst, func, cfg, isa);
} else {
expand_br_table_conds(inst, func, cfg, isa);
}
}
/// Expand br_table to jump table.
fn expand_br_table_jt(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
isa: &TargetIsa,
) {
use ir::condcodes::IntCC;
let (arg, default_ebb, table) = match func.dfg[inst] {
ir::InstructionData::BranchTable {
opcode: ir::Opcode::BrTable,
arg,
destination,
table,
} => (arg, destination, table),
_ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
};
let table_size = func.jump_tables[table].len();
let table_is_fully_dense = func.jump_tables[table].fully_dense();
let addr_ty = isa.pointer_type();
let entry_ty = I32;
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
// Bounds check
let oob = pos
.ins()
.icmp_imm(IntCC::UnsignedGreaterThanOrEqual, arg, table_size as i64);
pos.ins().brnz(oob, default_ebb, &[]);
let base_addr = pos.ins().jump_table_base(addr_ty, table);
let entry = pos
.ins()
.jump_table_entry(addr_ty, arg, base_addr, entry_ty.bytes() as u8, table);
// If the table isn't fully dense, zero-check the entry.
if !table_is_fully_dense {
pos.ins().brz(entry, default_ebb, &[]);
}
let addr = pos.ins().iadd(base_addr, entry);
pos.ins().indirect_jump_table_br(addr, table);
let ebb = pos.current_ebb().unwrap();
pos.remove_inst();
cfg.recompute_ebb(pos.func, ebb);
}
/// Expand br_table to series of conditionals.
fn expand_br_table_conds(
inst: ir::Inst,
func: &mut ir::Function,
cfg: &mut ControlFlowGraph,
@@ -177,17 +244,17 @@ fn expand_br_table(
) {
use ir::condcodes::IntCC;
let (arg, table) = match func.dfg[inst] {
let (arg, default_ebb, table) = match func.dfg[inst] {
ir::InstructionData::BranchTable {
opcode: ir::Opcode::BrTable,
arg,
destination,
table,
} => (arg, table),
} => (arg, destination, table),
_ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
};
// This is a poor man's jump table using just a sequence of conditional branches.
// TODO: Lower into a jump table load and indirect branch.
let table_size = func.jump_tables[table].len();
let mut pos = FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
@@ -199,7 +266,9 @@ fn expand_br_table(
}
}
// `br_table` falls through when nothing matches.
// `br_table` jumps to the default destination if nothing matches
pos.ins().jump(default_ebb, &[]);
let ebb = pos.current_ebb().unwrap();
pos.remove_inst();
cfg.recompute_ebb(pos.func, ebb);

10
lib/codegen/src/regalloc/coloring.rs
View File

@@ -900,11 +900,15 @@ impl<'a> Context<'a> {
let lr = &self.liveness[value];
lr.is_livein(ebb, ctx)
}
Table(jt) => {
Table(jt, ebb) => {
let lr = &self.liveness[value];
!lr.is_local() && self.cur.func.jump_tables[jt]
!lr.is_local()
&& (ebb.map_or(false, |ebb| lr.is_livein(ebb, ctx)) || self
.cur
.func
.jump_tables[jt]
.entries()
.any(|(_, ebb)| lr.is_livein(ebb, ctx))
.any(|(_, ebb)| lr.is_livein(ebb, ctx)))
}
}
}

3
lib/codegen/src/settings.rs
View File

@@ -380,7 +380,8 @@ mod tests {
allones_funcaddrs = false\n\
probestack_enabled = true\n\
probestack_func_adjusts_sp = false\n\
probestack_size_log2 = 12\n"
probestack_size_log2 = 12\n\
jump_tables_enabled = true\n"
);
assert_eq!(f.opt_level(), super::OptLevel::Default);
assert_eq!(f.enable_simd(), true);

7
lib/codegen/src/verifier/flags.rs
View File

@@ -135,7 +135,12 @@ impl<'a> FlagsVerifier<'a> {
merge(&mut live_val, val, inst, errors)?;
}
}
BranchInfo::Table(jt) => {
BranchInfo::Table(jt, dest) => {
if let Some(dest) = dest {
if let Some(val) = self.livein[dest].expand() {
merge(&mut live_val, val, inst, errors)?;
}
}
for (_, dest) in self.func.jump_tables[jt].entries() {
if let Some(val) = self.livein[dest].expand() {
merge(&mut live_val, val, inst, errors)?;

14
lib/codegen/src/verifier/locations.rs
View File

@@ -332,9 +332,21 @@ impl<'a> LocationVerifier<'a> {
}
}
}
Table(jt) => {
Table(jt, ebb) => {
for d in divert.all() {
let lr = &liveness[d.value];
if let Some(ebb) = ebb {
if lr.is_livein(ebb, liveness.context(&self.func.layout)) {
return fatal!(
errors,
inst,
"{} is diverted to {} and live in to {}",
d.value,
d.to.display(&self.reginfo),
ebb
);
}
}
for (_, ebb) in self.func.jump_tables[jt].entries() {
if lr.is_livein(ebb, liveness.context(&self.func.layout)) {
return fatal!(

19
lib/codegen/src/verifier/mod.rs
View File

@@ -657,7 +657,10 @@ impl<'a> Verifier<'a> {
self.verify_ebb(inst, destination, errors)?;
self.verify_value_list(inst, args, errors)?;
}
BranchTable { table, .. } => {
BranchTable { table, .. }
| BranchTableBase { table, .. }
| BranchTableEntry { table, .. }
| IndirectJump { table, .. } => {
self.verify_jump_table(inst, table, errors)?;
}
Call {
@@ -1213,7 +1216,19 @@ impl<'a> Verifier<'a> {
.map(|&v| self.func.dfg.value_type(v));
self.typecheck_variable_args_iterator(inst, iter, errors)?;
}
BranchInfo::Table(table) => {
BranchInfo::Table(table, ebb) => {
if let Some(ebb) = ebb {
let arg_count = self.func.dfg.num_ebb_params(ebb);
if arg_count != 0 {
return nonfatal!(
errors,
inst,
"takes no arguments, but had target {} with {} arguments",
ebb,
arg_count
);
}
}
for (_, ebb) in self.func.jump_tables[table].entries() {
let arg_count = self.func.dfg.num_ebb_params(ebb);
if arg_count != 0 {

12
lib/codegen/src/write.rs
View File

@@ -443,7 +443,17 @@ pub fn write_operands(
write!(w, " {} {}, {}, {}", cond, args[0], args[1], destination)?;
write_ebb_args(w, &args[2..])
}
BranchTable { arg, table, .. } => write!(w, " {}, {}", arg, table),
BranchTable {
arg,
destination,
table,
..
} => write!(w, " {}, {}, {}", arg, destination, table),
BranchTableBase { table, .. } => write!(w, " {}", table),
BranchTableEntry {
args, imm, table, ..
} => write!(w, " {}, {}, {}, {}", args[0], args[1], imm, table),
IndirectJump { arg, table, .. } => write!(w, " {}, {}", arg, table),
Call {
func_ref, ref args, ..
} => write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool))),

23
lib/filetests/src/test_binemit.rs
View File

@@ -4,7 +4,7 @@
//! functions and compares the results to the expected output.
use cranelift_codegen::binemit;
use cranelift_codegen::binemit::RegDiversions;
use cranelift_codegen::binemit::{CodeSink, RegDiversions};
use cranelift_codegen::dbg::DisplayList;
use cranelift_codegen::ir;
use cranelift_codegen::ir::entities::AnyEntity;
@@ -30,6 +30,7 @@ pub fn subtest(parsed: &TestCommand) -> SubtestResult<Box<SubTest>> {
/// Code sink that generates text.
struct TextSink {
code_size: binemit::CodeOffset,
offset: binemit::CodeOffset,
text: String,
}
@@ -38,6 +39,7 @@ impl TextSink {
/// Create a new empty TextSink.
pub fn new() -> Self {
Self {
code_size: 0,
offset: 0,
text: String::new(),
}
@@ -93,6 +95,10 @@ impl binemit::CodeSink for TextSink {
fn trap(&mut self, code: ir::TrapCode, _srcloc: ir::SourceLoc) {
write!(self.text, "{} ", code).unwrap();
}
fn begin_rodata(&mut self) {
self.code_size = self.offset
}
}
impl SubTest for TestBinEmit {
@@ -281,6 +287,21 @@ impl SubTest for TestBinEmit {
}
}
sink.begin_rodata();
for (jt, jt_data) in func.jump_tables.iter() {
let jt_offset = func.jt_offsets[jt];
for idx in 0..jt_data.len() {
match jt_data.get_entry(idx) {
Some(ebb) => {
let rel_offset: i32 = func.offsets[ebb] as i32 - jt_offset as i32;
sink.put4(rel_offset as u32)
}
None => sink.put4(0),
}
}
}
if sink.offset != code_size {
return Err(format!(
"Expected code size {}, got {}",

1
lib/filetests/src/test_compile.rs
View File

@@ -104,4 +104,5 @@ impl binemit::CodeSink for SizeSink {
}
fn reloc_jt(&mut self, _reloc: binemit::Reloc, _jt: ir::JumpTable) {}
fn trap(&mut self, _code: ir::TrapCode, _srcloc: ir::SourceLoc) {}
fn begin_rodata(&mut self) {}
}

106
lib/frontend/src/ssa.rs
View File

@@ -10,7 +10,7 @@ use cranelift_codegen::entity::{EntityRef, PrimaryMap, SecondaryMap};
use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
use cranelift_codegen::ir::instructions::BranchInfo;
use cranelift_codegen::ir::types::{F32, F64};
use cranelift_codegen::ir::{Ebb, Function, Inst, InstBuilder, Type, Value};
use cranelift_codegen::ir::{Ebb, Function, Inst, InstBuilder, InstructionData, Type, Value};
use cranelift_codegen::packed_option::PackedOption;
use cranelift_codegen::packed_option::ReservedValue;
use std::mem;
@@ -647,7 +647,7 @@ impl SSABuilder {
func.dfg.append_inst_arg(jump_inst, val);
None
}
BranchInfo::Table(jt) => {
BranchInfo::Table(jt, default_ebb) => {
// In the case of a jump table, the situation is tricky because br_table doesn't
// support arguments.
// We have to split the critical edge
@@ -656,6 +656,21 @@ impl SSABuilder {
let middle_block = self.declare_ebb_header_block(middle_ebb);
self.blocks[middle_block].add_predecessor(jump_inst_block, jump_inst);
self.mark_ebb_header_block_sealed(middle_block);
if let Some(default_ebb) = default_ebb {
if dest_ebb == default_ebb {
match func.dfg[jump_inst] {
InstructionData::BranchTable {
destination: ref mut dest,
..
} => {
*dest = middle_ebb;
}
_ => panic!("should not happen"),
}
}
}
for old_dest in func.jump_tables[jt].as_mut_slice() {
if *old_dest == PackedOption::from(dest_ebb) {
*old_dest = PackedOption::from(middle_ebb);
@@ -986,20 +1001,31 @@ mod tests {
#[test]
fn br_table_with_args() {
// This tests the on-demand splitting of critical edges for br_table with jump arguments
let mut func = Function::new();
let mut ssa = SSABuilder::new();
let ebb0 = func.dfg.make_ebb();
let ebb1 = func.dfg.make_ebb();
//
// Here is the pseudo-program we want to translate:
//
// function %f {
// jt = jump_table ebb2, 0, ebb1
// ebb0:
// x = 0;
// br_table x ebb1
// x = 1
// jump ebb1
// x = 1;
// br_table x, ebb2, jt
// ebb1:
// x = 2
// jump ebb2
// ebb2:
// x = x + 1
// return
//
// }
let mut func = Function::new();
let mut ssa = SSABuilder::new();
let mut jump_table = JumpTableData::new();
let ebb0 = func.dfg.make_ebb();
let ebb1 = func.dfg.make_ebb();
let ebb2 = func.dfg.make_ebb();
// ebb0:
// x = 1;
let block0 = ssa.declare_ebb_header_block(ebb0);
ssa.seal_ebb_header_block(ebb0, &mut func);
let x_var = Variable::new(0);
@@ -1007,42 +1033,60 @@ mod tests {
let mut cur = FuncCursor::new(&mut func);
cur.insert_ebb(ebb0);
cur.insert_ebb(ebb1);
cur.insert_ebb(ebb2);
cur.goto_bottom(ebb0);
cur.ins().iconst(I32, 1)
};
ssa.def_var(x_var, x1, block0);
let mut data = JumpTableData::new();
data.push_entry(ebb1);
data.set_entry(2, ebb1);
let jt = func.create_jump_table(data);
// jt = jump_table ebb2, 0, ebb1
jump_table.push_entry(ebb2);
jump_table.set_entry(2, ebb1);
let jt = func.create_jump_table(jump_table);
// ebb0:
// ...
// br_table x, ebb2, jt
ssa.use_var(&mut func, x_var, I32, block0).0;
let br_table = {
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb0);
cur.ins().br_table(x1, jt)
cur.ins().br_table(x1, ebb2, jt)
};
let block1 = ssa.declare_ebb_body_block(block0);
let x3 = {
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb0);
// ebb1:
// x = 2
// jump ebb2
let block1 = ssa.declare_ebb_header_block(ebb1);
ssa.seal_ebb_header_block(ebb1, &mut func);
let x2 = {
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb1);
cur.ins().iconst(I32, 2)
};
ssa.def_var(x_var, x3, block1);
ssa.def_var(x_var, x2, block1);
let jump_inst = {
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb0);
cur.ins().jump(ebb1, &[])
};
let block2 = ssa.declare_ebb_header_block(ebb1);
ssa.declare_ebb_predecessor(ebb1, block1, jump_inst);
ssa.declare_ebb_predecessor(ebb1, block0, br_table);
ssa.seal_ebb_header_block(ebb1, &mut func);
let x4 = ssa.use_var(&mut func, x_var, I32, block2).0;
{
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb1);
cur.ins().iadd_imm(x4, 1)
cur.ins().jump(ebb2, &[])
};
// ebb2:
// x = x + 1
// return
let block3 = ssa.declare_ebb_header_block(ebb2);
ssa.declare_ebb_predecessor(ebb2, block1, jump_inst);
ssa.declare_ebb_predecessor(ebb2, block0, br_table);
ssa.seal_ebb_header_block(ebb2, &mut func);
let block4 = ssa.declare_ebb_body_block(block3);
let x3 = ssa.use_var(&mut func, x_var, I32, block4).0;
let x4 = {
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb2);
cur.ins().iadd_imm(x3, 1)
};
ssa.def_var(x_var, x4, block4);
{
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb1);
let mut cur = FuncCursor::new(&mut func).at_bottom(ebb2);
cur.ins().return_(&[])
};
let flags = settings::Flags::new(settings::builder());
match verify_function(&func, &flags) {
Ok(()) => {}

12
lib/frontend/src/switch.rs
View File

@@ -156,8 +156,7 @@ impl Switch {
bx.switch_to_block(jt_ebb);
let discr = bx.ins().iadd_imm(val, (first_index as i64).wrapping_neg());
bx.ins().br_table(discr, jump_table);
bx.ins().jump(otherwise, &[]);
bx.ins().br_table(discr, otherwise, jump_table);
}
}
@@ -256,8 +255,7 @@ ebb0:
ebb3:
v3 = iadd_imm.i32 v1, 0
br_table v3, jt0
jump ebb0"
br_table v3, ebb0, jt0"
);
}
@@ -308,13 +306,11 @@ ebb8:
ebb11:
v7 = iadd_imm.i32 v1, 0
br_table v7, jt0
jump ebb0
br_table v7, ebb0, jt0
ebb10:
v8 = iadd_imm.i32 v1, -10
br_table v8, jt1
jump ebb0"
br_table v8, ebb0, jt1"
);
}

37
lib/reader/src/parser.rs
View File

@@ -2225,9 +2225,44 @@ impl<'a> Parser<'a> {
InstructionFormat::BranchTable => {
let arg = self.match_value("expected SSA value operand")?;
self.match_token(Token::Comma, "expected ',' between operands")?;
let ebb_num = self.match_ebb("expected branch destination EBB")?;
self.match_token(Token::Comma, "expected ',' between operands")?;
let table = self.match_jt()?;
ctx.check_jt(table, self.loc)?;
InstructionData::BranchTable { opcode, arg, table }
InstructionData::BranchTable {
opcode,
arg,
destination: ebb_num,
table,
}
}
InstructionFormat::BranchTableBase => {
let table = self.match_jt()?;
ctx.check_jt(table, self.loc)?;
InstructionData::BranchTableBase { opcode, table }
}
InstructionFormat::BranchTableEntry => {
let index = self.match_value("expected SSA value operand")?;
self.match_token(Token::Comma, "expected ',' between operands")?;
let base = self.match_value("expected SSA value operand")?;
self.match_token(Token::Comma, "expected ',' between operands")?;
let imm = self.match_uimm8("expected width")?;
self.match_token(Token::Comma, "expected ',' between operands")?;
let table = self.match_jt()?;
ctx.check_jt(table, self.loc)?;
InstructionData::BranchTableEntry {
opcode,
args: [index, base],
imm,
table,
}
}
InstructionFormat::IndirectJump => {
let arg = self.match_value("expected SSA value operand")?;
self.match_token(Token::Comma, "expected ',' between operands")?;
let table = self.match_jt()?;
ctx.check_jt(table, self.loc)?;
InstructionData::IndirectJump { opcode, arg, table }
}
InstructionFormat::InsertLane => {
let lhs = self.match_value("expected SSA value first operand")?;

47
lib/serde/src/serde_clif_json.rs
View File

@@ -117,6 +117,22 @@ pub enum SerInstData {
destination: String,
},
BranchTable {
opcode: String,
arg: String,
destination: String,
table: String,
},
BranchTableEntry {
opcode: String,
args: [String; 2],
imm: String,
table: String,
},
BranchTableBase {
opcode: String,
table: String,
},
IndirectJump {
opcode: String,
arg: String,
table: String,
@@ -445,7 +461,36 @@ pub fn get_inst_data(inst_index: Inst, func: &Function) -> SerInstData {
destination: destination.to_string(),
}
}
InstructionData::BranchTable { opcode, arg, table } => SerInstData::BranchTable {
InstructionData::BranchTable {
opcode,
arg,
destination,
table,
} => SerInstData::BranchTable {
opcode: opcode.to_string(),
arg: arg.to_string(),
destination: destination.to_string(),
table: table.to_string(),
},
InstructionData::BranchTableBase { opcode, table } => SerInstData::BranchTableBase {
opcode: opcode.to_string(),
table: table.to_string(),
},
InstructionData::BranchTableEntry {
opcode,
args,
imm,
table,
} => {
let hold_args = [args[0].to_string(), args[1].to_string()];
SerInstData::BranchTableEntry {
opcode: opcode.to_string(),
args: hold_args,
imm: imm.to_string(),
table: table.to_string(),
}
}
InstructionData::IndirectJump { opcode, arg, table } => SerInstData::IndirectJump {
opcode: opcode.to_string(),
arg: arg.to_string(),
table: table.to_string(),

7
lib/wasm/src/code_translator.rs
View File

@@ -283,14 +283,13 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
data.push_entry(ebb);
}
let jt = builder.create_jump_table(data);
builder.ins().br_table(val, jt);
let ebb = {
let i = state.control_stack.len() - 1 - (default as usize);
let frame = &mut state.control_stack[i];
frame.set_branched_to_exit();
frame.br_destination()
};
builder.ins().jump(ebb, &[]);
builder.ins().br_table(val, ebb, jt);
} else {
// Here we have jump arguments, but Cranelift's br_table doesn't support them
// We then proceed to split the edges going out of the br_table
@@ -309,14 +308,14 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
data.push_entry(branch_ebb);
}
let jt = builder.create_jump_table(data);
builder.ins().br_table(val, jt);
let default_ebb = {
let i = state.control_stack.len() - 1 - (default as usize);
let frame = &mut state.control_stack[i];
frame.set_branched_to_exit();
frame.br_destination()
};
builder.ins().jump(default_ebb, state.peekn(return_count));
dest_ebb_sequence.push((default as usize, default_ebb));
builder.ins().br_table(val, default_ebb, jt);
for (depth, dest_ebb) in dest_ebb_sequence {
builder.switch_to_block(dest_ebb);
builder.seal_block(dest_ebb);