From 79cea5e18b43fe2c4a089fd27e35627f66392925 Mon Sep 17 00:00:00 2001 From: Tyler McMullen Date: Wed, 3 Oct 2018 11:04:21 -0600 Subject: [PATCH] Implement jump tables (#453) * Add 'jump_table_entry' and 'indirect_jump' instructions. * Update CodeSink to keep track of code size. Pretty up clif-util's disassembly output. * Only disassemble the machine portion of output. Pretty print the read-only data after it. * Update switch frontend code to use new br_table instruction w/ default. --- cranelift/filetests/isa/x86/binary64.clif | 38 +++++++ .../filetests/isa/x86/legalize-br-table.clif | 4 +- cranelift/filetests/parser/branch.clif | 12 +- cranelift/filetests/verifier/type_check.clif | 6 +- cranelift/filetests/wasm/control.clif | 4 +- cranelift/src/compile.rs | 62 +++++++++- lib/codegen/meta-python/base/formats.py | 5 +- lib/codegen/meta-python/base/instructions.py | 41 ++++++- lib/codegen/meta-python/base/legalize.py | 4 +- lib/codegen/meta-python/base/settings.py | 9 ++ lib/codegen/meta-python/cdsl/instructions.py | 3 + lib/codegen/meta-python/cdsl/isa.py | 2 +- lib/codegen/meta-python/isa/x86/encodings.py | 11 ++ lib/codegen/meta-python/isa/x86/recipes.py | 45 ++++++++ lib/codegen/src/binemit/memorysink.rs | 7 ++ lib/codegen/src/binemit/mod.rs | 19 ++++ lib/codegen/src/binemit/relaxation.rs | 7 ++ lib/codegen/src/cfg_printer.rs | 7 +- lib/codegen/src/dominator_tree.rs | 25 +++-- lib/codegen/src/flowgraph.rs | 5 +- lib/codegen/src/ir/function.rs | 7 +- lib/codegen/src/ir/instructions.rs | 11 +- lib/codegen/src/ir/jumptable.rs | 5 + lib/codegen/src/ir/mod.rs | 3 + lib/codegen/src/isa/x86/binemit.rs | 8 +- lib/codegen/src/legalizer/mod.rs | 77 ++++++++++++- lib/codegen/src/regalloc/coloring.rs | 12 +- lib/codegen/src/settings.rs | 3 +- lib/codegen/src/verifier/flags.rs | 7 +- lib/codegen/src/verifier/locations.rs | 14 ++- lib/codegen/src/verifier/mod.rs | 19 +++- lib/codegen/src/write.rs | 12 +- lib/filetests/src/test_binemit.rs | 23 +++- lib/filetests/src/test_compile.rs | 1 + lib/frontend/src/ssa.rs | 106 +++++++++++++----- lib/frontend/src/switch.rs | 12 +- lib/reader/src/parser.rs | 37 +++++- lib/serde/src/serde_clif_json.rs | 47 +++++++- lib/wasm/src/code_translator.rs | 7 +- 39 files changed, 627 insertions(+), 100 deletions(-) diff --git a/cranelift/filetests/isa/x86/binary64.clif b/cranelift/filetests/isa/x86/binary64.clif index b50e0d33db..9ea203b799 100644 --- a/cranelift/filetests/isa/x86/binary64.clif +++ b/cranelift/filetests/isa/x86/binary64.clif @@ -1402,3 +1402,41 @@ ebb0: trap user0 ; bin: user0 0f 0b } + +; Tests for i64 jump table instructions. +function %I64_JT(i64 [%rdi]) { + jt0 = jump_table ebb1, ebb2, ebb3 + +ebb0(v0: i64 [%rdi]): + ; Note: The next two lines will need to change whenever instructions are + ; added or removed from this test. + [-, %rax] v1 = jump_table_base.i64 jt0 ; bin: 48 8d 05 00000039 + [-, %r10] v2 = jump_table_base.i64 jt0 ; bin: 4c 8d 15 00000032 + + [-, %rbx] v10 = iconst.i64 1 + [-, %r13] v11 = iconst.i64 2 + + [-, %rax] v20 = jump_table_entry.i64 v10, v1, 4, jt0 ; bin: 48 63 04 98 + [-, %rax] v21 = jump_table_entry.i64 v10, v2, 4, jt0 ; bin: 49 63 04 9a + [-, %rax] v22 = jump_table_entry.i64 v11, v1, 4, jt0 ; bin: 4a 63 04 a8 + [-, %rax] v23 = jump_table_entry.i64 v11, v2, 4, jt0 ; bin: 4b 63 04 aa + + [-, %r10] v30 = jump_table_entry.i64 v10, v1, 4, jt0 ; bin: 4c 63 14 98 + [-, %r10] v31 = jump_table_entry.i64 v10, v2, 4, jt0 ; bin: 4d 63 14 9a + [-, %r10] v32 = jump_table_entry.i64 v11, v1, 4, jt0 ; bin: 4e 63 14 a8 + [-, %r10] v33 = jump_table_entry.i64 v11, v2, 4, jt0 ; bin: 4f 63 14 aa + + fallthrough ebb10 + +ebb10: + indirect_jump_table_br v10, jt0 ; bin: ff e3 +ebb11: + indirect_jump_table_br v11, jt0 ; bin: 41 ff e5 + +ebb1: + fallthrough ebb2 +ebb2: + fallthrough ebb3 +ebb3: + trap user0 +} diff --git a/cranelift/filetests/isa/x86/legalize-br-table.clif b/cranelift/filetests/isa/x86/legalize-br-table.clif index 3047acd9d1..b600de147c 100644 --- a/cranelift/filetests/isa/x86/legalize-br-table.clif +++ b/cranelift/filetests/isa/x86/legalize-br-table.clif @@ -9,7 +9,9 @@ function u0:0(i64) system_v { ebb0(v0: i64): v1 = stack_addr.i64 ss0 v2 = load.i8 v1 - br_table v2, jt0 + br_table v2, ebb2, jt0 + +ebb2: jump ebb1 ebb1: diff --git a/cranelift/filetests/parser/branch.clif b/cranelift/filetests/parser/branch.clif index 2969c8f384..ba68fc590d 100644 --- a/cranelift/filetests/parser/branch.clif +++ b/cranelift/filetests/parser/branch.clif @@ -85,21 +85,22 @@ function %jumptable(i32) { jt2 = jump_table 0, 0, ebb10, ebb40, ebb20, ebb30 ebb10(v3: i32): - br_table v3, jt2 - trap user1 + br_table v3, ebb50, jt2 + ebb20: trap user2 ebb30: trap user3 ebb40: trap user4 +ebb50: + trap user1 } ; sameln: function %jumptable(i32) fast { ; check: jt2 = jump_table 0, 0, ebb10, ebb40, ebb20, ebb30 ; check: jt200 = jump_table 0 ; check: ebb10(v3: i32): -; nextln: br_table v3, jt2 -; nextln: trap user1 +; nextln: br_table v3, ebb50, jt2 ; nextln: ; nextln: ebb20: ; nextln: trap user2 @@ -109,4 +110,7 @@ ebb40: ; nextln: ; nextln: ebb40: ; nextln: trap user4 +; nextln: +; nextln: ebb50: +; nextln: trap user1 ; nextln: } diff --git a/cranelift/filetests/verifier/type_check.clif b/cranelift/filetests/verifier/type_check.clif index 9cd9c9f8b0..9fa8fa9cca 100644 --- a/cranelift/filetests/verifier/type_check.clif +++ b/cranelift/filetests/verifier/type_check.clif @@ -71,10 +71,12 @@ function %jump_table_args() { jt1 = jump_table ebb1 ebb0: v0 = iconst.i32 0 - br_table v0, jt1 ; error: takes no arguments, but had target ebb1 with 1 arguments - return + br_table v0, ebb2, jt1 ; error: takes no arguments, but had target ebb1 with 1 arguments + ebb1(v5: i32): return + ebb2: + return } function %jump_args() { diff --git a/cranelift/filetests/wasm/control.clif b/cranelift/filetests/wasm/control.clif index a34bbeb935..2e9f9a17ad 100644 --- a/cranelift/filetests/wasm/control.clif +++ b/cranelift/filetests/wasm/control.clif @@ -51,7 +51,9 @@ function %br_table(i32) { jt0 = jump_table ebb3, ebb1, 0, ebb2 ebb0(v0: i32): - br_table v0, jt0 + br_table v0, ebb4, jt0 + +ebb4: trap oob ebb1: diff --git a/cranelift/src/compile.rs b/cranelift/src/compile.rs index c49b5d3aef..708763e7b3 100644 --- a/cranelift/src/compile.rs +++ b/cranelift/src/compile.rs @@ -97,12 +97,20 @@ fn handle_module( context.func = func; // Compile and encode the result to machine code. + let total_size = context + .compile(isa) + .map_err(|err| pretty_error(&context.func, Some(isa), err))?; + let mut mem = Vec::new(); + mem.resize(total_size as usize, 0); + let mut relocs = PrintRelocs { flag_print }; let mut traps = PrintTraps { flag_print }; - context - .compile_and_emit(isa, &mut mem, &mut relocs, &mut traps) - .map_err(|err| pretty_error(&context.func, Some(isa), err))?; + let mut code_sink: binemit::MemoryCodeSink; + unsafe { + code_sink = binemit::MemoryCodeSink::new(mem.as_mut_ptr(), &mut relocs, &mut traps); + } + isa.emit_function_to_memory(&context.func, &mut code_sink); if flag_print { println!("{}", context.func.display(isa)); @@ -121,17 +129,41 @@ fn handle_module( } println!(); - print_disassembly(isa, &mem)?; + print_disassembly(isa, &mem[0..code_sink.code_size as usize])?; + print_readonly_data(&mem[code_sink.code_size as usize..total_size as usize]); } } Ok(()) } +fn print_readonly_data(mem: &[u8]) { + if mem.len() == 0 { + return; + } + + println!("\nFollowed by {} bytes of read-only data:", mem.len()); + + for (i, byte) in mem.iter().enumerate() { + if i % 16 == 0 { + if i != 0 { + println!(); + } + print!("{:4}: ", i); + } + if i % 4 == 0 { + print!(" "); + } + print!("{:02x} ", byte); + } + println!(); +} + cfg_if! { if #[cfg(feature = "disas")] { use capstone::prelude::*; use target_lexicon::Architecture; + use std::fmt::Write; fn get_disassembler(isa: &TargetIsa) -> Result { let cs = match isa.triple().architecture { @@ -168,10 +200,28 @@ cfg_if! { fn print_disassembly(isa: &TargetIsa, mem: &[u8]) -> Result<(), String> { let mut cs = get_disassembler(isa)?; - println!("\nDisassembly:"); + println!("\nDisassembly of {} bytes:", mem.len()); let insns = cs.disasm_all(&mem, 0x0).unwrap(); for i in insns.iter() { - println!("{}", i); + let mut line = String::new(); + + write!(&mut line, "{:4x}:\t", i.address()).unwrap(); + + let mut bytes_str = String::new(); + for b in i.bytes() { + write!(&mut bytes_str, "{:02x} ", b).unwrap(); + } + write!(&mut line, "{:21}\t", bytes_str).unwrap(); + + if let Some(s) = i.mnemonic() { + write!(&mut line, "{}\t", s).unwrap(); + } + + if let Some(s) = i.op_str() { + write!(&mut line, "{}", s).unwrap(); + } + + println!("{}", line); } Ok(()) } diff --git a/lib/codegen/meta-python/base/formats.py b/lib/codegen/meta-python/base/formats.py index c008306409..b63863eed0 100644 --- a/lib/codegen/meta-python/base/formats.py +++ b/lib/codegen/meta-python/base/formats.py @@ -50,7 +50,10 @@ Branch = InstructionFormat(VALUE, ebb, VARIABLE_ARGS) BranchInt = InstructionFormat(intcc, VALUE, ebb, VARIABLE_ARGS) BranchFloat = InstructionFormat(floatcc, VALUE, ebb, VARIABLE_ARGS) BranchIcmp = InstructionFormat(intcc, VALUE, VALUE, ebb, VARIABLE_ARGS) -BranchTable = InstructionFormat(VALUE, entities.jump_table) +BranchTable = InstructionFormat(VALUE, ebb, entities.jump_table) +BranchTableEntry = InstructionFormat(VALUE, VALUE, uimm8, entities.jump_table) +BranchTableBase = InstructionFormat(entities.jump_table) +IndirectJump = InstructionFormat(VALUE, entities.jump_table) Call = InstructionFormat(func_ref, VARIABLE_ARGS) CallIndirect = InstructionFormat(sig_ref, VALUE, VARIABLE_ARGS) diff --git a/lib/codegen/meta-python/base/instructions.py b/lib/codegen/meta-python/base/instructions.py index 83459a17bc..c22b6b9b77 100644 --- a/lib/codegen/meta-python/base/instructions.py +++ b/lib/codegen/meta-python/base/instructions.py @@ -130,6 +130,8 @@ brff = Instruction( ins=(Cond, f, EBB, args), is_branch=True) x = Operand('x', iB, doc='index into jump table') +Entry = TypeVar('Entry', 'A scalar integer type', ints=True) +entry = Operand('entry', Entry, doc='entry of jump table') JT = Operand('JT', entities.jump_table) br_table = Instruction( 'br_table', r""" @@ -137,12 +139,47 @@ br_table = Instruction( Use ``x`` as an unsigned index into the jump table ``JT``. If a jump table entry is found, branch to the corresponding EBB. If no entry was - found fall through to the next instruction. + found or the index is out-of-bounds, branch to the given default EBB. Note that this branch instruction can't pass arguments to the targeted blocks. Split critical edges as needed to work around this. """, - ins=(x, JT), is_branch=True) + ins=(x, EBB, JT), is_branch=True, is_terminator=True) + +Size = Operand('Size', uimm8, 'Size in bytes') +jump_table_entry = Instruction( + 'jump_table_entry', r""" + Get an entry from a jump table. + + Load a serialized ``entry`` from a jump table ``JT`` at a given index + ``addr`` with a specific ``Size``. The retrieved entry may need to be + decoded after loading, depending upon the jump table type used. + + Currently, the only type supported is entries which are relative to the + base of the jump table. + """, + ins=(x, addr, Size, JT), outs=entry) + +jump_table_base = Instruction( + 'jump_table_base', r""" + Get the absolute base address of a jump table. + + This is used for jump tables wherein the entries are stored relative to + the base of jump table. In order to use these, generated code should first + load an entry using ``jump_table_entry``, then use this instruction to add + the relative base back to it. + """, + ins=JT, outs=addr) + +indirect_jump_table_br = Instruction( + 'indirect_jump_table_br', r""" + Branch indirectly via a jump table entry. + + Unconditionally jump via a jump table entry that was previously loaded + with the ``jump_table_entry`` instruction. + """, + ins=(addr, JT), + is_branch=True, is_indirect_branch=True, is_terminator=True) code = Operand('code', trapcode) trap = Instruction( diff --git a/lib/codegen/meta-python/base/legalize.py b/lib/codegen/meta-python/base/legalize.py index ec82ed9d3f..a1c6883356 100644 --- a/lib/codegen/meta-python/base/legalize.py +++ b/lib/codegen/meta-python/base/legalize.py @@ -306,10 +306,10 @@ widen.legalize( for int_ty in [types.i8, types.i16]: widen.legalize( - br_table.bind(int_ty)(x, y), + br_table.bind(int_ty)(x, y, z), Rtl( b << uextend.i32(x), - br_table(b, y), + br_table(b, y, z), ) ) diff --git a/lib/codegen/meta-python/base/settings.py b/lib/codegen/meta-python/base/settings.py index 7e116b1668..e43e455693 100644 --- a/lib/codegen/meta-python/base/settings.py +++ b/lib/codegen/meta-python/base/settings.py @@ -166,4 +166,13 @@ probestack_size_log2 = NumSetting( """, default=12) +# +# Jump table options. +# +jump_tables_enabled = BoolSetting( + """ + Enable the use of jump tables in generated machine code. + """, + default=True) + group.close(globals()) diff --git a/lib/codegen/meta-python/cdsl/instructions.py b/lib/codegen/meta-python/cdsl/instructions.py index f972f82bc7..e4c7186333 100644 --- a/lib/codegen/meta-python/cdsl/instructions.py +++ b/lib/codegen/meta-python/cdsl/instructions.py @@ -89,6 +89,7 @@ class Instruction(object): :param constraints: Tuple of instruction-specific TypeConstraints. :param is_terminator: This is a terminator instruction. :param is_branch: This is a branch instruction. + :param is_indirect_branch: This is an indirect branch instruction. :param is_call: This is a call instruction. :param is_return: This is a return instruction. :param can_trap: This instruction can trap. @@ -103,6 +104,8 @@ class Instruction(object): ATTRIBS = { 'is_terminator': 'True for instructions that terminate the EBB.', 'is_branch': 'True for all branch or jump instructions.', + 'is_indirect_branch': + 'True for all indirect branch or jump instructions.', 'is_call': 'Is this a call instruction?', 'is_return': 'Is this a return instruction?', 'can_load': 'Can this instruction read from memory?', diff --git a/lib/codegen/meta-python/cdsl/isa.py b/lib/codegen/meta-python/cdsl/isa.py index bcff050b1b..5c57c52fbd 100644 --- a/lib/codegen/meta-python/cdsl/isa.py +++ b/lib/codegen/meta-python/cdsl/isa.py @@ -469,7 +469,7 @@ class Encoding(object): "Format {} must match recipe: {}".format( self.inst.format, recipe.format)) - if self.inst.is_branch: + if self.inst.is_branch and not self.inst.is_indirect_branch: assert recipe.branch_range, ( 'Recipe {} for {} must have a branch_range' .format(recipe, self.inst.name)) diff --git a/lib/codegen/meta-python/isa/x86/encodings.py b/lib/codegen/meta-python/isa/x86/encodings.py index a5101261c6..5db4e63b9a 100644 --- a/lib/codegen/meta-python/isa/x86/encodings.py +++ b/lib/codegen/meta-python/isa/x86/encodings.py @@ -507,6 +507,17 @@ enc_both(base.brz.b1, r.t8jccd_abcd, 0x84) enc_both(base.brnz.b1, r.t8jccb_abcd, 0x75) enc_both(base.brnz.b1, r.t8jccd_abcd, 0x85) +# +# Jump tables +# +X86_64.enc(base.jump_table_entry.i64.any.any, *r.jt_entry.rex(0x63, w=1)) +X86_32.enc(base.jump_table_entry.i32.any.any, *r.jt_entry(0x8b)) + +X86_64.enc(base.jump_table_base.i64, *r.jt_base.rex(0x8d, w=1)) +X86_32.enc(base.jump_table_base.i32, *r.jt_base(0x8d)) + +enc_x86_64(base.indirect_jump_table_br.i64, r.indirect_jmp, 0xff, rrr=4) +X86_32.enc(base.indirect_jump_table_br.i32, *r.indirect_jmp(0xff, rrr=4)) # # Trap as ud2 # diff --git a/lib/codegen/meta-python/isa/x86/recipes.py b/lib/codegen/meta-python/isa/x86/recipes.py index c2baa515e7..f033cc2090 100644 --- a/lib/codegen/meta-python/isa/x86/recipes.py +++ b/lib/codegen/meta-python/isa/x86/recipes.py @@ -14,6 +14,7 @@ from base.formats import IntCompare, IntCompareImm, FloatCompare from base.formats import IntCond, FloatCond from base.formats import IntSelect, IntCondTrap, FloatCondTrap from base.formats import Jump, Branch, BranchInt, BranchFloat +from base.formats import BranchTableEntry, BranchTableBase, IndirectJump from base.formats import Ternary, FuncAddr, UnaryGlobalValue from base.formats import RegMove, RegSpill, RegFill, CopySpecial from base.formats import LoadComplex, StoreComplex @@ -276,6 +277,18 @@ def floatccs(iform): return Or(*(IsEqual(iform.cond, cc) for cc in supported_floatccs)) +def valid_scale(iform): + # type: (InstructionFormat) -> PredNode + """ + Return an instruction predicate that checks if `iform.imm` is a valid + `scale` for a SIB byte. + """ + return Or(IsEqual(iform.imm, 1), + IsEqual(iform.imm, 2), + IsEqual(iform.imm, 4), + IsEqual(iform.imm, 8)) + + # A null unary instruction that takes a GPR register. Can be used for identity # copies and no-op conversions. null = EncRecipe('null', Unary, size=0, ins=GPR, outs=0, emit='') @@ -1473,6 +1486,38 @@ brfd = TailRecipe( disp4(destination, func, sink); ''') +indirect_jmp = TailRecipe( + 'indirect_jmp', IndirectJump, size=1, ins=GPR, outs=(), + clobbers_flags=False, + emit=''' + PUT_OP(bits, rex1(in_reg0), sink); + modrm_r_bits(in_reg0, bits, sink); + ''') + +jt_entry = TailRecipe( + 'jt_entry', BranchTableEntry, size=2, + ins=(GPR_DEREF_SAFE, GPR_ZERO_DEREF_SAFE), + outs=(GPR), + clobbers_flags=False, + instp=valid_scale(BranchTableEntry), + emit=''' + PUT_OP(bits, rex3(in_reg1, out_reg0, in_reg0), sink); + modrm_sib(out_reg0, sink); + sib(imm.trailing_zeros() as u8, in_reg0, in_reg1, sink); + ''') + +jt_base = TailRecipe( + 'jt_base', BranchTableBase, size=5, ins=(), outs=(GPR), + clobbers_flags=False, + emit=''' + // No reloc is needed here as the jump table is emitted directly after + // the function body. + PUT_OP(bits, rex2(0, out_reg0), sink); + modrm_riprel(out_reg0, sink); + + jt_disp4(table, func, sink); + ''') + # # Test flags and set a register. # diff --git a/lib/codegen/src/binemit/memorysink.rs b/lib/codegen/src/binemit/memorysink.rs index d3b98b63cb..c7ec60de89 100644 --- a/lib/codegen/src/binemit/memorysink.rs +++ b/lib/codegen/src/binemit/memorysink.rs @@ -32,6 +32,8 @@ use std::ptr::write_unaligned; pub struct MemoryCodeSink<'a> { data: *mut u8, offset: isize, + /// Size of the machine code portion of output + pub code_size: isize, relocs: &'a mut RelocSink, traps: &'a mut TrapSink, } @@ -49,6 +51,7 @@ impl<'a> MemoryCodeSink<'a> { MemoryCodeSink { data, offset: 0, + code_size: 0, relocs, traps, } @@ -131,6 +134,10 @@ impl<'a> CodeSink for MemoryCodeSink<'a> { let ofs = self.offset(); self.traps.trap(ofs, srcloc, code); } + + fn begin_rodata(&mut self) { + self.code_size = self.offset; + } } /// A `TrapSink` implementation that does nothing, which is convenient when diff --git a/lib/codegen/src/binemit/mod.rs b/lib/codegen/src/binemit/mod.rs index fd8cff031a..42b16d442b 100644 --- a/lib/codegen/src/binemit/mod.rs +++ b/lib/codegen/src/binemit/mod.rs @@ -94,6 +94,9 @@ pub trait CodeSink { /// Add trap information for the current offset. fn trap(&mut self, TrapCode, SourceLoc); + + /// Code output is complete, read-only data may follow. + fn begin_rodata(&mut self); } /// Report a bad encoding error. @@ -123,4 +126,20 @@ where emit_inst(func, inst, &mut divert, sink); } } + + sink.begin_rodata(); + + // output jump tables + for (jt, jt_data) in func.jump_tables.iter() { + let jt_offset = func.jt_offsets[jt]; + for idx in 0..jt_data.len() { + match jt_data.get_entry(idx) { + Some(ebb) => { + let rel_offset: i32 = func.offsets[ebb] as i32 - jt_offset as i32; + sink.put4(rel_offset as u32) + } + None => sink.put4(0), + } + } + } } diff --git a/lib/codegen/src/binemit/relaxation.rs b/lib/codegen/src/binemit/relaxation.rs index ad13dc21c1..3b45a1c7b3 100644 --- a/lib/codegen/src/binemit/relaxation.rs +++ b/lib/codegen/src/binemit/relaxation.rs @@ -95,6 +95,13 @@ pub fn relax_branches(func: &mut Function, isa: &TargetIsa) -> CodegenResult CFGPrinter<'a> { BranchInfo::SingleDest(dest, _) => { write!(w, " | <{}>{} {}", inst, idata.opcode(), dest)? } - BranchInfo::Table(table) => { - write!(w, " | <{}>{} {}", inst, idata.opcode(), table)? + BranchInfo::Table(table, dest) => { + write!(w, " | <{}>{} {}", inst, idata.opcode(), table)?; + if let Some(dest) = dest { + write!(w, " {}", dest)? + } } BranchInfo::NotABranch => {} } diff --git a/lib/codegen/src/dominator_tree.rs b/lib/codegen/src/dominator_tree.rs index f8ca5d546c..9883e34418 100644 --- a/lib/codegen/src/dominator_tree.rs +++ b/lib/codegen/src/dominator_tree.rs @@ -345,18 +345,13 @@ impl DominatorTree { fn push_successors(&mut self, func: &Function, ebb: Ebb) { for inst in func.layout.ebb_insts(ebb) { match func.dfg.analyze_branch(inst) { - BranchInfo::SingleDest(succ, _) => { - if self.nodes[succ].rpo_number == 0 { - self.nodes[succ].rpo_number = SEEN; - self.stack.push(succ); - } - } - BranchInfo::Table(jt) => { + BranchInfo::SingleDest(succ, _) => self.push_if_unseen(succ), + BranchInfo::Table(jt, dest) => { for (_, succ) in func.jump_tables[jt].entries() { - if self.nodes[succ].rpo_number == 0 { - self.nodes[succ].rpo_number = SEEN; - self.stack.push(succ); - } + self.push_if_unseen(succ); + } + if let Some(dest) = dest { + self.push_if_unseen(dest); } } BranchInfo::NotABranch => {} @@ -364,6 +359,14 @@ impl DominatorTree { } } + /// Push `ebb` onto `self.stack` if it has not already been seen. + fn push_if_unseen(&mut self, ebb: Ebb) { + if self.nodes[ebb].rpo_number == 0 { + self.nodes[ebb].rpo_number = SEEN; + self.stack.push(ebb); + } + } + /// Build a dominator tree from a control flow graph using Keith D. Cooper's /// "Simple, Fast Dominator Algorithm." fn compute_domtree(&mut self, func: &Function, cfg: &ControlFlowGraph) { diff --git a/lib/codegen/src/flowgraph.rs b/lib/codegen/src/flowgraph.rs index 97c3f932f5..48b328cb51 100644 --- a/lib/codegen/src/flowgraph.rs +++ b/lib/codegen/src/flowgraph.rs @@ -125,7 +125,10 @@ impl ControlFlowGraph { BranchInfo::SingleDest(dest, _) => { self.add_edge(ebb, inst, dest); } - BranchInfo::Table(jt) => { + BranchInfo::Table(jt, dest) => { + if let Some(dest) = dest { + self.add_edge(ebb, inst, dest); + } for (_, dest) in func.jump_tables[jt].entries() { self.add_edge(ebb, inst, dest); } diff --git a/lib/codegen/src/ir/function.rs b/lib/codegen/src/ir/function.rs index d15de64c3a..26e860dbe4 100644 --- a/lib/codegen/src/ir/function.rs +++ b/lib/codegen/src/ir/function.rs @@ -11,7 +11,8 @@ use ir::{ Ebb, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, JumpTable, JumpTableData, SigRef, StackSlot, StackSlotData, Table, TableData, }; -use ir::{EbbOffsets, InstEncodings, JumpTables, SourceLocs, StackSlots, ValueLocations}; +use ir::{EbbOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations}; +use ir::{JumpTableOffsets, JumpTables}; use isa::{EncInfo, Encoding, Legalize, TargetIsa}; use settings::CallConv; use std::fmt; @@ -64,6 +65,9 @@ pub struct Function { /// in the textual IR format. pub offsets: EbbOffsets, + /// Code offsets of Jump Table headers. + pub jt_offsets: JumpTableOffsets, + /// Source locations. /// /// Track the original source location for each instruction. The source locations are not @@ -87,6 +91,7 @@ impl Function { encodings: SecondaryMap::new(), locations: SecondaryMap::new(), offsets: SecondaryMap::new(), + jt_offsets: SecondaryMap::new(), srclocs: SecondaryMap::new(), } } diff --git a/lib/codegen/src/ir/instructions.rs b/lib/codegen/src/ir/instructions.rs index e709aa01c1..84e34b1f39 100644 --- a/lib/codegen/src/ir/instructions.rs +++ b/lib/codegen/src/ir/instructions.rs @@ -194,7 +194,10 @@ impl InstructionData { ref args, .. } => BranchInfo::SingleDest(destination, &args.as_slice(pool)[2..]), - InstructionData::BranchTable { table, .. } => BranchInfo::Table(table), + InstructionData::BranchTable { + table, destination, .. + } => BranchInfo::Table(table, Some(destination)), + InstructionData::IndirectJump { table, .. } => BranchInfo::Table(table, None), _ => { debug_assert!(!self.opcode().is_branch()); BranchInfo::NotABranch @@ -213,7 +216,7 @@ impl InstructionData { | InstructionData::BranchInt { destination, .. } | InstructionData::BranchFloat { destination, .. } | InstructionData::BranchIcmp { destination, .. } => Some(destination), - InstructionData::BranchTable { .. } => None, + InstructionData::BranchTable { .. } | InstructionData::IndirectJump { .. } => None, _ => { debug_assert!(!self.opcode().is_branch()); None @@ -284,8 +287,8 @@ pub enum BranchInfo<'a> { /// This is a branch or jump to a single destination EBB, possibly taking value arguments. SingleDest(Ebb, &'a [Value]), - /// This is a jump table branch which can have many destination EBBs. - Table(JumpTable), + /// This is a jump table branch which can have many destination EBBs and maybe one default EBB. + Table(JumpTable, Option), } /// Information about call instructions. diff --git a/lib/codegen/src/ir/jumptable.rs b/lib/codegen/src/ir/jumptable.rs index 17706d5959..aff6503a57 100644 --- a/lib/codegen/src/ir/jumptable.rs +++ b/lib/codegen/src/ir/jumptable.rs @@ -45,6 +45,11 @@ impl JumpTableData { self.table.len() } + /// Boolean that is false if the table has missing entries. + pub fn fully_dense(&self) -> bool { + self.holes == 0 + } + /// Set a table entry. /// /// The table will grow as needed to fit `idx`. diff --git a/lib/codegen/src/ir/mod.rs b/lib/codegen/src/ir/mod.rs index a9c035703e..716bf99b85 100644 --- a/lib/codegen/src/ir/mod.rs +++ b/lib/codegen/src/ir/mod.rs @@ -62,5 +62,8 @@ pub type InstEncodings = SecondaryMap; /// Code offsets for EBBs. pub type EbbOffsets = SecondaryMap; +/// Code offsets for Jump Tables. +pub type JumpTableOffsets = SecondaryMap; + /// Source locations for instructions. pub type SourceLocs = SecondaryMap; diff --git a/lib/codegen/src/isa/x86/binemit.rs b/lib/codegen/src/isa/x86/binemit.rs index 6b9c709bf2..9095512eda 100644 --- a/lib/codegen/src/isa/x86/binemit.rs +++ b/lib/codegen/src/isa/x86/binemit.rs @@ -3,7 +3,7 @@ use super::registers::RU; use binemit::{bad_encoding, CodeSink, Reloc}; use ir::condcodes::{CondCode, FloatCC, IntCC}; -use ir::{Ebb, Function, Inst, InstructionData, Opcode, TrapCode}; +use ir::{Ebb, Function, Inst, InstructionData, JumpTable, Opcode, TrapCode}; use isa::{RegUnit, StackBase, StackBaseMask, StackRef}; use regalloc::RegDiversions; @@ -249,6 +249,7 @@ fn sib_noindex(base: RegUnit, sink: &mut CS) { sink.put1(b); } +/// Emit a SIB byte with a scale, base, and index. fn sib(scale: u8, index: RegUnit, base: RegUnit, sink: &mut CS) { // SIB SS_III_BBB. debug_assert_eq!(scale & !0x03, 0, "Scale out of range"); @@ -332,3 +333,8 @@ fn disp4(destination: Ebb, func: &Function, sink: &mut CS let delta = func.offsets[destination].wrapping_sub(sink.offset() + 4); sink.put4(delta); } + +fn jt_disp4(jt: JumpTable, func: &Function, sink: &mut CS) { + let delta = func.jt_offsets[jt].wrapping_sub(sink.offset() + 4); + sink.put4(delta); +} diff --git a/lib/codegen/src/legalizer/mod.rs b/lib/codegen/src/legalizer/mod.rs index a7645fb643..9fb3a59554 100644 --- a/lib/codegen/src/legalizer/mod.rs +++ b/lib/codegen/src/legalizer/mod.rs @@ -16,6 +16,7 @@ use bitset::BitSet; use cursor::{Cursor, FuncCursor}; use flowgraph::ControlFlowGraph; +use ir::types::I32; use ir::{self, InstBuilder, MemFlags}; use isa::TargetIsa; use timing; @@ -170,6 +171,72 @@ fn expand_cond_trap( /// Jump tables. fn expand_br_table( + inst: ir::Inst, + func: &mut ir::Function, + cfg: &mut ControlFlowGraph, + isa: &TargetIsa, +) { + if isa.flags().jump_tables_enabled() { + expand_br_table_jt(inst, func, cfg, isa); + } else { + expand_br_table_conds(inst, func, cfg, isa); + } +} + +/// Expand br_table to jump table. +fn expand_br_table_jt( + inst: ir::Inst, + func: &mut ir::Function, + cfg: &mut ControlFlowGraph, + isa: &TargetIsa, +) { + use ir::condcodes::IntCC; + + let (arg, default_ebb, table) = match func.dfg[inst] { + ir::InstructionData::BranchTable { + opcode: ir::Opcode::BrTable, + arg, + destination, + table, + } => (arg, destination, table), + _ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)), + }; + + let table_size = func.jump_tables[table].len(); + let table_is_fully_dense = func.jump_tables[table].fully_dense(); + let addr_ty = isa.pointer_type(); + let entry_ty = I32; + + let mut pos = FuncCursor::new(func).at_inst(inst); + pos.use_srcloc(inst); + + // Bounds check + let oob = pos + .ins() + .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, arg, table_size as i64); + + pos.ins().brnz(oob, default_ebb, &[]); + + let base_addr = pos.ins().jump_table_base(addr_ty, table); + let entry = pos + .ins() + .jump_table_entry(addr_ty, arg, base_addr, entry_ty.bytes() as u8, table); + + // If the table isn't fully dense, zero-check the entry. + if !table_is_fully_dense { + pos.ins().brz(entry, default_ebb, &[]); + } + + let addr = pos.ins().iadd(base_addr, entry); + pos.ins().indirect_jump_table_br(addr, table); + + let ebb = pos.current_ebb().unwrap(); + pos.remove_inst(); + cfg.recompute_ebb(pos.func, ebb); +} + +/// Expand br_table to series of conditionals. +fn expand_br_table_conds( inst: ir::Inst, func: &mut ir::Function, cfg: &mut ControlFlowGraph, @@ -177,17 +244,17 @@ fn expand_br_table( ) { use ir::condcodes::IntCC; - let (arg, table) = match func.dfg[inst] { + let (arg, default_ebb, table) = match func.dfg[inst] { ir::InstructionData::BranchTable { opcode: ir::Opcode::BrTable, arg, + destination, table, - } => (arg, table), + } => (arg, destination, table), _ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)), }; // This is a poor man's jump table using just a sequence of conditional branches. - // TODO: Lower into a jump table load and indirect branch. let table_size = func.jump_tables[table].len(); let mut pos = FuncCursor::new(func).at_inst(inst); pos.use_srcloc(inst); @@ -199,7 +266,9 @@ fn expand_br_table( } } - // `br_table` falls through when nothing matches. + // `br_table` jumps to the default destination if nothing matches + pos.ins().jump(default_ebb, &[]); + let ebb = pos.current_ebb().unwrap(); pos.remove_inst(); cfg.recompute_ebb(pos.func, ebb); diff --git a/lib/codegen/src/regalloc/coloring.rs b/lib/codegen/src/regalloc/coloring.rs index d9df465f5c..0859bec657 100644 --- a/lib/codegen/src/regalloc/coloring.rs +++ b/lib/codegen/src/regalloc/coloring.rs @@ -900,11 +900,15 @@ impl<'a> Context<'a> { let lr = &self.liveness[value]; lr.is_livein(ebb, ctx) } - Table(jt) => { + Table(jt, ebb) => { let lr = &self.liveness[value]; - !lr.is_local() && self.cur.func.jump_tables[jt] - .entries() - .any(|(_, ebb)| lr.is_livein(ebb, ctx)) + !lr.is_local() + && (ebb.map_or(false, |ebb| lr.is_livein(ebb, ctx)) || self + .cur + .func + .jump_tables[jt] + .entries() + .any(|(_, ebb)| lr.is_livein(ebb, ctx))) } } } diff --git a/lib/codegen/src/settings.rs b/lib/codegen/src/settings.rs index 81f1ae2102..743bd29268 100644 --- a/lib/codegen/src/settings.rs +++ b/lib/codegen/src/settings.rs @@ -380,7 +380,8 @@ mod tests { allones_funcaddrs = false\n\ probestack_enabled = true\n\ probestack_func_adjusts_sp = false\n\ - probestack_size_log2 = 12\n" + probestack_size_log2 = 12\n\ + jump_tables_enabled = true\n" ); assert_eq!(f.opt_level(), super::OptLevel::Default); assert_eq!(f.enable_simd(), true); diff --git a/lib/codegen/src/verifier/flags.rs b/lib/codegen/src/verifier/flags.rs index cf9c924244..39cdb30660 100644 --- a/lib/codegen/src/verifier/flags.rs +++ b/lib/codegen/src/verifier/flags.rs @@ -135,7 +135,12 @@ impl<'a> FlagsVerifier<'a> { merge(&mut live_val, val, inst, errors)?; } } - BranchInfo::Table(jt) => { + BranchInfo::Table(jt, dest) => { + if let Some(dest) = dest { + if let Some(val) = self.livein[dest].expand() { + merge(&mut live_val, val, inst, errors)?; + } + } for (_, dest) in self.func.jump_tables[jt].entries() { if let Some(val) = self.livein[dest].expand() { merge(&mut live_val, val, inst, errors)?; diff --git a/lib/codegen/src/verifier/locations.rs b/lib/codegen/src/verifier/locations.rs index 73997d0a92..c1ed8bd1b9 100644 --- a/lib/codegen/src/verifier/locations.rs +++ b/lib/codegen/src/verifier/locations.rs @@ -332,9 +332,21 @@ impl<'a> LocationVerifier<'a> { } } } - Table(jt) => { + Table(jt, ebb) => { for d in divert.all() { let lr = &liveness[d.value]; + if let Some(ebb) = ebb { + if lr.is_livein(ebb, liveness.context(&self.func.layout)) { + return fatal!( + errors, + inst, + "{} is diverted to {} and live in to {}", + d.value, + d.to.display(&self.reginfo), + ebb + ); + } + } for (_, ebb) in self.func.jump_tables[jt].entries() { if lr.is_livein(ebb, liveness.context(&self.func.layout)) { return fatal!( diff --git a/lib/codegen/src/verifier/mod.rs b/lib/codegen/src/verifier/mod.rs index 40f8302592..01baa66c11 100644 --- a/lib/codegen/src/verifier/mod.rs +++ b/lib/codegen/src/verifier/mod.rs @@ -657,7 +657,10 @@ impl<'a> Verifier<'a> { self.verify_ebb(inst, destination, errors)?; self.verify_value_list(inst, args, errors)?; } - BranchTable { table, .. } => { + BranchTable { table, .. } + | BranchTableBase { table, .. } + | BranchTableEntry { table, .. } + | IndirectJump { table, .. } => { self.verify_jump_table(inst, table, errors)?; } Call { @@ -1213,7 +1216,19 @@ impl<'a> Verifier<'a> { .map(|&v| self.func.dfg.value_type(v)); self.typecheck_variable_args_iterator(inst, iter, errors)?; } - BranchInfo::Table(table) => { + BranchInfo::Table(table, ebb) => { + if let Some(ebb) = ebb { + let arg_count = self.func.dfg.num_ebb_params(ebb); + if arg_count != 0 { + return nonfatal!( + errors, + inst, + "takes no arguments, but had target {} with {} arguments", + ebb, + arg_count + ); + } + } for (_, ebb) in self.func.jump_tables[table].entries() { let arg_count = self.func.dfg.num_ebb_params(ebb); if arg_count != 0 { diff --git a/lib/codegen/src/write.rs b/lib/codegen/src/write.rs index 743540bf67..24b8499f39 100644 --- a/lib/codegen/src/write.rs +++ b/lib/codegen/src/write.rs @@ -443,7 +443,17 @@ pub fn write_operands( write!(w, " {} {}, {}, {}", cond, args[0], args[1], destination)?; write_ebb_args(w, &args[2..]) } - BranchTable { arg, table, .. } => write!(w, " {}, {}", arg, table), + BranchTable { + arg, + destination, + table, + .. + } => write!(w, " {}, {}, {}", arg, destination, table), + BranchTableBase { table, .. } => write!(w, " {}", table), + BranchTableEntry { + args, imm, table, .. + } => write!(w, " {}, {}, {}, {}", args[0], args[1], imm, table), + IndirectJump { arg, table, .. } => write!(w, " {}, {}", arg, table), Call { func_ref, ref args, .. } => write!(w, " {}({})", func_ref, DisplayValues(args.as_slice(pool))), diff --git a/lib/filetests/src/test_binemit.rs b/lib/filetests/src/test_binemit.rs index ef8e93936a..2f0db746bd 100644 --- a/lib/filetests/src/test_binemit.rs +++ b/lib/filetests/src/test_binemit.rs @@ -4,7 +4,7 @@ //! functions and compares the results to the expected output. use cranelift_codegen::binemit; -use cranelift_codegen::binemit::RegDiversions; +use cranelift_codegen::binemit::{CodeSink, RegDiversions}; use cranelift_codegen::dbg::DisplayList; use cranelift_codegen::ir; use cranelift_codegen::ir::entities::AnyEntity; @@ -30,6 +30,7 @@ pub fn subtest(parsed: &TestCommand) -> SubtestResult> { /// Code sink that generates text. struct TextSink { + code_size: binemit::CodeOffset, offset: binemit::CodeOffset, text: String, } @@ -38,6 +39,7 @@ impl TextSink { /// Create a new empty TextSink. pub fn new() -> Self { Self { + code_size: 0, offset: 0, text: String::new(), } @@ -93,6 +95,10 @@ impl binemit::CodeSink for TextSink { fn trap(&mut self, code: ir::TrapCode, _srcloc: ir::SourceLoc) { write!(self.text, "{} ", code).unwrap(); } + + fn begin_rodata(&mut self) { + self.code_size = self.offset + } } impl SubTest for TestBinEmit { @@ -281,6 +287,21 @@ impl SubTest for TestBinEmit { } } + sink.begin_rodata(); + + for (jt, jt_data) in func.jump_tables.iter() { + let jt_offset = func.jt_offsets[jt]; + for idx in 0..jt_data.len() { + match jt_data.get_entry(idx) { + Some(ebb) => { + let rel_offset: i32 = func.offsets[ebb] as i32 - jt_offset as i32; + sink.put4(rel_offset as u32) + } + None => sink.put4(0), + } + } + } + if sink.offset != code_size { return Err(format!( "Expected code size {}, got {}", diff --git a/lib/filetests/src/test_compile.rs b/lib/filetests/src/test_compile.rs index 1fb6d0e4c0..4ff112eab9 100644 --- a/lib/filetests/src/test_compile.rs +++ b/lib/filetests/src/test_compile.rs @@ -104,4 +104,5 @@ impl binemit::CodeSink for SizeSink { } fn reloc_jt(&mut self, _reloc: binemit::Reloc, _jt: ir::JumpTable) {} fn trap(&mut self, _code: ir::TrapCode, _srcloc: ir::SourceLoc) {} + fn begin_rodata(&mut self) {} } diff --git a/lib/frontend/src/ssa.rs b/lib/frontend/src/ssa.rs index 2cb6839b76..d13cc0531a 100644 --- a/lib/frontend/src/ssa.rs +++ b/lib/frontend/src/ssa.rs @@ -10,7 +10,7 @@ use cranelift_codegen::entity::{EntityRef, PrimaryMap, SecondaryMap}; use cranelift_codegen::ir::immediates::{Ieee32, Ieee64}; use cranelift_codegen::ir::instructions::BranchInfo; use cranelift_codegen::ir::types::{F32, F64}; -use cranelift_codegen::ir::{Ebb, Function, Inst, InstBuilder, Type, Value}; +use cranelift_codegen::ir::{Ebb, Function, Inst, InstBuilder, InstructionData, Type, Value}; use cranelift_codegen::packed_option::PackedOption; use cranelift_codegen::packed_option::ReservedValue; use std::mem; @@ -647,7 +647,7 @@ impl SSABuilder { func.dfg.append_inst_arg(jump_inst, val); None } - BranchInfo::Table(jt) => { + BranchInfo::Table(jt, default_ebb) => { // In the case of a jump table, the situation is tricky because br_table doesn't // support arguments. // We have to split the critical edge @@ -656,6 +656,21 @@ impl SSABuilder { let middle_block = self.declare_ebb_header_block(middle_ebb); self.blocks[middle_block].add_predecessor(jump_inst_block, jump_inst); self.mark_ebb_header_block_sealed(middle_block); + + if let Some(default_ebb) = default_ebb { + if dest_ebb == default_ebb { + match func.dfg[jump_inst] { + InstructionData::BranchTable { + destination: ref mut dest, + .. + } => { + *dest = middle_ebb; + } + _ => panic!("should not happen"), + } + } + } + for old_dest in func.jump_tables[jt].as_mut_slice() { if *old_dest == PackedOption::from(dest_ebb) { *old_dest = PackedOption::from(middle_ebb); @@ -986,20 +1001,31 @@ mod tests { #[test] fn br_table_with_args() { // This tests the on-demand splitting of critical edges for br_table with jump arguments - let mut func = Function::new(); - let mut ssa = SSABuilder::new(); - let ebb0 = func.dfg.make_ebb(); - let ebb1 = func.dfg.make_ebb(); + // // Here is the pseudo-program we want to translate: + // + // function %f { + // jt = jump_table ebb2, 0, ebb1 // ebb0: - // x = 0; - // br_table x ebb1 - // x = 1 - // jump ebb1 + // x = 1; + // br_table x, ebb2, jt // ebb1: + // x = 2 + // jump ebb2 + // ebb2: // x = x + 1 // return - // + // } + + let mut func = Function::new(); + let mut ssa = SSABuilder::new(); + let mut jump_table = JumpTableData::new(); + let ebb0 = func.dfg.make_ebb(); + let ebb1 = func.dfg.make_ebb(); + let ebb2 = func.dfg.make_ebb(); + + // ebb0: + // x = 1; let block0 = ssa.declare_ebb_header_block(ebb0); ssa.seal_ebb_header_block(ebb0, &mut func); let x_var = Variable::new(0); @@ -1007,42 +1033,60 @@ mod tests { let mut cur = FuncCursor::new(&mut func); cur.insert_ebb(ebb0); cur.insert_ebb(ebb1); + cur.insert_ebb(ebb2); cur.goto_bottom(ebb0); cur.ins().iconst(I32, 1) }; ssa.def_var(x_var, x1, block0); - let mut data = JumpTableData::new(); - data.push_entry(ebb1); - data.set_entry(2, ebb1); - let jt = func.create_jump_table(data); + + // jt = jump_table ebb2, 0, ebb1 + jump_table.push_entry(ebb2); + jump_table.set_entry(2, ebb1); + let jt = func.create_jump_table(jump_table); + + // ebb0: + // ... + // br_table x, ebb2, jt ssa.use_var(&mut func, x_var, I32, block0).0; let br_table = { let mut cur = FuncCursor::new(&mut func).at_bottom(ebb0); - cur.ins().br_table(x1, jt) + cur.ins().br_table(x1, ebb2, jt) }; - let block1 = ssa.declare_ebb_body_block(block0); - let x3 = { - let mut cur = FuncCursor::new(&mut func).at_bottom(ebb0); + + // ebb1: + // x = 2 + // jump ebb2 + let block1 = ssa.declare_ebb_header_block(ebb1); + ssa.seal_ebb_header_block(ebb1, &mut func); + let x2 = { + let mut cur = FuncCursor::new(&mut func).at_bottom(ebb1); cur.ins().iconst(I32, 2) }; - ssa.def_var(x_var, x3, block1); + ssa.def_var(x_var, x2, block1); let jump_inst = { - let mut cur = FuncCursor::new(&mut func).at_bottom(ebb0); - cur.ins().jump(ebb1, &[]) - }; - let block2 = ssa.declare_ebb_header_block(ebb1); - ssa.declare_ebb_predecessor(ebb1, block1, jump_inst); - ssa.declare_ebb_predecessor(ebb1, block0, br_table); - ssa.seal_ebb_header_block(ebb1, &mut func); - let x4 = ssa.use_var(&mut func, x_var, I32, block2).0; - { let mut cur = FuncCursor::new(&mut func).at_bottom(ebb1); - cur.ins().iadd_imm(x4, 1) + cur.ins().jump(ebb2, &[]) }; + + // ebb2: + // x = x + 1 + // return + let block3 = ssa.declare_ebb_header_block(ebb2); + ssa.declare_ebb_predecessor(ebb2, block1, jump_inst); + ssa.declare_ebb_predecessor(ebb2, block0, br_table); + ssa.seal_ebb_header_block(ebb2, &mut func); + let block4 = ssa.declare_ebb_body_block(block3); + let x3 = ssa.use_var(&mut func, x_var, I32, block4).0; + let x4 = { + let mut cur = FuncCursor::new(&mut func).at_bottom(ebb2); + cur.ins().iadd_imm(x3, 1) + }; + ssa.def_var(x_var, x4, block4); { - let mut cur = FuncCursor::new(&mut func).at_bottom(ebb1); + let mut cur = FuncCursor::new(&mut func).at_bottom(ebb2); cur.ins().return_(&[]) }; + let flags = settings::Flags::new(settings::builder()); match verify_function(&func, &flags) { Ok(()) => {} diff --git a/lib/frontend/src/switch.rs b/lib/frontend/src/switch.rs index 829ae24b7a..7e8db1183d 100644 --- a/lib/frontend/src/switch.rs +++ b/lib/frontend/src/switch.rs @@ -156,8 +156,7 @@ impl Switch { bx.switch_to_block(jt_ebb); let discr = bx.ins().iadd_imm(val, (first_index as i64).wrapping_neg()); - bx.ins().br_table(discr, jump_table); - bx.ins().jump(otherwise, &[]); + bx.ins().br_table(discr, otherwise, jump_table); } } @@ -256,8 +255,7 @@ ebb0: ebb3: v3 = iadd_imm.i32 v1, 0 - br_table v3, jt0 - jump ebb0" + br_table v3, ebb0, jt0" ); } @@ -308,13 +306,11 @@ ebb8: ebb11: v7 = iadd_imm.i32 v1, 0 - br_table v7, jt0 - jump ebb0 + br_table v7, ebb0, jt0 ebb10: v8 = iadd_imm.i32 v1, -10 - br_table v8, jt1 - jump ebb0" + br_table v8, ebb0, jt1" ); } diff --git a/lib/reader/src/parser.rs b/lib/reader/src/parser.rs index 1594d183b6..8809774047 100644 --- a/lib/reader/src/parser.rs +++ b/lib/reader/src/parser.rs @@ -2225,9 +2225,44 @@ impl<'a> Parser<'a> { InstructionFormat::BranchTable => { let arg = self.match_value("expected SSA value operand")?; self.match_token(Token::Comma, "expected ',' between operands")?; + let ebb_num = self.match_ebb("expected branch destination EBB")?; + self.match_token(Token::Comma, "expected ',' between operands")?; let table = self.match_jt()?; ctx.check_jt(table, self.loc)?; - InstructionData::BranchTable { opcode, arg, table } + InstructionData::BranchTable { + opcode, + arg, + destination: ebb_num, + table, + } + } + InstructionFormat::BranchTableBase => { + let table = self.match_jt()?; + ctx.check_jt(table, self.loc)?; + InstructionData::BranchTableBase { opcode, table } + } + InstructionFormat::BranchTableEntry => { + let index = self.match_value("expected SSA value operand")?; + self.match_token(Token::Comma, "expected ',' between operands")?; + let base = self.match_value("expected SSA value operand")?; + self.match_token(Token::Comma, "expected ',' between operands")?; + let imm = self.match_uimm8("expected width")?; + self.match_token(Token::Comma, "expected ',' between operands")?; + let table = self.match_jt()?; + ctx.check_jt(table, self.loc)?; + InstructionData::BranchTableEntry { + opcode, + args: [index, base], + imm, + table, + } + } + InstructionFormat::IndirectJump => { + let arg = self.match_value("expected SSA value operand")?; + self.match_token(Token::Comma, "expected ',' between operands")?; + let table = self.match_jt()?; + ctx.check_jt(table, self.loc)?; + InstructionData::IndirectJump { opcode, arg, table } } InstructionFormat::InsertLane => { let lhs = self.match_value("expected SSA value first operand")?; diff --git a/lib/serde/src/serde_clif_json.rs b/lib/serde/src/serde_clif_json.rs index 7ad03df646..536dda6a3b 100644 --- a/lib/serde/src/serde_clif_json.rs +++ b/lib/serde/src/serde_clif_json.rs @@ -117,6 +117,22 @@ pub enum SerInstData { destination: String, }, BranchTable { + opcode: String, + arg: String, + destination: String, + table: String, + }, + BranchTableEntry { + opcode: String, + args: [String; 2], + imm: String, + table: String, + }, + BranchTableBase { + opcode: String, + table: String, + }, + IndirectJump { opcode: String, arg: String, table: String, @@ -445,7 +461,36 @@ pub fn get_inst_data(inst_index: Inst, func: &Function) -> SerInstData { destination: destination.to_string(), } } - InstructionData::BranchTable { opcode, arg, table } => SerInstData::BranchTable { + InstructionData::BranchTable { + opcode, + arg, + destination, + table, + } => SerInstData::BranchTable { + opcode: opcode.to_string(), + arg: arg.to_string(), + destination: destination.to_string(), + table: table.to_string(), + }, + InstructionData::BranchTableBase { opcode, table } => SerInstData::BranchTableBase { + opcode: opcode.to_string(), + table: table.to_string(), + }, + InstructionData::BranchTableEntry { + opcode, + args, + imm, + table, + } => { + let hold_args = [args[0].to_string(), args[1].to_string()]; + SerInstData::BranchTableEntry { + opcode: opcode.to_string(), + args: hold_args, + imm: imm.to_string(), + table: table.to_string(), + } + } + InstructionData::IndirectJump { opcode, arg, table } => SerInstData::IndirectJump { opcode: opcode.to_string(), arg: arg.to_string(), table: table.to_string(), diff --git a/lib/wasm/src/code_translator.rs b/lib/wasm/src/code_translator.rs index 5a74db5bad..9b2f62777b 100644 --- a/lib/wasm/src/code_translator.rs +++ b/lib/wasm/src/code_translator.rs @@ -283,14 +283,13 @@ pub fn translate_operator( data.push_entry(ebb); } let jt = builder.create_jump_table(data); - builder.ins().br_table(val, jt); let ebb = { let i = state.control_stack.len() - 1 - (default as usize); let frame = &mut state.control_stack[i]; frame.set_branched_to_exit(); frame.br_destination() }; - builder.ins().jump(ebb, &[]); + builder.ins().br_table(val, ebb, jt); } else { // Here we have jump arguments, but Cranelift's br_table doesn't support them // We then proceed to split the edges going out of the br_table @@ -309,14 +308,14 @@ pub fn translate_operator( data.push_entry(branch_ebb); } let jt = builder.create_jump_table(data); - builder.ins().br_table(val, jt); let default_ebb = { let i = state.control_stack.len() - 1 - (default as usize); let frame = &mut state.control_stack[i]; frame.set_branched_to_exit(); frame.br_destination() }; - builder.ins().jump(default_ebb, state.peekn(return_count)); + dest_ebb_sequence.push((default as usize, default_ebb)); + builder.ins().br_table(val, default_ebb, jt); for (depth, dest_ebb) in dest_ebb_sequence { builder.switch_to_block(dest_ebb); builder.seal_block(dest_ebb);