diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs index 97211e2243..be0c5c5da8 100644 --- a/cranelift/codegen/src/legalizer/mod.rs +++ b/cranelift/codegen/src/legalizer/mod.rs @@ -150,29 +150,45 @@ fn expand_cond_trap( // Split the EBB after `inst`: // // trapnz arg + // .. // // Becomes: // - // brz arg, new_ebb - // trap - // new_ebb: + // brz arg, new_ebb_resume + // jump new_ebb_trap // + // new_ebb_trap: + // trap + // + // new_ebb_resume: + // .. let old_ebb = func.layout.pp_ebb(inst); - let new_ebb = func.dfg.make_ebb(); + let new_ebb_trap = func.dfg.make_ebb(); + let new_ebb_resume = func.dfg.make_ebb(); + + // Replace trap instruction by the inverted condition. if trapz { - func.dfg.replace(inst).brnz(arg, new_ebb, &[]); + func.dfg.replace(inst).brnz(arg, new_ebb_resume, &[]); } else { - func.dfg.replace(inst).brz(arg, new_ebb, &[]); + func.dfg.replace(inst).brz(arg, new_ebb_resume, &[]); } + // Add jump instruction after the inverted branch. let mut pos = FuncCursor::new(func).after_inst(inst); pos.use_srcloc(inst); + pos.ins().jump(new_ebb_trap, &[]); + + // Insert the new label and the unconditional trap terminator. + pos.insert_ebb(new_ebb_trap); pos.ins().trap(code); - pos.insert_ebb(new_ebb); + + // Insert the new label and resume the execution when the trap fails. + pos.insert_ebb(new_ebb_resume); // Finally update the CFG. cfg.recompute_ebb(pos.func, old_ebb); - cfg.recompute_ebb(pos.func, new_ebb); + cfg.recompute_ebb(pos.func, new_ebb_resume); + cfg.recompute_ebb(pos.func, new_ebb_trap); } /// Jump tables. diff --git a/cranelift/filetests/filetests/isa/x86/legalize-custom.clif b/cranelift/filetests/filetests/isa/x86/legalize-custom.clif index 667c08804c..51e7ca487d 100644 --- a/cranelift/filetests/filetests/isa/x86/legalize-custom.clif +++ b/cranelift/filetests/filetests/isa/x86/legalize-custom.clif @@ -33,6 +33,8 @@ ebb0(v1: i32): return ; check: ebb0(v1: i32 ; check: brnz v2, $(new=$EBB) + ; check: jump $(trap=$EBB) + ; check: $trap: ; nextln: trap user7 ; check: $new: ; nextln: return @@ -45,6 +47,8 @@ ebb0(v1: i32): return ; check: ebb0(v1: i32 ; check: brz v2, $(new=$EBB) + ; check: jump $(trap=$EBB) + ; check: $trap: ; nextln: trap user9 ; check: $new: ; nextln: return diff --git a/cranelift/filetests/filetests/isa/x86/legalize-heaps.clif b/cranelift/filetests/filetests/isa/x86/legalize-heaps.clif index 5f60c7b43f..c2f1ccb2d9 100644 --- a/cranelift/filetests/filetests/isa/x86/legalize-heaps.clif +++ b/cranelift/filetests/filetests/isa/x86/legalize-heaps.clif @@ -2,6 +2,7 @@ test legalizer target x86_64 ; Test legalization for various forms of heap addresses. +; regex: EBB=ebb\d+ function %heap_addrs(i32, i64, i64 vmctx) { gv4 = vmctx @@ -37,9 +38,11 @@ ebb0(v0: i32, v1: i64, v3: i64): v5 = heap_addr.i64 heap1, v0, 0 ; check: v14 = icmp_imm ugt v0, 0x0001_0000 - ; check: brz v14, ebb1 - ; check: trap heap_oob - ; check: ebb1: + ; check: brz v14, $(resume_1=$EBB) + ; nextln: jump $(trap_1=$EBB) + ; check: $trap_1: + ; nextln: trap heap_oob + ; check: $resume_1: ; check: v15 = uextend.i64 v0 ; check: v16 = iadd_imm.i64 v3, 64 ; check: v5 = iadd v16, v15 @@ -47,17 +50,21 @@ ebb0(v0: i32, v1: i64, v3: i64): v6 = heap_addr.i64 heap2, v1, 0 ; check: v19 = iconst.i64 0x0001_0000_0000 ; check: v17 = icmp.i64 ugt v1, v19 - ; check: brz v17, ebb2 - ; check: trap heap_oob - ; check: ebb2: + ; check: brz v17, $(resume_2=$EBB) + ; nextln: jump $(trap_2=$EBB) + ; check: $trap_2: + ; nextln: trap heap_oob + ; check: $resume_2: ; check: v18 = iadd_imm.i64 v3, 64 ; check: v6 = iadd v18, v1 v7 = heap_addr.i64 heap3, v1, 0 ; check: v20 = icmp_imm.i64 ugt v1, 0x0001_0000 - ; check: brz v20, ebb3 - ; check: trap heap_oob - ; check: ebb3: + ; check: brz v20, $(resume_3=$EBB) + ; nextln: jump $(trap_3=$EBB) + ; check: $trap_3: + ; nextln: trap heap_oob + ; check: $resume_3: ; check: v21 = iadd_imm.i64 v3, 64 ; check: v7 = iadd v21, v1 @@ -65,9 +72,11 @@ ebb0(v0: i32, v1: i64, v3: i64): ; check: v22 = load.i32 notrap aligned v3+88 ; check: v23 = iadd_imm v22, 0 ; check: v24 = icmp.i32 ugt v0, v23 - ; check: brz v24, ebb4 - ; check: trap heap_oob - ; check: ebb4: + ; check: brz v24, $(resume_4=$EBB) + ; nextln: jump $(trap_4=$EBB) + ; check: $trap_4: + ; nextln: trap heap_oob + ; check: $resume_4: ; check: v25 = uextend.i64 v0 ; check: v26 = iadd_imm.i64 v3, 72 ; check: v8 = iadd v26, v25 @@ -76,9 +85,11 @@ ebb0(v0: i32, v1: i64, v3: i64): ; check: v27 = load.i32 notrap aligned v3+88 ; check: v28 = iadd_imm v27, 0 ; check: v29 = icmp.i32 ugt v0, v28 - ; check: brz v29, ebb5 - ; check: trap heap_oob - ; check: ebb5: + ; check: brz v29, $(resume_5=$EBB) + ; nextln: jump $(trap_5=$EBB) + ; check: $trap_5: + ; nextln: trap heap_oob + ; check: $resume_5: ; check: v30 = uextend.i64 v0 ; check: v31 = iadd_imm.i64 v3, 72 ; check: v9 = iadd v31, v30 @@ -87,9 +98,11 @@ ebb0(v0: i32, v1: i64, v3: i64): ; check: v32 = iadd_imm.i64 v3, 80 ; check: v33 = iadd_imm v32, 0 ; check: v34 = icmp.i64 ugt v1, v33 - ; check: brz v34, ebb6 - ; check: trap heap_oob - ; check: ebb6: + ; check: brz v34, $(resume_6=$EBB) + ; nextln: jump $(trap_6=$EBB) + ; check: $trap_6: + ; nextln: trap heap_oob + ; check: $resume_6: ; check: v35 = iadd_imm.i64 v3, 72 ; check: v10 = iadd v35, v1 @@ -97,9 +110,11 @@ ebb0(v0: i32, v1: i64, v3: i64): ; check: v36 = iadd_imm.i64 v3, 80 ; check: v37 = iadd_imm v36, 0 ; check: v38 = icmp.i64 ugt v1, v37 - ; check: brz v38, ebb7 - ; check: trap heap_oob - ; check: ebb7: + ; check: brz v38, $(resume_7=$EBB) + ; nextln: jump $(trap_7=$EBB) + ; check: $trap_7: + ; nextln: trap heap_oob + ; check: $resume_7: ; check: v39 = iadd_imm.i64 v3, 72 ; check: v11 = iadd v39, v1 diff --git a/cranelift/filetests/filetests/isa/x86/legalize-memory.clif b/cranelift/filetests/filetests/isa/x86/legalize-memory.clif index eb24523b14..7b506d3876 100644 --- a/cranelift/filetests/filetests/isa/x86/legalize-memory.clif +++ b/cranelift/filetests/filetests/isa/x86/legalize-memory.clif @@ -100,6 +100,8 @@ ebb0(v0: i32, v999: i64): ; Boundscheck code ; check: $(oob=$V) = icmp ; nextln: brz $oob, $(ok=$EBB) + ; nextln: jump $(trap_oob=$EBB) + ; check: $trap_oob: ; nextln: trap heap_oob ; check: $ok: ; Checks here are assuming that no pipehole opts fold the load offsets. diff --git a/cranelift/filetests/filetests/isa/x86/legalize-tables.clif b/cranelift/filetests/filetests/isa/x86/legalize-tables.clif index 5995c230e0..762f8a1038 100644 --- a/cranelift/filetests/filetests/isa/x86/legalize-tables.clif +++ b/cranelift/filetests/filetests/isa/x86/legalize-tables.clif @@ -2,6 +2,7 @@ test legalizer target x86_64 ; Test legalization for various forms of table addresses. +; regex: EBB=ebb\d+ function %table_addrs(i32, i64, i64 vmctx) { gv4 = vmctx @@ -23,9 +24,11 @@ ebb0(v0: i32, v1: i64, v3: i64): v4 = table_addr.i64 table0, v0, +0 ; check: v8 = load.i32 notrap aligned v3+88 ; check: v9 = icmp uge v0, v8 - ; check: brz v9, ebb1 - ; check: trap table_oob - ; check: ebb1: + ; check: brz v9, $(resume_1=$EBB) + ; nextln: jump $(trap_1=$EBB) + ; check: $trap_1: + ; nextln: trap table_oob + ; check: $resume_1: ; check: v10 = uextend.i64 v0 ; check: v11 = iadd_imm.i64 v3, 72 ; check: v4 = iadd v11, v10 @@ -33,9 +36,11 @@ ebb0(v0: i32, v1: i64, v3: i64): v5 = table_addr.i64 table1, v0, +0 ; check: v12 = load.i32 notrap aligned v3+88 ; check: v13 = icmp.i32 uge v0, v12 - ; check: brz v13, ebb2 - ; check: trap table_oob - ; check: ebb2: + ; check: brz v13, $(resume_2=$EBB) + ; nextln: jump $(trap_2=$EBB) + ; check: $trap_2: + ; nextln: trap table_oob + ; check: $resume_2: ; check: v14 = uextend.i64 v0 ; check: v15 = iadd_imm.i64 v3, 72 ; check: v16 = ishl_imm v14, 4 @@ -44,18 +49,22 @@ ebb0(v0: i32, v1: i64, v3: i64): v6 = table_addr.i64 table2, v1, +0 ; check: v17 = iadd_imm.i64 v3, 80 ; check: v18 = icmp.i64 uge v1, v17 - ; check: brz v18, ebb3 - ; check: trap table_oob - ; check: ebb3: + ; check: brz v18, $(resume_3=$EBB) + ; nextln: jump $(trap_3=$EBB) + ; check: $trap_3: + ; nextln: trap table_oob + ; check: $resume_3: ; check: v19 = iadd_imm.i64 v3, 72 ; check: v6 = iadd v19, v1 v7 = table_addr.i64 table3, v1, +0 ; check: v20 = iadd_imm.i64 v3, 80 ; check: v21 = icmp.i64 uge v1, v20 - ; check: brz v21, ebb4 - ; check: trap table_oob - ; check: ebb4: + ; check: brz v21, $(resume_4=$EBB) + ; nextln: jump $(trap_4=$EBB) + ; check: $trap_4: + ; nextln: trap table_oob + ; check: $resume_4: ; check: v22 = iadd_imm.i64 v3, 72 ; check: v23 = ishl_imm.i64 v1, 4 ; check: v7 = iadd v22, v23