diff --git a/Cargo.lock b/Cargo.lock index 204f77947f..da38baa7ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -570,6 +570,7 @@ dependencies = [ "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", + "cranelift-control", "cranelift-entity", "cranelift-isle", "criterion", @@ -596,6 +597,13 @@ dependencies = [ name = "cranelift-codegen-shared" version = "0.96.0" +[[package]] +name = "cranelift-control" +version = "0.96.0" +dependencies = [ + "arbitrary", +] + [[package]] name = "cranelift-entity" version = "0.96.0" @@ -610,6 +618,7 @@ dependencies = [ "anyhow", "cranelift", "cranelift-codegen", + "cranelift-control", "cranelift-frontend", "cranelift-interpreter", "cranelift-jit", @@ -685,6 +694,7 @@ dependencies = [ "anyhow", "cranelift", "cranelift-codegen", + "cranelift-control", "cranelift-entity", "cranelift-frontend", "cranelift-module", @@ -704,6 +714,7 @@ version = "0.96.0" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", "hashbrown 0.13.1", ] @@ -722,6 +733,7 @@ version = "0.96.0" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", "cranelift-entity", "cranelift-frontend", "cranelift-module", @@ -3585,6 +3597,7 @@ version = "9.0.0" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", "cranelift-entity", "cranelift-frontend", "cranelift-native", @@ -3605,6 +3618,7 @@ version = "9.0.0" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", "cranelift-native", "gimli", "object", @@ -3684,6 +3698,7 @@ dependencies = [ "component-fuzz-util", "component-test-util", "cranelift-codegen", + "cranelift-control", "cranelift-filetests", "cranelift-fuzzgen", "cranelift-interpreter", diff --git a/Cargo.toml b/Cargo.toml index 0c53ebd033..03423e8b8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,6 +162,7 @@ cranelift-object = { path = "cranelift/object", version = "0.96.0" } cranelift-jit = { path = "cranelift/jit", version = "0.96.0" } cranelift-fuzzgen = { path = "cranelift/fuzzgen" } cranelift-bforest = { path = "cranelift/bforest", version = "0.96.0" } +cranelift-control = { path = "cranelift/control", version = "0.96.0" } cranelift = { path = "cranelift/umbrella", version = "0.96.0" } winch-codegen = { path = "winch/codegen", version = "=0.7.0" } diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 11edf16d3f..b22ec38bf5 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -19,6 +19,7 @@ capstone = { workspace = true, optional = true } cranelift-codegen-shared = { path = "./shared", version = "0.96.0" } cranelift-entity = { workspace = true } cranelift-bforest = { workspace = true } +cranelift-control = { workspace = true } hashbrown = { workspace = true, features = ["raw"] } target-lexicon = { workspace = true } log = { workspace = true } diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index db0ff0434a..b5b10eaa76 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -33,6 +33,7 @@ use crate::{timing, CompileError}; #[cfg(feature = "souper-harvest")] use alloc::string::String; use alloc::vec::Vec; +use cranelift_control::ControlPlane; #[cfg(feature = "souper-harvest")] use crate::souper_harvest::do_souper_harvest; @@ -124,8 +125,9 @@ impl Context { &mut self, isa: &dyn TargetIsa, mem: &mut Vec, + ctrl_plane: &mut ControlPlane, ) -> CompileResult<&CompiledCode> { - let compiled_code = self.compile(isa)?; + let compiled_code = self.compile(isa, ctrl_plane)?; mem.extend_from_slice(compiled_code.code_buffer()); Ok(compiled_code) } @@ -133,14 +135,18 @@ impl Context { /// Internally compiles the function into a stencil. /// /// Public only for testing and fuzzing purposes. - pub fn compile_stencil(&mut self, isa: &dyn TargetIsa) -> CodegenResult { + pub fn compile_stencil( + &mut self, + isa: &dyn TargetIsa, + ctrl_plane: &mut ControlPlane, + ) -> CodegenResult { let _tt = timing::compile(); self.verify_if(isa)?; self.optimize(isa)?; - isa.compile_function(&self.func, &self.domtree, self.want_disasm) + isa.compile_function(&self.func, &self.domtree, self.want_disasm, ctrl_plane) } /// Optimize the function, performing all compilation steps up to @@ -212,11 +218,17 @@ impl Context { /// code sink. /// /// Returns information about the function's code and read-only data. - pub fn compile(&mut self, isa: &dyn TargetIsa) -> CompileResult<&CompiledCode> { - let stencil = self.compile_stencil(isa).map_err(|error| CompileError { - inner: error, - func: &self.func, - })?; + pub fn compile( + &mut self, + isa: &dyn TargetIsa, + ctrl_plane: &mut ControlPlane, + ) -> CompileResult<&CompiledCode> { + let stencil = self + .compile_stencil(isa, ctrl_plane) + .map_err(|error| CompileError { + inner: error, + func: &self.func, + })?; Ok(self .compiled_code .insert(stencil.apply_params(&self.func.params))) diff --git a/cranelift/codegen/src/incremental_cache.rs b/cranelift/codegen/src/incremental_cache.rs index fdd7ca1615..8c7638e25e 100644 --- a/cranelift/codegen/src/incremental_cache.rs +++ b/cranelift/codegen/src/incremental_cache.rs @@ -32,6 +32,7 @@ use crate::{isa::TargetIsa, timing}; use crate::{trace, CompileError, Context}; use alloc::borrow::{Cow, ToOwned as _}; use alloc::string::ToString as _; +use cranelift_control::ControlPlane; impl Context { /// Compile the function, as in `compile`, but tries to reuse compiled artifacts from former @@ -40,6 +41,7 @@ impl Context { &mut self, isa: &dyn TargetIsa, cache_store: &mut dyn CacheKvStore, + ctrl_plane: &mut ControlPlane, ) -> CompileResult<(&CompiledCode, bool)> { let cache_key_hash = { let _tt = timing::try_incremental_cache(); @@ -52,7 +54,7 @@ impl Context { let info = compiled_code.code_info(); if isa.flags().enable_incremental_compilation_cache_checks() { - let actual_result = self.compile(isa)?; + let actual_result = self.compile(isa, ctrl_plane)?; assert_eq!(*actual_result, compiled_code); assert_eq!(actual_result.code_info(), info); // no need to set `compiled_code` here, it's set by `compile()`. @@ -71,10 +73,12 @@ impl Context { cache_key_hash }; - let stencil = self.compile_stencil(isa).map_err(|err| CompileError { - inner: err, - func: &self.func, - })?; + let stencil = self + .compile_stencil(isa, ctrl_plane) + .map_err(|err| CompileError { + inner: err, + func: &self.func, + })?; let stencil = { let _tt = timing::store_incremental_cache(); diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 808cae255f..da0cb5b63a 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -1,5 +1,6 @@ //! AArch64 ISA: binary code emission. +use cranelift_control::ControlPlane; use regalloc2::Allocation; use crate::binemit::{Reloc, StackMap}; @@ -638,15 +639,19 @@ pub struct EmitState { stack_map: Option, /// Current source-code location corresponding to instruction to be emitted. cur_srcloc: RelSourceLoc, + /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and + /// optimized away at compiletime. See [cranelift_control]. + ctrl_plane: ControlPlane, } impl MachInstEmitState for EmitState { - fn new(abi: &Callee) -> Self { + fn new(abi: &Callee, ctrl_plane: ControlPlane) -> Self { EmitState { virtual_sp_offset: 0, nominal_sp_to_fp: abi.frame_size() as i64, stack_map: None, cur_srcloc: Default::default(), + ctrl_plane, } } @@ -657,6 +662,14 @@ impl MachInstEmitState for EmitState { fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } + + fn ctrl_plane_mut(&mut self) -> &mut ControlPlane { + &mut self.ctrl_plane + } + + fn take_ctrl_plane(self) -> ControlPlane { + self.ctrl_plane + } } impl EmitState { @@ -1511,7 +1524,7 @@ impl MachInstEmit for Inst { let again_label = sink.get_label(); // again: - sink.bind_label(again_label); + sink.bind_label(again_label, &mut state.ctrl_plane); let srcloc = state.cur_srcloc(); if !srcloc.is_default() && !flags.notrap() { @@ -1713,7 +1726,7 @@ impl MachInstEmit for Inst { let out_label = sink.get_label(); // again: - sink.bind_label(again_label); + sink.bind_label(again_label, &mut state.ctrl_plane); let srcloc = state.cur_srcloc(); if !srcloc.is_default() && !flags.notrap() { @@ -1762,7 +1775,7 @@ impl MachInstEmit for Inst { sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19); // out: - sink.bind_label(out_label); + sink.bind_label(out_label, &mut state.ctrl_plane); } &Inst::LoadAcquire { access_ty, @@ -3007,13 +3020,13 @@ impl MachInstEmit for Inst { sink.put4(enc_jump26(0b000101, 0 /* will be fixed up later */)); // else: - sink.bind_label(else_label); + sink.bind_label(else_label, &mut state.ctrl_plane); // mov rd, rn sink.put4(enc_vecmov(/* 16b = */ true, rd, rn)); // out: - sink.bind_label(out_label); + sink.bind_label(out_label, &mut state.ctrl_plane); } &Inst::MovToNZCV { rn } => { let rn = allocs.next(rn); @@ -3464,8 +3477,8 @@ impl MachInstEmit for Inst { dest: BranchTarget::Label(jump_around_label), }; jmp.emit(&[], sink, emit_info, state); - sink.emit_island(needed_space + 4); - sink.bind_label(jump_around_label); + sink.emit_island(needed_space + 4, &mut state.ctrl_plane); + sink.bind_label(jump_around_label, &mut state.ctrl_plane); } } @@ -3579,7 +3592,7 @@ impl MachInstEmit for Inst { // out at this time. let loop_start = sink.get_label(); - sink.bind_label(loop_start); + sink.bind_label(loop_start, &mut state.ctrl_plane); Inst::AluRRImm12 { alu_op: ALUOp::Sub, @@ -3614,7 +3627,7 @@ impl MachInstEmit for Inst { kind: CondBrKind::Cond(Cond::Gt), } .emit(&[], sink, emit_info, state); - sink.bind_label(loop_end); + sink.bind_label(loop_end, &mut state.ctrl_plane); } } diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index fa5ab5068f..3e57cd9768 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -7811,7 +7811,7 @@ fn test_aarch64_binemit() { let mut buffer = MachBuffer::new(); insn.emit(&[], &mut buffer, &emit_info, &mut Default::default()); - let buffer = buffer.finish(); + let buffer = buffer.finish(&mut Default::default()); let actual_encoding = &buffer.stringify_code_bytes(); assert_eq!(expected_encoding, actual_encoding); } diff --git a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs index f3b69597f9..0f2300a1df 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs @@ -91,7 +91,9 @@ mod tests { Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), )); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) @@ -130,7 +132,9 @@ mod tests { let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index 65c628b430..a5403ca5c3 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -15,6 +15,7 @@ use crate::result::CodegenResult; use crate::settings as shared_settings; use alloc::{boxed::Box, vec::Vec}; use core::fmt; +use cranelift_control::ControlPlane; use regalloc2::MachineEnv; use target_lexicon::{Aarch64Architecture, Architecture, OperatingSystem, Triple}; @@ -72,6 +73,7 @@ impl TargetIsa for AArch64Backend { func: &Function, domtree: &DominatorTree, want_disasm: bool, + ctrl_plane: &mut ControlPlane, ) -> CodegenResult { let (vcode, regalloc_result) = self.compile_vcode(func, domtree)?; @@ -79,10 +81,11 @@ impl TargetIsa for AArch64Backend { ®alloc_result, want_disasm, self.flags.machine_code_cfg_info(), + ctrl_plane, ); let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; - let buffer = emit_result.buffer.finish(); + let buffer = emit_result.buffer.finish(ctrl_plane); let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 3baf230f7a..da778c9b61 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -57,6 +57,7 @@ use crate::CodegenResult; use alloc::{boxed::Box, sync::Arc, vec::Vec}; use core::fmt; use core::fmt::{Debug, Formatter}; +use cranelift_control::ControlPlane; use target_lexicon::{triple, Architecture, PointerWidth, Triple}; // This module is made public here for benchmarking purposes. No guarantees are @@ -273,6 +274,7 @@ pub trait TargetIsa: fmt::Display + Send + Sync { func: &Function, domtree: &DominatorTree, want_disasm: bool, + ctrl_plane: &mut ControlPlane, ) -> CodegenResult; #[cfg(feature = "unwind")] diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index 3d2585930d..0db271046b 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -6,6 +6,7 @@ use crate::ir::TrapCode; use crate::isa::riscv64::inst::*; use crate::isa::riscv64::inst::{zero_reg, AluOPRRR}; use crate::machinst::{AllocationConsumer, Reg, Writable}; +use cranelift_control::ControlPlane; use regalloc2::Allocation; pub struct EmitInfo { @@ -110,6 +111,9 @@ pub struct EmitState { stack_map: Option, /// Current source-code location corresponding to instruction to be emitted. cur_srcloc: RelSourceLoc, + /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and + /// optimized away at compiletime. See [cranelift_control]. + ctrl_plane: ControlPlane, } impl EmitState { @@ -127,12 +131,16 @@ impl EmitState { } impl MachInstEmitState for EmitState { - fn new(abi: &Callee) -> Self { + fn new( + abi: &Callee, + ctrl_plane: ControlPlane, + ) -> Self { EmitState { virtual_sp_offset: 0, nominal_sp_to_fp: abi.frame_size() as i64, stack_map: None, cur_srcloc: RelSourceLoc::default(), + ctrl_plane, } } @@ -143,6 +151,14 @@ impl MachInstEmitState for EmitState { fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } + + fn ctrl_plane_mut(&mut self) -> &mut ControlPlane { + &mut self.ctrl_plane + } + + fn take_ctrl_plane(self) -> ControlPlane { + self.ctrl_plane + } } impl Inst { @@ -997,7 +1013,7 @@ impl MachInstEmit for Inst { // we need to emit a jump table here to support that jump. let distance = (targets.len() * 2 * Inst::INSTRUCTION_SIZE as usize) as u32; if sink.island_needed(distance) { - sink.emit_island(distance); + sink.emit_island(distance, &mut state.ctrl_plane); } // Emit the jumps back to back @@ -1125,13 +1141,13 @@ impl MachInstEmit for Inst { insts .drain(..) .for_each(|i: Inst| i.emit(&[], sink, emit_info, state)); - sink.bind_label(label_false); + sink.bind_label(label_false, &mut state.ctrl_plane); // select second value1 insts.extend(gen_moves(&dst[..], y.regs())); insts .into_iter() .for_each(|i| i.emit(&[], sink, emit_info, state)); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::Jalr { rd, base, offset } => { let rd = allocs.next_writable(rd); @@ -1167,13 +1183,13 @@ impl MachInstEmit for Inst { .into_iter() .for_each(|i| i.emit(&[], sink, emit_info, state)); - sink.bind_label(label_true); + sink.bind_label(label_true, &mut state.ctrl_plane); Inst::load_imm12(rd, Imm12::TRUE).emit(&[], sink, emit_info, state); Inst::Jal { dest: BranchTarget::offset(Inst::INSTRUCTION_SIZE * 2), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_false); + sink.bind_label(label_false, &mut state.ctrl_plane); Inst::load_imm12(rd, Imm12::FALSE).emit(&[], sink, emit_info, state); } &Inst::AtomicCas { @@ -1204,7 +1220,7 @@ impl MachInstEmit for Inst { // fail: let fail_label = sink.get_label(); let cas_lebel = sink.get_label(); - sink.bind_label(cas_lebel); + sink.bind_label(cas_lebel, &mut state.ctrl_plane); Inst::Atomic { op: AtomicOP::load_op(ty), rd: dst, @@ -1274,7 +1290,7 @@ impl MachInstEmit for Inst { }, } .emit(&[], sink, emit_info, state); - sink.bind_label(fail_label); + sink.bind_label(fail_label, &mut state.ctrl_plane); } &Inst::AtomicRmwLoop { offset, @@ -1291,7 +1307,7 @@ impl MachInstEmit for Inst { let t0 = allocs.next_writable(t0); let dst = allocs.next_writable(dst); let retry = sink.get_label(); - sink.bind_label(retry); + sink.bind_label(retry, &mut state.ctrl_plane); // load old value. Inst::Atomic { op: AtomicOP::load_op(ty), @@ -1419,9 +1435,9 @@ impl MachInstEmit for Inst { dest: BranchTarget::Label(label_select_done), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_select_dst); + sink.bind_label(label_select_dst, &mut state.ctrl_plane); Inst::gen_move(t0, dst.to_reg(), I64).emit(&[], sink, emit_info, state); - sink.bind_label(label_select_done); + sink.bind_label(label_select_done, &mut state.ctrl_plane); Inst::Atomic { op: AtomicOP::load_op(ty), rd: writable_spilltmp_reg2(), @@ -1534,13 +1550,13 @@ impl MachInstEmit for Inst { .for_each(|i| i.emit(&[], sink, emit_info, state)); }; //here is true , use x. - sink.bind_label(label_true); + sink.bind_label(label_true, &mut state.ctrl_plane); gen_move(&dst, &x, sink, state); Inst::gen_jump(label_done).emit(&[], sink, emit_info, state); // here is false use y - sink.bind_label(label_false); + sink.bind_label(label_false, &mut state.ctrl_plane); gen_move(&dst, &y, sink, state); - sink.bind_label(label_done); + sink.bind_label(label_done, &mut state.ctrl_plane); } &Inst::Csr { csr_op, @@ -1585,9 +1601,9 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // here condition is true , use rs1 - sink.bind_label(label_true); + sink.bind_label(label_true, &mut state.ctrl_plane); Inst::gen_move(rd, rs1, I64).emit(&[], sink, emit_info, state); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::FcvtToInt { is_sat, @@ -1747,7 +1763,7 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // here is nan , move 0 into rd register - sink.bind_label(label_nan); + sink.bind_label(label_nan, &mut state.ctrl_plane); if is_sat { Inst::load_imm12(rd, Imm12::from_bits(0)).emit(&[], sink, emit_info, state); } else { @@ -1758,7 +1774,7 @@ impl MachInstEmit for Inst { .emit(&[], sink, emit_info, state); } // bind jump_over - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::LoadExtName { @@ -1812,12 +1828,12 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // trap - sink.bind_label(label_trap); + sink.bind_label(label_trap, &mut state.ctrl_plane); Inst::Udf { trap_code: trap_code, } .emit(&[], sink, emit_info, state); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::TrapIf { test, trap_code } => { let test = allocs.next(test); @@ -1834,12 +1850,12 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // trap - sink.bind_label(label_trap); + sink.bind_label(label_trap, &mut state.ctrl_plane); Inst::Udf { trap_code: trap_code, } .emit(&[], sink, emit_info, state); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::Udf { trap_code } => { sink.add_trap(trap_code); @@ -1874,7 +1890,7 @@ impl MachInstEmit for Inst { .emit(&[], sink, emit_info, state); // here select x. - sink.bind_label(label_select_x); + sink.bind_label(label_select_x, &mut state.ctrl_plane); gen_moves(&rd[..], x.regs()) .into_iter() .for_each(|i| i.emit(&[], sink, emit_info, state)); @@ -1884,11 +1900,11 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // here select y. - sink.bind_label(label_select_y); + sink.bind_label(label_select_y, &mut state.ctrl_plane); gen_moves(&rd[..], y.regs()) .into_iter() .for_each(|i| i.emit(&[], sink, emit_info, state)); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::AtomicLoad { rd, ty, p } => { let p = allocs.next(p); @@ -2049,7 +2065,7 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // here is nan. - sink.bind_label(label_nan); + sink.bind_label(label_nan, &mut state.ctrl_plane); Inst::FpuRRR { alu_op: if ty == F32 { FpuOPRRR::FaddS @@ -2067,9 +2083,9 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // here select origin x. - sink.bind_label(label_x); + sink.bind_label(label_x, &mut state.ctrl_plane); Inst::gen_move(rd, rs, ty).emit(&[], sink, emit_info, state); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::FloatSelectPseudo { op, @@ -2114,9 +2130,9 @@ impl MachInstEmit for Inst { dest: BranchTarget::Label(label_jump_over), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_rs2); + sink.bind_label(label_rs2, &mut state.ctrl_plane); Inst::gen_move(rd, rs2, ty).emit(&[], sink, emit_info, state); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::FloatSelect { @@ -2224,7 +2240,7 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // - sink.bind_label(label_done); + sink.bind_label(label_done, &mut state.ctrl_plane); } // we have the reuslt,jump over. Inst::Jal { @@ -2232,7 +2248,7 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); // here is nan. - sink.bind_label(label_nan); + sink.bind_label(label_nan, &mut state.ctrl_plane); op.snan_bits(tmp, ty) .into_iter() .for_each(|i| i.emit(&[], sink, emit_info, state)); @@ -2244,7 +2260,7 @@ impl MachInstEmit for Inst { rs: tmp.to_reg(), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_jump_over); + sink.bind_label(label_jump_over, &mut state.ctrl_plane); } &Inst::Popcnt { sum, @@ -2277,7 +2293,7 @@ impl MachInstEmit for Inst { .emit(&[], sink, emit_info, state); let label_done = sink.get_label(); let label_loop = sink.get_label(); - sink.bind_label(label_loop); + sink.bind_label(label_loop, &mut state.ctrl_plane); Inst::CondBr { taken: BranchTarget::Label(label_done), not_taken: BranchTarget::zero(), @@ -2315,7 +2331,7 @@ impl MachInstEmit for Inst { imm12: Imm12::from_bits(1), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_over); + sink.bind_label(label_over, &mut state.ctrl_plane); } // set step and tmp. { @@ -2338,7 +2354,7 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); } - sink.bind_label(label_done); + sink.bind_label(label_done, &mut state.ctrl_plane); } &Inst::Rev8 { rs, rd, tmp, step } => { let rs = allocs.next(rs); @@ -2352,7 +2368,7 @@ impl MachInstEmit for Inst { Inst::load_imm12(step, Imm12::from_bits(56)).emit(&[], sink, emit_info, state); let label_done = sink.get_label(); let label_loop = sink.get_label(); - sink.bind_label(label_loop); + sink.bind_label(label_loop, &mut state.ctrl_plane); Inst::CondBr { taken: BranchTarget::Label(label_done), not_taken: BranchTarget::zero(), @@ -2408,7 +2424,7 @@ impl MachInstEmit for Inst { } } .emit(&[], sink, emit_info, state); - sink.bind_label(label_done); + sink.bind_label(label_done, &mut state.ctrl_plane); } &Inst::Cltz { sum, @@ -2444,7 +2460,7 @@ impl MachInstEmit for Inst { } let label_done = sink.get_label(); let label_loop = sink.get_label(); - sink.bind_label(label_loop); + sink.bind_label(label_loop, &mut state.ctrl_plane); Inst::CondBr { taken: BranchTarget::Label(label_done), not_taken: BranchTarget::zero(), @@ -2507,7 +2523,7 @@ impl MachInstEmit for Inst { } .emit(&[], sink, emit_info, state); } - sink.bind_label(label_done); + sink.bind_label(label_done, &mut state.ctrl_plane); } &Inst::Brev8 { rs, @@ -2549,7 +2565,7 @@ impl MachInstEmit for Inst { let label_done = sink.get_label(); let label_loop = sink.get_label(); - sink.bind_label(label_loop); + sink.bind_label(label_loop, &mut state.ctrl_plane); Inst::CondBr { taken: BranchTarget::Label(label_done), not_taken: BranchTarget::zero(), @@ -2587,7 +2603,7 @@ impl MachInstEmit for Inst { rs2: tmp2.to_reg(), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_over); + sink.bind_label(label_over, &mut state.ctrl_plane); } // set step and tmp. { @@ -2645,7 +2661,7 @@ impl MachInstEmit for Inst { dest: BranchTarget::Label(label_over), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_sll_1); + sink.bind_label(label_sll_1, &mut state.ctrl_plane); Inst::AluRRImm12 { alu_op: AluOPRRI::Slli, rd: tmp2, @@ -2653,14 +2669,14 @@ impl MachInstEmit for Inst { imm12: Imm12::from_bits(1), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_over); + sink.bind_label(label_over, &mut state.ctrl_plane); } Inst::Jal { dest: BranchTarget::Label(label_loop), } .emit(&[], sink, emit_info, state); } - sink.bind_label(label_done); + sink.bind_label(label_done, &mut state.ctrl_plane); } &Inst::StackProbeLoop { guard_size, @@ -2681,7 +2697,7 @@ impl MachInstEmit for Inst { let loop_start = sink.get_label(); let label_done = sink.get_label(); - sink.bind_label(loop_start); + sink.bind_label(loop_start, &mut state.ctrl_plane); Inst::CondBr { taken: BranchTarget::Label(label_done), not_taken: BranchTarget::zero(), @@ -2719,7 +2735,7 @@ impl MachInstEmit for Inst { dest: BranchTarget::Label(loop_start), } .emit(&[], sink, emit_info, state); - sink.bind_label(label_done); + sink.bind_label(label_done, &mut state.ctrl_plane); } }; let end_off = sink.cur_offset(); diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs b/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs index 261f9c2b50..32f49c8427 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs @@ -2077,7 +2077,7 @@ fn test_riscv64_binemit() { let mut buffer = MachBuffer::new(); unit.inst .emit(&[], &mut buffer, &emit_info, &mut Default::default()); - let buffer = buffer.finish(); + let buffer = buffer.finish(&mut Default::default()); if buffer.data() != unit.code.to_le_bytes() { { let gnu = DebugRTypeInst::from_bs(&unit.code.to_le_bytes()); @@ -2304,7 +2304,7 @@ fn riscv64_worst_case_instruction_size() { for i in candidates { let mut buffer = MachBuffer::new(); i.emit(&[], &mut buffer, &emit_info, &mut Default::default()); - let buffer = buffer.finish(); + let buffer = buffer.finish(&mut Default::default()); let length = buffer.data().len() as u32; if length > max.0 { let length = buffer.data().len() as u32; diff --git a/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs index 1803ecfec1..b7457dd923 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs @@ -89,7 +89,9 @@ mod tests { Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), )); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) @@ -129,7 +131,9 @@ mod tests { let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs index 69711e0c82..f567481e92 100644 --- a/cranelift/codegen/src/isa/riscv64/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/mod.rs @@ -15,6 +15,7 @@ use crate::result::CodegenResult; use crate::settings as shared_settings; use alloc::{boxed::Box, vec::Vec}; use core::fmt; +use cranelift_control::ControlPlane; use regalloc2::MachineEnv; use target_lexicon::{Architecture, Triple}; mod abi; @@ -72,6 +73,7 @@ impl TargetIsa for Riscv64Backend { func: &Function, domtree: &DominatorTree, want_disasm: bool, + ctrl_plane: &mut ControlPlane, ) -> CodegenResult { let (vcode, regalloc_result) = self.compile_vcode(func, domtree)?; @@ -80,10 +82,11 @@ impl TargetIsa for Riscv64Backend { ®alloc_result, want_disasm, self.flags.machine_code_cfg_info(), + ctrl_plane, ); let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; - let buffer = emit_result.buffer.finish(); + let buffer = emit_result.buffer.finish(ctrl_plane); let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index 4ac01db596..4754928415 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -8,6 +8,7 @@ use crate::isa::s390x::settings as s390x_settings; use crate::machinst::{Reg, RegClass}; use crate::trace; use core::convert::TryFrom; +use cranelift_control::ControlPlane; use regalloc2::Allocation; /// Debug macro for testing that a regpair is valid: that the high register is even, and the low @@ -1347,15 +1348,19 @@ pub struct EmitState { stack_map: Option, /// Current source-code location corresponding to instruction to be emitted. cur_srcloc: RelSourceLoc, + /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and + /// optimized away at compiletime. See [cranelift_control]. + ctrl_plane: ControlPlane, } impl MachInstEmitState for EmitState { - fn new(abi: &Callee) -> Self { + fn new(abi: &Callee, ctrl_plane: ControlPlane) -> Self { EmitState { virtual_sp_offset: 0, initial_sp_offset: abi.frame_size() as i64, stack_map: None, cur_srcloc: Default::default(), + ctrl_plane, } } @@ -1366,6 +1371,14 @@ impl MachInstEmitState for EmitState { fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } + + fn ctrl_plane_mut(&mut self) -> &mut ControlPlane { + &mut self.ctrl_plane + } + + fn take_ctrl_plane(self) -> ControlPlane { + self.ctrl_plane + } } impl EmitState { @@ -2137,7 +2150,7 @@ impl Inst { let done_label = sink.get_label(); // Emit label at the start of the loop. - sink.bind_label(loop_label); + sink.bind_label(loop_label, &mut state.ctrl_plane); for inst in (&body).into_iter() { match &inst { @@ -2160,7 +2173,7 @@ impl Inst { inst.emit(&[], sink, emit_info, state); // Emit label at the end of the loop. - sink.bind_label(done_label); + sink.bind_label(done_label, &mut state.ctrl_plane); } &Inst::CondBreak { .. } => unreachable!(), // Only valid inside a Loop. &Inst::AtomicCas32 { @@ -3646,7 +3659,7 @@ impl Inst { // The first entry is the default target, which is not emitted // into the jump table, so we skip it here. It is only in the // list so MachTerminator will see the potential target. - sink.bind_label(table_label); + sink.bind_label(table_label, &mut state.ctrl_plane); let jt_off = sink.cur_offset(); for &target in targets.iter().skip(1) { let word_off = sink.cur_offset(); diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs index c33f48964d..3f74c740bc 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs @@ -13363,6 +13363,7 @@ fn test_s390x_binemit() { let mut isa_flag_builder = s390x_settings::builder(); isa_flag_builder.enable("arch13").unwrap(); let isa_flags = s390x_settings::Flags::new(&flags, &isa_flag_builder); + let ctrl_plane = &mut Default::default(); let emit_info = EmitInfo::new(isa_flags); for (insn, expected_encoding, expected_printing) in insns { @@ -13380,16 +13381,16 @@ fn test_s390x_binemit() { // Label 0 before the instruction. let label0 = buffer.get_label(); - buffer.bind_label(label0); + buffer.bind_label(label0, ctrl_plane); // Emit the instruction. insn.emit(&[], &mut buffer, &emit_info, &mut Default::default()); // Label 1 after the instruction. let label1 = buffer.get_label(); - buffer.bind_label(label1); + buffer.bind_label(label1, ctrl_plane); - let buffer = buffer.finish(); + let buffer = buffer.finish(ctrl_plane); let actual_encoding = &buffer.stringify_code_bytes(); assert_eq!(expected_encoding, actual_encoding); } diff --git a/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs index 9f29b7d414..52d57118ed 100644 --- a/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs @@ -122,7 +122,9 @@ mod tests { Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), )); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) @@ -164,7 +166,9 @@ mod tests { Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), )); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs index 9ba81d14ac..d2705e78cc 100644 --- a/cranelift/codegen/src/isa/s390x/mod.rs +++ b/cranelift/codegen/src/isa/s390x/mod.rs @@ -15,6 +15,7 @@ use crate::result::CodegenResult; use crate::settings as shared_settings; use alloc::{boxed::Box, vec::Vec}; use core::fmt; +use cranelift_control::ControlPlane; use regalloc2::MachineEnv; use target_lexicon::{Architecture, Triple}; @@ -72,14 +73,20 @@ impl TargetIsa for S390xBackend { func: &Function, domtree: &DominatorTree, want_disasm: bool, + ctrl_plane: &mut ControlPlane, ) -> CodegenResult { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func, domtree)?; - let emit_result = vcode.emit(®alloc_result, want_disasm, flags.machine_code_cfg_info()); + let emit_result = vcode.emit( + ®alloc_result, + want_disasm, + flags.machine_code_cfg_info(), + ctrl_plane, + ); let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; - let buffer = emit_result.buffer.finish(); + let buffer = emit_result.buffer.finish(ctrl_plane); let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; diff --git a/cranelift/codegen/src/isa/x64/encoding/vex.rs b/cranelift/codegen/src/isa/x64/encoding/vex.rs index 760c7199d1..a25e9c17a2 100644 --- a/cranelift/codegen/src/isa/x64/encoding/vex.rs +++ b/cranelift/codegen/src/isa/x64/encoding/vex.rs @@ -357,7 +357,7 @@ mod tests { .imm(0x17) .encode(&mut sink); - let bytes = sink.finish().data; + let bytes = sink.finish(&mut Default::default()).data; assert_eq!(bytes.as_slice(), [0xc5, 0xf1, 0x73, 0xfa, 0x17]); } @@ -385,7 +385,7 @@ mod tests { .imm_reg(c) .encode(&mut sink); - let bytes = sink.finish().data; + let bytes = sink.finish(&mut Default::default()).data; assert_eq!(bytes.as_slice(), [0xc4, 0xe3, 0x69, 0x4b, 0xcb, 0x40]); } @@ -410,7 +410,7 @@ mod tests { .imm(4) .encode(&mut sink); - let bytes = sink.finish().data; + let bytes = sink.finish(&mut Default::default()).data; assert_eq!(bytes.as_slice(), [0xc4, 0x41, 0x24, 0xc2, 0xd4, 0x04]); } @@ -434,7 +434,7 @@ mod tests { .rm(src2) .encode(&mut sink); - let bytes = sink.finish().data; + let bytes = sink.finish(&mut Default::default()).data; assert_eq!(bytes.as_slice(), [0xc5, 0xf0, 0x55, 0xd0]); } @@ -462,7 +462,7 @@ mod tests { .rm(src2) .encode(&mut sink); - let bytes = sink.finish().data; + let bytes = sink.finish(&mut Default::default()).data; assert_eq!(bytes.as_slice(), [0xc4, 0xc1, 0x70, 0x55, 0x55, 0x0a]); } @@ -492,7 +492,7 @@ mod tests { .rm(src2) .encode(&mut sink); - let bytes = sink.finish().data; + let bytes = sink.finish(&mut Default::default()).data; assert_eq!(bytes.as_slice(), [0xc4, 0xa1, 0x70, 0x55, 0x54, 0xa8, 100]); } } diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index e2ca40cfe4..c2be0409e7 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -620,7 +620,7 @@ pub(crate) fn emit( // Here the `idiv` is executed, which is different depending on the // size - sink.bind_label(do_op); + sink.bind_label(do_op, &mut state.ctrl_plane); let inst = match size { OperandSize::Size8 => Inst::div8( DivSignedness::Signed, @@ -642,7 +642,7 @@ pub(crate) fn emit( }; inst.emit(&[], sink, info, state); - sink.bind_label(done_label); + sink.bind_label(done_label, &mut state.ctrl_plane); } Inst::Imm { @@ -1291,7 +1291,7 @@ pub(crate) fn emit( let inst = Inst::xmm_unary_rm_r(op, consequent, Writable::from_reg(dst)); inst.emit(&[], sink, info, state); - sink.bind_label(next); + sink.bind_label(next, &mut state.ctrl_plane); } Inst::Push64 { src } => { @@ -1395,7 +1395,7 @@ pub(crate) fn emit( // Emit the main loop! let loop_start = sink.get_label(); - sink.bind_label(loop_start); + sink.bind_label(loop_start, &mut state.ctrl_plane); // sub rsp, GUARD_SIZE let inst = Inst::alu_rmi_r( @@ -1666,7 +1666,7 @@ pub(crate) fn emit( inst.emit(&[], sink, info, state); // Emit jump table (table of 32-bit offsets). - sink.bind_label(start_of_jumptable); + sink.bind_label(start_of_jumptable, &mut state.ctrl_plane); let jt_off = sink.cur_offset(); for &target in targets.iter().chain(std::iter::once(default_target)) { let word_off = sink.cur_offset(); @@ -1698,7 +1698,7 @@ pub(crate) fn emit( one_way_jmp(sink, cc1.invert(), else_label); one_way_jmp(sink, *cc2, trap_label); - sink.bind_label(else_label); + sink.bind_label(else_label, &mut state.ctrl_plane); } Inst::TrapIfOr { @@ -2750,18 +2750,18 @@ pub(crate) fn emit( // x86's min/max are not symmetric; if either operand is a NaN, they return the // read-only operand: perform an addition between the two operands, which has the // desired NaN propagation effects. - sink.bind_label(propagate_nan); + sink.bind_label(propagate_nan, &mut state.ctrl_plane); let inst = Inst::xmm_rm_r(add_op, RegMem::reg(lhs), Writable::from_reg(dst)); inst.emit(&[], sink, info, state); one_way_jmp(sink, CC::P, done); - sink.bind_label(do_min_max); + sink.bind_label(do_min_max, &mut state.ctrl_plane); let inst = Inst::xmm_rm_r(min_max_op, RegMem::reg(lhs), Writable::from_reg(dst)); inst.emit(&[], sink, info, state); - sink.bind_label(done); + sink.bind_label(done, &mut state.ctrl_plane); } Inst::XmmRmRImm { @@ -3028,7 +3028,7 @@ pub(crate) fn emit( let inst = Inst::jmp_known(done); inst.emit(&[], sink, info, state); - sink.bind_label(handle_negative); + sink.bind_label(handle_negative, &mut state.ctrl_plane); // Divide x by two to get it in range for the signed conversion, keep the LSB, and // scale it back up on the FP side. @@ -3081,7 +3081,7 @@ pub(crate) fn emit( let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst), Writable::from_reg(dst)); inst.emit(&[], sink, info, state); - sink.bind_label(done); + sink.bind_label(done, &mut state.ctrl_plane); } Inst::CvtFloatToSintSeq { @@ -3183,7 +3183,7 @@ pub(crate) fn emit( let inst = Inst::jmp_known(done); inst.emit(&[], sink, info, state); - sink.bind_label(not_nan); + sink.bind_label(not_nan, &mut state.ctrl_plane); // If the input was positive, saturate to INT_MAX. @@ -3281,7 +3281,7 @@ pub(crate) fn emit( inst.emit(&[], sink, info, state); } - sink.bind_label(done); + sink.bind_label(done, &mut state.ctrl_plane); } Inst::CvtFloatToUintSeq { @@ -3391,7 +3391,7 @@ pub(crate) fn emit( let inst = Inst::jmp_known(done); inst.emit(&[], sink, info, state); - sink.bind_label(not_nan); + sink.bind_label(not_nan, &mut state.ctrl_plane); } else { // Trap. let inst = Inst::trap_if(CC::P, TrapCode::BadConversionToInteger); @@ -3430,7 +3430,7 @@ pub(crate) fn emit( // Now handle large inputs. - sink.bind_label(handle_large); + sink.bind_label(handle_large, &mut state.ctrl_plane); let inst = Inst::gen_move(Writable::from_reg(tmp_xmm2), src, types::F64); inst.emit(&[], sink, info, state); @@ -3463,7 +3463,7 @@ pub(crate) fn emit( let inst = Inst::jmp_known(done); inst.emit(&[], sink, info, state); - sink.bind_label(next_is_large); + sink.bind_label(next_is_large, &mut state.ctrl_plane); } else { let inst = Inst::trap_if(CC::L, TrapCode::IntegerOverflow); inst.emit(&[], sink, info, state); @@ -3490,7 +3490,7 @@ pub(crate) fn emit( inst.emit(&[], sink, info, state); } - sink.bind_label(done); + sink.bind_label(done, &mut state.ctrl_plane); } Inst::LoadExtName { dst, name, offset } => { @@ -3600,7 +3600,7 @@ pub(crate) fn emit( i1.emit(&[], sink, info, state); // again: - sink.bind_label(again_label); + sink.bind_label(again_label, &mut state.ctrl_plane); // movq %rax, %r_temp let i2 = Inst::mov_r_r(OperandSize::Size64, dst_old.to_reg(), temp); diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index ee759d1a10..ef6721b8b1 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -5098,6 +5098,7 @@ fn test_x64_emit() { // ======================================================== // Actually run the tests! + let ctrl_plane = &mut Default::default(); let mut flag_builder = settings::builder(); flag_builder.enable("is_pic").unwrap(); let flags = settings::Flags::new(flag_builder); @@ -5126,9 +5127,9 @@ fn test_x64_emit() { // Allow one label just after the instruction (so the offset is 0). let label = buffer.get_label(); - buffer.bind_label(label); + buffer.bind_label(label, ctrl_plane); - let buffer = buffer.finish(); + let buffer = buffer.finish(ctrl_plane); let actual_encoding = &buffer.stringify_code_bytes(); assert_eq!(expected_encoding, actual_encoding, "{}", expected_printing); } diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 723cffb10a..984cbbe8fb 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -10,6 +10,7 @@ use crate::{machinst::*, trace}; use crate::{settings, CodegenError, CodegenResult}; use alloc::boxed::Box; use alloc::vec::Vec; +use cranelift_control::ControlPlane; use regalloc2::{Allocation, PRegSet, VReg}; use smallvec::{smallvec, SmallVec}; use std::fmt; @@ -2547,6 +2548,9 @@ pub struct EmitState { stack_map: Option, /// Current source location. cur_srcloc: RelSourceLoc, + /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and + /// optimized away at compiletime. See [cranelift_control]. + ctrl_plane: ControlPlane, } /// Constant state used during emissions of a sequence of instructions. @@ -2583,12 +2587,13 @@ impl MachInstEmit for Inst { } impl MachInstEmitState for EmitState { - fn new(abi: &Callee) -> Self { + fn new(abi: &Callee, ctrl_plane: ControlPlane) -> Self { EmitState { virtual_sp_offset: 0, nominal_sp_to_fp: abi.frame_size() as i64, stack_map: None, cur_srcloc: Default::default(), + ctrl_plane, } } @@ -2599,6 +2604,14 @@ impl MachInstEmitState for EmitState { fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } + + fn ctrl_plane_mut(&mut self) -> &mut ControlPlane { + &mut self.ctrl_plane + } + + fn take_ctrl_plane(self) -> ControlPlane { + self.ctrl_plane + } } impl EmitState { diff --git a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs index 6a11262104..fbda8dfcf6 100644 --- a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs @@ -118,7 +118,9 @@ mod tests { Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), )); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) @@ -157,7 +159,9 @@ mod tests { let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); - let code = context.compile(&*isa).expect("expected compilation"); + let code = context + .compile(&*isa, &mut Default::default()) + .expect("expected compilation"); let fde = match code .create_unwind_info(isa.as_ref()) diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index dd84f4395a..029c7f9f57 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -17,6 +17,7 @@ use crate::result::{CodegenError, CodegenResult}; use crate::settings::{self as shared_settings, Flags}; use alloc::{boxed::Box, vec::Vec}; use core::fmt; +use cranelift_control::ControlPlane; use regalloc2::MachineEnv; use target_lexicon::Triple; @@ -66,6 +67,7 @@ impl TargetIsa for X64Backend { func: &Function, domtree: &DominatorTree, want_disasm: bool, + ctrl_plane: &mut ControlPlane, ) -> CodegenResult { let (vcode, regalloc_result) = self.compile_vcode(func, domtree)?; @@ -73,10 +75,11 @@ impl TargetIsa for X64Backend { ®alloc_result, want_disasm, self.flags.machine_code_cfg_info(), + ctrl_plane, ); let frame_size = emit_result.frame_size; let value_labels_ranges = emit_result.value_labels_ranges; - let buffer = emit_result.buffer.finish(); + let buffer = emit_result.buffer.finish(ctrl_plane); let sized_stackslot_offsets = emit_result.sized_stackslot_offsets; let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets; diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index a5c82c7619..39c0f14809 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -92,8 +92,8 @@ pub use crate::machinst::buffer::{ MachCallSite, MachReloc, MachSrcLoc, MachStackMap, MachTextSectionBuilder, MachTrap, }; pub use crate::machinst::{ - CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, Reg, - TextSectionBuilder, Writable, + CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, + MachInstEmitState, Reg, TextSectionBuilder, Writable, }; mod alias_analysis; diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 895f5e5ffd..bbb03dabde 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -148,6 +148,7 @@ use crate::machinst::{ }; use crate::timing; use crate::trace; +use cranelift_control::ControlPlane; use cranelift_entity::{entity_impl, SecondaryMap}; use smallvec::SmallVec; use std::convert::TryFrom; @@ -523,7 +524,7 @@ impl MachBuffer { } /// Bind a label to the current offset. A label can only be bound once. - pub fn bind_label(&mut self, label: MachLabel) { + pub fn bind_label(&mut self, label: MachLabel, ctrl_plane: &mut ControlPlane) { trace!( "MachBuffer: bind label {:?} at offset {}", label, @@ -542,7 +543,7 @@ impl MachBuffer { // offset and added it to the list (which contains all labels at the // current offset). - self.optimize_branches(); + self.optimize_branches(ctrl_plane); // Post-invariant: by `optimize_branches()` (see argument there). } @@ -773,7 +774,11 @@ impl MachBuffer { // fixup record referring to that last branch is removed. } - fn optimize_branches(&mut self) { + fn optimize_branches(&mut self, ctrl_plane: &mut ControlPlane) { + if ctrl_plane.get_decision() { + return; + } + self.lazily_clear_labels_at_tail(); // Invariants valid at this point. @@ -1141,13 +1146,18 @@ impl MachBuffer { /// Should only be called if `island_needed()` returns true, i.e., if we /// actually reach a deadline. It's not necessarily a problem to do so /// otherwise but it may result in unnecessary work during emission. - pub fn emit_island(&mut self, distance: CodeOffset) { - self.emit_island_maybe_forced(false, distance); + pub fn emit_island(&mut self, distance: CodeOffset, ctrl_plane: &mut ControlPlane) { + self.emit_island_maybe_forced(false, distance, ctrl_plane); } /// Same as `emit_island`, but an internal API with a `force_veneers` /// argument to force all veneers to always get emitted for debugging. - fn emit_island_maybe_forced(&mut self, force_veneers: bool, distance: CodeOffset) { + fn emit_island_maybe_forced( + &mut self, + force_veneers: bool, + distance: CodeOffset, + ctrl_plane: &mut ControlPlane, + ) { // We're going to purge fixups, so no latest-branch editing can happen // anymore. self.latest_branches.clear(); @@ -1190,7 +1200,7 @@ impl MachBuffer { self.start_srcloc(loc); } self.align_to(I::LabelUse::ALIGN); - self.bind_label(label); + self.bind_label(label, ctrl_plane); self.add_trap(code); if let Some(map) = stack_map { let extent = StackMapExtent::UpcomingBytes(I::TRAP_OPCODE.len() as u32); @@ -1204,7 +1214,7 @@ impl MachBuffer { for MachLabelConstant { label, align, data } in mem::take(&mut self.pending_constants) { self.align_to(align); - self.bind_label(label); + self.bind_label(label, ctrl_plane); self.put_data(&data[..]); } @@ -1328,7 +1338,11 @@ impl MachBuffer { self.use_label_at_offset(veneer_fixup_off, label, veneer_label_use); } - fn finish_emission_maybe_forcing_veneers(&mut self, force_veneers: bool) { + fn finish_emission_maybe_forcing_veneers( + &mut self, + force_veneers: bool, + ctrl_plane: &mut ControlPlane, + ) { while !self.pending_constants.is_empty() || !self.pending_traps.is_empty() || !self.fixup_records.is_empty() @@ -1336,7 +1350,7 @@ impl MachBuffer { // `emit_island()` will emit any pending veneers and constants, and // as a side-effect, will also take care of any fixups with resolved // labels eagerly. - self.emit_island_maybe_forced(force_veneers, u32::MAX); + self.emit_island_maybe_forced(force_veneers, u32::MAX, ctrl_plane); } // Ensure that all labels have been fixed up after the last island is emitted. This is a @@ -1346,14 +1360,14 @@ impl MachBuffer { } /// Finish any deferred emissions and/or fixups. - pub fn finish(mut self) -> MachBufferFinalized { + pub fn finish(mut self, ctrl_plane: &mut ControlPlane) -> MachBufferFinalized { let _tt = timing::vcode_emit_finish(); // Do any optimizations on branches at tail of buffer, as if we // had bound one last label. - self.optimize_branches(); + self.optimize_branches(ctrl_plane); - self.finish_emission_maybe_forcing_veneers(false); + self.finish_emission_maybe_forcing_veneers(false, ctrl_plane); let mut srclocs = self.srclocs; srclocs.sort_by_key(|entry| entry.start); @@ -1713,19 +1727,28 @@ impl MachTextSectionBuilder { } impl TextSectionBuilder for MachTextSectionBuilder { - fn append(&mut self, labeled: bool, func: &[u8], align: u32) -> u64 { + fn append( + &mut self, + labeled: bool, + func: &[u8], + align: u32, + ctrl_plane: &mut ControlPlane, + ) -> u64 { // Conditionally emit an island if it's necessary to resolve jumps // between functions which are too far away. let size = func.len() as u32; if self.force_veneers || self.buf.island_needed(size) { - self.buf.emit_island_maybe_forced(self.force_veneers, size); + self.buf + .emit_island_maybe_forced(self.force_veneers, size, ctrl_plane); } self.buf.align_to(align); let pos = self.buf.cur_offset(); if labeled { - self.buf - .bind_label(MachLabel::from_block(BlockIndex::new(self.next_func))); + self.buf.bind_label( + MachLabel::from_block(BlockIndex::new(self.next_func)), + ctrl_plane, + ); self.next_func += 1; } self.buf.put_data(func); @@ -1748,13 +1771,13 @@ impl TextSectionBuilder for MachTextSectionBuilder { self.force_veneers = true; } - fn finish(&mut self) -> Vec { + fn finish(&mut self, ctrl_plane: &mut ControlPlane) -> Vec { // Double-check all functions were pushed. assert_eq!(self.next_func, self.buf.label_offsets.len()); // Finish up any veneers, if necessary. self.buf - .finish_emission_maybe_forcing_veneers(self.force_veneers); + .finish_emission_maybe_forcing_veneers(self.force_veneers, ctrl_plane); // We don't need the data any more, so return it to the caller. mem::take(&mut self.buf.data).into_vec() @@ -1770,7 +1793,7 @@ mod test { use crate::ir::UserExternalNameRef; use crate::isa::aarch64::inst::xreg; use crate::isa::aarch64::inst::{BranchTarget, CondBrKind, EmitInfo, Inst}; - use crate::machinst::MachInstEmit; + use crate::machinst::{MachInstEmit, MachInstEmitState}; use crate::settings; use std::default::Default; use std::vec::Vec; @@ -1786,14 +1809,14 @@ mod test { fn test_elide_jump_to_next() { let info = EmitInfo::new(settings::Flags::new(settings::builder())); let mut buf = MachBuffer::new(); - let mut state = Default::default(); + let mut state = ::State::default(); buf.reserve_labels_for_blocks(2); - buf.bind_label(label(0)); + buf.bind_label(label(0), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(1) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(1)); - let buf = buf.finish(); + buf.bind_label(label(1), state.ctrl_plane_mut()); + let buf = buf.finish(state.ctrl_plane_mut()); assert_eq!(0, buf.total_size()); } @@ -1801,11 +1824,11 @@ mod test { fn test_elide_trivial_jump_blocks() { let info = EmitInfo::new(settings::Flags::new(settings::builder())); let mut buf = MachBuffer::new(); - let mut state = Default::default(); + let mut state = ::State::default(); buf.reserve_labels_for_blocks(4); - buf.bind_label(label(0)); + buf.bind_label(label(0), state.ctrl_plane_mut()); let inst = Inst::CondBr { kind: CondBrKind::NotZero(xreg(0)), taken: target(1), @@ -1813,17 +1836,17 @@ mod test { }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(1)); + buf.bind_label(label(1), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(3) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(2)); + buf.bind_label(label(2), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(3) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(3)); + buf.bind_label(label(3), state.ctrl_plane_mut()); - let buf = buf.finish(); + let buf = buf.finish(state.ctrl_plane_mut()); assert_eq!(0, buf.total_size()); } @@ -1831,11 +1854,11 @@ mod test { fn test_flip_cond() { let info = EmitInfo::new(settings::Flags::new(settings::builder())); let mut buf = MachBuffer::new(); - let mut state = Default::default(); + let mut state = ::State::default(); buf.reserve_labels_for_blocks(4); - buf.bind_label(label(0)); + buf.bind_label(label(0), state.ctrl_plane_mut()); let inst = Inst::CondBr { kind: CondBrKind::Zero(xreg(0)), taken: target(1), @@ -1843,19 +1866,19 @@ mod test { }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(1)); + buf.bind_label(label(1), state.ctrl_plane_mut()); let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(2)); + buf.bind_label(label(2), state.ctrl_plane_mut()); let inst = Inst::Udf { trap_code: TrapCode::Interrupt, }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(3)); + buf.bind_label(label(3), state.ctrl_plane_mut()); - let buf = buf.finish(); + let buf = buf.finish(state.ctrl_plane_mut()); let mut buf2 = MachBuffer::new(); let mut state = Default::default(); @@ -1867,7 +1890,7 @@ mod test { let inst = Inst::Nop4; inst.emit(&[], &mut buf2, &info, &mut state); - let buf2 = buf2.finish(); + let buf2 = buf2.finish(state.ctrl_plane_mut()); assert_eq!(buf.data, buf2.data); } @@ -1876,11 +1899,11 @@ mod test { fn test_island() { let info = EmitInfo::new(settings::Flags::new(settings::builder())); let mut buf = MachBuffer::new(); - let mut state = Default::default(); + let mut state = ::State::default(); buf.reserve_labels_for_blocks(4); - buf.bind_label(label(0)); + buf.bind_label(label(0), state.ctrl_plane_mut()); let inst = Inst::CondBr { kind: CondBrKind::NotZero(xreg(0)), taken: target(2), @@ -1888,24 +1911,24 @@ mod test { }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(1)); + buf.bind_label(label(1), state.ctrl_plane_mut()); while buf.cur_offset() < 2000000 { if buf.island_needed(0) { - buf.emit_island(0); + buf.emit_island(0, state.ctrl_plane_mut()); } let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); } - buf.bind_label(label(2)); + buf.bind_label(label(2), state.ctrl_plane_mut()); let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(3)); + buf.bind_label(label(3), state.ctrl_plane_mut()); let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); - let buf = buf.finish(); + let buf = buf.finish(state.ctrl_plane_mut()); assert_eq!(2000000 + 8, buf.total_size()); @@ -1934,7 +1957,7 @@ mod test { }; inst.emit(&[], &mut buf2, &info, &mut state); - let buf2 = buf2.finish(); + let buf2 = buf2.finish(state.ctrl_plane_mut()); assert_eq!(&buf.data[0..8], &buf2.data[..]); } @@ -1943,25 +1966,25 @@ mod test { fn test_island_backward() { let info = EmitInfo::new(settings::Flags::new(settings::builder())); let mut buf = MachBuffer::new(); - let mut state = Default::default(); + let mut state = ::State::default(); buf.reserve_labels_for_blocks(4); - buf.bind_label(label(0)); + buf.bind_label(label(0), state.ctrl_plane_mut()); let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(1)); + buf.bind_label(label(1), state.ctrl_plane_mut()); let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(2)); + buf.bind_label(label(2), state.ctrl_plane_mut()); while buf.cur_offset() < 2000000 { let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); } - buf.bind_label(label(3)); + buf.bind_label(label(3), state.ctrl_plane_mut()); let inst = Inst::CondBr { kind: CondBrKind::NotZero(xreg(0)), taken: target(0), @@ -1969,7 +1992,7 @@ mod test { }; inst.emit(&[], &mut buf, &info, &mut state); - let buf = buf.finish(); + let buf = buf.finish(state.ctrl_plane_mut()); assert_eq!(2000000 + 12, buf.total_size()); @@ -1986,7 +2009,7 @@ mod test { }; inst.emit(&[], &mut buf2, &info, &mut state); - let buf2 = buf2.finish(); + let buf2 = buf2.finish(state.ctrl_plane_mut()); assert_eq!(&buf.data[2000000..], &buf2.data[..]); } @@ -2028,11 +2051,11 @@ mod test { let info = EmitInfo::new(settings::Flags::new(settings::builder())); let mut buf = MachBuffer::new(); - let mut state = Default::default(); + let mut state = ::State::default(); buf.reserve_labels_for_blocks(8); - buf.bind_label(label(0)); + buf.bind_label(label(0), state.ctrl_plane_mut()); let inst = Inst::CondBr { kind: CondBrKind::Zero(xreg(0)), taken: target(1), @@ -2040,38 +2063,38 @@ mod test { }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(1)); + buf.bind_label(label(1), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(3) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(2)); + buf.bind_label(label(2), state.ctrl_plane_mut()); let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); inst.emit(&[], &mut buf, &info, &mut state); let inst = Inst::Jump { dest: target(0) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(3)); + buf.bind_label(label(3), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(4) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(4)); + buf.bind_label(label(4), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(5) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(5)); + buf.bind_label(label(5), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(7) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(6)); + buf.bind_label(label(6), state.ctrl_plane_mut()); let inst = Inst::Nop4; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(7)); + buf.bind_label(label(7), state.ctrl_plane_mut()); let inst = Inst::Ret { rets: vec![] }; inst.emit(&[], &mut buf, &info, &mut state); - let buf = buf.finish(); + let buf = buf.finish(state.ctrl_plane_mut()); let golden_data = vec![ 0xa0, 0x00, 0x00, 0xb4, // cbz x0, 0x14 @@ -2104,31 +2127,31 @@ mod test { // b label0 let info = EmitInfo::new(settings::Flags::new(settings::builder())); let mut buf = MachBuffer::new(); - let mut state = Default::default(); + let mut state = ::State::default(); buf.reserve_labels_for_blocks(5); - buf.bind_label(label(0)); + buf.bind_label(label(0), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(1) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(1)); + buf.bind_label(label(1), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(2) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(2)); + buf.bind_label(label(2), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(3) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(3)); + buf.bind_label(label(3), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(4) }; inst.emit(&[], &mut buf, &info, &mut state); - buf.bind_label(label(4)); + buf.bind_label(label(4), state.ctrl_plane_mut()); let inst = Inst::Jump { dest: target(1) }; inst.emit(&[], &mut buf, &info, &mut state); - let buf = buf.finish(); + let buf = buf.finish(state.ctrl_plane_mut()); let golden_data = vec![ 0x00, 0x00, 0x00, 0x14, // b 0 @@ -2140,10 +2163,11 @@ mod test { #[test] fn metadata_records() { let mut buf = MachBuffer::::new(); + let ctrl_plane = &mut Default::default(); buf.reserve_labels_for_blocks(1); - buf.bind_label(label(0)); + buf.bind_label(label(0), ctrl_plane); buf.put1(1); buf.add_trap(TrapCode::HeapOutOfBounds); buf.put1(2); @@ -2163,7 +2187,7 @@ mod test { ); buf.put1(4); - let buf = buf.finish(); + let buf = buf.finish(ctrl_plane); assert_eq!(buf.data(), &[1, 2, 3, 4]); assert_eq!( diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index 895ea38626..237497afac 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -52,6 +52,7 @@ use crate::settings::Flags; use crate::value_label::ValueLabelsRanges; use alloc::vec::Vec; use core::fmt::Debug; +use cranelift_control::ControlPlane; use cranelift_entity::PrimaryMap; use regalloc2::{Allocation, VReg}; use smallvec::{smallvec, SmallVec}; @@ -272,13 +273,21 @@ pub trait MachInstEmit: MachInst { /// emitting a function body. pub trait MachInstEmitState: Default + Clone + Debug { /// Create a new emission state given the ABI object. - fn new(abi: &Callee) -> Self; + fn new(abi: &Callee, ctrl_plane: ControlPlane) -> Self; /// Update the emission state before emitting an instruction that is a /// safepoint. fn pre_safepoint(&mut self, _stack_map: StackMap) {} /// Update the emission state to indicate instructions are associated with a /// particular RelSourceLoc. fn pre_sourceloc(&mut self, _srcloc: RelSourceLoc) {} + /// The emission state holds ownership of a control plane, so it doesn't + /// have to be passed around explicitly too much. `ctrl_plane_mut` may + /// be used if temporary access to the control plane is needed by some + /// other function that doesn't have access to the emission state. + fn ctrl_plane_mut(&mut self) -> &mut ControlPlane; + /// Used to continue using a control plane after the emission state is + /// not needed anymore. + fn take_ctrl_plane(self) -> ControlPlane; } /// The result of a `MachBackend::compile_function()` call. Contains machine @@ -474,7 +483,13 @@ pub trait TextSectionBuilder { /// /// This function returns the offset at which the data was placed in the /// text section. - fn append(&mut self, labeled: bool, data: &[u8], align: u32) -> u64; + fn append( + &mut self, + labeled: bool, + data: &[u8], + align: u32, + ctrl_plane: &mut ControlPlane, + ) -> u64; /// Attempts to resolve a relocation for this function. /// @@ -497,7 +512,7 @@ pub trait TextSectionBuilder { /// Completes this text section, filling out any final details, and returns /// the bytes of the text section. - fn finish(&mut self) -> Vec; + fn finish(&mut self, ctrl_plane: &mut ControlPlane) -> Vec; } /// Expected unwind info type. diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index 81aaf54862..4cfa3b8c59 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -26,6 +26,7 @@ use crate::timing; use crate::trace; use crate::CodegenError; use crate::ValueLocRange; +use cranelift_control::ControlPlane; use regalloc2::{ Edit, Function as RegallocFunction, InstOrEdit, InstRange, Operand, OperandKind, PRegSet, RegClass, VReg, @@ -763,6 +764,7 @@ impl VCode { regalloc: ®alloc2::Output, want_disasm: bool, want_metadata: bool, + ctrl_plane: &mut ControlPlane, ) -> EmitResult where I: VCodeInst, @@ -813,7 +815,7 @@ impl VCode { let mut cur_srcloc = None; let mut last_offset = None; let mut inst_offsets = vec![]; - let mut state = I::State::new(&self.abi); + let mut state = I::State::new(&self.abi, std::mem::take(ctrl_plane)); let mut disasm = String::new(); @@ -874,7 +876,7 @@ impl VCode { // Now emit the regular block body. - buffer.bind_label(MachLabel::from_block(block)); + buffer.bind_label(MachLabel::from_block(block), state.ctrl_plane_mut()); if want_disasm { writeln!(&mut disasm, "block{}:", block.index()).unwrap(); @@ -1054,11 +1056,14 @@ impl VCode { let worst_case_next_bb = I::worst_case_size() * (next_block_size + next_block_ra_insertions); if buffer.island_needed(worst_case_next_bb) { - buffer.emit_island(worst_case_next_bb); + buffer.emit_island(worst_case_next_bb, ctrl_plane); } } } + // emission state is not needed anymore, move control plane back out + *ctrl_plane = state.take_ctrl_plane(); + // Emit the constants used by the function. let mut alignment = 1; for (constant, data) in self.constants.iter() { diff --git a/cranelift/control/Cargo.toml b/cranelift/control/Cargo.toml new file mode 100644 index 0000000000..d820c24234 --- /dev/null +++ b/cranelift/control/Cargo.toml @@ -0,0 +1,20 @@ +[package] +authors = ["The Cranelift Project Developers"] +name = "cranelift-control" +version = "0.96.0" +description = "White-box fuzz testing framework" +license = "Apache-2.0 WITH LLVM-exception" +repository = "https://github.com/bytecodealliance/wasmtime" +readme = "README.md" +keywords = ["fuzz", "test"] +edition.workspace = true + +[dependencies] +arbitrary = { version = "1.1.0" } + +[features] + +# Turn on chaos mode. +# Without this feature, a zero-sized dummy will be compiled +# for the control plane. +chaos = [] diff --git a/cranelift/control/LICENSE b/cranelift/control/LICENSE new file mode 100644 index 0000000000..f9d81955f4 --- /dev/null +++ b/cranelift/control/LICENSE @@ -0,0 +1,220 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/cranelift/control/README.md b/cranelift/control/README.md new file mode 100644 index 0000000000..3eecb3d47b --- /dev/null +++ b/cranelift/control/README.md @@ -0,0 +1,4 @@ +This crate contains the control plane for "chaos mode". It can be used to +inject pseudo-random perturbations into specific sections in the code while +fuzzing. Its compilation is feature-gated to prevent any performance +impact on release builds. diff --git a/cranelift/control/src/chaos.rs b/cranelift/control/src/chaos.rs new file mode 100644 index 0000000000..98db39979d --- /dev/null +++ b/cranelift/control/src/chaos.rs @@ -0,0 +1,28 @@ +use arbitrary::Arbitrary; + +/// The control plane of chaos mode. +/// Please see the [crate-level documentation](crate). +#[derive(Debug, Clone, Default)] +pub struct ControlPlane { + data: Vec, +} + +impl Arbitrary<'_> for ControlPlane { + fn arbitrary<'a>(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + Ok(Self { + data: u.arbitrary()?, + }) + } +} + +impl ControlPlane { + /// Returns a pseudo-random boolean if the control plane was constructed + /// with `arbitrary`. + /// + /// The default value `false` will always be returned if the + /// pseudo-random data is exhausted or the control plane was constructed + /// with `default`. + pub fn get_decision(&mut self) -> bool { + self.data.pop().unwrap_or_default() + } +} diff --git a/cranelift/control/src/lib.rs b/cranelift/control/src/lib.rs new file mode 100644 index 0000000000..15e6e582a7 --- /dev/null +++ b/cranelift/control/src/lib.rs @@ -0,0 +1,30 @@ +//! # Cranelift Control +//! +//! This is the home of the control plane of chaos mode, a compilation feature +//! intended to be turned on for certain fuzz targets. When the feature is +//! turned off, as is normally the case, [ControlPlane] will be a zero-sized +//! type and optimized away. +//! +//! While the feature is turned on, the struct [ControlPlane] +//! provides functionality to tap into pseudo-randomness at specific locations +//! in the code. It may be used for targeted fuzzing of compiler internals, +//! e.g. manipulate heuristic optimizations, clobber undefined register bits +//! etc. +//! +//! There are two ways to acquire a [ControlPlane]: +//! - [arbitrary] for the real deal +//! - [default] for an "empty" control plane which always returns default +//! values +//! +//! [arbitrary]: ControlPlane#method.arbitrary +//! [default]: ControlPlane#method.default + +#[cfg(not(feature = "chaos"))] +mod zero_sized; +#[cfg(not(feature = "chaos"))] +pub use zero_sized::*; + +#[cfg(feature = "chaos")] +mod chaos; +#[cfg(feature = "chaos")] +pub use chaos::*; diff --git a/cranelift/control/src/zero_sized.rs b/cranelift/control/src/zero_sized.rs new file mode 100644 index 0000000000..73b2c58ec1 --- /dev/null +++ b/cranelift/control/src/zero_sized.rs @@ -0,0 +1,28 @@ +//! Shims for ControlPlane when chaos mode is disabled. Enables +//! unconditional use of the type and its methods throughout cranelift. + +/// A shim for ControlPlane when chaos mode is disabled. +/// Please see the [crate-level documentation](crate). +#[derive(Debug, Clone, Default)] +pub struct ControlPlane { + /// prevent direct instantiation (use `default` instead) + _private: (), +} + +/// A shim for ControlPlane's `Arbitrary` implementation when chaos mode is +/// disabled. It doesn't consume any bytes and always returns a default +/// control plane. +impl arbitrary::Arbitrary<'_> for ControlPlane { + fn arbitrary<'a>(_u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + Ok(Self::default()) + } +} + +impl ControlPlane { + /// Returns a pseudo-random boolean. This variant is used when chaos + /// mode is disabled. It always returns `false`. + #[inline] + pub fn get_decision(&mut self) -> bool { + false + } +} diff --git a/cranelift/filetests/Cargo.toml b/cranelift/filetests/Cargo.toml index 7ed94698ce..789dfa5c66 100644 --- a/cranelift/filetests/Cargo.toml +++ b/cranelift/filetests/Cargo.toml @@ -17,6 +17,7 @@ cranelift-native = { workspace = true } cranelift-reader = { workspace = true } cranelift-jit = { workspace = true, features = ["selinux-fix"] } cranelift-module = { workspace = true } +cranelift-control = { workspace = true } file-per-thread-logger = "0.1.2" filecheck = "0.5.0" gimli = { workspace = true } diff --git a/cranelift/filetests/src/function_runner.rs b/cranelift/filetests/src/function_runner.rs index 1bc1594518..945e0ee333 100644 --- a/cranelift/filetests/src/function_runner.rs +++ b/cranelift/filetests/src/function_runner.rs @@ -7,6 +7,7 @@ use cranelift_codegen::ir::{ }; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; use cranelift_codegen::{ir, settings, CodegenError, Context}; +use cranelift_control::ControlPlane; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use cranelift_jit::{JITBuilder, JITModule}; use cranelift_module::{FuncId, Linkage, Module, ModuleError}; @@ -49,6 +50,7 @@ struct DefinedFunction { /// "outside-of-function" functionality, see `cranelift_jit::backend::JITBackend`. /// /// ``` +/// # let ctrl_plane = &mut Default::default(); /// use cranelift_filetests::TestFileCompiler; /// use cranelift_reader::parse_functions; /// use cranelift_codegen::data_value::DataValue; @@ -57,8 +59,8 @@ struct DefinedFunction { /// let func = parse_functions(code).unwrap().into_iter().nth(0).unwrap(); /// let mut compiler = TestFileCompiler::with_default_host_isa().unwrap(); /// compiler.declare_function(&func).unwrap(); -/// compiler.define_function(func.clone()).unwrap(); -/// compiler.create_trampoline_for_function(&func).unwrap(); +/// compiler.define_function(func.clone(), ctrl_plane).unwrap(); +/// compiler.create_trampoline_for_function(&func, ctrl_plane).unwrap(); /// let compiled = compiler.compile().unwrap(); /// let trampoline = compiled.get_trampoline(&func).unwrap(); /// @@ -116,16 +118,24 @@ impl TestFileCompiler { /// Declares and compiles all functions in `functions`. Additionally creates a trampoline for /// each one of them. - pub fn add_functions(&mut self, functions: &[Function]) -> Result<()> { + pub fn add_functions( + &mut self, + functions: &[Function], + ctrl_planes: Vec, + ) -> Result<()> { // Declare all functions in the file, so that they may refer to each other. for func in functions { self.declare_function(func)?; } + let ctrl_planes = ctrl_planes + .into_iter() + .chain(std::iter::repeat(ControlPlane::default())); + // Define all functions and trampolines - for func in functions { - self.define_function(func.clone())?; - self.create_trampoline_for_function(func)?; + for (func, ref mut ctrl_plane) in functions.iter().zip(ctrl_planes) { + self.define_function(func.clone(), ctrl_plane)?; + self.create_trampoline_for_function(func, ctrl_plane)?; } Ok(()) @@ -141,7 +151,7 @@ impl TestFileCompiler { .cloned() .collect::>(); - self.add_functions(&functions[..])?; + self.add_functions(&functions[..], Vec::new())?; Ok(()) } @@ -217,7 +227,7 @@ impl TestFileCompiler { } /// Defines the body of a function - pub fn define_function(&mut self, func: Function) -> Result<()> { + pub fn define_function(&mut self, func: Function, ctrl_plane: &mut ControlPlane) -> Result<()> { let defined_func = self .defined_functions .get(&func.name) @@ -225,13 +235,17 @@ impl TestFileCompiler { self.ctx.func = self.apply_func_rename(func, defined_func)?; self.module - .define_function(defined_func.func_id, &mut self.ctx)?; + .define_function(defined_func.func_id, &mut self.ctx, ctrl_plane)?; self.module.clear_context(&mut self.ctx); Ok(()) } /// Creates and registers a trampoline for a function if none exists. - pub fn create_trampoline_for_function(&mut self, func: &Function) -> Result<()> { + pub fn create_trampoline_for_function( + &mut self, + func: &Function, + ctrl_plane: &mut ControlPlane, + ) -> Result<()> { if !self.defined_functions.contains_key(&func.name) { anyhow::bail!("Undeclared function {} found!", &func.name); } @@ -246,7 +260,7 @@ impl TestFileCompiler { let trampoline = make_trampoline(name.clone(), &func.signature, self.module.isa()); self.declare_function(&trampoline)?; - self.define_function(trampoline)?; + self.define_function(trampoline, ctrl_plane)?; self.trampolines.insert(func.signature.clone(), name); @@ -504,6 +518,7 @@ mod test { return v1 }", ); + let ctrl_plane = &mut ControlPlane::default(); // extract function let test_file = parse_test(code.as_str(), ParseOptions::default()).unwrap(); @@ -513,8 +528,12 @@ mod test { // execute function let mut compiler = TestFileCompiler::with_default_host_isa().unwrap(); compiler.declare_function(&function).unwrap(); - compiler.define_function(function.clone()).unwrap(); - compiler.create_trampoline_for_function(&function).unwrap(); + compiler + .define_function(function.clone(), ctrl_plane) + .unwrap(); + compiler + .create_trampoline_for_function(&function, ctrl_plane) + .unwrap(); let compiled = compiler.compile().unwrap(); let trampoline = compiled.get_trampoline(&function).unwrap(); let returned = trampoline.call(&[]); diff --git a/cranelift/filetests/src/test_compile.rs b/cranelift/filetests/src/test_compile.rs index 9342190861..4e42d9c15c 100644 --- a/cranelift/filetests/src/test_compile.rs +++ b/cranelift/filetests/src/test_compile.rs @@ -53,7 +53,7 @@ impl SubTest for TestCompile { comp_ctx.set_disasm(true); let compiled_code = comp_ctx - .compile(isa) + .compile(isa, &mut Default::default()) .map_err(|e| crate::pretty_anyhow_error(&e.func, e.inner))?; let total_size = compiled_code.code_info().total_size; diff --git a/cranelift/filetests/src/test_unwind.rs b/cranelift/filetests/src/test_unwind.rs index 5174b91da0..0c43d30410 100644 --- a/cranelift/filetests/src/test_unwind.rs +++ b/cranelift/filetests/src/test_unwind.rs @@ -39,7 +39,9 @@ impl SubTest for TestUnwind { let isa = context.isa.expect("unwind needs an ISA"); let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned()); - let code = comp_ctx.compile(isa).expect("failed to compile function"); + let code = comp_ctx + .compile(isa, &mut Default::default()) + .expect("failed to compile function"); let mut text = String::new(); match code.create_unwind_info(isa).expect("unwind info") { diff --git a/cranelift/filetests/src/test_wasm.rs b/cranelift/filetests/src/test_wasm.rs index 0ecba3510a..c3bcc25ca3 100644 --- a/cranelift/filetests/src/test_wasm.rs +++ b/cranelift/filetests/src/test_wasm.rs @@ -58,7 +58,7 @@ pub fn run(path: &Path, wat: &str) -> Result<()> { let mut ctx = cranelift_codegen::Context::for_function(func.clone()); ctx.set_disasm(true); let code = ctx - .compile(isa) + .compile(isa, &mut Default::default()) .map_err(|e| crate::pretty_anyhow_error(&e.func, e.inner))?; writeln!(&mut actual, "function {}:", func.name).unwrap(); writeln!(&mut actual, "{}", code.vcode.as_ref().unwrap()).unwrap(); diff --git a/cranelift/jit/Cargo.toml b/cranelift/jit/Cargo.toml index 3b0f1eea5f..ad3c0f2f8f 100644 --- a/cranelift/jit/Cargo.toml +++ b/cranelift/jit/Cargo.toml @@ -14,6 +14,7 @@ cranelift-module = { workspace = true } cranelift-native = { workspace = true } cranelift-codegen = { workspace = true, features = ["std"] } cranelift-entity = { workspace = true } +cranelift-control = { workspace = true } anyhow = { workspace = true } region = "2.2.0" libc = { version = "0.2.42" } diff --git a/cranelift/jit/examples/jit-minimal.rs b/cranelift/jit/examples/jit-minimal.rs index 3ebf7536f9..2d273430ab 100644 --- a/cranelift/jit/examples/jit-minimal.rs +++ b/cranelift/jit/examples/jit-minimal.rs @@ -51,7 +51,10 @@ fn main() { bcx.seal_all_blocks(); bcx.finalize(); } - module.define_function(func_a, &mut ctx).unwrap(); + let ctrl_plane = &mut Default::default(); + module + .define_function(func_a, &mut ctx, ctrl_plane) + .unwrap(); module.clear_context(&mut ctx); ctx.func.signature = sig_b; @@ -74,7 +77,9 @@ fn main() { bcx.seal_all_blocks(); bcx.finalize(); } - module.define_function(func_b, &mut ctx).unwrap(); + module + .define_function(func_b, &mut ctx, ctrl_plane) + .unwrap(); module.clear_context(&mut ctx); // Perform linking. diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index e3d00a1872..5d0008f9d3 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -5,6 +5,7 @@ use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; use cranelift_codegen::settings::Configurable; use cranelift_codegen::{self, ir, settings, MachReloc}; use cranelift_codegen::{binemit::Reloc, CodegenError}; +use cranelift_control::ControlPlane; use cranelift_entity::SecondaryMap; use cranelift_module::{ DataContext, DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleCompiledFunction, @@ -682,6 +683,7 @@ impl Module for JITModule { &mut self, id: FuncId, ctx: &mut cranelift_codegen::Context, + ctrl_plane: &mut ControlPlane, ) -> ModuleResult { info!("defining function {}: {}", id, ctx.func.display()); let decl = self.declarations.get_function_decl(id); @@ -694,7 +696,7 @@ impl Module for JITModule { } // work around borrow-checker to allow reuse of ctx below - let res = ctx.compile(self.isa())?; + let res = ctx.compile(self.isa(), ctrl_plane)?; let alignment = res.alignment as u64; let compiled_code = ctx.compiled_code().unwrap(); diff --git a/cranelift/jit/tests/basic.rs b/cranelift/jit/tests/basic.rs index 357d5ef533..8714acaf71 100644 --- a/cranelift/jit/tests/basic.rs +++ b/cranelift/jit/tests/basic.rs @@ -57,7 +57,9 @@ fn define_simple_function(module: &mut JITModule) -> FuncId { bcx.ins().return_(&[]); } - module.define_function(func_id, &mut ctx).unwrap(); + module + .define_function(func_id, &mut ctx, &mut Default::default()) + .unwrap(); func_id } @@ -207,7 +209,9 @@ fn libcall_function() { bcx.ins().return_(&[]); } - module.define_function(func_id, &mut ctx).unwrap(); + module + .define_function(func_id, &mut ctx, &mut Default::default()) + .unwrap(); module.finalize_definitions().unwrap(); } diff --git a/cranelift/module/Cargo.toml b/cranelift/module/Cargo.toml index 5494ccfade..16419af867 100644 --- a/cranelift/module/Cargo.toml +++ b/cranelift/module/Cargo.toml @@ -12,6 +12,7 @@ edition.workspace = true [dependencies] cranelift-codegen = { workspace = true } +cranelift-control = { workspace = true } hashbrown = { workspace = true, optional = true } anyhow = { workspace = true } diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index d6fa93b834..0e3ccf9179 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -14,6 +14,7 @@ use cranelift_codegen::ir::Function; use cranelift_codegen::settings::SetError; use cranelift_codegen::{binemit, MachReloc}; use cranelift_codegen::{ir, isa, CodegenError, CompileError, Context}; +use cranelift_control::ControlPlane; use std::borrow::ToOwned; use std::string::String; @@ -658,6 +659,7 @@ pub trait Module { &mut self, func: FuncId, ctx: &mut Context, + ctrl_plane: &mut ControlPlane, ) -> ModuleResult; /// Define a function, taking the function body from the given `bytes`. @@ -760,8 +762,9 @@ impl Module for &mut M { &mut self, func: FuncId, ctx: &mut Context, + ctrl_plane: &mut ControlPlane, ) -> ModuleResult { - (**self).define_function(func, ctx) + (**self).define_function(func, ctx, ctrl_plane) } fn define_function_bytes( diff --git a/cranelift/object/Cargo.toml b/cranelift/object/Cargo.toml index 33a5c36bca..5b75fd86d9 100644 --- a/cranelift/object/Cargo.toml +++ b/cranelift/object/Cargo.toml @@ -12,6 +12,7 @@ edition.workspace = true [dependencies] cranelift-module = { workspace = true } cranelift-codegen = { workspace = true, features = ["std"] } +cranelift-control = { workspace = true } object = { workspace = true, features = ["write"] } target-lexicon = { workspace = true } anyhow = { workspace = true } diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index 24bd04d13b..b864a5b97e 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -8,6 +8,7 @@ use cranelift_codegen::{ binemit::{Addend, CodeOffset, Reloc}, CodegenError, }; +use cranelift_control::ControlPlane; use cranelift_module::{ DataContext, DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleCompiledFunction, ModuleDeclarations, ModuleError, ModuleExtName, ModuleReloc, ModuleResult, @@ -319,11 +320,12 @@ impl Module for ObjectModule { &mut self, func_id: FuncId, ctx: &mut cranelift_codegen::Context, + ctrl_plane: &mut ControlPlane, ) -> ModuleResult { info!("defining function {}: {}", func_id, ctx.func.display()); let mut code: Vec = Vec::new(); - let res = ctx.compile_and_emit(self.isa(), &mut code)?; + let res = ctx.compile_and_emit(self.isa(), &mut code, ctrl_plane)?; let alignment = res.alignment as u64; self.define_function_bytes( diff --git a/cranelift/object/tests/basic.rs b/cranelift/object/tests/basic.rs index d918f55e34..b89395b757 100644 --- a/cranelift/object/tests/basic.rs +++ b/cranelift/object/tests/basic.rs @@ -2,6 +2,7 @@ use cranelift_codegen::ir::*; use cranelift_codegen::isa::CallConv; use cranelift_codegen::settings; use cranelift_codegen::{ir::types::I16, Context}; +use cranelift_control::ControlPlane; use cranelift_entity::EntityRef; use cranelift_frontend::*; use cranelift_module::*; @@ -31,7 +32,7 @@ fn error_on_incompatible_sig_in_declare_function() { .unwrap(); // Make sure this is an error } -fn define_simple_function(module: &mut ObjectModule) -> FuncId { +fn define_simple_function(module: &mut ObjectModule, ctrl_plane: &mut ControlPlane) -> FuncId { let sig = Signature { params: vec![], returns: vec![], @@ -52,7 +53,9 @@ fn define_simple_function(module: &mut ObjectModule) -> FuncId { bcx.ins().return_(&[]); } - module.define_function(func_id, &mut ctx).unwrap(); + module + .define_function(func_id, &mut ctx, ctrl_plane) + .unwrap(); func_id } @@ -60,6 +63,7 @@ fn define_simple_function(module: &mut ObjectModule) -> FuncId { #[test] #[should_panic(expected = "Result::unwrap()` on an `Err` value: DuplicateDefinition(\"abc\")")] fn panic_on_define_after_finalize() { + let ctrl_plane = &mut ControlPlane::default(); let flag_builder = settings::builder(); let isa_builder = cranelift_codegen::isa::lookup_by_name("x86_64-unknown-linux-gnu").unwrap(); let isa = isa_builder @@ -68,8 +72,8 @@ fn panic_on_define_after_finalize() { let mut module = ObjectModule::new(ObjectBuilder::new(isa, "foo", default_libcall_names()).unwrap()); - define_simple_function(&mut module); - define_simple_function(&mut module); + define_simple_function(&mut module, ctrl_plane); + define_simple_function(&mut module, ctrl_plane); } #[test] @@ -192,7 +196,9 @@ fn libcall_function() { bcx.ins().return_(&[]); } - module.define_function(func_id, &mut ctx).unwrap(); + module + .define_function(func_id, &mut ctx, &mut ControlPlane::default()) + .unwrap(); module.finish(); } diff --git a/cranelift/src/bugpoint.rs b/cranelift/src/bugpoint.rs index da3de4fd70..90db62f106 100644 --- a/cranelift/src/bugpoint.rs +++ b/cranelift/src/bugpoint.rs @@ -1041,9 +1041,11 @@ impl<'a> CrashCheckContext<'a> { std::panic::set_hook(Box::new(|_| {})); // silence panics let res = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _ = self - .context - .compile_and_emit(self.isa, &mut self.code_memory); + let _ = self.context.compile_and_emit( + self.isa, + &mut self.code_memory, + &mut Default::default(), + ); })) { Ok(()) => CheckResult::Succeed, Err(err) => CheckResult::Crash(get_panic_string(err)), diff --git a/cranelift/src/compile.rs b/cranelift/src/compile.rs index be9315c8d0..fcbb0b382d 100644 --- a/cranelift/src/compile.rs +++ b/cranelift/src/compile.rs @@ -102,7 +102,7 @@ fn handle_module( // Compile and encode the result to machine code. let compiled_code = context - .compile_and_emit(isa, &mut mem) + .compile_and_emit(isa, &mut mem, &mut Default::default()) .map_err(|err| anyhow::anyhow!("{}", pretty_error(&err.func, err.inner)))?; let code_info = compiled_code.code_info(); @@ -113,7 +113,7 @@ fn handle_module( cranelift_module::Linkage::Export, &context.func.signature, )?; - module.define_function(fid, &mut context)?; + module.define_function(fid, &mut context, &mut Default::default())?; } if options.print { diff --git a/cranelift/src/wasm.rs b/cranelift/src/wasm.rs index 7456b0554e..f6af1221bf 100644 --- a/cranelift/src/wasm.rs +++ b/cranelift/src/wasm.rs @@ -257,7 +257,7 @@ fn handle_module(options: &Options, path: &Path, name: &str, fisa: FlagsOrIsa) - (vec![], vec![], vec![]) } else { let compiled_code = context - .compile_and_emit(isa, &mut mem) + .compile_and_emit(isa, &mut mem, &mut Default::default()) .map_err(|err| anyhow::anyhow!("{}", pretty_error(&err.func, err.inner)))?; let code_info = compiled_code.code_info(); diff --git a/crates/cranelift-shared/Cargo.toml b/crates/cranelift-shared/Cargo.toml index 9621ad9dd2..78446e42ad 100644 --- a/crates/cranelift-shared/Cargo.toml +++ b/crates/cranelift-shared/Cargo.toml @@ -12,6 +12,7 @@ edition.workspace = true anyhow = { workspace = true } wasmtime-environ = { workspace = true } cranelift-codegen = { workspace = true } +cranelift-control = { workspace = true } cranelift-native = { workspace = true } target-lexicon = { workspace = true } gimli = { workspace = true } diff --git a/crates/cranelift-shared/src/obj.rs b/crates/cranelift-shared/src/obj.rs index 44901f8456..25b32c8520 100644 --- a/crates/cranelift-shared/src/obj.rs +++ b/crates/cranelift-shared/src/obj.rs @@ -19,6 +19,7 @@ use cranelift_codegen::binemit::Reloc; use cranelift_codegen::ir::LibCall; use cranelift_codegen::isa::unwind::{systemv, UnwindInfo}; use cranelift_codegen::TextSectionBuilder; +use cranelift_control::ControlPlane; use gimli::write::{Address, EhFrame, EndianVec, FrameTable, Writer}; use gimli::RunTimeEndian; use object::write::{Object, SectionId, StandardSegment, Symbol, SymbolId, SymbolSection}; @@ -59,6 +60,8 @@ pub struct ModuleTextBuilder<'a> { /// Note that this isn't typically used. It's only used for SSE-disabled /// builds without SIMD on x86_64 right now. libcall_symbols: HashMap, + + ctrl_plane: ControlPlane, } impl<'a> ModuleTextBuilder<'a> { @@ -88,6 +91,7 @@ impl<'a> ModuleTextBuilder<'a> { unwind_info: Default::default(), text, libcall_symbols: HashMap::default(), + ctrl_plane: ControlPlane::default(), } } @@ -115,6 +119,7 @@ impl<'a> ModuleTextBuilder<'a> { true, &body, self.compiler.function_alignment().max(alignment), + &mut self.ctrl_plane, ); let symbol_id = self.obj.add_symbol(Symbol { @@ -227,7 +232,8 @@ impl<'a> ModuleTextBuilder<'a> { if padding == 0 { return; } - self.text.append(false, &vec![0; padding], 1); + self.text + .append(false, &vec![0; padding], 1, &mut self.ctrl_plane); } /// Indicates that the text section has been written completely and this @@ -237,7 +243,7 @@ impl<'a> ModuleTextBuilder<'a> { /// necessary. pub fn finish(mut self) { // Finish up the text section now that we're done adding functions. - let text = self.text.finish(); + let text = self.text.finish(&mut self.ctrl_plane); self.obj .section_mut(self.text_section) .set_data(text, self.compiler.page_size_align()); diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index 7c31add30a..40a0a9bfc2 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -19,6 +19,7 @@ cranelift-codegen = { workspace = true } cranelift-frontend = { workspace = true } cranelift-entity = { workspace = true } cranelift-native = { workspace = true } +cranelift-control = { workspace = true } wasmtime-cranelift-shared = { workspace = true } wasmparser = { workspace = true } target-lexicon = { workspace = true } diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index 5733a998ed..7594a34143 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -612,7 +612,7 @@ fn compile_uncached<'a>( ) -> Result<(&'a CompiledCode, Vec), CompileError> { let mut code_buf = Vec::new(); let compiled_code = context - .compile_and_emit(isa, &mut code_buf) + .compile_and_emit(isa, &mut code_buf, &mut Default::default()) .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; Ok((compiled_code, code_buf)) } diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index f9774cac38..53575417f2 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -17,6 +17,7 @@ cranelift-filetests = { workspace = true } cranelift-interpreter = { workspace = true } cranelift-fuzzgen = { workspace = true } cranelift-native = { workspace = true } +cranelift-control = { workspace = true } libfuzzer-sys = { version = "0.4.0", features = ["arbitrary-derive"] } target-lexicon = { workspace = true } smallvec = { workspace = true } @@ -36,6 +37,7 @@ component-fuzz-util = { workspace = true } [features] default = ['fuzz-spec-interpreter'] fuzz-spec-interpreter = ['wasmtime-fuzzing/fuzz-spec-interpreter'] +chaos = ["cranelift-control/chaos"] [[bin]] name = "compile" diff --git a/fuzz/fuzz_targets/cranelift-fuzzgen.rs b/fuzz/fuzz_targets/cranelift-fuzzgen.rs index addf4f71f4..2cadc524ff 100644 --- a/fuzz/fuzz_targets/cranelift-fuzzgen.rs +++ b/fuzz/fuzz_targets/cranelift-fuzzgen.rs @@ -5,6 +5,7 @@ use cranelift_codegen::ir::Signature; use cranelift_codegen::ir::UserExternalName; use cranelift_codegen::ir::UserFuncName; use cranelift_codegen::Context; +use cranelift_control::ControlPlane; use libfuzzer_sys::arbitrary; use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::arbitrary::Unstructured; @@ -146,6 +147,9 @@ pub struct TestCase { /// Functions under test /// By convention the first function is the main function. pub functions: Vec, + /// Control planes for function compilation. + /// There should be an equal amount as functions to compile. + pub ctrl_planes: Vec, /// Generate multiple test inputs for each test case. /// This allows us to get more coverage per compilation, which may be somewhat expensive. pub inputs: Vec, @@ -191,6 +195,7 @@ impl TestCase { // the start. let func_count = gen.u.int_in_range(gen.config.testcase_funcs.clone())?; let mut functions: Vec = Vec::with_capacity(func_count); + let mut ctrl_planes: Vec = Vec::with_capacity(func_count); for i in (0..func_count).rev() { // Function name must be in a different namespace than TESTFILE_NAMESPACE (0) let fname = UserFuncName::user(1, i as u32); @@ -212,6 +217,8 @@ impl TestCase { ALLOWED_LIBCALLS.to_vec(), )?; functions.push(func); + + ctrl_planes.push(ControlPlane::arbitrary(gen.u)?); } // Now reverse the functions so that the main function is at the start. functions.reverse(); @@ -222,6 +229,7 @@ impl TestCase { Ok(TestCase { isa, functions, + ctrl_planes, inputs, compare_against_host, }) @@ -241,6 +249,7 @@ impl TestCase { TestCase { isa: self.isa.clone(), functions: optimized_functions, + ctrl_planes: self.ctrl_planes.clone(), inputs: self.inputs.clone(), compare_against_host: false, } @@ -368,7 +377,9 @@ fuzz_target!(|testcase: TestCase| { }); } else { let mut compiler = TestFileCompiler::new(testcase.isa.clone()); - compiler.add_functions(&testcase.functions[..]).unwrap(); + compiler + .add_functions(&testcase.functions[..], testcase.ctrl_planes.clone()) + .unwrap(); let compiled = compiler.compile().unwrap(); let trampoline = compiled.get_trampoline(testcase.main()).unwrap(); diff --git a/fuzz/fuzz_targets/cranelift-icache.rs b/fuzz/fuzz_targets/cranelift-icache.rs index aa4dc7c793..f493eb7258 100644 --- a/fuzz/fuzz_targets/cranelift-icache.rs +++ b/fuzz/fuzz_targets/cranelift-icache.rs @@ -108,7 +108,7 @@ fuzz_target!(|func: FunctionWithIsa| { let cache_key_hash = icache::compute_cache_key(&*isa, &func); let mut context = Context::for_function(func.clone()); - let prev_stencil = match context.compile_stencil(&*isa) { + let prev_stencil = match context.compile_stencil(&*isa, &mut Default::default()) { Ok(stencil) => stencil, Err(_) => return, }; @@ -199,7 +199,7 @@ fuzz_target!(|func: FunctionWithIsa| { context = Context::for_function(func.clone()); - let after_mutation_result = match context.compile(&*isa) { + let after_mutation_result = match context.compile(&*isa, &mut Default::default()) { Ok(info) => info, Err(_) => return, }; diff --git a/scripts/publish.rs b/scripts/publish.rs index 3ff02789ec..786d352fd8 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -25,6 +25,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "cranelift-codegen-shared", "cranelift-codegen-meta", "cranelift-egraph", + "cranelift-control", "cranelift-codegen", "cranelift-reader", "cranelift-serde", @@ -99,6 +100,7 @@ const PUBLIC_CRATES: &[&str] = &[ "cranelift-codegen-shared", "cranelift-codegen-meta", "cranelift-egraph", + "cranelift-control", "cranelift-codegen", "cranelift-reader", "cranelift-serde", diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rs index 34a19cfb09..9407fc7f71 100644 --- a/winch/codegen/src/isa/aarch64/asm.rs +++ b/winch/codegen/src/isa/aarch64/asm.rs @@ -9,7 +9,7 @@ use cranelift_codegen::{ emit::{EmitInfo, EmitState}, ALUOp, AMode, ExtendOp, Imm12, Inst, PairAMode, }, - settings, Final, MachBuffer, MachBufferFinalized, MachInstEmit, Writable, + settings, Final, MachBuffer, MachBufferFinalized, MachInstEmit, MachInstEmitState, Writable, }; /// An Aarch64 instruction operand. @@ -58,8 +58,8 @@ impl Assembler { impl Assembler { /// Return the emitted code. - pub fn finalize(self) -> MachBufferFinalized { - let stencil = self.buffer.finish(); + pub fn finalize(mut self) -> MachBufferFinalized { + let stencil = self.buffer.finish(self.emit_state.ctrl_plane_mut()); stencil.apply_base_srcloc(Default::default()) } diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs index d582f16ba8..cd9ffa42ab 100644 --- a/winch/codegen/src/isa/x64/asm.rs +++ b/winch/codegen/src/isa/x64/asm.rs @@ -15,7 +15,7 @@ use cranelift_codegen::{ }, settings as x64_settings, CallInfo, EmitInfo, EmitState, Inst, }, - settings, Final, MachBuffer, MachBufferFinalized, MachInstEmit, Writable, + settings, Final, MachBuffer, MachBufferFinalized, MachInstEmit, MachInstEmitState, Writable, }; use super::{address::Address, regs}; @@ -98,8 +98,8 @@ impl Assembler { } /// Return the emitted code. - pub fn finalize(self) -> MachBufferFinalized { - let stencil = self.buffer.finish(); + pub fn finalize(mut self) -> MachBufferFinalized { + let stencil = self.buffer.finish(self.emit_state.ctrl_plane_mut()); stencil.apply_base_srcloc(Default::default()) }