From 8a9b1a902591bcd5ae2e7af588e0968a66dc8e3e Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Fri, 12 Aug 2022 18:47:43 +0200 Subject: [PATCH] Implement an incremental compilation cache for Cranelift (#4551) This is the implementation of https://github.com/bytecodealliance/wasmtime/issues/4155, using the "inverted API" approach suggested by @cfallin (thanks!) in Cranelift, and trait object to provide a backend for an all-included experience in Wasmtime. After the suggestion of Chris, `Function` has been split into mostly two parts: - on the one hand, `FunctionStencil` contains all the fields required during compilation, and that act as a compilation cache key: if two function stencils are the same, then the result of their compilation (`CompiledCodeBase`) will be the same. This makes caching trivial, as the only thing to cache is the `FunctionStencil`. - on the other hand, `FunctionParameters` contain the... function parameters that are required to finalize the result of compilation into a `CompiledCode` (aka `CompiledCodeBase`) with proper final relocations etc., by applying fixups and so on. Most changes are here to accomodate those requirements, in particular that `FunctionStencil` should be `Hash`able to be used as a key in the cache: - most source locations are now relative to a base source location in the function, and as such they're encoded as `RelSourceLoc` in the `FunctionStencil`. This required changes so that there's no need to explicitly mark a `SourceLoc` as the base source location, it's automatically detected instead the first time a non-default `SourceLoc` is set. - user-defined external names in the `FunctionStencil` (aka before this patch `ExternalName::User { namespace, index }`) are now references into an external table of `UserExternalNameRef -> UserExternalName`, present in the `FunctionParameters`, and must be explicitly declared using `Function::declare_imported_user_function`. - some refactorings have been made for function names: - `ExternalName` was used as the type for a `Function`'s name; while it thus allowed `ExternalName::Libcall` in this place, this would have been quite confusing to use it there. Instead, a new enum `UserFuncName` is introduced for this name, that's either a user-defined function name (the above `UserExternalName`) or a test case name. - The future of `ExternalName` is likely to become a full reference into the `FunctionParameters`'s mapping, instead of being "either a handle for user-defined external names, or the thing itself for other variants". I'm running out of time to do this, and this is not trivial as it implies touching ISLE which I'm less familiar with. The cache computes a sha256 hash of the `FunctionStencil`, and uses this as the cache key. No equality check (using `PartialEq`) is performed in addition to the hash being the same, as we hope that this is sufficient data to avoid collisions. A basic fuzz target has been introduced that tries to do the bare minimum: - check that a function successfully compiled and cached will be also successfully reloaded from the cache, and returns the exact same function. - check that a trivial modification in the external mapping of `UserExternalNameRef -> UserExternalName` hits the cache, and that other modifications don't hit the cache. - This last check is less efficient and less likely to happen, so probably should be rethought a bit. Thanks to both @alexcrichton and @cfallin for your very useful feedback on Zulip. Some numbers show that for a large wasm module we're using internally, this is a 20% compile-time speedup, because so many `FunctionStencil`s are the same, even within a single module. For a group of modules that have a lot of code in common, we get hit rates up to 70% when they're used together. When a single function changes in a wasm module, every other function is reloaded; that's still slower than I expect (between 10% and 50% of the overall compile time), so there's likely room for improvement. Fixes #4155. --- Cargo.lock | 4 + cranelift/codegen/Cargo.toml | 9 + cranelift/codegen/meta/src/gen_inst.rs | 2 +- cranelift/codegen/meta/src/shared/settings.rs | 14 + cranelift/codegen/src/binemit/mod.rs | 2 +- cranelift/codegen/src/context.rs | 111 ++++---- cranelift/codegen/src/cursor.rs | 5 +- cranelift/codegen/src/incremental_cache.rs | 254 +++++++++++++++++ cranelift/codegen/src/ir/constant.rs | 31 +- cranelift/codegen/src/ir/dfg.rs | 22 +- cranelift/codegen/src/ir/dynamic_type.rs | 2 +- cranelift/codegen/src/ir/entities.rs | 6 + cranelift/codegen/src/ir/extfunc.rs | 45 ++- cranelift/codegen/src/ir/extname.rs | 266 +++++++++++++---- cranelift/codegen/src/ir/function.rs | 268 ++++++++++++++---- cranelift/codegen/src/ir/globalvalue.rs | 4 +- cranelift/codegen/src/ir/heap.rs | 4 +- cranelift/codegen/src/ir/jumptable.rs | 2 +- cranelift/codegen/src/ir/layout.rs | 8 +- cranelift/codegen/src/ir/libcall.rs | 6 +- cranelift/codegen/src/ir/mod.rs | 15 +- cranelift/codegen/src/ir/sourceloc.rs | 55 ++++ cranelift/codegen/src/ir/stackslot.rs | 6 +- cranelift/codegen/src/ir/table.rs | 2 +- .../codegen/src/isa/aarch64/inst/emit.rs | 32 +-- cranelift/codegen/src/isa/aarch64/inst/mod.rs | 4 +- .../src/isa/aarch64/inst/unwind/systemv.rs | 8 +- cranelift/codegen/src/isa/aarch64/mod.rs | 19 +- cranelift/codegen/src/isa/mod.rs | 8 +- cranelift/codegen/src/isa/s390x/inst/args.rs | 2 +- cranelift/codegen/src/isa/s390x/inst/emit.rs | 24 +- cranelift/codegen/src/isa/s390x/inst/mod.rs | 12 +- .../src/isa/s390x/inst/unwind/systemv.rs | 8 +- cranelift/codegen/src/isa/s390x/lower/isle.rs | 1 + cranelift/codegen/src/isa/s390x/mod.rs | 18 +- cranelift/codegen/src/isa/x64/abi.rs | 4 +- .../codegen/src/isa/x64/inst/emit_tests.rs | 51 ++-- cranelift/codegen/src/isa/x64/inst/mod.rs | 10 +- .../src/isa/x64/inst/unwind/systemv.rs | 8 +- cranelift/codegen/src/isa/x64/mod.rs | 32 ++- cranelift/codegen/src/legalizer/heap.rs | 4 +- cranelift/codegen/src/lib.rs | 5 +- cranelift/codegen/src/machinst/blockorder.rs | 5 +- cranelift/codegen/src/machinst/buffer.rs | 129 +++++++-- cranelift/codegen/src/machinst/lower.rs | 21 +- cranelift/codegen/src/machinst/mod.rs | 43 ++- cranelift/codegen/src/machinst/vcode.rs | 19 +- cranelift/codegen/src/settings.rs | 1 + cranelift/codegen/src/timing.rs | 2 + cranelift/codegen/src/verifier/mod.rs | 4 +- cranelift/codegen/src/write.rs | 15 +- cranelift/entity/src/list.rs | 16 +- cranelift/entity/src/map.rs | 2 +- .../filetests/isa/aarch64/call-pauth.clif | 2 +- .../filetests/filetests/isa/aarch64/call.clif | 64 ++--- .../filetests/isa/aarch64/reftypes.clif | 2 +- .../filetests/isa/aarch64/stack-limit.clif | 4 +- .../filetests/isa/aarch64/symbol-value.clif | 2 +- .../filetests/isa/aarch64/tls-elf-gd.clif | 2 +- .../filetests/isa/s390x/struct-arg.clif | 8 +- .../filetests/isa/s390x/tls_elf.clif | 4 +- .../filetests/isa/x64/struct-arg.clif | 6 +- .../filetests/filetests/isa/x64/tls_coff.clif | 3 +- .../filetests/filetests/isa/x64/tls_elf.clif | 2 +- .../filetests/filetests/parser/memory.clif | 2 +- cranelift/filetests/src/function_runner.rs | 6 +- cranelift/frontend/src/frontend.rs | 56 ++-- cranelift/frontend/src/lib.rs | 8 +- cranelift/frontend/src/switch.rs | 4 +- cranelift/fuzzgen/src/config.rs | 4 + cranelift/fuzzgen/src/function_generator.rs | 68 ++++- cranelift/fuzzgen/src/lib.rs | 23 +- cranelift/interpreter/src/environment.rs | 6 +- cranelift/jit/examples/jit-minimal.rs | 7 +- cranelift/jit/src/backend.rs | 101 ++++--- cranelift/jit/src/compiled_blob.rs | 14 +- cranelift/jit/tests/basic.rs | 7 +- cranelift/module/src/data_context.rs | 34 ++- cranelift/module/src/lib.rs | 2 +- cranelift/module/src/module.rs | 152 +++++++--- cranelift/object/src/backend.rs | 26 +- cranelift/object/tests/basic.rs | 7 +- cranelift/reader/src/lexer.rs | 2 + cranelift/reader/src/parser.rs | 105 ++++++- cranelift/src/bugpoint.rs | 6 +- cranelift/src/compile.rs | 1 + cranelift/src/disasm.rs | 11 +- cranelift/src/wasm.rs | 1 + cranelift/wasm/src/environ/dummy.rs | 20 +- cranelift/wasm/src/func_translator.rs | 6 +- crates/cranelift/Cargo.toml | 1 + crates/cranelift/src/builder.rs | 13 +- crates/cranelift/src/compiler.rs | 184 ++++++++++-- crates/cranelift/src/compiler/component.rs | 21 +- crates/cranelift/src/func_environ.rs | 11 +- crates/environ/src/compilation.rs | 21 ++ crates/wasmtime/Cargo.toml | 3 + crates/wasmtime/src/config.rs | 20 ++ crates/wasmtime/src/engine.rs | 1 + fuzz/Cargo.toml | 8 +- fuzz/README.md | 4 + fuzz/fuzz_targets/cranelift-icache.rs | 148 ++++++++++ tests/all/module.rs | 6 +- 103 files changed, 2176 insertions(+), 693 deletions(-) create mode 100644 cranelift/codegen/src/incremental_cache.rs create mode 100644 fuzz/fuzz_targets/cranelift-icache.rs diff --git a/Cargo.lock b/Cargo.lock index 226ec05fdf..76caf07c2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -542,6 +542,7 @@ dependencies = [ "miette", "regalloc2", "serde", + "sha2", "smallvec", "souper-ir", "target-lexicon", @@ -2646,6 +2647,9 @@ name = "smallvec" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +dependencies = [ + "serde", +] [[package]] name = "smawk" diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index dbc919cf6d..a0f3eb8136 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -25,6 +25,7 @@ gimli = { version = "0.26.0", default-features = false, features = ["write"], op smallvec = { version = "1.6.1" } regalloc2 = { version = "0.3.2", features = ["checker"] } souper-ir = { version = "2.1.0", optional = true } +sha2 = { version = "0.9.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary # machine code. Integration tests that need external dependencies can be @@ -82,6 +83,14 @@ enable-serde = [ "serde", "cranelift-entity/enable-serde", "regalloc2/enable-serde", + "smallvec/serde" +] + +# Enable the incremental compilation cache for hot-reload use cases. +incremental-cache = [ + "enable-serde", + "bincode", + "sha2" ] # Enable support for the Souper harvester. diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs index eb2a6dfd20..d85f6dc69b 100644 --- a/cranelift/codegen/meta/src/gen_inst.rs +++ b/cranelift/codegen/meta/src/gen_inst.rs @@ -66,7 +66,7 @@ fn gen_formats(formats: &[&InstructionFormat], fmt: &mut Formatter) { /// 16 bytes on 64-bit architectures. If more space is needed to represent an instruction, use a /// `ValueList` to store the additional information out of line. fn gen_instruction_data(formats: &[&InstructionFormat], fmt: &mut Formatter) { - fmt.line("#[derive(Clone, Debug)]"); + fmt.line("#[derive(Clone, Debug, PartialEq, Hash)]"); fmt.line(r#"#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]"#); fmt.line("#[allow(missing_docs)]"); fmt.line("pub enum InstructionData {"); diff --git a/cranelift/codegen/meta/src/shared/settings.rs b/cranelift/codegen/meta/src/shared/settings.rs index cf4473a1cd..9cdfcc19a8 100644 --- a/cranelift/codegen/meta/src/shared/settings.rs +++ b/cranelift/codegen/meta/src/shared/settings.rs @@ -327,5 +327,19 @@ pub(crate) fn define() -> SettingGroup { true, ); + settings.add_bool( + "enable_incremental_compilation_cache_checks", + "Enable additional checks for debugging the incremental compilation cache.", + r#" + Enables additional checks that are useful during development of the incremental + compilation cache. This should be mostly useful for Cranelift hackers, as well as for + helping to debug false incremental cache positives for embedders. + + This option is disabled by default and requires enabling the "incremental-cache" Cargo + feature in cranelift-codegen. + "#, + false, + ); + settings.build() } diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index a9d43d51e3..713d89bec8 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -103,7 +103,7 @@ impl fmt::Display for Reloc { /// The code starts at offset 0 and is followed optionally by relocatable jump tables and copyable /// (raw binary) read-only data. Any padding between sections is always part of the section that /// precedes the boundary between the sections. -#[derive(PartialEq)] +#[derive(Debug, PartialEq)] pub struct CodeInfo { /// Number of bytes in total. pub total_size: CodeOffset, diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index fb7b8bb37d..1003e57337 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -18,7 +18,7 @@ use crate::isa::TargetIsa; use crate::legalizer::simple_legalize; use crate::licm::do_licm; use crate::loop_analysis::LoopAnalysis; -use crate::machinst::CompiledCode; +use crate::machinst::{CompiledCode, CompiledCodeStencil}; use crate::nan_canonicalization::do_nan_canonicalization; use crate::remove_constant_phis::do_remove_constant_phis; use crate::result::{CodegenResult, CompileResult}; @@ -50,7 +50,7 @@ pub struct Context { pub loop_analysis: LoopAnalysis, /// Result of MachBackend compilation, if computed. - compiled_code: Option, + pub(crate) compiled_code: Option, /// Flag: do we want a disassembly with the CompiledCode? pub want_disasm: bool, @@ -109,7 +109,7 @@ impl Context { /// `Vec`. The machine code is not relocated. Instead, any relocations can be obtained /// from `compiled_code()`. /// - /// This function calls `compile` and `emit_to_memory`, taking care to resize `mem` as + /// This function calls `compile`, taking care to resize `mem` as /// needed, so it provides a safe interface. /// /// Returns information about the function's code and read-only data. @@ -126,6 +126,53 @@ impl Context { Ok(compiled_code) } + /// Internally compiles the function into a stencil. + /// + /// Public only for testing and fuzzing purposes. + pub fn compile_stencil(&mut self, isa: &dyn TargetIsa) -> CodegenResult { + let _tt = timing::compile(); + + self.verify_if(isa)?; + + let opt_level = isa.flags().opt_level(); + log::trace!( + "Compiling (opt level {:?}):\n{}", + opt_level, + self.func.display() + ); + + self.compute_cfg(); + if opt_level != OptLevel::None { + self.preopt(isa)?; + } + if isa.flags().enable_nan_canonicalization() { + self.canonicalize_nans(isa)?; + } + + self.legalize(isa)?; + if opt_level != OptLevel::None { + self.compute_domtree(); + self.compute_loop_analysis(); + self.licm(isa)?; + self.simple_gvn(isa)?; + } + + self.compute_domtree(); + self.eliminate_unreachable_code(isa)?; + if opt_level != OptLevel::None { + self.dce(isa)?; + } + + self.remove_constant_phis(isa)?; + + if opt_level != OptLevel::None && isa.flags().enable_alias_analysis() { + self.replace_redundant_loads()?; + self.simple_gvn(isa)?; + } + + isa.compile_function(&self.func, self.want_disasm) + } + /// Compile the function. /// /// Run the function through all the passes necessary to generate code for the target ISA @@ -135,57 +182,13 @@ impl Context { /// Returns information about the function's code and read-only data. pub fn compile(&mut self, isa: &dyn TargetIsa) -> CompileResult<&CompiledCode> { let _tt = timing::compile(); - - let mut inner = || { - self.verify_if(isa)?; - - let opt_level = isa.flags().opt_level(); - log::trace!( - "Compiling (opt level {:?}):\n{}", - opt_level, - self.func.display() - ); - - self.compute_cfg(); - if opt_level != OptLevel::None { - self.preopt(isa)?; - } - if isa.flags().enable_nan_canonicalization() { - self.canonicalize_nans(isa)?; - } - - self.legalize(isa)?; - if opt_level != OptLevel::None { - self.compute_domtree(); - self.compute_loop_analysis(); - self.licm(isa)?; - self.simple_gvn(isa)?; - } - - self.compute_domtree(); - self.eliminate_unreachable_code(isa)?; - if opt_level != OptLevel::None { - self.dce(isa)?; - } - - self.remove_constant_phis(isa)?; - - if opt_level != OptLevel::None && isa.flags().enable_alias_analysis() { - self.replace_redundant_loads()?; - self.simple_gvn(isa)?; - } - - let result = isa.compile_function(&self.func, self.want_disasm)?; - self.compiled_code = Some(result); - Ok(()) - }; - - inner() - .map(|_| self.compiled_code.as_ref().unwrap()) - .map_err(|error| CompileError { - inner: error, - func: &self.func, - }) + let stencil = self.compile_stencil(isa).map_err(|error| CompileError { + inner: error, + func: &self.func, + })?; + Ok(self + .compiled_code + .insert(stencil.apply_params(&self.func.params))) } /// If available, return information about the code layout in the diff --git a/cranelift/codegen/src/cursor.rs b/cranelift/codegen/src/cursor.rs index 2dc8ce7a2b..0a6d74d21b 100644 --- a/cranelift/codegen/src/cursor.rs +++ b/cranelift/codegen/src/cursor.rs @@ -589,7 +589,7 @@ impl<'f> FuncCursor<'f> { /// Use the source location of `inst` for future instructions. pub fn use_srcloc(&mut self, inst: ir::Inst) { - self.srcloc = self.func.srclocs[inst]; + self.srcloc = self.func.srcloc(inst); } /// Create an instruction builder that inserts an instruction at the current position. @@ -612,6 +612,7 @@ impl<'f> Cursor for FuncCursor<'f> { } fn set_srcloc(&mut self, srcloc: ir::SourceLoc) { + self.func.params.ensure_base_srcloc(srcloc); self.srcloc = srcloc; } @@ -658,7 +659,7 @@ impl<'c, 'f> ir::InstInserterBase<'c> for &'c mut FuncCursor<'f> { } self.insert_inst(inst); if !self.srcloc.is_default() { - self.func.srclocs[inst] = self.srcloc; + self.func.set_srcloc(inst, self.srcloc); } &mut self.func.dfg } diff --git a/cranelift/codegen/src/incremental_cache.rs b/cranelift/codegen/src/incremental_cache.rs new file mode 100644 index 0000000000..61702bd776 --- /dev/null +++ b/cranelift/codegen/src/incremental_cache.rs @@ -0,0 +1,254 @@ +//! This module provides a set of primitives that allow implementing an incremental cache on top of +//! Cranelift, making it possible to reuse previous compiled artifacts for functions that have been +//! compiled previously. +//! +//! This set of operation is experimental and can be enabled using the Cargo feature +//! `incremental-cache`. +//! +//! This can bring speedups in different cases: change-code-and-immediately-recompile iterations +//! get faster, modules sharing lots of code can reuse each other's artifacts, etc. +//! +//! The three main primitives are the following: +//! - `compute_cache_key` is used to compute the cache key associated to a `Function`. This is +//! basically the content of the function, modulo a few things the caching system is resilient to. +//! - `serialize_compiled` is used to serialize the result of a compilation, so it can be reused +//! later on by... +//! - `try_finish_recompile`, which reads binary blobs serialized with `serialize_compiled`, +//! re-creating the compilation artifact from those. +//! +//! The `CacheStore` trait and `Context::compile_with_cache` method are provided as +//! high-level, easy-to-use facilities to make use of that cache, and show an example of how to use +//! the above three primitives to form a full incremental caching system. + +use core::fmt; + +use crate::alloc::string::String; +use crate::alloc::vec::Vec; +use crate::ir::function::{FunctionStencil, VersionMarker}; +use crate::ir::Function; +use crate::machinst::{CompiledCode, CompiledCodeStencil}; +use crate::result::CompileResult; +use crate::{isa::TargetIsa, timing}; +use crate::{trace, CompileError, Context}; +use alloc::borrow::{Cow, ToOwned as _}; +use alloc::string::ToString as _; + +impl Context { + /// Compile the function, as in `compile`, but tries to reuse compiled artifacts from former + /// compilations using the provided cache store. + pub fn compile_with_cache( + &mut self, + isa: &dyn TargetIsa, + cache_store: &mut dyn CacheKvStore, + ) -> CompileResult<(&CompiledCode, bool)> { + let cache_key_hash = { + let _tt = timing::try_incremental_cache(); + + let cache_key_hash = compute_cache_key(isa, &mut self.func); + + if let Some(blob) = cache_store.get(&cache_key_hash.0) { + match try_finish_recompile(&self.func, &blob) { + Ok(compiled_code) => { + let info = compiled_code.code_info(); + + if isa.flags().enable_incremental_compilation_cache_checks() { + let actual_result = self.compile(isa)?; + assert_eq!(*actual_result, compiled_code); + assert_eq!(actual_result.code_info(), info); + // no need to set `compiled_code` here, it's set by `compile()`. + return Ok((actual_result, true)); + } + + let compiled_code = self.compiled_code.insert(compiled_code); + return Ok((compiled_code, true)); + } + Err(err) => { + trace!("error when finishing recompilation: {err}"); + } + } + } + + cache_key_hash + }; + + let stencil = self.compile_stencil(isa).map_err(|err| CompileError { + inner: err, + func: &self.func, + })?; + + let stencil = { + let _tt = timing::store_incremental_cache(); + let (stencil, res) = serialize_compiled(stencil); + if let Ok(blob) = res { + cache_store.insert(&cache_key_hash.0, blob); + } + stencil + }; + + let compiled_code = self + .compiled_code + .insert(stencil.apply_params(&self.func.params)); + + Ok((compiled_code, false)) + } +} + +/// Backing storage for an incremental compilation cache, when enabled. +pub trait CacheKvStore { + /// Given a cache key hash, retrieves the associated opaque serialized data. + fn get(&self, key: &[u8]) -> Option>; + + /// Given a new cache key and a serialized blob obtained from `serialize_compiled`, stores it + /// in the cache store. + fn insert(&mut self, key: &[u8], val: Vec); +} + +/// Hashed `CachedKey`, to use as an identifier when looking up whether a function has already been +/// compiled or not. +#[derive(Clone, Hash, PartialEq, Eq)] +pub struct CacheKeyHash([u8; 32]); + +impl std::fmt::Display for CacheKeyHash { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "CacheKeyHash:{:?}", self.0) + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct CachedFunc { + stencil: CompiledCodeStencil, + version_marker: VersionMarker, +} + +/// Key for caching a single function's compilation. +/// +/// If two functions get the same `CacheKey`, then we can reuse the compiled artifacts, modulo some +/// fixups. +/// +/// Note: the key will be invalidated across different versions of cranelift, as the +/// `FunctionStencil` contains a `VersionMarker` itself. +#[derive(Hash)] +struct CacheKey<'a> { + stencil: &'a FunctionStencil, + parameters: CompileParameters, +} + +#[derive(Clone, PartialEq, Hash, serde::Serialize, serde::Deserialize)] +struct CompileParameters { + isa: String, + triple: String, + flags: String, + isa_flags: Vec, +} + +impl CompileParameters { + fn from_isa(isa: &dyn TargetIsa) -> Self { + Self { + isa: isa.name().to_owned(), + triple: isa.triple().to_string(), + flags: isa.flags().to_string(), + isa_flags: isa + .isa_flags() + .into_iter() + .map(|v| v.value_string()) + .collect(), + } + } +} + +impl<'a> CacheKey<'a> { + /// Creates a new cache store key for a function. + /// + /// This is a bit expensive to compute, so it should be cached and reused as much as possible. + fn new(isa: &dyn TargetIsa, f: &'a mut Function) -> Self { + // Make sure the blocks and instructions are sequenced the same way as we might + // have serialized them earlier. This is the symmetric of what's done in + // `try_load`. + f.stencil.layout.full_renumber(); + CacheKey { + stencil: &f.stencil, + parameters: CompileParameters::from_isa(isa), + } + } +} + +/// Compute a cache key, and hash it on your behalf. +/// +/// Since computing the `CacheKey` is a bit expensive, it should be done as least as possible. +pub fn compute_cache_key(isa: &dyn TargetIsa, func: &mut Function) -> CacheKeyHash { + use core::hash::{Hash as _, Hasher}; + use sha2::Digest as _; + + struct Sha256Hasher(sha2::Sha256); + + impl Hasher for Sha256Hasher { + fn finish(&self) -> u64 { + panic!("Sha256Hasher doesn't support finish!"); + } + fn write(&mut self, bytes: &[u8]) { + self.0.update(bytes); + } + } + + let cache_key = CacheKey::new(isa, func); + + let mut hasher = Sha256Hasher(sha2::Sha256::new()); + cache_key.hash(&mut hasher); + let hash: [u8; 32] = hasher.0.finalize().into(); + + CacheKeyHash(hash) +} + +/// Given a function that's been successfully compiled, serialize it to a blob that the caller may +/// store somewhere for future use by `try_finish_recompile`. +/// +/// As this function requires ownership on the `CompiledCodeStencil`, it gives it back at the end +/// of the function call. The value is left untouched. +pub fn serialize_compiled( + result: CompiledCodeStencil, +) -> (CompiledCodeStencil, Result, bincode::Error>) { + let cached = CachedFunc { + stencil: result, + version_marker: VersionMarker, + }; + let result = bincode::serialize(&cached); + (cached.stencil, result) +} + +/// An error returned when recompiling failed. +#[derive(Debug)] +pub enum RecompileError { + /// The version embedded in the cache entry isn't the same as cranelift's current version. + VersionMismatch, + /// An error occurred while deserializing the cache entry. + Deserialize(bincode::Error), +} + +impl fmt::Display for RecompileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RecompileError::VersionMismatch => write!(f, "cranelift version mismatch",), + RecompileError::Deserialize(err) => { + write!(f, "bincode failed during deserialization: {err}") + } + } + } +} + +/// Given a function that's been precompiled and its entry in the caching storage, try to shortcut +/// compilation of the given function. +/// +/// Precondition: the bytes must have retrieved from a cache store entry which hash value +/// is strictly the same as the `Function`'s computed hash retrieved from `compute_cache_key`. +pub fn try_finish_recompile(func: &Function, bytes: &[u8]) -> Result { + match bincode::deserialize::(bytes) { + Ok(result) => { + if result.version_marker != func.stencil.version_marker { + Err(RecompileError::VersionMismatch) + } else { + Ok(result.stencil.apply_params(&func.params)) + } + } + Err(err) => Err(RecompileError::Deserialize(err)), + } +} diff --git a/cranelift/codegen/src/ir/constant.rs b/cranelift/codegen/src/ir/constant.rs index 3cd88d5546..1c540c0c37 100644 --- a/cranelift/codegen/src/ir/constant.rs +++ b/cranelift/codegen/src/ir/constant.rs @@ -10,7 +10,6 @@ use crate::ir::immediates::{IntoBytes, V128Imm}; use crate::ir::Constant; -use crate::HashMap; use alloc::collections::BTreeMap; use alloc::vec::Vec; use core::fmt; @@ -27,7 +26,7 @@ use serde::{Deserialize, Serialize}; /// WebAssembly values, which are [little-endian by design]. /// /// [little-endian by design]: https://github.com/WebAssembly/design/blob/master/Portability.md -#[derive(Clone, Hash, Eq, PartialEq, Debug, Default)] +#[derive(Clone, Hash, Eq, PartialEq, Debug, Default, PartialOrd, Ord)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ConstantData(Vec); @@ -169,16 +168,20 @@ impl FromStr for ConstantData { /// Maintains the mapping between a constant handle (i.e. [`Constant`](crate::ir::Constant)) and /// its constant data (i.e. [`ConstantData`](crate::ir::ConstantData)). -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ConstantPool { /// This mapping maintains the insertion order as long as Constants are created with /// sequentially increasing integers. + /// + /// It is important that, by construction, no entry in that list gets removed. If that ever + /// need to happen, don't forget to update the `Constant` generation scheme. handles_to_values: BTreeMap, - /// This mapping is unordered (no need for lexicographic ordering) but allows us to map - /// constant data back to handles. - values_to_handles: HashMap, + /// Mapping of hashed `ConstantData` to the index into the other hashmap. + /// + /// This allows for deduplication of entries into the `handles_to_values` mapping. + values_to_handles: BTreeMap, } impl ConstantPool { @@ -186,7 +189,7 @@ impl ConstantPool { pub fn new() -> Self { Self { handles_to_values: BTreeMap::new(), - values_to_handles: HashMap::new(), + values_to_handles: BTreeMap::new(), } } @@ -200,13 +203,13 @@ impl ConstantPool { /// data is inserted that is a duplicate of previous constant data, the existing handle will be /// returned. pub fn insert(&mut self, constant_value: ConstantData) -> Constant { - if self.values_to_handles.contains_key(&constant_value) { - *self.values_to_handles.get(&constant_value).unwrap() - } else { - let constant_handle = Constant::new(self.len()); - self.set(constant_handle, constant_value); - constant_handle + if let Some(cst) = self.values_to_handles.get(&constant_value) { + return *cst; } + + let constant_handle = Constant::new(self.len()); + self.set(constant_handle, constant_value); + constant_handle } /// Retrieve the constant data given a handle. @@ -250,7 +253,7 @@ impl ConstantPool { /// Return the combined size of all of the constant values in the pool. pub fn byte_size(&self) -> usize { - self.values_to_handles.keys().map(|c| c.len()).sum() + self.handles_to_values.values().map(|c| c.len()).sum() } } diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs index 65b97cbb71..9173c5fed8 100644 --- a/cranelift/codegen/src/ir/dfg.rs +++ b/cranelift/codegen/src/ir/dfg.rs @@ -4,22 +4,22 @@ use crate::entity::{self, PrimaryMap, SecondaryMap}; use crate::ir; use crate::ir::builder::ReplaceBuilder; use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes}; -use crate::ir::extfunc::ExtFuncData; use crate::ir::instructions::{BranchInfo, CallInfo, InstructionData}; use crate::ir::{types, ConstantData, ConstantPool, Immediate}; use crate::ir::{ - Block, DynamicType, FuncRef, Inst, SigRef, Signature, SourceLoc, Type, Value, - ValueLabelAssignments, ValueList, ValueListPool, + Block, DynamicType, FuncRef, Inst, SigRef, Signature, Type, Value, ValueLabelAssignments, + ValueList, ValueListPool, }; +use crate::ir::{ExtFuncData, RelSourceLoc}; use crate::packed_option::ReservedValue; use crate::write::write_operands; -use crate::HashMap; use core::fmt; use core::iter; use core::mem; use core::ops::{Index, IndexMut}; use core::u16; +use alloc::collections::BTreeMap; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -30,7 +30,7 @@ use serde::{Deserialize, Serialize}; /// The layout of blocks in the function and of instructions in each block is recorded by the /// `Layout` data structure which forms the other half of the function representation. /// -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DataFlowGraph { /// Data about all of the instructions in the function, including opcodes and operands. @@ -76,7 +76,7 @@ pub struct DataFlowGraph { pub ext_funcs: PrimaryMap, /// Saves Value labels. - pub values_labels: Option>, + pub values_labels: Option>, /// Constants used within the function pub constants: ConstantPool, @@ -154,13 +154,13 @@ impl DataFlowGraph { /// Starts collection of debug information. pub fn collect_debug_info(&mut self) { if self.values_labels.is_none() { - self.values_labels = Some(HashMap::new()); + self.values_labels = Some(Default::default()); } } /// Inserts a `ValueLabelAssignments::Alias` for `to_alias` if debug info /// collection is enabled. - pub fn add_value_label_alias(&mut self, to_alias: Value, from: SourceLoc, value: Value) { + pub fn add_value_label_alias(&mut self, to_alias: Value, from: RelSourceLoc, value: Value) { if let Some(values_labels) = self.values_labels.as_mut() { values_labels.insert(to_alias, ir::ValueLabelAssignments::Alias { from, value }); } @@ -435,7 +435,7 @@ impl ValueDef { } /// Internal table storage for extended values. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] enum ValueData { /// Value is defined by an instruction. @@ -457,7 +457,7 @@ enum ValueData { /// ```plain /// | tag:2 | type:14 | num:16 | index:32 | /// ``` -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] struct ValueDataPacked(u64); @@ -1056,7 +1056,7 @@ impl DataFlowGraph { /// Parameters on a basic block are values that dominate everything in the block. All /// branches to this block must provide matching arguments, and the arguments to the entry block must /// match the function arguments. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] struct BlockData { /// List of parameters to this block. diff --git a/cranelift/codegen/src/ir/dynamic_type.rs b/cranelift/codegen/src/ir/dynamic_type.rs index 85589cef67..91b13af98b 100644 --- a/cranelift/codegen/src/ir/dynamic_type.rs +++ b/cranelift/codegen/src/ir/dynamic_type.rs @@ -9,7 +9,7 @@ use crate::ir::Type; use serde::{Deserialize, Serialize}; /// A dynamic type object which has a base vector type and a scaling factor. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DynamicTypeData { /// Base vector type, this is the minimum size of the type. diff --git a/cranelift/codegen/src/ir/entities.rs b/cranelift/codegen/src/ir/entities.rs index 2be7014685..ea113397ad 100644 --- a/cranelift/codegen/src/ir/entities.rs +++ b/cranelift/codegen/src/ir/entities.rs @@ -328,6 +328,12 @@ impl FuncRef { } } +/// A reference to an `UserExternalName`, declared with `Function::declare_imported_user_function`. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UserExternalNameRef(u32); +entity_impl!(UserExternalNameRef, "userextname"); + /// An opaque reference to a function [`Signature`](super::Signature). /// /// `SigRef`s are used to declare a function with diff --git a/cranelift/codegen/src/ir/extfunc.rs b/cranelift/codegen/src/ir/extfunc.rs index 3491849ff9..07bc52c8c1 100644 --- a/cranelift/codegen/src/ir/extfunc.rs +++ b/cranelift/codegen/src/ir/extfunc.rs @@ -14,6 +14,8 @@ use core::str::FromStr; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; +use super::function::FunctionParameters; + /// Function signature. /// /// The function signature describes the types of formal parameters and return values along with @@ -301,7 +303,7 @@ impl FromStr for ArgumentPurpose { /// An external function. /// /// Information about a function that can be called directly with a direct `call` instruction. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ExtFuncData { /// Name of the external function. @@ -324,15 +326,6 @@ pub struct ExtFuncData { pub colocated: bool, } -impl fmt::Display for ExtFuncData { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.colocated { - write!(f, "colocated ")?; - } - write!(f, "{} {}", self.name, self.signature) - } -} - impl ExtFuncData { /// Return an estimate of the distance to the referred-to function symbol. pub fn reloc_distance(&self) -> RelocDistance { @@ -342,6 +335,38 @@ impl ExtFuncData { RelocDistance::Far } } + + /// Returns a displayable version of the `ExtFuncData`, with or without extra context to + /// prettify the output. + pub fn display<'a>( + &'a self, + params: Option<&'a FunctionParameters>, + ) -> DisplayableExtFuncData<'a> { + DisplayableExtFuncData { + ext_func: self, + params, + } + } +} + +/// A displayable `ExtFuncData`, with extra context to prettify the output. +pub struct DisplayableExtFuncData<'a> { + ext_func: &'a ExtFuncData, + params: Option<&'a FunctionParameters>, +} + +impl<'a> fmt::Display for DisplayableExtFuncData<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.ext_func.colocated { + write!(f, "colocated ")?; + } + write!( + f, + "{} {}", + self.ext_func.name.display(self.params), + self.ext_func.signature + ) + } } #[cfg(test)] diff --git a/cranelift/codegen/src/ir/extname.rs b/cranelift/codegen/src/ir/extname.rs index 54c56a4247..64b6d174a5 100644 --- a/cranelift/codegen/src/ir/extname.rs +++ b/cranelift/codegen/src/ir/extname.rs @@ -9,10 +9,108 @@ use core::cmp; use core::fmt::{self, Write}; use core::str::FromStr; +use cranelift_entity::EntityRef as _; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; -const TESTCASE_NAME_LENGTH: usize = 16; +use super::entities::UserExternalNameRef; +use super::function::FunctionParameters; + +pub(crate) const TESTCASE_NAME_LENGTH: usize = 16; + +/// An explicit name for a user-defined function, be it defined in code or in CLIF text. +/// +/// This is used both for naming a function (for debugging purposes) and for declaring external +/// functions. In the latter case, this becomes an `ExternalName`, which gets embedded in +/// relocations later, etc. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub enum UserFuncName { + /// A user-defined name, with semantics left to the user. + User(UserExternalName), + /// A name for a test case, mostly intended for Cranelift testing. + Testcase(TestcaseName), +} + +impl UserFuncName { + /// Creates a new external name from a sequence of bytes. Caller is expected + /// to guarantee bytes are only ascii alphanumeric or `_`. + pub fn testcase>(v: T) -> Self { + Self::Testcase(TestcaseName::new(v)) + } + + /// Create a new external name from a user-defined external function reference. + pub fn user(namespace: u32, index: u32) -> Self { + Self::User(UserExternalName { namespace, index }) + } +} + +impl Default for UserFuncName { + fn default() -> Self { + UserFuncName::User(UserExternalName::default()) + } +} + +impl fmt::Display for UserFuncName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + UserFuncName::User(user) => user.fmt(f), + UserFuncName::Testcase(testcase) => testcase.fmt(f), + } + } +} + +/// An external name in a user-defined symbol table. +/// +/// Cranelift does not interpret these numbers in any way, so they can represent arbitrary values. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UserExternalName { + /// Arbitrary. + pub namespace: u32, + /// Arbitrary. + pub index: u32, +} + +impl fmt::Display for UserExternalName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "u{}:{}", self.namespace, self.index) + } +} + +/// A name for a test case. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct TestcaseName { + /// How many of the bytes in `ascii` are valid? + length: u8, + /// Ascii bytes of the name. + ascii: [u8; TESTCASE_NAME_LENGTH], +} + +impl fmt::Display for TestcaseName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('%')?; + for byte in self.ascii.iter().take(self.length as usize) { + f.write_char(*byte as char)?; + } + Ok(()) + } +} + +impl TestcaseName { + pub(crate) fn new>(v: T) -> Self { + let vec = v.as_ref(); + let len = cmp::min(vec.len(), TESTCASE_NAME_LENGTH); + let mut bytes = [0u8; TESTCASE_NAME_LENGTH]; + bytes[0..len].copy_from_slice(&vec[0..len]); + + Self { + length: len as u8, + ascii: bytes, + } + } +} /// The name of an external is either a reference to a user-defined symbol /// table, or a short sequence of ascii bytes so that test cases do not have @@ -25,31 +123,26 @@ const TESTCASE_NAME_LENGTH: usize = 16; /// External names can also serve as a primitive testing and debugging tool. /// In particular, many `.clif` test files use function names to identify /// functions. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum ExternalName { - /// A name in a user-defined symbol table. Cranelift does not interpret - /// these numbers in any way. - User { - /// Arbitrary. - namespace: u32, - /// Arbitrary. - index: u32, - }, + /// A reference to a name in a user-defined symbol table. + User(UserExternalNameRef), /// A test case function name of up to a hardcoded amount of ascii /// characters. This is not intended to be used outside test cases. - TestCase { - /// How many of the bytes in `ascii` are valid? - length: u8, - /// Ascii bytes of the name. - ascii: [u8; TESTCASE_NAME_LENGTH], - }, + TestCase(TestcaseName), /// A well-known runtime library function. LibCall(LibCall), /// A well-known symbol. KnownSymbol(KnownSymbol), } +impl Default for ExternalName { + fn default() -> Self { + Self::User(UserExternalNameRef::new(0)) + } +} + impl ExternalName { /// Creates a new external name from a sequence of bytes. Caller is expected /// to guarantee bytes are only ascii alphanumeric or `_`. @@ -60,53 +153,56 @@ impl ExternalName { /// # use cranelift_codegen::ir::ExternalName; /// // Create `ExternalName` from a string. /// let name = ExternalName::testcase("hello"); - /// assert_eq!(name.to_string(), "%hello"); + /// assert_eq!(name.display(None).to_string(), "%hello"); /// ``` pub fn testcase>(v: T) -> Self { - let vec = v.as_ref(); - let len = cmp::min(vec.len(), TESTCASE_NAME_LENGTH); - let mut bytes = [0u8; TESTCASE_NAME_LENGTH]; - bytes[0..len].copy_from_slice(&vec[0..len]); - - Self::TestCase { - length: len as u8, - ascii: bytes, - } + Self::TestCase(TestcaseName::new(v)) } - /// Create a new external name from user-provided integer indices. + /// Create a new external name from a user-defined external function reference. /// /// # Examples /// ```rust - /// # use cranelift_codegen::ir::ExternalName; - /// // Create `ExternalName` from integer indices - /// let name = ExternalName::user(123, 456); - /// assert_eq!(name.to_string(), "u123:456"); + /// # use cranelift_codegen::ir::{ExternalName, UserExternalNameRef}; + /// let user_func_ref: UserExternalNameRef = Default::default(); // usually obtained with `Function::declare_imported_user_function()` + /// let name = ExternalName::user(user_func_ref); + /// assert_eq!(name.display(None).to_string(), "userextname0"); /// ``` - pub fn user(namespace: u32, index: u32) -> Self { - Self::User { namespace, index } + pub fn user(func_ref: UserExternalNameRef) -> Self { + Self::User(func_ref) + } + + /// Returns a display for the current `ExternalName`, with extra context to prettify the + /// output. + pub fn display<'a>( + &'a self, + params: Option<&'a FunctionParameters>, + ) -> DisplayableExternalName<'a> { + DisplayableExternalName { name: self, params } } } -impl Default for ExternalName { - fn default() -> Self { - Self::user(0, 0) - } +/// An `ExternalName` that has enough context to be displayed. +pub struct DisplayableExternalName<'a> { + name: &'a ExternalName, + params: Option<&'a FunctionParameters>, } -impl fmt::Display for ExternalName { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Self::User { namespace, index } => write!(f, "u{}:{}", namespace, index), - Self::TestCase { length, ascii } => { - f.write_char('%')?; - for byte in ascii.iter().take(length as usize) { - f.write_char(*byte as char)?; +impl<'a> fmt::Display for DisplayableExternalName<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.name { + ExternalName::User(func_ref) => { + if let Some(params) = self.params { + let name = ¶ms.user_named_funcs()[*func_ref]; + write!(f, "u{}:{}", name.namespace, name.index) + } else { + // Best effort. + write!(f, "{}", *func_ref) } - Ok(()) } - Self::LibCall(lc) => write!(f, "%{}", lc), - Self::KnownSymbol(ks) => write!(f, "%{}", ks), + ExternalName::TestCase(testcase) => testcase.fmt(f), + ExternalName::LibCall(lc) => write!(f, "%{}", lc), + ExternalName::KnownSymbol(ks) => write!(f, "%{}", ks), } } } @@ -133,33 +229,83 @@ impl FromStr for ExternalName { #[cfg(test)] mod tests { use super::ExternalName; - use crate::ir::LibCall; + use crate::ir::{ + entities::UserExternalNameRef, function::FunctionParameters, LibCall, UserExternalName, + }; use alloc::string::ToString; use core::u32; + use cranelift_entity::EntityRef as _; #[test] fn display_testcase() { - assert_eq!(ExternalName::testcase("").to_string(), "%"); - assert_eq!(ExternalName::testcase("x").to_string(), "%x"); - assert_eq!(ExternalName::testcase("x_1").to_string(), "%x_1"); + assert_eq!(ExternalName::testcase("").display(None).to_string(), "%"); + assert_eq!(ExternalName::testcase("x").display(None).to_string(), "%x"); assert_eq!( - ExternalName::testcase("longname12345678").to_string(), + ExternalName::testcase("x_1").display(None).to_string(), + "%x_1" + ); + assert_eq!( + ExternalName::testcase("longname12345678") + .display(None) + .to_string(), "%longname12345678" ); // Constructor will silently drop bytes beyond the 16th assert_eq!( - ExternalName::testcase("longname123456789").to_string(), + ExternalName::testcase("longname123456789") + .display(None) + .to_string(), "%longname12345678" ); } #[test] fn display_user() { - assert_eq!(ExternalName::user(0, 0).to_string(), "u0:0"); - assert_eq!(ExternalName::user(1, 1).to_string(), "u1:1"); assert_eq!( - ExternalName::user(u32::MAX, u32::MAX).to_string(), - "u4294967295:4294967295" + ExternalName::user(UserExternalNameRef::new(0)) + .display(None) + .to_string(), + "userextname0" + ); + assert_eq!( + ExternalName::user(UserExternalNameRef::new(1)) + .display(None) + .to_string(), + "userextname1" + ); + assert_eq!( + ExternalName::user(UserExternalNameRef::new((u32::MAX - 1) as _)) + .display(None) + .to_string(), + "userextname4294967294" + ); + + let mut func_params = FunctionParameters::new(); + + // ref 0 + func_params.ensure_user_func_name(UserExternalName { + namespace: 13, + index: 37, + }); + + // ref 1 + func_params.ensure_user_func_name(UserExternalName { + namespace: 2, + index: 4, + }); + + assert_eq!( + ExternalName::user(UserExternalNameRef::new(0)) + .display(Some(&func_params)) + .to_string(), + "u13:37" + ); + + assert_eq!( + ExternalName::user(UserExternalNameRef::new(1)) + .display(Some(&func_params)) + .to_string(), + "u2:4" ); } @@ -170,7 +316,9 @@ mod tests { Ok(ExternalName::LibCall(LibCall::FloorF32)) ); assert_eq!( - ExternalName::LibCall(LibCall::FloorF32).to_string(), + ExternalName::LibCall(LibCall::FloorF32) + .display(None) + .to_string(), "%FloorF32" ); } diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 856c8f5f0d..5ed3cb553f 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -11,11 +11,12 @@ use crate::ir::{ ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, InstructionData, JumpTable, JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, Type, }; -use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature}; +use crate::ir::{DataFlowGraph, Layout, Signature}; use crate::ir::{DynamicStackSlots, SourceLocs, StackSlots}; use crate::isa::CallConv; use crate::value_label::ValueLabelsRanges; use crate::write::write_function; +use crate::HashMap; #[cfg(feature = "enable-serde")] use alloc::string::String; use core::fmt; @@ -27,9 +28,13 @@ use serde::ser::Serializer; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; +use super::entities::UserExternalNameRef; +use super::extname::UserFuncName; +use super::{RelSourceLoc, SourceLoc, UserExternalName}; + /// A version marker used to ensure that serialized clif ir is never deserialized with a /// different version of Cranelift. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Hash)] pub struct VersionMarker; #[cfg(feature = "enable-serde")] @@ -60,21 +65,99 @@ impl<'de> Deserialize<'de> for VersionMarker { } } -/// -/// Functions can be cloned, but it is not a very fast operation. -/// The clone will have all the same entity numbers as the original. +/// Function parameters used when creating this function, and that will become applied after +/// compilation to materialize the final `CompiledCode`. #[derive(Clone)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] -pub struct Function { +pub struct FunctionParameters { + /// The first `SourceLoc` appearing in the function, serving as a base for every relative + /// source loc in the function. + base_srcloc: Option, + + /// External user-defined function references. + user_named_funcs: PrimaryMap, + + /// Inverted mapping of `user_named_funcs`, to deduplicate internally. + user_ext_name_to_ref: HashMap, +} + +impl FunctionParameters { + /// Creates a new `FunctionParameters` with the given name. + pub fn new() -> Self { + Self { + base_srcloc: None, + user_named_funcs: Default::default(), + user_ext_name_to_ref: Default::default(), + } + } + + /// Returns the base `SourceLoc`. + /// + /// If it was never explicitly set with `ensure_base_srcloc`, will return an invalid + /// `SourceLoc`. + pub fn base_srcloc(&self) -> SourceLoc { + self.base_srcloc.unwrap_or_default() + } + + /// Sets the base `SourceLoc`, if not set yet, and returns the base value. + pub fn ensure_base_srcloc(&mut self, srcloc: SourceLoc) -> SourceLoc { + match self.base_srcloc { + Some(val) => val, + None => { + self.base_srcloc = Some(srcloc); + srcloc + } + } + } + + /// Retrieve a `UserExternalNameRef` for the given name, or add a new one. + /// + /// This method internally deduplicates same `UserExternalName` so they map to the same + /// reference. + pub fn ensure_user_func_name(&mut self, name: UserExternalName) -> UserExternalNameRef { + if let Some(reff) = self.user_ext_name_to_ref.get(&name) { + *reff + } else { + let reff = self.user_named_funcs.push(name.clone()); + self.user_ext_name_to_ref.insert(name, reff); + reff + } + } + + /// Resets an already existing user function name to a new value. + pub fn reset_user_func_name(&mut self, index: UserExternalNameRef, name: UserExternalName) { + if let Some(prev_name) = self.user_named_funcs.get_mut(index) { + self.user_ext_name_to_ref.remove(prev_name); + *prev_name = name.clone(); + self.user_ext_name_to_ref.insert(name, index); + } + } + + /// Returns the internal mapping of `UserExternalNameRef` to `UserExternalName`. + pub fn user_named_funcs(&self) -> &PrimaryMap { + &self.user_named_funcs + } + + fn clear(&mut self) { + self.base_srcloc = None; + self.user_named_funcs.clear(); + self.user_ext_name_to_ref.clear(); + } +} + +/// Function fields needed when compiling a function. +/// +/// Additionally, these fields can be the same for two functions that would be compiled the same +/// way, and finalized by applying `FunctionParameters` onto their `CompiledCodeStencil`. +#[derive(Clone, PartialEq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct FunctionStencil { /// A version marker used to ensure that serialized clif ir is never deserialized with a /// different version of Cranelift. // Note: This must be the first field to ensure that Serde will deserialize it before // attempting to deserialize other fields that are potentially changed between versions. pub version_marker: VersionMarker, - /// Name of this function. Mostly used by `.clif` files. - pub name: ExternalName, - /// Signature of this function. pub signature: Signature, @@ -106,7 +189,7 @@ pub struct Function { /// /// Track the original source location for each instruction. The source locations are not /// interpreted by Cranelift, only preserved. - pub srclocs: SourceLocs, + srclocs: SourceLocs, /// An optional global value which represents an expression evaluating to /// the stack limit for this function. This `GlobalValue` will be @@ -116,28 +199,8 @@ pub struct Function { pub stack_limit: Option, } -impl Function { - /// Create a function with the given name and signature. - pub fn with_name_signature(name: ExternalName, sig: Signature) -> Self { - Self { - version_marker: VersionMarker, - name, - signature: sig, - sized_stack_slots: StackSlots::new(), - dynamic_stack_slots: DynamicStackSlots::new(), - global_values: PrimaryMap::new(), - heaps: PrimaryMap::new(), - tables: PrimaryMap::new(), - jump_tables: PrimaryMap::new(), - dfg: DataFlowGraph::new(), - layout: Layout::new(), - srclocs: SecondaryMap::new(), - stack_limit: None, - } - } - - /// Clear all data structures in this function. - pub fn clear(&mut self) { +impl FunctionStencil { + fn clear(&mut self) { self.signature.clear(CallConv::Fast); self.sized_stack_slots.clear(); self.dynamic_stack_slots.clear(); @@ -151,11 +214,6 @@ impl Function { self.stack_limit = None; } - /// Create a new empty, anonymous function with a Fast calling convention. - pub fn new() -> Self { - Self::with_name_signature(ExternalName::default(), Signature::new(CallConv::Fast)) - } - /// Creates a jump table in the function, to be used by `br_table` instructions. pub fn create_jump_table(&mut self, data: JumpTableData) -> JumpTable { self.jump_tables.push(data) @@ -178,11 +236,6 @@ impl Function { self.dfg.signatures.push(signature) } - /// Declare an external function import. - pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef { - self.dfg.ext_funcs.push(data) - } - /// Declares a global value accessible to the function. pub fn create_global_value(&mut self, data: GlobalValueData) -> GlobalValue { self.global_values.push(data) @@ -218,19 +271,6 @@ impl Function { self.tables.push(data) } - /// Return an object that can display this function with correct ISA-specific annotations. - pub fn display(&self) -> DisplayFunction<'_> { - DisplayFunction(self, Default::default()) - } - - /// Return an object that can display this function with correct ISA-specific annotations. - pub fn display_with<'a>( - &'a self, - annotations: DisplayFunctionAnnotations<'a>, - ) -> DisplayFunction<'a> { - DisplayFunction(self, annotations) - } - /// Find a presumed unique special-purpose function parameter value. /// /// Returns the value of the last `purpose` parameter, or `None` if no such parameter exists. @@ -260,8 +300,8 @@ impl Function { /// Rewrite the branch destination to `new_dest` if the destination matches `old_dest`. /// Does nothing if called with a non-jump or non-branch instruction. /// - /// Unlike [change_branch_destination](Function::change_branch_destination), this method rewrite the destinations of - /// multi-destination branches like `br_table`. + /// Unlike [change_branch_destination](FunctionStencil::change_branch_destination), this method + /// rewrite the destinations of multi-destination branches like `br_table`. pub fn rewrite_branch_destination(&mut self, inst: Inst, old_dest: Block, new_dest: Block) { match self.dfg.analyze_branch(inst) { BranchInfo::SingleDest(dest, ..) => { @@ -356,6 +396,120 @@ impl Function { pub fn fixed_stack_size(&self) -> u32 { self.sized_stack_slots.values().map(|ss| ss.size).sum() } + + /// Returns the list of relative source locations for this function. + pub(crate) fn rel_srclocs(&self) -> &SecondaryMap { + &self.srclocs + } +} + +/// Functions can be cloned, but it is not a very fast operation. +/// The clone will have all the same entity numbers as the original. +#[derive(Clone)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct Function { + /// Name of this function. + /// + /// Mostly used by `.clif` files, only there for debugging / naming purposes. + pub name: UserFuncName, + + /// All the fields required for compiling a function, independently of details irrelevant to + /// compilation and that are stored in the `FunctionParameters` `params` field instead. + pub stencil: FunctionStencil, + + /// All the parameters that can be applied onto the function stencil, that is, that don't + /// matter when caching compilation artifacts. + pub params: FunctionParameters, +} + +impl core::ops::Deref for Function { + type Target = FunctionStencil; + + fn deref(&self) -> &Self::Target { + &self.stencil + } +} + +impl core::ops::DerefMut for Function { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stencil + } +} + +impl Function { + /// Create a function with the given name and signature. + pub fn with_name_signature(name: UserFuncName, sig: Signature) -> Self { + Self { + name, + stencil: FunctionStencil { + version_marker: VersionMarker, + signature: sig, + sized_stack_slots: StackSlots::new(), + dynamic_stack_slots: DynamicStackSlots::new(), + global_values: PrimaryMap::new(), + heaps: PrimaryMap::new(), + tables: PrimaryMap::new(), + jump_tables: PrimaryMap::new(), + dfg: DataFlowGraph::new(), + layout: Layout::new(), + srclocs: SecondaryMap::new(), + stack_limit: None, + }, + params: FunctionParameters::new(), + } + } + + /// Clear all data structures in this function. + pub fn clear(&mut self) { + self.stencil.clear(); + self.params.clear(); + self.name = UserFuncName::default(); + } + + /// Create a new empty, anonymous function with a Fast calling convention. + pub fn new() -> Self { + Self::with_name_signature(Default::default(), Signature::new(CallConv::Fast)) + } + + /// Return an object that can display this function with correct ISA-specific annotations. + pub fn display(&self) -> DisplayFunction<'_> { + DisplayFunction(self, Default::default()) + } + + /// Return an object that can display this function with correct ISA-specific annotations. + pub fn display_with<'a>( + &'a self, + annotations: DisplayFunctionAnnotations<'a>, + ) -> DisplayFunction<'a> { + DisplayFunction(self, annotations) + } + + /// Sets an absolute source location for the given instruction. + /// + /// If no base source location has been set yet, records it at the same time. + pub fn set_srcloc(&mut self, inst: Inst, srcloc: SourceLoc) { + let base = self.params.ensure_base_srcloc(srcloc); + self.stencil.srclocs[inst] = RelSourceLoc::from_base_offset(base, srcloc); + } + + /// Returns an absolute source location for the given instruction. + pub fn srcloc(&self, inst: Inst) -> SourceLoc { + let base = self.params.base_srcloc(); + self.stencil.srclocs[inst].expand(base) + } + + /// Declare a user-defined external function import, to be referenced in `ExtFuncData::User` later. + pub fn declare_imported_user_function( + &mut self, + name: UserExternalName, + ) -> UserExternalNameRef { + self.params.ensure_user_func_name(name) + } + + /// Declare an external function import. + pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef { + self.stencil.dfg.ext_funcs.push(data) + } } /// Additional annotations for function display. diff --git a/cranelift/codegen/src/ir/globalvalue.rs b/cranelift/codegen/src/ir/globalvalue.rs index 8ec39bf0a4..6e26931fed 100644 --- a/cranelift/codegen/src/ir/globalvalue.rs +++ b/cranelift/codegen/src/ir/globalvalue.rs @@ -10,7 +10,7 @@ use core::fmt; use serde::{Deserialize, Serialize}; /// Information about a global value declaration. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum GlobalValueData { /// Value is the address of the VM context struct. @@ -151,7 +151,7 @@ impl fmt::Display for GlobalValueData { "symbol {}{}{}", if colocated { "colocated " } else { "" }, if tls { "tls " } else { "" }, - name + name.display(None) )?; let offset_val: i64 = offset.into(); if offset_val > 0 { diff --git a/cranelift/codegen/src/ir/heap.rs b/cranelift/codegen/src/ir/heap.rs index 91aabccaa2..7d62915af5 100644 --- a/cranelift/codegen/src/ir/heap.rs +++ b/cranelift/codegen/src/ir/heap.rs @@ -8,7 +8,7 @@ use core::fmt; use serde::{Deserialize, Serialize}; /// Information about a heap declaration. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct HeapData { /// The address of the start of the heap's storage. @@ -29,7 +29,7 @@ pub struct HeapData { } /// Style of heap including style-specific information. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum HeapStyle { /// A dynamic heap can be relocated to a different base address when it is grown. diff --git a/cranelift/codegen/src/ir/jumptable.rs b/cranelift/codegen/src/ir/jumptable.rs index bf05169d36..7897dff845 100644 --- a/cranelift/codegen/src/ir/jumptable.rs +++ b/cranelift/codegen/src/ir/jumptable.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; /// Contents of a jump table. /// /// All jump tables use 0-based indexing and are densely populated. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct JumpTableData { // Table entries. diff --git a/cranelift/codegen/src/ir/layout.rs b/cranelift/codegen/src/ir/layout.rs index ec411b4b17..7162c848c5 100644 --- a/cranelift/codegen/src/ir/layout.rs +++ b/cranelift/codegen/src/ir/layout.rs @@ -25,7 +25,7 @@ use core::iter::{IntoIterator, Iterator}; /// While data dependencies are not recorded, instruction ordering does affect control /// dependencies, so part of the semantics of the program are determined by the layout. /// -#[derive(Clone)] +#[derive(Debug, Clone, PartialEq, Hash)] pub struct Layout { /// Linked list nodes for the layout order of blocks Forms a doubly linked list, terminated in /// both ends by `None`. @@ -311,7 +311,7 @@ impl Layout { /// /// This doesn't affect the position of anything, but it gives more room in the internal /// sequence numbers for inserting instructions later. - fn full_renumber(&mut self) { + pub(crate) fn full_renumber(&mut self) { let _tt = timing::layout_renumber(); let mut seq = 0; let mut next_block = self.first_block; @@ -486,7 +486,7 @@ impl Layout { /// A single node in the linked-list of blocks. // Whenever you add new fields here, don't forget to update the custom serializer for `Layout` too. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq, Hash)] struct BlockNode { prev: PackedOption, next: PackedOption, @@ -748,7 +748,7 @@ impl Layout { } } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq, Hash)] struct InstNode { /// The Block containing this instruction, or `None` if the instruction is not yet inserted. block: PackedOption, diff --git a/cranelift/codegen/src/ir/libcall.rs b/cranelift/codegen/src/ir/libcall.rs index 82971f4a22..e7ef344bb3 100644 --- a/cranelift/codegen/src/ir/libcall.rs +++ b/cranelift/codegen/src/ir/libcall.rs @@ -1,7 +1,9 @@ //! Naming well-known routines in the runtime library. -use crate::ir::{types, AbiParam, ExternalName, FuncRef, Function, Opcode, Signature, Type}; -use crate::isa::CallConv; +use crate::{ + ir::{types, AbiParam, ExternalName, FuncRef, Function, Opcode, Signature, Type}, + isa::CallConv, +}; use core::fmt; use core::str::FromStr; #[cfg(feature = "enable-serde")] diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index 15e14660e4..5dc5ad612d 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -38,12 +38,12 @@ pub use crate::ir::dfg::{DataFlowGraph, ValueDef}; pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes}; pub use crate::ir::entities::{ Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, Immediate, Inst, - JumpTable, SigRef, StackSlot, Table, Value, + JumpTable, SigRef, StackSlot, Table, UserExternalNameRef, Value, }; pub use crate::ir::extfunc::{ AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature, }; -pub use crate::ir::extname::ExternalName; +pub use crate::ir::extname::{ExternalName, UserExternalName, UserFuncName}; pub use crate::ir::function::{DisplayFunctionAnnotations, Function}; pub use crate::ir::globalvalue::GlobalValueData; pub use crate::ir::heap::{HeapData, HeapStyle}; @@ -56,6 +56,7 @@ pub use crate::ir::layout::Layout; pub use crate::ir::libcall::{get_probestack_funcref, LibCall}; pub use crate::ir::memflags::{Endianness, MemFlags}; pub use crate::ir::progpoint::{ExpandedProgramPoint, ProgramOrder, ProgramPoint}; +pub use crate::ir::sourceloc::RelSourceLoc; pub use crate::ir::sourceloc::SourceLoc; pub use crate::ir::stackslot::{ DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKind, StackSlots, @@ -71,7 +72,7 @@ use crate::entity::{entity_impl, PrimaryMap, SecondaryMap}; pub type JumpTables = PrimaryMap; /// Source locations for instructions. -pub type SourceLocs = SecondaryMap; +pub(crate) type SourceLocs = SecondaryMap; /// Marked with a label value. #[derive(Copy, Clone, PartialEq, Eq, Hash)] @@ -80,18 +81,18 @@ pub struct ValueLabel(u32); entity_impl!(ValueLabel, "val"); /// A label of a Value. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ValueLabelStart { /// Source location when it is in effect - pub from: SourceLoc, + pub from: RelSourceLoc, /// The label index. pub label: ValueLabel, } /// Value label assignements: label starts or value aliases. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum ValueLabelAssignments { /// Original value labels assigned at transform. @@ -100,7 +101,7 @@ pub enum ValueLabelAssignments { /// A value alias to original value. Alias { /// Source location when it is in effect - from: SourceLoc, + from: RelSourceLoc, /// The label index. value: Value, diff --git a/cranelift/codegen/src/ir/sourceloc.rs b/cranelift/codegen/src/ir/sourceloc.rs index ccab62f89b..57e7a4160c 100644 --- a/cranelift/codegen/src/ir/sourceloc.rs +++ b/cranelift/codegen/src/ir/sourceloc.rs @@ -51,6 +51,61 @@ impl fmt::Display for SourceLoc { } } +/// Source location relative to another base source location. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct RelSourceLoc(u32); + +impl RelSourceLoc { + /// Create a new relative source location with the given bits. + pub fn new(bits: u32) -> Self { + Self(bits) + } + + /// Creates a new `RelSourceLoc` based on the given base and offset. + /// + /// # Panics + /// + /// Panics if the offset is smaller than the base. + pub fn from_base_offset(base: SourceLoc, offset: SourceLoc) -> Self { + if base.is_default() || offset.is_default() { + Self::default() + } else { + Self(offset.bits().wrapping_sub(base.bits())) + } + } + + /// Expands the relative source location into an absolute one, using the given base. + pub fn expand(&self, base: SourceLoc) -> SourceLoc { + if self.is_default() || base.is_default() { + Default::default() + } else { + SourceLoc::new(self.0.wrapping_add(base.bits())) + } + } + + /// Is this the default relative source location? + pub fn is_default(self) -> bool { + self == Default::default() + } +} + +impl Default for RelSourceLoc { + fn default() -> Self { + Self(!0) + } +} + +impl fmt::Display for RelSourceLoc { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.is_default() { + write!(f, "@-") + } else { + write!(f, "@+{:04x}", self.0) + } + } +} + #[cfg(test)] mod tests { use crate::ir::SourceLoc; diff --git a/cranelift/codegen/src/ir/stackslot.rs b/cranelift/codegen/src/ir/stackslot.rs index e4db80d5d7..aa77a7ac7e 100644 --- a/cranelift/codegen/src/ir/stackslot.rs +++ b/cranelift/codegen/src/ir/stackslot.rs @@ -27,7 +27,7 @@ use serde::{Deserialize, Serialize}; pub type StackSize = u32; /// The kind of a stack slot. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum StackSlotKind { /// An explicit stack slot. This is a chunk of stack memory for use by the `stack_load` @@ -62,7 +62,7 @@ impl fmt::Display for StackSlotKind { } /// Contents of a stack slot. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct StackSlotData { /// The kind of stack slot. @@ -100,7 +100,7 @@ impl fmt::Display for StackSlotData { } /// Contents of a dynamic stack slot. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DynamicStackSlotData { /// The kind of stack slot. diff --git a/cranelift/codegen/src/ir/table.rs b/cranelift/codegen/src/ir/table.rs index 713d1f5df7..6acfb14fa1 100644 --- a/cranelift/codegen/src/ir/table.rs +++ b/cranelift/codegen/src/ir/table.rs @@ -8,7 +8,7 @@ use core::fmt; use serde::{Deserialize, Serialize}; /// Information about a table declaration. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct TableData { /// Global value giving the address of the start of the table. diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index c36e118c24..6ec2358fe4 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -3,7 +3,7 @@ use regalloc2::Allocation; use crate::binemit::{CodeOffset, Reloc, StackMap}; -use crate::ir::types::*; +use crate::ir::{types::*, RelSourceLoc}; use crate::ir::{LibCall, MemFlags, TrapCode}; use crate::isa::aarch64::inst::*; use crate::machinst::{ty_bits, Reg, RegClass, Writable}; @@ -617,7 +617,7 @@ pub struct EmitState { /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. stack_map: Option, /// Current source-code location corresponding to instruction to be emitted. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, } impl MachInstEmitState for EmitState { @@ -626,7 +626,7 @@ impl MachInstEmitState for EmitState { virtual_sp_offset: 0, nominal_sp_to_fp: abi.frame_size() as i64, stack_map: None, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), } } @@ -634,7 +634,7 @@ impl MachInstEmitState for EmitState { self.stack_map = Some(stack_map); } - fn pre_sourceloc(&mut self, srcloc: SourceLoc) { + fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } } @@ -648,7 +648,7 @@ impl EmitState { self.stack_map = None; } - fn cur_srcloc(&self) -> SourceLoc { + fn cur_srcloc(&self) -> RelSourceLoc { self.cur_srcloc } } @@ -954,7 +954,7 @@ impl MachInstEmit for Inst { }; let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1074,7 +1074,7 @@ impl MachInstEmit for Inst { }; let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual store instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1151,7 +1151,7 @@ impl MachInstEmit for Inst { let rt2 = allocs.next(rt2); let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual store instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1183,7 +1183,7 @@ impl MachInstEmit for Inst { let rt2 = allocs.next(rt2.to_reg()); let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1223,7 +1223,7 @@ impl MachInstEmit for Inst { let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1269,7 +1269,7 @@ impl MachInstEmit for Inst { let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual store instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1417,7 +1417,7 @@ impl MachInstEmit for Inst { // again: sink.bind_label(again_label); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } sink.put4(enc_ldaxr(ty, x27wr, x25)); // ldaxr x27, [x25] @@ -1541,7 +1541,7 @@ impl MachInstEmit for Inst { } let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } if op == AtomicRMWLoopOp::Xchg { @@ -1603,7 +1603,7 @@ impl MachInstEmit for Inst { // again: sink.bind_label(again_label); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } // ldaxr x27, [x25] @@ -1630,7 +1630,7 @@ impl MachInstEmit for Inst { sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25] @@ -2662,7 +2662,7 @@ impl MachInstEmit for Inst { let (q, size) = size.enc_size(); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 6e45beb66b..241d0f79f3 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -4,7 +4,7 @@ use crate::binemit::{Addend, CodeOffset, Reloc}; use crate::ir::types::{ B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64, }; -use crate::ir::{types, ExternalName, MemFlags, Opcode, SourceLoc, Type}; +use crate::ir::{types, ExternalName, MemFlags, Opcode, Type}; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; @@ -2704,7 +2704,7 @@ impl Inst { &Inst::EmitIsland { needed_space } => format!("emit_island {}", needed_space), &Inst::ElfTlsGetAddr { ref symbol } => { - format!("x0 = elf_tls_get_addr {}", symbol) + format!("x0 = elf_tls_get_addr {}", symbol.display(None)) } &Inst::Unwind { ref inst } => { format!("unwind {:?}", inst) diff --git a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs index f5d86252b3..02c3e9dcf5 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs @@ -70,8 +70,7 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { mod tests { use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ - types, AbiParam, ExternalName, Function, InstBuilder, Signature, StackSlotData, - StackSlotKind, + types, AbiParam, Function, InstBuilder, Signature, StackSlotData, StackSlotKind, }; use crate::isa::{lookup, CallConv}; use crate::settings::{builder, Flags}; @@ -108,8 +107,7 @@ mod tests { } fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + let mut func = Function::with_name_signature(Default::default(), Signature::new(call_conv)); let block0 = func.dfg.make_block(); let mut pos = FuncCursor::new(&mut func); @@ -153,7 +151,7 @@ mod tests { fn create_multi_return_function(call_conv: CallConv) -> Function { let mut sig = Signature::new(call_conv); sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(Default::default(), sig); let block0 = func.dfg.make_block(); let v0 = func.dfg.append_block_param(block0, types::I32); diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index 4d96b80340..264c2d5053 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -7,7 +7,8 @@ use crate::isa::aarch64::settings as aarch64_settings; use crate::isa::unwind::systemv; use crate::isa::{Builder as IsaBuilder, TargetIsa}; use crate::machinst::{ - compile, CompiledCode, MachTextSectionBuilder, Reg, TextSectionBuilder, VCode, + compile, CompiledCode, CompiledCodeStencil, MachTextSectionBuilder, Reg, TextSectionBuilder, + VCode, }; use crate::result::CodegenResult; use crate::settings as shared_settings; @@ -65,7 +66,11 @@ impl AArch64Backend { } impl TargetIsa for AArch64Backend { - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult { + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func, flags.clone())?; @@ -80,7 +85,7 @@ impl TargetIsa for AArch64Backend { log::debug!("disassembly:\n{}", disasm); } - Ok(CompiledCode { + Ok(CompiledCodeStencil { buffer, frame_size, disasm: emit_result.disasm, @@ -204,7 +209,7 @@ mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::types::*; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, JumpTableData, Signature}; + use crate::ir::{AbiParam, Function, InstBuilder, JumpTableData, Signature, UserFuncName}; use crate::isa::CallConv; use crate::settings; use crate::settings::Configurable; @@ -213,7 +218,7 @@ mod test { #[test] fn test_compile_function() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -252,7 +257,7 @@ mod test { #[test] fn test_branch_lowering() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -320,7 +325,7 @@ mod test { #[test] fn test_br_table() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 5904f51b20..f3eb30a7cc 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -49,7 +49,7 @@ use crate::flowgraph; use crate::ir::{self, Function}; #[cfg(feature = "unwind")] use crate::isa::unwind::systemv::RegisterMappingError; -use crate::machinst::{CompiledCode, TextSectionBuilder, UnwindInfoKind}; +use crate::machinst::{CompiledCode, CompiledCodeStencil, TextSectionBuilder, UnwindInfoKind}; use crate::settings; use crate::settings::SetResult; use crate::CodegenResult; @@ -230,7 +230,11 @@ pub trait TargetIsa: fmt::Display + Send + Sync { fn dynamic_vector_bytes(&self, dynamic_ty: ir::Type) -> u32; /// Compile the given function. - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult; + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult; #[cfg(feature = "unwind")] /// Map a regalloc::Reg to its corresponding DWARF register. diff --git a/cranelift/codegen/src/isa/s390x/inst/args.rs b/cranelift/codegen/src/isa/s390x/inst/args.rs index 7a0905641b..d69fe2d60c 100644 --- a/cranelift/codegen/src/isa/s390x/inst/args.rs +++ b/cranelift/codegen/src/isa/s390x/inst/args.rs @@ -326,7 +326,7 @@ impl PrettyPrint for MemArg { &MemArg::Label { target } => target.to_string(), &MemArg::Symbol { ref name, offset, .. - } => format!("{} + {}", name, offset), + } => format!("{} + {}", name.display(None), offset), // Eliminated by `mem_finalize()`. &MemArg::InitialSPOffset { .. } | &MemArg::NominalSPOffset { .. } diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index 62f3736fe1..346fb306e0 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -1,8 +1,8 @@ //! S390x ISA: binary code emission. use crate::binemit::{Reloc, StackMap}; -use crate::ir::MemFlags; -use crate::ir::{SourceLoc, TrapCode}; +use crate::ir::TrapCode; +use crate::ir::{MemFlags, RelSourceLoc}; use crate::isa::s390x::inst::*; use crate::isa::s390x::settings as s390x_settings; use crate::machinst::reg::count_operands; @@ -133,7 +133,7 @@ pub fn mem_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -201,7 +201,7 @@ pub fn mem_rs_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -243,7 +243,7 @@ pub fn mem_imm8_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -281,7 +281,7 @@ pub fn mem_imm16_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -308,7 +308,7 @@ pub fn mem_mem_emit( ) { if add_trap && (dst.can_trap() || src.can_trap()) { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if srcloc != Default::default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -343,7 +343,7 @@ pub fn mem_vrx_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -1256,7 +1256,7 @@ pub struct EmitState { /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. stack_map: Option, /// Current source-code location corresponding to instruction to be emitted. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, } impl MachInstEmitState for EmitState { @@ -1265,7 +1265,7 @@ impl MachInstEmitState for EmitState { virtual_sp_offset: 0, initial_sp_offset: abi.frame_size() as i64, stack_map: None, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), } } @@ -1273,7 +1273,7 @@ impl MachInstEmitState for EmitState { self.stack_map = Some(stack_map); } - fn pre_sourceloc(&mut self, srcloc: SourceLoc) { + fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } } @@ -1287,7 +1287,7 @@ impl EmitState { self.stack_map = None; } - fn cur_srcloc(&self) -> SourceLoc { + fn cur_srcloc(&self) -> RelSourceLoc { self.cur_srcloc } } diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index 60e148cd8d..d13a69f204 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -2932,10 +2932,12 @@ impl Inst { let link = pretty_print_reg(link.to_reg(), allocs); let tls_symbol = match &info.tls_symbol { None => "".to_string(), - Some(SymbolReloc::TlsGd { name }) => format!(":tls_gdcall:{}", name), + Some(SymbolReloc::TlsGd { name }) => { + format!(":tls_gdcall:{}", name.display(None)) + } _ => unreachable!(), }; - format!("brasl {}, {}{}", link, info.dest, tls_symbol) + format!("brasl {}, {}{}", link, info.dest.display(None), tls_symbol) } &Inst::CallInd { link, ref info, .. } => { let link = pretty_print_reg(link.to_reg(), allocs); @@ -3003,8 +3005,10 @@ impl Inst { let rd = pretty_print_reg(rd.to_reg(), allocs); let tmp = pretty_print_reg(writable_spilltmp_reg().to_reg(), &mut empty_allocs); let symbol = match &**symbol_reloc { - SymbolReloc::Absolute { name, offset } => format!("{} + {}", name, offset), - SymbolReloc::TlsGd { name } => format!("{}@tlsgd", name), + SymbolReloc::Absolute { name, offset } => { + format!("{} + {}", name.display(None), offset) + } + SymbolReloc::TlsGd { name } => format!("{}@tlsgd", name.display(None)), }; format!("bras {}, 12 ; data {} ; lg {}, 0({})", tmp, symbol, rd, tmp) } diff --git a/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs index 152dabe44b..f787a95d02 100644 --- a/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs @@ -101,8 +101,7 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { mod tests { use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ - types, AbiParam, ExternalName, Function, InstBuilder, Signature, StackSlotData, - StackSlotKind, + types, AbiParam, Function, InstBuilder, Signature, StackSlotData, StackSlotKind, }; use crate::isa::{lookup, CallConv}; use crate::settings::{builder, Flags}; @@ -139,8 +138,7 @@ mod tests { } fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + let mut func = Function::with_name_signature(Default::default(), Signature::new(call_conv)); let block0 = func.dfg.make_block(); let mut pos = FuncCursor::new(&mut func); @@ -187,7 +185,7 @@ mod tests { ) -> Function { let mut sig = Signature::new(call_conv); sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(Default::default(), sig); let block0 = func.dfg.make_block(); let v0 = func.dfg.append_block_param(block0, types::I32); diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 0ffba418d1..b8587a44ad 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -3,6 +3,7 @@ // Pull in the ISLE generated code. pub mod generated_code; +use crate::ir::ExternalName; // Types that the generated ISLE code uses via `use super::*`. use crate::isa::s390x::abi::{S390xMachineDeps, REG_SAVE_AREA_SIZE}; use crate::isa::s390x::inst::{ diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs index 123209b8f9..e2165c585c 100644 --- a/cranelift/codegen/src/isa/s390x/mod.rs +++ b/cranelift/codegen/src/isa/s390x/mod.rs @@ -7,7 +7,8 @@ use crate::isa::s390x::settings as s390x_settings; use crate::isa::unwind::systemv::RegisterMappingError; use crate::isa::{Builder as IsaBuilder, TargetIsa}; use crate::machinst::{ - compile, CompiledCode, MachTextSectionBuilder, Reg, TextSectionBuilder, VCode, + compile, CompiledCode, CompiledCodeStencil, MachTextSectionBuilder, Reg, TextSectionBuilder, + VCode, }; use crate::result::CodegenResult; use crate::settings as shared_settings; @@ -63,7 +64,11 @@ impl S390xBackend { } impl TargetIsa for S390xBackend { - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult { + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func)?; @@ -78,7 +83,7 @@ impl TargetIsa for S390xBackend { log::debug!("disassembly:\n{}", disasm); } - Ok(CompiledCode { + Ok(CompiledCodeStencil { buffer, frame_size, disasm: emit_result.disasm, @@ -186,7 +191,8 @@ mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::types::*; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; + use crate::ir::UserFuncName; + use crate::ir::{AbiParam, Function, InstBuilder, Signature}; use crate::isa::CallConv; use crate::settings; use crate::settings::Configurable; @@ -195,7 +201,7 @@ mod test { #[test] fn test_compile_function() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -233,7 +239,7 @@ mod test { #[test] fn test_branch_lowering() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 12c5fcf5a6..d0c1137f6f 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -1,7 +1,7 @@ //! Implementation of the standard x64 ABI. -use crate::ir::types::*; -use crate::ir::{self, types, ExternalName, LibCall, MemFlags, Opcode, Signature, TrapCode, Type}; +use crate::ir::{self, types, LibCall, MemFlags, Opcode, Signature, TrapCode, Type}; +use crate::ir::{types::*, ExternalName}; use crate::isa; use crate::isa::{unwind::UnwindInst, x64::inst::*, x64::settings as x64_settings, CallConv}; use crate::machinst::abi_impl::*; diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 087613d52c..61b7f33e36 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -13,9 +13,11 @@ //! -- isa::x64::inst::emit_tests::test_x64_emit use super::*; +use crate::ir::UserExternalNameRef; use crate::isa::x64; use alloc::boxed::Box; use alloc::vec::Vec; +use cranelift_entity::EntityRef as _; impl Inst { fn neg(size: OperandSize, src: Writable) -> Inst { @@ -3395,17 +3397,14 @@ fn test_x64_emit() { // CallKnown insns.push(( Inst::call_known( - ExternalName::User { - namespace: 0, - index: 0, - }, + ExternalName::User(UserExternalNameRef::new(0)), smallvec![], smallvec![], PRegSet::default(), Opcode::Call, ), "E800000000", - "call User { namespace: 0, index: 0 }", + "call User(userextname0)", )); // ======================================================== @@ -3449,38 +3448,29 @@ fn test_x64_emit() { insns.push(( Inst::LoadExtName { dst: Writable::from_reg(r11), - name: Box::new(ExternalName::User { - namespace: 0, - index: 0, - }), + name: Box::new(ExternalName::User(UserExternalNameRef::new(0))), offset: 0, }, "4C8B1D00000000", - "load_ext_name u0:0+0, %r11", + "load_ext_name userextname0+0, %r11", )); insns.push(( Inst::LoadExtName { dst: Writable::from_reg(r11), - name: Box::new(ExternalName::User { - namespace: 0, - index: 0, - }), + name: Box::new(ExternalName::User(UserExternalNameRef::new(0))), offset: 0x12345678, }, "4C8B1D000000004981C378563412", - "load_ext_name u0:0+305419896, %r11", + "load_ext_name userextname0+305419896, %r11", )); insns.push(( Inst::LoadExtName { dst: Writable::from_reg(r11), - name: Box::new(ExternalName::User { - namespace: 0, - index: 0, - }), + name: Box::new(ExternalName::User(UserExternalNameRef::new(0))), offset: -0x12345678, }, "4C8B1D000000004981EB78563412", - "load_ext_name u0:0+-305419896, %r11", + "load_ext_name userextname0+-305419896, %r11", )); // ======================================================== @@ -4690,35 +4680,26 @@ fn test_x64_emit() { insns.push(( Inst::ElfTlsGetAddr { - symbol: ExternalName::User { - namespace: 0, - index: 0, - }, + symbol: ExternalName::User(UserExternalNameRef::new(0)), }, "66488D3D00000000666648E800000000", - "%rax = elf_tls_get_addr User { namespace: 0, index: 0 }", + "%rax = elf_tls_get_addr User(userextname0)", )); insns.push(( Inst::MachOTlsGetAddr { - symbol: ExternalName::User { - namespace: 0, - index: 0, - }, + symbol: ExternalName::User(UserExternalNameRef::new(0)), }, "488B3D00000000FF17", - "%rax = macho_tls_get_addr User { namespace: 0, index: 0 }", + "%rax = macho_tls_get_addr User(userextname0)", )); insns.push(( Inst::CoffTlsGetAddr { - symbol: ExternalName::User { - namespace: 0, - index: 0, - }, + symbol: ExternalName::User(UserExternalNameRef::new(0)), }, "8B050000000065488B0C2558000000488B04C1488D8000000000", - "%rax = coff_tls_get_addr User { namespace: 0, index: 0 }", + "%rax = coff_tls_get_addr User(userextname0)", )); // ======================================================== diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 581a55ef7c..fbe790a0f7 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1,7 +1,7 @@ //! This module defines x86_64-specific machine instruction types. use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; -use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type}; +use crate::ir::{types, ExternalName, Opcode, RelSourceLoc, TrapCode, Type}; use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::inst::regs::pretty_print_reg; use crate::isa::x64::settings as x64_settings; @@ -1624,7 +1624,7 @@ impl PrettyPrint for Inst { format!( "{} {}+{}, {}", ljustify("load_ext_name".into()), - name, + name.display(None), offset, dst, ) @@ -2424,7 +2424,7 @@ pub struct EmitState { /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. stack_map: Option, /// Current source location. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, } /// Constant state used during emissions of a sequence of instructions. @@ -2465,7 +2465,7 @@ impl MachInstEmitState for EmitState { virtual_sp_offset: 0, nominal_sp_to_fp: abi.frame_size() as i64, stack_map: None, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), } } @@ -2473,7 +2473,7 @@ impl MachInstEmitState for EmitState { self.stack_map = Some(stack_map); } - fn pre_sourceloc(&mut self, srcloc: SourceLoc) { + fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } } diff --git a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs index d3970a575a..335c97a107 100644 --- a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs @@ -97,8 +97,7 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { mod tests { use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ - types, AbiParam, ExternalName, Function, InstBuilder, Signature, StackSlotData, - StackSlotKind, + types, AbiParam, Function, InstBuilder, Signature, StackSlotData, StackSlotKind, }; use crate::isa::{lookup, CallConv}; use crate::settings::{builder, Flags}; @@ -135,8 +134,7 @@ mod tests { } fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + let mut func = Function::with_name_signature(Default::default(), Signature::new(call_conv)); let block0 = func.dfg.make_block(); let mut pos = FuncCursor::new(&mut func); @@ -177,7 +175,7 @@ mod tests { fn create_multi_return_function(call_conv: CallConv) -> Function { let mut sig = Signature::new(call_conv); sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(Default::default(), sig); let block0 = func.dfg.make_block(); let v0 = func.dfg.append_block_param(block0, types::I32); diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 8c7415d88a..1205b661af 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -8,8 +8,10 @@ use crate::ir::{condcodes::IntCC, Function, Type}; use crate::isa::unwind::systemv; use crate::isa::x64::{inst::regs::create_reg_env_systemv, settings as x64_settings}; use crate::isa::Builder as IsaBuilder; -use crate::machinst::Reg; -use crate::machinst::{compile, CompiledCode, MachTextSectionBuilder, TextSectionBuilder, VCode}; +use crate::machinst::{ + compile, CompiledCode, CompiledCodeStencil, MachTextSectionBuilder, Reg, TextSectionBuilder, + VCode, +}; use crate::result::{CodegenError, CodegenResult}; use crate::settings::{self as shared_settings, Flags}; use alloc::{boxed::Box, vec::Vec}; @@ -57,7 +59,11 @@ impl X64Backend { } impl TargetIsa for X64Backend { - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult { + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func, flags.clone())?; @@ -72,7 +78,7 @@ impl TargetIsa for X64Backend { log::trace!("disassembly:\n{}", disasm); } - Ok(CompiledCode { + Ok(CompiledCodeStencil { buffer, frame_size, disasm: emit_result.disasm, @@ -201,8 +207,8 @@ fn isa_constructor( mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; - use crate::ir::{types::*, SourceLoc, ValueLabel, ValueLabelStart}; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, JumpTableData, Signature}; + use crate::ir::{types::*, RelSourceLoc, SourceLoc, UserFuncName, ValueLabel, ValueLabelStart}; + use crate::ir::{AbiParam, Function, InstBuilder, JumpTableData, Signature}; use crate::isa::CallConv; use crate::settings; use crate::settings::Configurable; @@ -217,7 +223,7 @@ mod test { /// well do the test here, where we have a backend to use. #[test] fn test_cold_blocks() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -271,35 +277,35 @@ mod test { pos.func.dfg.values_labels.as_mut().unwrap().insert( v0, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(1), + from: RelSourceLoc::new(1), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v1, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(2), + from: RelSourceLoc::new(2), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v2, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(3), + from: RelSourceLoc::new(3), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v3, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(4), + from: RelSourceLoc::new(4), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v4, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(5), + from: RelSourceLoc::new(5), label: ValueLabel::new(1), }]), ); @@ -371,7 +377,7 @@ mod test { // expands during emission. #[test] fn br_table() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); diff --git a/cranelift/codegen/src/legalizer/heap.rs b/cranelift/codegen/src/legalizer/heap.rs index 91ae3da3c7..16d195932a 100644 --- a/cranelift/codegen/src/legalizer/heap.rs +++ b/cranelift/codegen/src/legalizer/heap.rs @@ -7,7 +7,7 @@ use crate::cursor::{Cursor, FuncCursor}; use crate::flowgraph::ControlFlowGraph; use crate::ir::condcodes::IntCC; use crate::ir::immediates::Uimm32; -use crate::ir::{self, InstBuilder}; +use crate::ir::{self, InstBuilder, RelSourceLoc}; use crate::isa::TargetIsa; /// Expand a `heap_addr` instruction according to the definition of the heap. @@ -212,7 +212,9 @@ fn cast_offset_to_pointer_ty( // Add debug value-label alias so that debuginfo can name the extended // value as the address let loc = pos.srcloc(); + let loc = RelSourceLoc::from_base_offset(pos.func.params.base_srcloc(), loc); pos.func + .stencil .dfg .add_value_label_alias(extended_offset, loc, offset); diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 593a8fc1e9..e244fbafc1 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -89,7 +89,7 @@ pub mod write; pub use crate::entity::packed_option; pub use crate::machinst::buffer::{MachCallSite, MachReloc, MachSrcLoc, MachStackMap, MachTrap}; -pub use crate::machinst::TextSectionBuilder; +pub use crate::machinst::{CompiledCode, TextSectionBuilder}; mod alias_analysis; mod bitset; @@ -116,6 +116,9 @@ mod souper_harvest; pub use crate::result::{CodegenError, CodegenResult, CompileError}; +#[cfg(feature = "incremental-cache")] +pub mod incremental_cache; + /// Even when trace logging is disabled, the trace macro has a significant performance cost so we /// disable it by default. #[macro_export] diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs index 32e6874d4a..4d1708dc4b 100644 --- a/cranelift/codegen/src/machinst/blockorder.rs +++ b/cranelift/codegen/src/machinst/blockorder.rs @@ -501,13 +501,14 @@ mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::types::*; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; + use crate::ir::UserFuncName; + use crate::ir::{AbiParam, Function, InstBuilder, Signature}; use crate::isa::CallConv; fn build_test_func(n_blocks: usize, edges: &[(usize, usize)]) -> Function { assert!(n_blocks > 0); - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); let mut func = Function::with_name_signature(name, sig); diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 084346cc4e..226feaf899 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -141,7 +141,8 @@ //! semantics below (grep for "Preserves execution semantics"). use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; -use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode}; +use crate::ir::function::FunctionParameters; +use crate::ir::{ExternalName, Opcode, RelSourceLoc, SourceLoc, TrapCode}; use crate::isa::unwind::UnwindInst; use crate::machinst::{ BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst, @@ -155,6 +156,42 @@ use std::mem; use std::string::String; use std::vec::Vec; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "enable-serde")] +pub trait CompilePhase { + type MachSrcLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone; + type SourceLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone; +} + +#[cfg(not(feature = "enable-serde"))] +pub trait CompilePhase { + type MachSrcLocType: core::fmt::Debug + PartialEq + Clone; + type SourceLocType: core::fmt::Debug + PartialEq + Clone; +} + +/// Status of a compiled artifact that needs patching before being used. +/// +/// Only used internally. +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct Stencil; + +/// Status of a compiled artifact ready to use. +#[derive(Clone, Debug, PartialEq)] +pub struct Final; + +impl CompilePhase for Stencil { + type MachSrcLocType = MachSrcLoc; + type SourceLocType = RelSourceLoc; +} + +impl CompilePhase for Final { + type MachSrcLocType = MachSrcLoc; + type SourceLocType = SourceLoc; +} + /// A buffer of output to be produced, fixed up, and then emitted to a CodeSink /// in bulk. /// @@ -174,14 +211,14 @@ pub struct MachBuffer { /// Any call site records referring to this code. call_sites: SmallVec<[MachCallSite; 16]>, /// Any source location mappings referring to this code. - srclocs: SmallVec<[MachSrcLoc; 64]>, + srclocs: SmallVec<[MachSrcLoc; 64]>, /// Any stack maps referring to this code. stack_maps: SmallVec<[MachStackMap; 8]>, /// Any unwind info at a given location. unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, /// The current source location in progress (after `start_srcloc()` and /// before `end_srcloc()`). This is a (start_offset, src_loc) tuple. - cur_srcloc: Option<(CodeOffset, SourceLoc)>, + cur_srcloc: Option<(CodeOffset, RelSourceLoc)>, /// Known label offsets; `UNKNOWN_LABEL_OFFSET` if unknown. label_offsets: SmallVec<[CodeOffset; 16]>, /// Label aliases: when one label points to an unconditional jump, and that @@ -229,23 +266,43 @@ pub struct MachBuffer { constant_labels: SecondaryMap, } +impl MachBufferFinalized { + pub(crate) fn apply_params(self, params: &FunctionParameters) -> MachBufferFinalized { + MachBufferFinalized { + data: self.data, + relocs: self.relocs, + traps: self.traps, + call_sites: self.call_sites, + srclocs: self + .srclocs + .into_iter() + .map(|srcloc| srcloc.apply_params(params)) + .collect(), + stack_maps: self.stack_maps, + unwind_info: self.unwind_info, + } + } +} + /// A `MachBuffer` once emission is completed: holds generated code and records, /// without fixups. This allows the type to be independent of the backend. -pub struct MachBufferFinalized { +#[derive(PartialEq, Debug, Clone)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MachBufferFinalized { /// The buffer contents, as raw bytes. - data: SmallVec<[u8; 1024]>, + pub(crate) data: SmallVec<[u8; 1024]>, /// Any relocations referring to this code. Note that only *external* /// relocations are tracked here; references to labels within the buffer are /// resolved before emission. - relocs: SmallVec<[MachReloc; 16]>, + pub(crate) relocs: SmallVec<[MachReloc; 16]>, /// Any trap records referring to this code. - traps: SmallVec<[MachTrap; 16]>, + pub(crate) traps: SmallVec<[MachTrap; 16]>, /// Any call site records referring to this code. - call_sites: SmallVec<[MachCallSite; 16]>, + pub(crate) call_sites: SmallVec<[MachCallSite; 16]>, /// Any source location mappings referring to this code. - srclocs: SmallVec<[MachSrcLoc; 64]>, + pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>, /// Any stack maps referring to this code. - stack_maps: SmallVec<[MachStackMap; 8]>, + pub(crate) stack_maps: SmallVec<[MachStackMap; 8]>, /// Any unwind info at a given location. pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, } @@ -1211,7 +1268,7 @@ impl MachBuffer { } /// Finish any deferred emissions and/or fixups. - pub fn finish(mut self) -> MachBufferFinalized { + pub fn finish(mut self) -> MachBufferFinalized { let _tt = timing::vcode_emit_finish(); // Do any optimizations on branches at tail of buffer, as if we @@ -1305,7 +1362,7 @@ impl MachBuffer { /// Set the `SourceLoc` for code from this offset until the offset at the /// next call to `end_srcloc()`. - pub fn start_srcloc(&mut self, loc: SourceLoc) { + pub fn start_srcloc(&mut self, loc: RelSourceLoc) { self.cur_srcloc = Some((self.cur_offset(), loc)); } @@ -1351,9 +1408,9 @@ impl MachBuffer { } } -impl MachBufferFinalized { +impl MachBufferFinalized { /// Get a list of source location mapping tuples in sorted-by-start-offset order. - pub fn get_srclocs_sorted(&self) -> &[MachSrcLoc] { + pub fn get_srclocs_sorted(&self) -> &[T::MachSrcLocType] { &self.srclocs[..] } @@ -1437,7 +1494,8 @@ struct MachLabelFixup { } /// A relocation resulting from a compilation. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachReloc { /// The offset at which the relocation applies, *relative to the /// containing section*. @@ -1451,7 +1509,8 @@ pub struct MachReloc { } /// A trap record resulting from a compilation. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachTrap { /// The offset at which the trap instruction occurs, *relative to the /// containing section*. @@ -1461,7 +1520,8 @@ pub struct MachTrap { } /// A call site record resulting from a compilation. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachCallSite { /// The offset of the call's return address, *relative to the containing section*. pub ret_addr: CodeOffset, @@ -1470,8 +1530,9 @@ pub struct MachCallSite { } /// A source-location mapping resulting from a compilation. -#[derive(Clone, Debug)] -pub struct MachSrcLoc { +#[derive(PartialEq, Debug, Clone)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MachSrcLoc { /// The start of the region of code corresponding to a source location. /// This is relative to the start of the function, not to the start of the /// section. @@ -1481,11 +1542,22 @@ pub struct MachSrcLoc { /// section. pub end: CodeOffset, /// The source location. - pub loc: SourceLoc, + pub loc: T::SourceLocType, +} + +impl MachSrcLoc { + fn apply_params(self, params: &FunctionParameters) -> MachSrcLoc { + MachSrcLoc { + start: self.start, + end: self.end, + loc: self.loc.expand(params.base_srcloc()), + } + } } /// Record of stack map metadata: stack offsets containing references. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachStackMap { /// The code offset at which this stack map applies. pub offset: CodeOffset, @@ -1599,7 +1671,10 @@ impl TextSectionBuilder for MachTextSectionBuilder { // We use an actual instruction definition to do tests, so we depend on the `arm64` feature here. #[cfg(all(test, feature = "arm64"))] mod test { + use cranelift_entity::EntityRef as _; + use super::*; + use crate::ir::UserExternalNameRef; use crate::isa::aarch64::inst::xreg; use crate::isa::aarch64::inst::{BranchTarget, CondBrKind, EmitInfo, Inst}; use crate::machinst::MachInstEmit; @@ -1982,9 +2057,17 @@ mod test { buf.add_trap(TrapCode::IntegerOverflow); buf.add_trap(TrapCode::IntegerDivisionByZero); buf.add_call_site(Opcode::Call); - buf.add_reloc(Reloc::Abs4, &ExternalName::user(0, 0), 0); + buf.add_reloc( + Reloc::Abs4, + &ExternalName::User(UserExternalNameRef::new(0)), + 0, + ); buf.put1(3); - buf.add_reloc(Reloc::Abs8, &ExternalName::user(1, 1), 1); + buf.add_reloc( + Reloc::Abs8, + &ExternalName::User(UserExternalNameRef::new(1)), + 1, + ); buf.put1(4); let buf = buf.finish(); diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index eab008925f..73e68c5afd 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -11,10 +11,11 @@ use crate::fx::{FxHashMap, FxHashSet}; use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit}; use crate::ir::{ types::{FFLAGS, IFLAGS}, - ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function, - GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Signature, - SourceLoc, Type, Value, ValueDef, ValueLabelAssignments, ValueLabelStart, + ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, Function, GlobalValue, + GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Signature, Type, Value, + ValueDef, ValueLabelAssignments, ValueLabelStart, }; +use crate::ir::{ExternalName, RelSourceLoc}; use crate::machinst::{ non_writable_value_regs, writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, Reg, VCode, VCodeBuilder, VCodeConstant, VCodeConstantData, @@ -813,10 +814,10 @@ impl<'func, I: VCodeInst> Lower<'func, I> { for &arg in self.f.dfg.block_params(block) { self.emit_value_label_marks_for_value(arg); } - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } - fn finish_ir_inst(&mut self, loc: SourceLoc) { + fn finish_ir_inst(&mut self, loc: RelSourceLoc) { self.vcode.set_srcloc(loc); // The VCodeBuilder builds in reverse order (and reverses at // the end), but `ir_insts` is in forward order, so reverse @@ -874,7 +875,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } self.vcode.add_succ(succ, &branch_arg_vregs[..]); } - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } fn collect_branches_and_targets( @@ -974,7 +975,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { self.vcode.add_succ(succ, &branch_arg_vregs[..]); self.emit(I::gen_jump(MachLabel::from_block(succ))); - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } // Original block body. @@ -986,7 +987,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { if bindex.index() == 0 { // Set up the function with arg vreg inits. self.gen_arg_setup(); - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } self.finish_bb(); @@ -1104,8 +1105,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } /// Get the source location for a given instruction. - pub fn srcloc(&self, ir_inst: Inst) -> SourceLoc { - self.f.srclocs[ir_inst] + pub fn srcloc(&self, ir_inst: Inst) -> RelSourceLoc { + self.f.rel_srclocs()[ir_inst] } /// Get the number of inputs to the given IR instruction. diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index fc7bb0abfa..a91dd9ef77 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -45,7 +45,8 @@ //! ``` use crate::binemit::{Addend, CodeInfo, CodeOffset, Reloc, StackMap}; -use crate::ir::{DynamicStackSlot, SourceLoc, StackSlot, Type}; +use crate::ir::function::FunctionParameters; +use crate::ir::{DynamicStackSlot, RelSourceLoc, StackSlot, Type}; use crate::result::CodegenResult; use crate::settings::Flags; use crate::value_label::ValueLabelsRanges; @@ -57,6 +58,9 @@ use regalloc2::{Allocation, VReg}; use smallvec::{smallvec, SmallVec}; use std::string::String; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + #[macro_use] pub mod isle; @@ -263,15 +267,17 @@ pub trait MachInstEmitState: Default + Clone + Debug { /// safepoint. fn pre_safepoint(&mut self, _stack_map: StackMap) {} /// Update the emission state to indicate instructions are associated with a - /// particular SourceLoc. - fn pre_sourceloc(&mut self, _srcloc: SourceLoc) {} + /// particular RelSourceLoc. + fn pre_sourceloc(&mut self, _srcloc: RelSourceLoc) {} } /// The result of a `MachBackend::compile_function()` call. Contains machine /// code (as bytes) and a disassembly, if requested. -pub struct CompiledCode { +#[derive(PartialEq, Debug, Clone)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct CompiledCodeBase { /// Machine code. - pub buffer: MachBufferFinalized, + pub buffer: MachBufferFinalized, /// Size of stack frame, in bytes. pub frame_size: u32, /// Disassembly, if requested. @@ -296,7 +302,23 @@ pub struct CompiledCode { pub bb_edges: Vec<(CodeOffset, CodeOffset)>, } -impl CompiledCode { +impl CompiledCodeStencil { + /// Apply function parameters to finalize a stencil into its final form. + pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode { + CompiledCode { + buffer: self.buffer.apply_params(params), + frame_size: self.frame_size, + disasm: self.disasm, + value_labels_ranges: self.value_labels_ranges, + sized_stackslot_offsets: self.sized_stackslot_offsets, + dynamic_stackslot_offsets: self.dynamic_stackslot_offsets, + bb_starts: self.bb_starts, + bb_edges: self.bb_edges, + } + } +} + +impl CompiledCodeBase { /// Get a `CodeInfo` describing section sizes from this compilation result. pub fn code_info(&self) -> CodeInfo { CodeInfo { @@ -310,6 +332,15 @@ impl CompiledCode { } } +/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it. +/// +/// Only used internally, in a transient manner, for the incremental compilation cache. +pub type CompiledCodeStencil = CompiledCodeBase; + +/// `CompiledCode` in its final form (i.e. after `FunctionParameters` have been applied), ready for +/// consumption. +pub type CompiledCode = CompiledCodeBase; + /// An object that can be used to create the text section of an executable. /// /// This primarily handles resolving relative relocations at diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index 1db50be4d1..c6e37b8517 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -19,9 +19,8 @@ use crate::fx::FxHashMap; use crate::fx::FxHashSet; -use crate::ir::{ - self, types, Constant, ConstantData, DynamicStackSlot, LabelValueLoc, SourceLoc, ValueLabel, -}; +use crate::ir::RelSourceLoc; +use crate::ir::{self, types, Constant, ConstantData, DynamicStackSlot, LabelValueLoc, ValueLabel}; use crate::machinst::*; use crate::timing; use crate::trace; @@ -90,7 +89,7 @@ pub struct VCode { /// Source locations for each instruction. (`SourceLoc` is a `u32`, so it is /// reasonable to keep one of these per instruction.) - srclocs: Vec, + srclocs: Vec, /// Entry block. entry: BlockIndex, @@ -261,7 +260,7 @@ pub struct VCodeBuilder { branch_block_arg_succ_start: usize, /// Current source location. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, /// Debug-value label in-progress map, keyed by label. For each /// label, we keep disjoint ranges mapping to vregs. We'll flatten @@ -296,7 +295,7 @@ impl VCodeBuilder { succ_start: 0, block_params_start: 0, branch_block_arg_succ_start: 0, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), debug_info: FxHashMap::default(), } } @@ -399,7 +398,7 @@ impl VCodeBuilder { } /// Set the current source location. - pub fn set_srcloc(&mut self, srcloc: SourceLoc) { + pub fn set_srcloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } @@ -847,8 +846,8 @@ impl VCode { // Is this the first block? Emit the prologue directly if so. if block == self.entry { trace!(" -> entry block"); - buffer.start_srcloc(SourceLoc::default()); - state.pre_sourceloc(SourceLoc::default()); + buffer.start_srcloc(Default::default()); + state.pre_sourceloc(Default::default()); for inst in &prologue_insts { do_emit(&inst, &[], &mut disasm, &mut buffer, &mut state); } @@ -919,7 +918,7 @@ impl VCode { buffer.start_srcloc(srcloc); cur_srcloc = Some(srcloc); } - state.pre_sourceloc(cur_srcloc.unwrap_or(SourceLoc::default())); + state.pre_sourceloc(cur_srcloc.unwrap_or_default()); // If this is a safepoint, compute a stack map // and pass it to the emit state. diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs index 9c3c7b2262..ef9556d021 100644 --- a/cranelift/codegen/src/settings.rs +++ b/cranelift/codegen/src/settings.rs @@ -548,6 +548,7 @@ probestack_func_adjusts_sp = false enable_jump_tables = true enable_heap_access_spectre_mitigation = true enable_table_access_spectre_mitigation = true +enable_incremental_compilation_cache_checks = false "# ); assert_eq!(f.opt_level(), super::OptLevel::None); diff --git a/cranelift/codegen/src/timing.rs b/cranelift/codegen/src/timing.rs index f21a68fa33..dcefde2da4 100644 --- a/cranelift/codegen/src/timing.rs +++ b/cranelift/codegen/src/timing.rs @@ -52,6 +52,8 @@ define_passes! { verify_flags: "Verify CPU flags", compile: "Compilation passes", + try_incremental_cache: "Try loading from incremental cache", + store_incremental_cache: "Store in incremental cache", flowgraph: "Control flow graph", domtree: "Dominator tree", loop_analysis: "Loop analysis", diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index e3be9b334a..938dfa9059 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -1845,8 +1845,8 @@ mod tests { imm: 0.into(), }); func.layout.append_inst(nullary_with_bad_opcode, block0); - func.layout.append_inst( - func.dfg.make_inst(InstructionData::Jump { + func.stencil.layout.append_inst( + func.stencil.dfg.make_inst(InstructionData::Jump { opcode: Opcode::Jump, destination: block0, args: EntityList::default(), diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs index d0b6ad8132..abdc5992ad 100644 --- a/cranelift/codegen/src/write.rs +++ b/cranelift/codegen/src/write.rs @@ -80,7 +80,12 @@ pub trait FuncWriter { for (fnref, ext_func) in &func.dfg.ext_funcs { if ext_func.signature != SigRef::reserved_value() { any = true; - self.write_entity_definition(w, func, fnref.into(), ext_func)?; + self.write_entity_definition( + w, + func, + fnref.into(), + &ext_func.display(Some(&func.params)), + )?; } } @@ -254,7 +259,7 @@ fn decorate_block( block: Block, ) -> fmt::Result { // Indent all instructions if any srclocs are present. - let indent = if func.srclocs.is_empty() { 4 } else { 36 }; + let indent = if func.rel_srclocs().is_empty() { 4 } else { 36 }; func_w.write_block_header(w, func, block, indent)?; for a in func.dfg.block_params(block).iter().cloned() { @@ -335,7 +340,7 @@ fn write_instruction( let mut s = String::with_capacity(16); // Source location goes first. - let srcloc = func.srclocs[inst]; + let srcloc = func.srcloc(inst); if !srcloc.is_default() { write!(s, "{} ", srcloc)?; } @@ -572,7 +577,7 @@ impl<'a> fmt::Display for DisplayValuesWithDelimiter<'a> { mod tests { use crate::cursor::{Cursor, CursorPosition, FuncCursor}; use crate::ir::types; - use crate::ir::{ExternalName, Function, InstBuilder, StackSlotData, StackSlotKind}; + use crate::ir::{Function, InstBuilder, StackSlotData, StackSlotKind, UserFuncName}; use alloc::string::ToString; #[test] @@ -580,7 +585,7 @@ mod tests { let mut f = Function::new(); assert_eq!(f.to_string(), "function u0:0() fast {\n}\n"); - f.name = ExternalName::testcase("foo"); + f.name = UserFuncName::testcase("foo"); assert_eq!(f.to_string(), "function %foo() fast {\n}\n"); f.create_sized_stack_slot(StackSlotData::new(StackSlotKind::ExplicitSlot, 4)); diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs index d4a057bf4e..4efed334ff 100644 --- a/cranelift/entity/src/list.rs +++ b/cranelift/entity/src/list.rs @@ -62,7 +62,7 @@ use serde::{Deserialize, Serialize}; /// /// The index stored in an `EntityList` points to part 2, the list elements. The value 0 is /// reserved for the empty list which isn't allocated in the vector. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct EntityList { index: u32, @@ -90,6 +90,20 @@ pub struct ListPool { free: Vec, } +impl PartialEq for ListPool { + fn eq(&self, other: &Self) -> bool { + // ignore the free list + self.data == other.data + } +} + +impl core::hash::Hash for ListPool { + fn hash(&self, state: &mut H) { + // ignore the free list + self.data.hash(state); + } +} + /// Lists are allocated in sizes that are powers of two, starting from 4. /// Each power of two is assigned a size class number, so the size is `4 << SizeClass`. type SizeClass = u8; diff --git a/cranelift/entity/src/map.rs b/cranelift/entity/src/map.rs index 67cdc11004..332cd061b7 100644 --- a/cranelift/entity/src/map.rs +++ b/cranelift/entity/src/map.rs @@ -23,7 +23,7 @@ use serde::{ /// /// The map does not track if an entry for a key has been inserted or not. Instead it behaves as if /// all keys have a default entry from the beginning. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Hash)] pub struct SecondaryMap where K: EntityRef, diff --git a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif index 2181f9e41e..0dfb996593 100644 --- a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif +++ b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif @@ -14,7 +14,7 @@ block0(v0: i64): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; autiasp ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/call.clif b/cranelift/filetests/filetests/isa/aarch64/call.clif index b1c6d211aa..45807c974d 100644 --- a/cranelift/filetests/filetests/isa/aarch64/call.clif +++ b/cranelift/filetests/filetests/isa/aarch64/call.clif @@ -14,7 +14,7 @@ block0(v0: i64): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; ret @@ -30,7 +30,7 @@ block0(v0: i32): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; ret @@ -54,7 +54,7 @@ block0(v0: i32): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; ret @@ -91,7 +91,7 @@ block0(v0: i8): ; movz x6, #42 ; movz x7, #42 ; strb w15, [sp] -; ldr x15, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; add sp, sp, #16 ; virtual_sp_offset_adjust -16 @@ -140,25 +140,25 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x9, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x9 ; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x11, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x13, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x13 ; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x1, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x1 ; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 @@ -184,25 +184,25 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x9, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x9 ; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x11, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x13, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x13 ; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x1, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x1 ; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 @@ -232,25 +232,25 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x9, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x9 ; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x11, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x13, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x13 ; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x1, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x1 ; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 @@ -283,7 +283,7 @@ block0(v0: i64): ; movz x0, #42 ; movz x2, #42 ; mov x1, x7 -; ldr x10, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x10, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x10 ; ldp fp, lr, [sp], #16 ; ret @@ -315,7 +315,7 @@ block0(v0: i64): ; movz x3, #42 ; movz x0, #42 ; mov x2, x7 -; ldr x10, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x10, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x10 ; ldp fp, lr, [sp], #16 ; ret @@ -347,7 +347,7 @@ block0(v0: i64): ; movz x2, #42 ; movz x0, #42 ; mov x1, x7 -; ldr x10, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x10, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x10 ; ldp fp, lr, [sp], #16 ; ret @@ -388,7 +388,7 @@ block0(v0: i128, v1: i64): ; mov x6, x14 ; str x13, [sp] ; str x15, [sp, #8] -; ldr x7, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x7, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x7 ; add sp, sp, #16 ; virtual_sp_offset_adjust -16 @@ -431,7 +431,7 @@ block0(v0: i128, v1: i64): ; mov x6, x14 ; str x13, [sp] ; str x15, [sp, #8] -; ldr x7, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x7, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x7 ; add sp, sp, #16 ; virtual_sp_offset_adjust -16 @@ -477,7 +477,7 @@ block0(v0: i64): ; mov fp, sp ; block0: ; mov x8, x0 -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; mov x0, x8 ; ldp fp, lr, [sp], #16 @@ -496,7 +496,7 @@ block0(v0: i64): ; str x24, [sp, #-16]! ; block0: ; mov x24, x8 -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; mov x8, x24 ; ldr x24, [sp], #16 diff --git a/cranelift/filetests/filetests/isa/aarch64/reftypes.clif b/cranelift/filetests/filetests/isa/aarch64/reftypes.clif index 7253ae6c9a..4920258760 100644 --- a/cranelift/filetests/filetests/isa/aarch64/reftypes.clif +++ b/cranelift/filetests/filetests/isa/aarch64/reftypes.clif @@ -69,7 +69,7 @@ block3(v7: r64, v8: r64): ; block0: ; str x1, [sp, #16] ; str x0, [sp, #8] -; ldr x3, 8 ; b 12 ; data TestCase { length: 1, ascii: [102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; mov x9, sp ; ldr x11, [sp, #8] diff --git a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif index 993d63c3cc..ceb56c34dc 100644 --- a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif +++ b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif @@ -42,7 +42,7 @@ block0(v0: i64): ; subs xzr, sp, x0, UXTX ; b.hs 8 ; udf ; block0: -; ldr x2, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x2, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x2 ; ldp fp, lr, [sp], #16 ; ret @@ -65,7 +65,7 @@ block0(v0: i64): ; subs xzr, sp, x16, UXTX ; b.hs 8 ; udf ; block0: -; ldr x2, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x2, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x2 ; ldp fp, lr, [sp], #16 ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif b/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif index b9eecef1bd..ba48ada8d0 100644 --- a/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif +++ b/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif @@ -11,6 +11,6 @@ block0: } ; block0: -; ldr x0, 8 ; b 12 ; data TestCase { length: 9, ascii: [109, 121, 95, 103, 108, 111, 98, 97, 108, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x0, 8 ; b 12 ; data TestCase(TestcaseName { length: 9, ascii: [109, 121, 95, 103, 108, 111, 98, 97, 108, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif b/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif index d1657b231a..11f87179df 100644 --- a/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif +++ b/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif @@ -19,7 +19,7 @@ block0(v0: i32): ; stp d8, d9, [sp, #-16]! ; block0: ; mov x25, x0 -; x0 = elf_tls_get_addr u1:0 +; x0 = elf_tls_get_addr userextname0 ; mov x1, x0 ; mov x0, x25 ; ldp d8, d9, [sp], #16 diff --git a/cranelift/filetests/filetests/isa/s390x/struct-arg.clif b/cranelift/filetests/filetests/isa/s390x/struct-arg.clif index cc28db4bc0..c05a17fd61 100644 --- a/cranelift/filetests/filetests/isa/s390x/struct-arg.clif +++ b/cranelift/filetests/filetests/isa/s390x/struct-arg.clif @@ -39,7 +39,7 @@ block0(v0: i64): ; block0: ; mvc 160(63,%r15), 0(%r2) ; la %r2, 160(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r14, %r15, 336(%r15) ; br %r14 @@ -57,7 +57,7 @@ block0(v0: i64, v1: i64): ; block0: ; mvc 160(63,%r15), 0(%r3) ; la %r3, 160(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r14, %r15, 336(%r15) ; br %r14 @@ -91,7 +91,7 @@ block0(v0: i64, v1: i64, v2: i64): ; mvc 416(63,%r15), 0(%r4) ; la %r3, 160(%r15) ; la %r4, 416(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r14, %r15, 592(%r15) ; br %r14 @@ -118,7 +118,7 @@ block0(v0: i64, v1: i64, v2: i64): ; lgr %r2, %r7 ; la %r3, 160(%r15) ; la %r4, 1184(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r7, %r15, 1304(%r15) ; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif index fa943e746e..e0f598a96f 100644 --- a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif +++ b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif @@ -15,8 +15,8 @@ block0(v0: i32): ; virtual_sp_offset_adjust 160 ; block0: ; larl %r12, %ElfGlobalOffsetTable + 0 -; bras %r1, 12 ; data u1:0@tlsgd ; lg %r2, 0(%r1) -; brasl %r14, %ElfTlsGetOffset:tls_gdcall:u1:0 +; bras %r1, 12 ; data userextname0@tlsgd ; lg %r2, 0(%r1) +; brasl %r14, %ElfTlsGetOffset:tls_gdcall:userextname0 ; ear %r3, %a0 ; sllg %r4, %r3, 32 ; ear %r4, %a1 diff --git a/cranelift/filetests/filetests/isa/x64/struct-arg.clif b/cranelift/filetests/filetests/isa/x64/struct-arg.clif index 5ae2c3fd03..bd877b4cae 100644 --- a/cranelift/filetests/filetests/isa/x64/struct-arg.clif +++ b/cranelift/filetests/filetests/isa/x64/struct-arg.clif @@ -54,7 +54,7 @@ block0(v0: i64): ; movl $64, %edx ; load_ext_name %Memcpy+0, %rcx ; call *%rcx -; call User { namespace: 0, index: 0 } +; call User(userextname0) ; addq %rsp, $64, %rsp ; virtual_sp_offset_adjust -64 ; movq %rbp, %rsp @@ -82,7 +82,7 @@ block0(v0: i64, v1: i64): ; load_ext_name %Memcpy+0, %rcx ; call *%rcx ; movq %r12, %rdi -; call User { namespace: 0, index: 0 } +; call User(userextname0) ; addq %rsp, $64, %rsp ; virtual_sp_offset_adjust -64 ; movq 0(%rsp), %r12 @@ -139,7 +139,7 @@ block0(v0: i64, v1: i64, v2: i64): ; load_ext_name %Memcpy+0, %rcx ; call *%rcx ; movq %r14, %rdi -; call User { namespace: 0, index: 0 } +; call User(userextname0) ; addq %rsp, $192, %rsp ; virtual_sp_offset_adjust -192 ; movq 0(%rsp), %rbx diff --git a/cranelift/filetests/filetests/isa/x64/tls_coff.clif b/cranelift/filetests/filetests/isa/x64/tls_coff.clif index ad05bd5a52..95421d8c78 100644 --- a/cranelift/filetests/filetests/isa/x64/tls_coff.clif +++ b/cranelift/filetests/filetests/isa/x64/tls_coff.clif @@ -14,7 +14,8 @@ block0(v0: i32): ; pushq %rbp ; movq %rsp, %rbp ; block0: -; %rax = coff_tls_get_addr User { namespace: 1, index: 0 } +; %rax = coff_tls_get_addr User(userextname0) ; movq %rbp, %rsp ; popq %rbp ; ret + diff --git a/cranelift/filetests/filetests/isa/x64/tls_elf.clif b/cranelift/filetests/filetests/isa/x64/tls_elf.clif index c7286e159f..cbee527f99 100644 --- a/cranelift/filetests/filetests/isa/x64/tls_elf.clif +++ b/cranelift/filetests/filetests/isa/x64/tls_elf.clif @@ -13,7 +13,7 @@ block0(v0: i32): ; pushq %rbp ; movq %rsp, %rbp ; block0: -; %rax = elf_tls_get_addr User { namespace: 1, index: 0 } +; %rax = elf_tls_get_addr User(userextname0) ; movq %rbp, %rsp ; popq %rbp ; ret diff --git a/cranelift/filetests/filetests/parser/memory.clif b/cranelift/filetests/filetests/parser/memory.clif index ecf872d64f..abe059c0fb 100644 --- a/cranelift/filetests/filetests/parser/memory.clif +++ b/cranelift/filetests/filetests/parser/memory.clif @@ -40,7 +40,7 @@ function %symbol() -> i32 { gv0 = symbol %something ; check: gv0 = symbol %something gv1 = symbol u8:9 - ; check: gv1 = symbol u8:9 + ; check: gv1 = symbol userextname0 block0: v0 = global_value.i32 gv0 ; check: v0 = global_value.i32 gv0 diff --git a/cranelift/filetests/src/function_runner.rs b/cranelift/filetests/src/function_runner.rs index 0502843b6b..b4484c72b4 100644 --- a/cranelift/filetests/src/function_runner.rs +++ b/cranelift/filetests/src/function_runner.rs @@ -78,11 +78,11 @@ impl SingleFunctionCompiler { let mut module = JITModule::new(builder); let mut ctx = module.make_context(); - let name = format!("{}", function.name); + let name = function.name.to_string(); let func_id = module.declare_function(&name, Linkage::Local, &function.signature)?; // Build and declare the trampoline in the module - let trampoline_name = format!("{}", trampoline.name); + let trampoline_name = trampoline.name.to_string(); let trampoline_id = module.declare_function(&trampoline_name, Linkage::Local, &trampoline.signature)?; @@ -260,7 +260,7 @@ fn make_trampoline(signature: &ir::Signature, isa: &dyn TargetIsa) -> Function { wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); // Add the `callee_address` parameter. wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); // Add the `values_vec` parameter. - let mut func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wrapper_sig); + let mut func = ir::Function::with_name_signature(ir::UserFuncName::default(), wrapper_sig); // The trampoline has a single block filled with loads, one call to callee_address, and some loads. let mut builder_context = FunctionBuilderContext::new(); diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs index 644c38e729..7a20d5c6e5 100644 --- a/cranelift/frontend/src/frontend.rs +++ b/cranelift/frontend/src/frontend.rs @@ -10,8 +10,8 @@ use cranelift_codegen::ir::{ types, AbiParam, Block, DataFlowGraph, DynamicStackSlot, DynamicStackSlotData, ExtFuncData, ExternalName, FuncRef, Function, GlobalValue, GlobalValueData, Heap, HeapData, Inst, InstBuilder, InstBuilderBase, InstructionData, JumpTable, JumpTableData, LibCall, MemFlags, - SigRef, Signature, StackSlot, StackSlotData, Type, Value, ValueLabel, ValueLabelAssignments, - ValueLabelStart, + RelSourceLoc, SigRef, Signature, StackSlot, StackSlotData, Type, Value, ValueLabel, + ValueLabelAssignments, ValueLabelStart, }; use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_codegen::packed_option::PackedOption; @@ -112,7 +112,7 @@ impl<'short, 'long> InstBuilderBase<'short> for FuncInstBuilder<'short, 'long> { self.builder.func.dfg.make_inst_results(inst, ctrl_typevar); self.builder.func.layout.append_inst(inst, self.block); if !self.builder.srcloc.is_default() { - self.builder.func.srclocs[inst] = self.builder.srcloc; + self.builder.func.set_srcloc(inst, self.builder.srcloc); } if data.opcode().is_branch() { @@ -469,11 +469,11 @@ impl<'a> FunctionBuilder<'a> { /// /// This will not do anything unless `func.dfg.collect_debug_info` is called first. pub fn set_val_label(&mut self, val: Value, label: ValueLabel) { - if let Some(values_labels) = self.func.dfg.values_labels.as_mut() { - use crate::hash_map::Entry; + if let Some(values_labels) = self.func.stencil.dfg.values_labels.as_mut() { + use alloc::collections::btree_map::Entry; let start = ValueLabelStart { - from: self.srcloc, + from: RelSourceLoc::from_base_offset(self.func.params.base_srcloc(), self.srcloc), label, }; @@ -574,9 +574,12 @@ impl<'a> FunctionBuilder<'a> { // These parameters count as "user" parameters here because they aren't // inserted by the SSABuilder. let user_param_count = &mut self.func_ctx.blocks[block].user_param_count; - for argtyp in &self.func.signature.params { + for argtyp in &self.func.stencil.signature.params { *user_param_count += 1; - self.func.dfg.append_block_param(block, argtyp.value_type); + self.func + .stencil + .dfg + .append_block_param(block, argtyp.value_type); } } @@ -587,9 +590,12 @@ impl<'a> FunctionBuilder<'a> { // These parameters count as "user" parameters here because they aren't // inserted by the SSABuilder. let user_param_count = &mut self.func_ctx.blocks[block].user_param_count; - for argtyp in &self.func.signature.returns { + for argtyp in &self.func.stencil.signature.returns { *user_param_count += 1; - self.func.dfg.append_block_param(block, argtyp.value_type); + self.func + .stencil + .dfg + .append_block_param(block, argtyp.value_type); } } @@ -1037,11 +1043,11 @@ impl<'a> FunctionBuilder<'a> { if let Some(small_type) = size.try_into().ok().and_then(Type::int_with_byte_size) { if let Equal | NotEqual = zero_cc { let mut left_flags = flags; - if size == left_align.get().into() { + if size == left_align.get() as u64 { left_flags.set_aligned(); } let mut right_flags = flags; - if size == right_align.get().into() { + if size == right_align.get() as u64 { right_flags.set_aligned(); } let left_val = self.ins().load(small_type, left_flags, left, 0); @@ -1105,10 +1111,8 @@ mod tests { use alloc::string::ToString; use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::condcodes::IntCC; - use cranelift_codegen::ir::types::*; - use cranelift_codegen::ir::{ - AbiParam, ExternalName, Function, InstBuilder, MemFlags, Signature, Value, - }; + use cranelift_codegen::ir::{types::*, UserFuncName}; + use cranelift_codegen::ir::{AbiParam, Function, InstBuilder, MemFlags, Signature, Value}; use cranelift_codegen::isa::{CallConv, TargetFrontendConfig, TargetIsa}; use cranelift_codegen::settings; use cranelift_codegen::verifier::verify_function; @@ -1120,7 +1124,7 @@ mod tests { sig.params.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1239,7 +1243,7 @@ mod tests { sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1288,7 +1292,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1342,7 +1346,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1399,7 +1403,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1439,7 +1443,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1498,7 +1502,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1711,7 +1715,7 @@ block0: sig.returns.push(AbiParam::new(B1)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1750,7 +1754,7 @@ block0: sig.returns.push(AbiParam::new(F32X4)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1804,7 +1808,7 @@ block0: let sig = Signature::new(CallConv::SystemV); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); diff --git a/cranelift/frontend/src/lib.rs b/cranelift/frontend/src/lib.rs index 73928d6d79..9a07687055 100644 --- a/cranelift/frontend/src/lib.rs +++ b/cranelift/frontend/src/lib.rs @@ -69,7 +69,7 @@ //! //! use cranelift_codegen::entity::EntityRef; //! use cranelift_codegen::ir::types::*; -//! use cranelift_codegen::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; +//! use cranelift_codegen::ir::{AbiParam, UserFuncName, Function, InstBuilder, Signature}; //! use cranelift_codegen::isa::CallConv; //! use cranelift_codegen::settings; //! use cranelift_codegen::verifier::verify_function; @@ -79,7 +79,7 @@ //! sig.returns.push(AbiParam::new(I32)); //! sig.params.push(AbiParam::new(I32)); //! let mut fn_builder_ctx = FunctionBuilderContext::new(); -//! let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); +//! let mut func = Function::with_name_signature(UserFuncName::user(0, 0), sig); //! { //! let mut builder = FunctionBuilder::new(&mut func, &mut fn_builder_ctx); //! @@ -189,9 +189,9 @@ extern crate alloc; extern crate std; #[cfg(not(feature = "std"))] -use hashbrown::{hash_map, HashMap}; +use hashbrown::HashMap; #[cfg(feature = "std")] -use std::collections::{hash_map, HashMap}; +use std::collections::HashMap; pub use crate::frontend::{FunctionBuilder, FunctionBuilderContext}; pub use crate::switch::Switch; diff --git a/cranelift/frontend/src/switch.rs b/cranelift/frontend/src/switch.rs index 3de30d1576..232337474f 100644 --- a/cranelift/frontend/src/switch.rs +++ b/cranelift/frontend/src/switch.rs @@ -14,13 +14,13 @@ type EntryIndex = u128; /// /// ```rust /// # use cranelift_codegen::ir::types::*; -/// # use cranelift_codegen::ir::{ExternalName, Function, Signature, InstBuilder}; +/// # use cranelift_codegen::ir::{UserFuncName, Function, Signature, InstBuilder}; /// # use cranelift_codegen::isa::CallConv; /// # use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Switch}; /// # /// # let mut sig = Signature::new(CallConv::SystemV); /// # let mut fn_builder_ctx = FunctionBuilderContext::new(); -/// # let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); +/// # let mut func = Function::with_name_signature(UserFuncName::user(0, 0), sig); /// # let mut builder = FunctionBuilder::new(&mut func, &mut fn_builder_ctx); /// # /// # let entry = builder.create_block(); diff --git a/cranelift/fuzzgen/src/config.rs b/cranelift/fuzzgen/src/config.rs index 0ea0ae1408..abc3b330aa 100644 --- a/cranelift/fuzzgen/src/config.rs +++ b/cranelift/fuzzgen/src/config.rs @@ -21,12 +21,15 @@ pub struct Config { /// decides to insert more. pub jump_tables_per_function: RangeInclusive, pub jump_table_entries: RangeInclusive, + /// The Switch API specializes either individual blocks or contiguous ranges. /// In `switch_cases` we decide to produce either a single block or a range. /// The size of the range is controlled by `switch_max_range_size`. pub switch_cases: RangeInclusive, pub switch_max_range_size: RangeInclusive, + pub funcrefs_per_function: RangeInclusive, + /// Stack slots. /// The combination of these two determines stack usage per function pub static_stack_slots_per_function: RangeInclusive, @@ -49,6 +52,7 @@ impl Default for Config { switch_cases: 0..=64, // Ranges smaller than 2 don't make sense. switch_max_range_size: 2..=32, + funcrefs_per_function: 0..=8, static_stack_slots_per_function: 0..=8, static_stack_slot_size: 0..=128, } diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs index 68e9af330b..522130d81d 100644 --- a/cranelift/fuzzgen/src/function_generator.rs +++ b/cranelift/fuzzgen/src/function_generator.rs @@ -2,14 +2,14 @@ use crate::codegen::ir::{ArgumentExtension, ArgumentPurpose, ValueList}; use crate::config::Config; use anyhow::Result; use arbitrary::{Arbitrary, Unstructured}; -use cranelift::codegen::ir::types::*; +use cranelift::codegen::ir::{types::*, FuncRef, LibCall, UserExternalName, UserFuncName}; use cranelift::codegen::ir::{ AbiParam, Block, ExternalName, Function, JumpTable, Opcode, Signature, StackSlot, Type, Value, }; use cranelift::codegen::isa::CallConv; use cranelift::frontend::{FunctionBuilder, FunctionBuilderContext, Switch, Variable}; use cranelift::prelude::{ - EntityRef, InstBuilder, IntCC, JumpTableData, StackSlotData, StackSlotKind, + EntityRef, ExtFuncData, InstBuilder, IntCC, JumpTableData, StackSlotData, StackSlotKind, }; use std::collections::HashMap; use std::ops::RangeInclusive; @@ -41,6 +41,25 @@ fn insert_opcode( Ok(()) } +fn insert_call( + fgen: &mut FunctionGenerator, + builder: &mut FunctionBuilder, + opcode: Opcode, + _args: &'static [Type], + _rets: &'static [Type], +) -> Result<()> { + assert_eq!(opcode, Opcode::Call, "only call handled at the moment"); + let (sig, func_ref) = fgen.u.choose(&fgen.func_refs)?.clone(); + + let actuals = fgen.generate_values_for_signature( + builder, + sig.params.iter().map(|abi_param| abi_param.value_type), + )?; + + builder.ins().call(func_ref, &actuals); + Ok(()) +} + fn insert_stack_load( fgen: &mut FunctionGenerator, builder: &mut FunctionBuilder, @@ -369,6 +388,8 @@ const OPCODE_SIGNATURES: &'static [( (Opcode::F64const, &[], &[F64], insert_const), // Bool Consts (Opcode::Bconst, &[], &[B1], insert_const), + // Call + (Opcode::Call, &[], &[], insert_call), ]; pub struct FunctionGenerator<'r, 'data> @@ -380,6 +401,8 @@ where vars: Vec<(Type, Variable)>, blocks: Vec<(Block, BlockSignature)>, jump_tables: Vec, + func_refs: Vec<(Signature, FuncRef)>, + next_func_index: u32, static_stack_slots: Vec, } @@ -394,7 +417,9 @@ where vars: vec![], blocks: vec![], jump_tables: vec![], + func_refs: vec![], static_stack_slots: vec![], + next_func_index: 0, } } @@ -739,6 +764,42 @@ where Ok(()) } + fn generate_funcrefs(&mut self, builder: &mut FunctionBuilder) -> Result<()> { + for _ in 0..self.param(&self.config.funcrefs_per_function)? { + let (ext_name, sig) = if self.u.arbitrary::()? { + let func_index = self.next_func_index; + self.next_func_index = self.next_func_index.wrapping_add(1); + let user_func_ref = builder + .func + .declare_imported_user_function(UserExternalName { + namespace: 0, + index: func_index, + }); + let name = ExternalName::User(user_func_ref); + let signature = self.generate_signature()?; + (name, signature) + } else { + // Use udivi64 as an example of a libcall function. + let mut signature = Signature::new(CallConv::Fast); + signature.params.push(AbiParam::new(I64)); + signature.params.push(AbiParam::new(I64)); + signature.returns.push(AbiParam::new(I64)); + (ExternalName::LibCall(LibCall::UdivI64), signature) + }; + + let sig_ref = builder.import_signature(sig.clone()); + let func_ref = builder.import_function(ExtFuncData { + name: ext_name, + signature: sig_ref, + colocated: self.u.arbitrary()?, + }); + + self.func_refs.push((sig, func_ref)); + } + + Ok(()) + } + fn generate_stack_slots(&mut self, builder: &mut FunctionBuilder) -> Result<()> { for _ in 0..self.param(&self.config.static_stack_slots_per_function)? { let bytes = self.param(&self.config.static_stack_slot_size)? as u32; @@ -865,7 +926,7 @@ where let sig = self.generate_signature()?; let mut fn_builder_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::user(0, 1), sig.clone()); + let mut func = Function::with_name_signature(UserFuncName::user(0, 1), sig.clone()); let mut builder = FunctionBuilder::new(&mut func, &mut fn_builder_ctx); @@ -873,6 +934,7 @@ where // Function preamble self.generate_jumptables(&mut builder)?; + self.generate_funcrefs(&mut builder)?; self.generate_stack_slots(&mut builder)?; // Main instruction generation loop diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs index 26151697a2..8dce6e0213 100644 --- a/cranelift/fuzzgen/src/lib.rs +++ b/cranelift/fuzzgen/src/lib.rs @@ -15,6 +15,19 @@ mod function_generator; pub type TestCaseInput = Vec; +/// Simple wrapper to generate a single Cranelift `Function`. +#[derive(Debug)] +pub struct SingleFunction(pub Function); + +impl<'a> Arbitrary<'a> for SingleFunction { + fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { + FuzzGen::new(u) + .generate_func() + .map_err(|_| arbitrary::Error::IncorrectFormat) + .map(Self) + } +} + pub struct TestCase { pub func: Function, /// Generate multiple test inputs for each test case. @@ -171,12 +184,14 @@ where Ok(ctx.func) } - pub fn generate_test(mut self) -> Result { + fn generate_func(&mut self) -> Result { let func = FunctionGenerator::new(&mut self.u, &self.config).generate()?; + self.run_func_passes(func) + } + + pub fn generate_test(mut self) -> Result { + let func = self.generate_func()?; let inputs = self.generate_test_inputs(&func.signature)?; - - let func = self.run_func_passes(func)?; - Ok(TestCase { func, inputs }) } } diff --git a/cranelift/interpreter/src/environment.rs b/cranelift/interpreter/src/environment.rs index 6c7a9a0b26..ab10dc9dba 100644 --- a/cranelift/interpreter/src/environment.rs +++ b/cranelift/interpreter/src/environment.rs @@ -63,18 +63,20 @@ impl<'a> FunctionStore<'a> { /// currently it retrieves the function name as a string and performs string matching. fn get_function_name(func_ref: FuncRef, function: &Function) -> String { function + .stencil .dfg .ext_funcs .get(func_ref) .expect("function to exist") .name + .display(Some(&function.params)) .to_string() } #[cfg(test)] mod tests { use super::*; - use cranelift_codegen::ir::{ExternalName, Signature}; + use cranelift_codegen::ir::{Signature, UserFuncName}; use cranelift_codegen::isa::CallConv; #[test] @@ -95,7 +97,7 @@ mod tests { #[test] fn from() { - let name = ExternalName::testcase("test"); + let name = UserFuncName::testcase("test"); let signature = Signature::new(CallConv::Fast); let func = &Function::with_name_signature(name, signature); let env: FunctionStore = func.into(); diff --git a/cranelift/jit/examples/jit-minimal.rs b/cranelift/jit/examples/jit-minimal.rs index dc24df7c86..f5e70e9b62 100644 --- a/cranelift/jit/examples/jit-minimal.rs +++ b/cranelift/jit/examples/jit-minimal.rs @@ -1,3 +1,4 @@ +use codegen::ir::UserFuncName; use cranelift::prelude::*; use cranelift_codegen::settings::{self, Configurable}; use cranelift_jit::{JITBuilder, JITModule}; @@ -35,7 +36,8 @@ fn main() { .unwrap(); ctx.func.signature = sig_a; - ctx.func.name = ExternalName::user(0, func_a.as_u32()); + ctx.func.name = UserFuncName::user(0, func_a.as_u32()); + { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); let block = bcx.create_block(); @@ -53,7 +55,8 @@ fn main() { module.clear_context(&mut ctx); ctx.func.signature = sig_b; - ctx.func.name = ExternalName::user(0, func_b.as_u32()); + ctx.func.name = UserFuncName::user(0, func_b.as_u32()); + { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); let block = bcx.create_block(); diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index 75e8b2338e..a24d723a40 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -8,7 +8,7 @@ use cranelift_codegen::{binemit::Reloc, CodegenError}; use cranelift_entity::SecondaryMap; use cranelift_module::{ DataContext, DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleCompiledFunction, - ModuleDeclarations, ModuleError, ModuleResult, + ModuleDeclarations, ModuleError, ModuleExtName, ModuleReloc, ModuleResult, }; use log::info; use std::cell::RefCell; @@ -275,9 +275,9 @@ impl JITModule { std::ptr::write(plt_ptr, plt_val); } - fn get_address(&self, name: &ir::ExternalName) -> *const u8 { + fn get_address(&self, name: &ModuleExtName) -> *const u8 { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { let (name, linkage) = if ModuleDeclarations::is_function(name) { if self.hotswap_enabled { return self.get_plt_address(name); @@ -309,12 +309,12 @@ impl JITModule { panic!("can't resolve symbol {}", name); } } - ir::ExternalName::LibCall(ref libcall) => { + ModuleExtName::LibCall(ref libcall) => { let sym = (self.libcall_names)(*libcall); self.lookup_symbol(&sym) .unwrap_or_else(|| panic!("can't resolve libcall {}", sym)) } - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), } } @@ -326,9 +326,9 @@ impl JITModule { unsafe { got_entry.as_ref() }.load(Ordering::SeqCst) } - fn get_got_address(&self, name: &ir::ExternalName) -> NonNull> { + fn get_got_address(&self, name: &ModuleExtName) -> NonNull> { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { if ModuleDeclarations::is_function(name) { let func_id = FuncId::from_name(name); self.function_got_entries[func_id].unwrap() @@ -337,17 +337,17 @@ impl JITModule { self.data_object_got_entries[data_id].unwrap() } } - ir::ExternalName::LibCall(ref libcall) => *self + ModuleExtName::LibCall(ref libcall) => *self .libcall_got_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)), - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), } } - fn get_plt_address(&self, name: &ir::ExternalName) -> *const u8 { + fn get_plt_address(&self, name: &ModuleExtName) -> *const u8 { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { if ModuleDeclarations::is_function(name) { let func_id = FuncId::from_name(name); self.function_plt_entries[func_id] @@ -358,13 +358,13 @@ impl JITModule { unreachable!("PLT relocations can only have functions as target"); } } - ir::ExternalName::LibCall(ref libcall) => self + ModuleExtName::LibCall(ref libcall) => self .libcall_plt_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) .as_ptr() .cast::(), - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), } } @@ -631,12 +631,16 @@ impl Module for JITModule { /// /// TODO: Coalesce redundant decls and signatures. /// TODO: Look into ways to reduce the risk of using a FuncRef in the wrong function. - fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { + fn declare_func_in_func(&mut self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { let decl = self.declarations.get_function_decl(func); let signature = in_func.import_signature(decl.signature.clone()); let colocated = !self.hotswap_enabled && decl.linkage.is_final(); + let user_name_ref = in_func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: func.as_u32(), + }); in_func.import_function(ir::ExtFuncData { - name: ir::ExternalName::user(0, func.as_u32()), + name: ir::ExternalName::user(user_name_ref), signature, colocated, }) @@ -648,24 +652,18 @@ impl Module for JITModule { fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue { let decl = self.declarations.get_data_decl(data); let colocated = !self.hotswap_enabled && decl.linkage.is_final(); + let user_name_ref = func.declare_imported_user_function(ir::UserExternalName { + namespace: 1, + index: data.as_u32(), + }); func.create_global_value(ir::GlobalValueData::Symbol { - name: ir::ExternalName::user(1, data.as_u32()), + name: ir::ExternalName::user(user_name_ref), offset: ir::immediates::Imm64::new(0), colocated, tls: decl.tls, }) } - /// TODO: Same as above. - fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { - ctx.import_function(ir::ExternalName::user(0, func.as_u32())) - } - - /// TODO: Same as above. - fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { - ctx.import_global_value(ir::ExternalName::user(1, data.as_u32())) - } - fn define_function( &mut self, id: FuncId, @@ -681,7 +679,10 @@ impl Module for JITModule { return Err(ModuleError::DuplicateDefinition(decl.name.to_owned())); } - let compiled_code = ctx.compile(self.isa())?; + // work around borrow-checker to allow reuse of ctx below + let _ = ctx.compile(self.isa())?; + let compiled_code = ctx.compiled_code().unwrap(); + let code_size = compiled_code.code_info().total_size; let size = code_size as usize; @@ -696,7 +697,12 @@ impl Module for JITModule { mem.copy_from_slice(compiled_code.code_buffer()); } - let relocs = compiled_code.buffer.relocs().to_vec(); + let relocs = compiled_code + .buffer + .relocs() + .iter() + .map(|reloc| ModuleReloc::from_mach_reloc(reloc, &ctx.func)) + .collect(); self.record_function_for_perf(ptr, size, &decl.name); self.compiled_functions[id] = Some(CompiledBlob { ptr, size, relocs }); @@ -714,16 +720,16 @@ impl Module for JITModule { .unwrap() .perform_relocations( |name| match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { unreachable!("non GOT or PLT relocation in function {} to {}", id, name) } - ir::ExternalName::LibCall(ref libcall) => self + ModuleExtName::LibCall(ref libcall) => self .libcall_plt_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) .as_ptr() .cast::(), - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), }, |name| self.get_got_address(name).as_ptr().cast(), |name| self.get_plt_address(name), @@ -738,6 +744,7 @@ impl Module for JITModule { fn define_function_bytes( &mut self, id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult { @@ -771,7 +778,10 @@ impl Module for JITModule { self.compiled_functions[id] = Some(CompiledBlob { ptr, size, - relocs: relocs.to_vec(), + relocs: relocs + .iter() + .map(|reloc| ModuleReloc::from_mach_reloc(reloc, func)) + .collect(), }); if self.isa.flags().is_pic() { @@ -866,6 +876,33 @@ impl Module for JITModule { Ok(()) } + + fn get_name(&self, name: &str) -> Option { + self.declarations().get_name(name) + } + + fn target_config(&self) -> cranelift_codegen::isa::TargetFrontendConfig { + self.isa().frontend_config() + } + + fn make_context(&self) -> cranelift_codegen::Context { + let mut ctx = cranelift_codegen::Context::new(); + ctx.func.signature.call_conv = self.isa().default_call_conv(); + ctx + } + + fn clear_context(&self, ctx: &mut cranelift_codegen::Context) { + ctx.clear(); + ctx.func.signature.call_conv = self.isa().default_call_conv(); + } + + fn make_signature(&self) -> ir::Signature { + ir::Signature::new(self.isa().default_call_conv()) + } + + fn clear_signature(&self, sig: &mut ir::Signature) { + sig.clear(self.isa().default_call_conv()); + } } #[cfg(not(windows))] diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index ee42eb8b0c..1d1d5cc4db 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -1,25 +1,25 @@ use cranelift_codegen::binemit::Reloc; -use cranelift_codegen::ir::ExternalName; -use cranelift_codegen::MachReloc; +use cranelift_module::ModuleExtName; +use cranelift_module::ModuleReloc; use std::convert::TryFrom; #[derive(Clone)] pub(crate) struct CompiledBlob { pub(crate) ptr: *mut u8, pub(crate) size: usize, - pub(crate) relocs: Vec, + pub(crate) relocs: Vec, } impl CompiledBlob { pub(crate) fn perform_relocations( &self, - get_address: impl Fn(&ExternalName) -> *const u8, - get_got_entry: impl Fn(&ExternalName) -> *const u8, - get_plt_entry: impl Fn(&ExternalName) -> *const u8, + get_address: impl Fn(&ModuleExtName) -> *const u8, + get_got_entry: impl Fn(&ModuleExtName) -> *const u8, + get_plt_entry: impl Fn(&ModuleExtName) -> *const u8, ) { use std::ptr::write_unaligned; - for &MachReloc { + for &ModuleReloc { kind, offset, ref name, diff --git a/cranelift/jit/tests/basic.rs b/cranelift/jit/tests/basic.rs index dc95a00e50..64957aa1f7 100644 --- a/cranelift/jit/tests/basic.rs +++ b/cranelift/jit/tests/basic.rs @@ -48,7 +48,7 @@ fn define_simple_function(module: &mut JITModule) -> FuncId { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); @@ -91,7 +91,7 @@ fn switch_error() { call_conv: CallConv::SystemV, }; - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(UserFuncName::default(), sig); let mut func_ctx = FunctionBuilderContext::new(); { @@ -179,7 +179,8 @@ fn libcall_function() { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); + let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); diff --git a/cranelift/module/src/data_context.rs b/cranelift/module/src/data_context.rs index 0c5fd1a1a1..3ec4079968 100644 --- a/cranelift/module/src/data_context.rs +++ b/cranelift/module/src/data_context.rs @@ -3,12 +3,14 @@ use cranelift_codegen::binemit::{Addend, CodeOffset, Reloc}; use cranelift_codegen::entity::PrimaryMap; use cranelift_codegen::ir; -use cranelift_codegen::MachReloc; use std::borrow::ToOwned; use std::boxed::Box; use std::string::String; use std::vec::Vec; +use crate::module::ModuleReloc; +use crate::ModuleExtName; + /// This specifies how data is to be initialized. #[derive(Clone, PartialEq, Eq, Debug)] pub enum Init { @@ -43,9 +45,9 @@ pub struct DataDescription { /// How the data should be initialized. pub init: Init, /// External function declarations. - pub function_decls: PrimaryMap, + pub function_decls: PrimaryMap, /// External data object declarations. - pub data_decls: PrimaryMap, + pub data_decls: PrimaryMap, /// Function addresses to write at specified offsets. pub function_relocs: Vec<(CodeOffset, ir::FuncRef)>, /// Data addresses to write at specified offsets. @@ -59,11 +61,14 @@ pub struct DataDescription { impl DataDescription { /// An iterator over all relocations of the data object. - pub fn all_relocs<'a>(&'a self, pointer_reloc: Reloc) -> impl Iterator + 'a { + pub fn all_relocs<'a>( + &'a self, + pointer_reloc: Reloc, + ) -> impl Iterator + 'a { let func_relocs = self .function_relocs .iter() - .map(move |&(offset, id)| MachReloc { + .map(move |&(offset, id)| ModuleReloc { kind: pointer_reloc, offset, name: self.function_decls[id].clone(), @@ -72,7 +77,7 @@ impl DataDescription { let data_relocs = self .data_relocs .iter() - .map(move |&(offset, id, addend)| MachReloc { + .map(move |&(offset, id, addend)| ModuleReloc { kind: pointer_reloc, offset, name: self.data_decls[id].clone(), @@ -144,7 +149,7 @@ impl DataContext { /// Users of the `Module` API generally should call /// `Module::declare_func_in_data` instead, as it takes care of generating /// the appropriate `ExternalName`. - pub fn import_function(&mut self, name: ir::ExternalName) -> ir::FuncRef { + pub fn import_function(&mut self, name: ModuleExtName) -> ir::FuncRef { self.description.function_decls.push(name) } @@ -155,7 +160,7 @@ impl DataContext { /// Users of the `Module` API generally should call /// `Module::declare_data_in_data` instead, as it takes care of generating /// the appropriate `ExternalName`. - pub fn import_global_value(&mut self, name: ir::ExternalName) -> ir::GlobalValue { + pub fn import_global_value(&mut self, name: ModuleExtName) -> ir::GlobalValue { self.description.data_decls.push(name) } @@ -181,8 +186,9 @@ impl DataContext { #[cfg(test)] mod tests { + use crate::ModuleExtName; + use super::{DataContext, Init}; - use cranelift_codegen::ir; #[test] fn basic_data_context() { @@ -198,11 +204,11 @@ mod tests { data_ctx.define_zeroinit(256); - let _func_a = data_ctx.import_function(ir::ExternalName::user(0, 0)); - let func_b = data_ctx.import_function(ir::ExternalName::user(0, 1)); - let func_c = data_ctx.import_function(ir::ExternalName::user(1, 0)); - let _data_a = data_ctx.import_global_value(ir::ExternalName::user(2, 2)); - let data_b = data_ctx.import_global_value(ir::ExternalName::user(2, 3)); + let _func_a = data_ctx.import_function(ModuleExtName::user(0, 0)); + let func_b = data_ctx.import_function(ModuleExtName::user(0, 1)); + let func_c = data_ctx.import_function(ModuleExtName::user(0, 2)); + let _data_a = data_ctx.import_global_value(ModuleExtName::user(0, 3)); + let data_b = data_ctx.import_global_value(ModuleExtName::user(0, 4)); data_ctx.write_function_addr(8, func_b); data_ctx.write_function_addr(16, func_c); diff --git a/cranelift/module/src/lib.rs b/cranelift/module/src/lib.rs index 5997e14709..4f9bde29d5 100644 --- a/cranelift/module/src/lib.rs +++ b/cranelift/module/src/lib.rs @@ -43,7 +43,7 @@ mod traps; pub use crate::data_context::{DataContext, DataDescription, Init}; pub use crate::module::{ DataId, FuncId, FuncOrDataId, Linkage, Module, ModuleCompiledFunction, ModuleDeclarations, - ModuleError, ModuleResult, + ModuleError, ModuleExtName, ModuleReloc, ModuleResult, }; pub use crate::traps::TrapSite; diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index def84f1018..1312f4880b 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -7,19 +7,57 @@ use super::HashMap; use crate::data_context::DataContext; +use core::fmt::Display; +use cranelift_codegen::binemit::{CodeOffset, Reloc}; use cranelift_codegen::entity::{entity_impl, PrimaryMap}; +use cranelift_codegen::ir::Function; use cranelift_codegen::{binemit, MachReloc}; use cranelift_codegen::{ir, isa, CodegenError, CompileError, Context}; use std::borrow::ToOwned; use std::string::String; +/// A module relocation. +#[derive(Clone)] +pub struct ModuleReloc { + /// The offset at which the relocation applies, *relative to the + /// containing section*. + pub offset: CodeOffset, + /// The kind of relocation. + pub kind: Reloc, + /// The external symbol / name to which this relocation refers. + pub name: ModuleExtName, + /// The addend to add to the symbol value. + pub addend: i64, +} + +impl ModuleReloc { + /// Converts a `MachReloc` produced from a `Function` into a `ModuleReloc`. + pub fn from_mach_reloc(mach_reloc: &MachReloc, func: &Function) -> Self { + let name = match mach_reloc.name { + ir::ExternalName::User(reff) => { + let name = &func.params.user_named_funcs()[reff]; + ModuleExtName::user(name.namespace, name.index) + } + ir::ExternalName::TestCase(_) => unimplemented!(), + ir::ExternalName::LibCall(libcall) => ModuleExtName::LibCall(libcall), + ir::ExternalName::KnownSymbol(ks) => ModuleExtName::KnownSymbol(ks), + }; + Self { + offset: mach_reloc.offset, + kind: mach_reloc.kind, + name, + addend: mach_reloc.addend, + } + } +} + /// A function identifier for use in the `Module` interface. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct FuncId(u32); entity_impl!(FuncId, "funcid"); /// Function identifiers are namespace 0 in `ir::ExternalName` -impl From for ir::ExternalName { +impl From for ModuleExtName { fn from(id: FuncId) -> Self { Self::User { namespace: 0, @@ -30,12 +68,12 @@ impl From for ir::ExternalName { impl FuncId { /// Get the `FuncId` for the function named by `name`. - pub fn from_name(name: &ir::ExternalName) -> FuncId { - if let ir::ExternalName::User { namespace, index } = *name { - debug_assert_eq!(namespace, 0); - FuncId::from_u32(index) + pub fn from_name(name: &ModuleExtName) -> FuncId { + if let ModuleExtName::User { namespace, index } = name { + debug_assert_eq!(*namespace, 0); + FuncId::from_u32(*index) } else { - panic!("unexpected ExternalName kind {}", name) + panic!("unexpected name in DataId::from_name") } } } @@ -46,7 +84,7 @@ pub struct DataId(u32); entity_impl!(DataId, "dataid"); /// Data identifiers are namespace 1 in `ir::ExternalName` -impl From for ir::ExternalName { +impl From for ModuleExtName { fn from(id: DataId) -> Self { Self::User { namespace: 1, @@ -57,12 +95,12 @@ impl From for ir::ExternalName { impl DataId { /// Get the `DataId` for the data object named by `name`. - pub fn from_name(name: &ir::ExternalName) -> DataId { - if let ir::ExternalName::User { namespace, index } = *name { - debug_assert_eq!(namespace, 1); - DataId::from_u32(index) + pub fn from_name(name: &ModuleExtName) -> DataId { + if let ModuleExtName::User { namespace, index } = name { + debug_assert_eq!(*namespace, 1); + DataId::from_u32(*index) } else { - panic!("unexpected ExternalName kind {}", name) + panic!("unexpected name in DataId::from_name") } } } @@ -134,8 +172,8 @@ pub enum FuncOrDataId { Data(DataId), } -/// Mapping to `ir::ExternalName` is trivial based on the `FuncId` and `DataId` mapping. -impl From for ir::ExternalName { +/// Mapping to `ModuleExtName` is trivial based on the `FuncId` and `DataId` mapping. +impl From for ModuleExtName { fn from(id: FuncOrDataId) -> Self { match id { FuncOrDataId::Func(funcid) => Self::from(funcid), @@ -277,6 +315,39 @@ impl DataDeclaration { } } +/// A translated `ExternalName` into something global we can handle. +#[derive(Clone)] +pub enum ModuleExtName { + /// User defined function, converted from `ExternalName::User`. + User { + /// Arbitrary. + namespace: u32, + /// Arbitrary. + index: u32, + }, + /// Call into a library function. + LibCall(ir::LibCall), + /// Symbols known to the linker. + KnownSymbol(ir::KnownSymbol), +} + +impl ModuleExtName { + /// Creates a user-defined external name. + pub fn user(namespace: u32, index: u32) -> Self { + Self::User { namespace, index } + } +} + +impl Display for ModuleExtName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::User { namespace, index } => write!(f, "u{}:{}", namespace, index), + Self::LibCall(lc) => write!(f, "%{}", lc), + Self::KnownSymbol(ks) => write!(f, "{}", ks), + } + } +} + /// This provides a view to the state of a module which allows `ir::ExternalName`s to be translated /// into `FunctionDeclaration`s and `DataDeclaration`s. #[derive(Debug, Default)] @@ -299,11 +370,12 @@ impl ModuleDeclarations { } /// Return whether `name` names a function, rather than a data object. - pub fn is_function(name: &ir::ExternalName) -> bool { - if let ir::ExternalName::User { namespace, .. } = *name { - namespace == 0 - } else { - panic!("unexpected ExternalName kind {}", name) + pub fn is_function(name: &ModuleExtName) -> bool { + match name { + ModuleExtName::User { namespace, .. } => *namespace == 0, + ModuleExtName::LibCall(_) | ModuleExtName::KnownSymbol(_) => { + panic!("unexpected module ext name") + } } } @@ -503,12 +575,16 @@ pub trait Module { /// /// TODO: Coalesce redundant decls and signatures. /// TODO: Look into ways to reduce the risk of using a FuncRef in the wrong function. - fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { - let decl = &self.declarations().functions[func]; - let signature = in_func.import_signature(decl.signature.clone()); + fn declare_func_in_func(&mut self, func_id: FuncId, func: &mut ir::Function) -> ir::FuncRef { + let decl = &self.declarations().functions[func_id]; + let signature = func.import_signature(decl.signature.clone()); + let user_name_ref = func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: func_id.as_u32(), + }); let colocated = decl.linkage.is_final(); - in_func.import_function(ir::ExtFuncData { - name: ir::ExternalName::user(0, func.as_u32()), + func.import_function(ir::ExtFuncData { + name: ir::ExternalName::user(user_name_ref), signature, colocated, }) @@ -520,8 +596,12 @@ pub trait Module { fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue { let decl = &self.declarations().data_objects[data]; let colocated = decl.linkage.is_final(); + let user_name_ref = func.declare_imported_user_function(ir::UserExternalName { + namespace: 1, + index: data.as_u32(), + }); func.create_global_value(ir::GlobalValueData::Symbol { - name: ir::ExternalName::user(1, data.as_u32()), + name: ir::ExternalName::user(user_name_ref), offset: ir::immediates::Imm64::new(0), colocated, tls: decl.tls, @@ -530,12 +610,12 @@ pub trait Module { /// TODO: Same as above. fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { - ctx.import_function(ir::ExternalName::user(0, func.as_u32())) + ctx.import_function(ModuleExtName::user(0, func.as_u32())) } /// TODO: Same as above. fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { - ctx.import_global_value(ir::ExternalName::user(1, data.as_u32())) + ctx.import_global_value(ModuleExtName::user(1, data.as_u32())) } /// Define a function, producing the function body from the given `Context`. @@ -558,7 +638,8 @@ pub trait Module { /// Returns the size of the function's code. fn define_function_bytes( &mut self, - func: FuncId, + func_id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult; @@ -627,7 +708,7 @@ impl Module for &mut M { (**self).declare_anonymous_data(writable, tls) } - fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { + fn declare_func_in_func(&mut self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { (**self).declare_func_in_func(func, in_func) } @@ -635,14 +716,6 @@ impl Module for &mut M { (**self).declare_data_in_func(data, func) } - fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { - (**self).declare_func_in_data(func, ctx) - } - - fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { - (**self).declare_data_in_data(data, ctx) - } - fn define_function( &mut self, func: FuncId, @@ -653,11 +726,12 @@ impl Module for &mut M { fn define_function_bytes( &mut self, - func: FuncId, + func_id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult { - (**self).define_function_bytes(func, bytes, relocs) + (**self).define_function_bytes(func_id, func, bytes, relocs) } fn define_data(&mut self, data: DataId, data_ctx: &DataContext) -> ModuleResult<()> { diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index 88cca41bbe..6f7a4e0af1 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -10,7 +10,7 @@ use cranelift_codegen::{ }; use cranelift_module::{ DataContext, DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleCompiledFunction, - ModuleDeclarations, ModuleError, ModuleResult, + ModuleDeclarations, ModuleError, ModuleExtName, ModuleReloc, ModuleResult, }; use log::info; use object::write::{ @@ -316,12 +316,18 @@ impl Module for ObjectModule { ctx.compile_and_emit(self.isa(), &mut code)?; - self.define_function_bytes(func_id, &code, ctx.compiled_code().unwrap().buffer.relocs()) + self.define_function_bytes( + func_id, + &ctx.func, + &code, + ctx.compiled_code().unwrap().buffer.relocs(), + ) } fn define_function_bytes( &mut self, func_id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult { @@ -364,7 +370,7 @@ impl Module for ObjectModule { if !relocs.is_empty() { let relocs = relocs .iter() - .map(|record| self.process_reloc(record)) + .map(|record| self.process_reloc(&ModuleReloc::from_mach_reloc(&record, func))) .collect(); self.relocs.push(SymbolRelocs { section, @@ -518,9 +524,9 @@ impl ObjectModule { /// This should only be called during finish because it creates /// symbols for missing libcalls. - fn get_symbol(&mut self, name: &ir::ExternalName) -> SymbolId { + fn get_symbol(&mut self, name: &ModuleExtName) -> SymbolId { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { if ModuleDeclarations::is_function(name) { let id = FuncId::from_name(name); self.functions[id].unwrap().0 @@ -529,7 +535,7 @@ impl ObjectModule { self.data_objects[id].unwrap().0 } } - ir::ExternalName::LibCall(ref libcall) => { + ModuleExtName::LibCall(ref libcall) => { let name = (self.libcall_names)(*libcall); if let Some(symbol) = self.object.symbol_id(name.as_bytes()) { symbol @@ -552,7 +558,7 @@ impl ObjectModule { } // These are "magic" names well-known to the linker. // They require special treatment. - ir::ExternalName::KnownSymbol(ref known_symbol) => { + ModuleExtName::KnownSymbol(ref known_symbol) => { if let Some(symbol) = self.known_symbols.get(known_symbol) { *symbol } else { @@ -582,11 +588,10 @@ impl ObjectModule { symbol } } - _ => panic!("invalid ExternalName {}", name), } } - fn process_reloc(&self, record: &MachReloc) -> ObjectRelocRecord { + fn process_reloc(&self, record: &ModuleReloc) -> ObjectRelocRecord { let mut addend = record.addend; let (kind, encoding, size) = match record.kind { Reloc::Abs4 => (RelocationKind::Absolute, RelocationEncoding::Generic, 32), @@ -696,6 +701,7 @@ impl ObjectModule { // FIXME reloc => unimplemented!("{:?}", reloc), }; + ObjectRelocRecord { offset: record.offset, name: record.name.clone(), @@ -762,7 +768,7 @@ struct SymbolRelocs { #[derive(Clone)] struct ObjectRelocRecord { offset: CodeOffset, - name: ir::ExternalName, + name: ModuleExtName, kind: RelocationKind, encoding: RelocationEncoding, size: u8, diff --git a/cranelift/object/tests/basic.rs b/cranelift/object/tests/basic.rs index 9ed9c351a2..d918f55e34 100644 --- a/cranelift/object/tests/basic.rs +++ b/cranelift/object/tests/basic.rs @@ -43,7 +43,7 @@ fn define_simple_function(module: &mut ObjectModule) -> FuncId { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); @@ -82,8 +82,7 @@ fn switch_error() { call_conv: CallConv::SystemV, }; - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); - + let mut func = Function::with_name_signature(UserFuncName::default(), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut func, &mut func_ctx); @@ -166,7 +165,7 @@ fn libcall_function() { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rs index 6b471be0f8..ba2b0dc5eb 100644 --- a/cranelift/reader/src/lexer.rs +++ b/cranelift/reader/src/lexer.rs @@ -47,6 +47,7 @@ pub enum Token<'a> { FuncRef(u32), // fn2 SigRef(u32), // sig2 UserRef(u32), // u345 + UserNameRef(u32), // userextname345 Name(&'a str), // %9arbitrary_alphanum, %x3, %0, %function ... String(&'a str), // "arbitrary quoted string with no escape" ... HexSequence(&'a str), // #89AF @@ -354,6 +355,7 @@ impl<'a> Lexer<'a> { "fn" => Some(Token::FuncRef(number)), "sig" => Some(Token::SigRef(number)), "u" => Some(Token::UserRef(number)), + "userextname" => Some(Token::UserNameRef(number)), _ => None, } } diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs index 78e50b0d88..00e7a5227c 100644 --- a/cranelift/reader/src/parser.rs +++ b/cranelift/reader/src/parser.rs @@ -9,19 +9,19 @@ use crate::sourcemap::SourceMap; use crate::testcommand::TestCommand; use crate::testfile::{Comment, Details, Feature, TestFile}; use cranelift_codegen::data_value::DataValue; -use cranelift_codegen::entity::EntityRef; -use cranelift_codegen::ir; +use cranelift_codegen::entity::{EntityRef, PrimaryMap}; use cranelift_codegen::ir::entities::{AnyEntity, DynamicType}; use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, Uimm32, Uimm64}; use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs}; use cranelift_codegen::ir::types::INVALID; use cranelift_codegen::ir::types::*; +use cranelift_codegen::ir::{self, UserExternalNameRef}; use cranelift_codegen::ir::{ AbiParam, ArgumentExtension, ArgumentPurpose, Block, Constant, ConstantData, DynamicStackSlot, DynamicStackSlotData, DynamicTypeData, ExtFuncData, ExternalName, FuncRef, Function, GlobalValue, GlobalValueData, Heap, HeapData, HeapStyle, JumpTable, JumpTableData, MemFlags, Opcode, SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, Table, TableData, Type, - Value, + UserFuncName, Value, }; use cranelift_codegen::isa::{self, CallConv}; use cranelift_codegen::packed_option::ReservedValue; @@ -225,6 +225,12 @@ pub struct Parser<'a> { /// Comments collected so far. comments: Vec>, + /// Maps inlined external names to a ref value, so they can be declared before parsing the rest + /// of the function later. + /// + /// This maintains backward compatibility with previous ways for declaring external names. + predeclared_external_names: PrimaryMap, + /// Default calling conventions; used when none is specified. default_calling_convention: CallConv, } @@ -508,6 +514,7 @@ impl<'a> Parser<'a> { gathered_comments: Vec::new(), comments: Vec::new(), default_calling_convention: CallConv::Fast, + predeclared_external_names: Default::default(), } } @@ -1282,7 +1289,7 @@ impl<'a> Parser<'a> { let location = self.loc; // function ::= "function" * name signature "{" preamble function-body "}" - let name = self.parse_external_name()?; + let name = self.parse_user_func_name()?; // function ::= "function" name * signature "{" preamble function-body "}" let sig = self.parse_signature()?; @@ -1307,6 +1314,16 @@ impl<'a> Parser<'a> { self.token(); self.claim_gathered_comments(AnyEntity::Function); + // Claim all the declared user-defined function names. + for (user_func_ref, user_external_name) in + std::mem::take(&mut self.predeclared_external_names) + { + let actual_ref = ctx + .function + .declare_imported_user_function(user_external_name); + assert_eq!(user_func_ref, actual_ref); + } + let details = Details { location, comments: self.take_comments(), @@ -1316,18 +1333,17 @@ impl<'a> Parser<'a> { Ok((ctx.function, details)) } - // Parse an external name. + // Parse a user-defined function name // // For example, in a function decl, the parser would be in this state: // // function ::= "function" * name signature { ... } // - fn parse_external_name(&mut self) -> ParseResult { + fn parse_user_func_name(&mut self) -> ParseResult { match self.token() { Some(Token::Name(s)) => { self.consume(); - s.parse() - .map_err(|_| self.error("invalid test case or libcall name")) + Ok(UserFuncName::testcase(s)) } Some(Token::UserRef(namespace)) => { self.consume(); @@ -1336,23 +1352,88 @@ impl<'a> Parser<'a> { self.consume(); match self.token() { Some(Token::Integer(index_str)) => { + self.consume(); let index: u32 = u32::from_str_radix(index_str, 10).map_err(|_| { self.error("the integer given overflows the u32 type") })?; - self.consume(); - Ok(ExternalName::user(namespace, index)) + Ok(UserFuncName::user(namespace, index)) } _ => err!(self.loc, "expected integer"), } } - _ => err!(self.loc, "expected colon"), + _ => { + err!(self.loc, "expected user function name in the form uX:Y") + } } } _ => err!(self.loc, "expected external name"), } } + // Parse an external name. + // + // For example, in a function reference decl, the parser would be in this state: + // + // fn0 = * name signature + // + fn parse_external_name(&mut self) -> ParseResult { + match self.token() { + Some(Token::Name(s)) => { + self.consume(); + s.parse() + .map_err(|_| self.error("invalid test case or libcall name")) + } + + Some(Token::UserNameRef(name_ref)) => { + self.consume(); + Ok(ExternalName::user(UserExternalNameRef::new( + name_ref as usize, + ))) + } + + Some(Token::UserRef(namespace)) => { + self.consume(); + if let Some(Token::Colon) = self.token() { + self.consume(); + match self.token() { + Some(Token::Integer(index_str)) => { + let index: u32 = u32::from_str_radix(index_str, 10).map_err(|_| { + self.error("the integer given overflows the u32 type") + })?; + self.consume(); + + // Deduplicate the reference (O(n), but should be fine for tests), + // to follow `FunctionParameters::declare_imported_user_function`, + // otherwise this will cause ref mismatches when asserted below. + let name_ref = self + .predeclared_external_names + .iter() + .find_map(|(reff, name)| { + if name.index == index && name.namespace == namespace { + Some(reff) + } else { + None + } + }) + .unwrap_or_else(|| { + self.predeclared_external_names + .push(ir::UserExternalName { namespace, index }) + }); + + Ok(ExternalName::user(name_ref)) + } + _ => err!(self.loc, "expected integer"), + } + } else { + err!(self.loc, "expected colon") + } + } + + _ => err!(self.loc, "expected external name"), + } + } + // Parse a function signature. // // signature ::= * "(" [paramlist] ")" ["->" retlist] [callconv] @@ -2249,7 +2330,7 @@ impl<'a> Parser<'a> { .expect("duplicate inst references created"); if !srcloc.is_default() { - ctx.function.srclocs[inst] = srcloc; + ctx.function.set_srcloc(inst, srcloc); } if results.len() != num_results { diff --git a/cranelift/src/bugpoint.rs b/cranelift/src/bugpoint.rs index dcc48245f2..a4673227a1 100644 --- a/cranelift/src/bugpoint.rs +++ b/cranelift/src/bugpoint.rs @@ -810,9 +810,9 @@ fn inst_count(func: &Function) -> usize { } fn resolve_aliases(func: &mut Function) { - for block in func.layout.blocks() { - for inst in func.layout.block_insts(block) { - func.dfg.resolve_aliases_in_arguments(inst); + for block in func.stencil.layout.blocks() { + for inst in func.stencil.layout.block_insts(block) { + func.stencil.dfg.resolve_aliases_in_arguments(inst); } } } diff --git a/cranelift/src/compile.rs b/cranelift/src/compile.rs index af03d8a4fe..2631c00331 100644 --- a/cranelift/src/compile.rs +++ b/cranelift/src/compile.rs @@ -81,6 +81,7 @@ fn handle_module(options: &Options, path: &Path, name: &str, fisa: FlagsOrIsa) - let result = context.compiled_code().unwrap(); print_all( isa, + &context.func.params, &mem, code_info.total_size, options.print, diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rs index c372707b4a..a950d6b98f 100644 --- a/cranelift/src/disasm.rs +++ b/cranelift/src/disasm.rs @@ -1,10 +1,11 @@ use anyhow::Result; use cfg_if::cfg_if; +use cranelift_codegen::ir::function::FunctionParameters; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::{MachReloc, MachStackMap, MachTrap}; use std::fmt::Write; -pub fn print_relocs(relocs: &[MachReloc]) -> String { +fn print_relocs(func_params: &FunctionParameters, relocs: &[MachReloc]) -> String { let mut text = String::new(); for &MachReloc { kind, @@ -16,7 +17,10 @@ pub fn print_relocs(relocs: &[MachReloc]) -> String { writeln!( text, "reloc_external: {} {} {} at {}", - kind, name, addend, offset + kind, + name.display(Some(func_params)), + addend, + offset ) .unwrap(); } @@ -172,6 +176,7 @@ cfg_if! { pub fn print_all( isa: &dyn TargetIsa, + func_params: &FunctionParameters, mem: &[u8], code_size: u32, print: bool, @@ -184,7 +189,7 @@ pub fn print_all( if print { println!( "\n{}\n{}\n{}", - print_relocs(relocs), + print_relocs(func_params, relocs), print_traps(traps), print_stack_maps(stack_maps) ); diff --git a/cranelift/src/wasm.rs b/cranelift/src/wasm.rs index f526e0c8a2..1ccf788f7d 100644 --- a/cranelift/src/wasm.rs +++ b/cranelift/src/wasm.rs @@ -313,6 +313,7 @@ fn handle_module(options: &Options, path: &Path, name: &str, fisa: FlagsOrIsa) - if let Some(total_size) = saved_size { print_all( isa, + &context.func.params, &mem, total_size, options.print, diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index 70833483ae..332269c437 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -16,8 +16,8 @@ use crate::{ use core::convert::TryFrom; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; -use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, InstBuilder}; +use cranelift_codegen::ir::{types::*, UserFuncName}; use cranelift_codegen::isa::{CallConv, TargetFrontendConfig}; use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap}; use cranelift_frontend::FunctionBuilder; @@ -26,11 +26,6 @@ use std::string::String; use std::vec::Vec; use wasmparser::{FuncValidator, FunctionBody, Operator, ValidatorResources, WasmFeatures}; -/// Compute a `ir::ExternalName` for a given wasm function index. -fn get_func_name(func_index: FuncIndex) -> ir::ExternalName { - ir::ExternalName::user(0, func_index.as_u32()) -} - /// A collection of names under which a given entity is exported. pub struct Exportable { /// A wasm entity. @@ -341,7 +336,11 @@ impl<'dummy_environment> FuncEnvironment for DummyFuncEnvironment<'dummy_environ // A real implementation would probably add a `vmctx` argument. // And maybe attempt some signature de-duplication. let signature = func.import_signature(self.vmctx_sig(sigidx)); - let name = get_func_name(index); + let name = + ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: index.as_u32(), + })); Ok(func.import_function(ir::ExtFuncData { name, signature, @@ -849,12 +848,15 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment { DummyFuncEnvironment::new(&self.info, self.expected_reachability.clone()); let func_index = FuncIndex::new(self.get_num_func_imports() + self.info.function_bodies.len()); - let name = get_func_name(func_index); + let sig = func_environ.vmctx_sig(self.get_func_type(func_index)); - let mut func = ir::Function::with_name_signature(name, sig); + let mut func = + ir::Function::with_name_signature(UserFuncName::user(0, func_index.as_u32()), sig); + if self.debug_info { func.collect_debug_info(); } + self.trans .translate_body(&mut validator, body, &mut func, &mut func_environ)?; func diff --git a/cranelift/wasm/src/func_translator.rs b/cranelift/wasm/src/func_translator.rs index 4404324b04..66aecf3b95 100644 --- a/cranelift/wasm/src/func_translator.rs +++ b/cranelift/wasm/src/func_translator.rs @@ -310,7 +310,7 @@ mod tests { let mut ctx = Context::new(); - ctx.func.name = ir::ExternalName::testcase("small1"); + ctx.func.name = ir::UserFuncName::testcase("small1"); ctx.func.signature.params.push(ir::AbiParam::new(I32)); ctx.func.signature.returns.push(ir::AbiParam::new(I32)); @@ -348,7 +348,7 @@ mod tests { let mut ctx = Context::new(); - ctx.func.name = ir::ExternalName::testcase("small2"); + ctx.func.name = ir::UserFuncName::testcase("small2"); ctx.func.signature.params.push(ir::AbiParam::new(I32)); ctx.func.signature.returns.push(ir::AbiParam::new(I32)); @@ -391,7 +391,7 @@ mod tests { let mut ctx = Context::new(); - ctx.func.name = ir::ExternalName::testcase("infloop"); + ctx.func.name = ir::UserFuncName::testcase("infloop"); ctx.func.signature.returns.push(ir::AbiParam::new(I32)); let (body, mut validator) = extract_func(&wasm); diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index 23df5dd712..58f18084d7 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -28,3 +28,4 @@ thiserror = "1.0.4" [features] all-arch = ["cranelift-codegen/all-arch"] component-model = ["wasmtime-environ/component-model"] +incremental-cache = ["cranelift-codegen/incremental-cache"] diff --git a/crates/cranelift/src/builder.rs b/crates/cranelift/src/builder.rs index c810e3498d..4fb2b6369a 100644 --- a/crates/cranelift/src/builder.rs +++ b/crates/cranelift/src/builder.rs @@ -7,12 +7,14 @@ use anyhow::Result; use cranelift_codegen::isa; use cranelift_codegen::settings::{self, Configurable, SetError}; use std::fmt; -use wasmtime_environ::{CompilerBuilder, Setting, SettingKind}; +use std::sync::Arc; +use wasmtime_environ::{CacheStore, CompilerBuilder, Setting, SettingKind}; struct Builder { flags: settings::Builder, isa_flags: isa::Builder, linkopts: LinkOptions, + cache_store: Option>, } #[derive(Clone, Default)] @@ -46,6 +48,7 @@ pub fn builder() -> Box { flags, isa_flags: cranelift_native::builder().expect("host machine is not a supported target"), linkopts: LinkOptions::default(), + cache_store: None, }) } @@ -103,6 +106,7 @@ impl CompilerBuilder for Builder { .finish(settings::Flags::new(self.flags.clone()))?; Ok(Box::new(crate::compiler::Compiler::new( isa, + self.cache_store.clone(), self.linkopts.clone(), ))) } @@ -123,6 +127,13 @@ impl CompilerBuilder for Builder { }) .collect() } + + fn enable_incremental_compilation( + &mut self, + cache_store: Arc, + ) { + self.cache_store = Some(cache_store); + } } impl fmt::Debug for Builder { diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index addb9177bb..3d6829f1cc 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -1,18 +1,20 @@ use crate::builder::LinkOptions; use crate::debug::{DwarfSectionRelocTarget, ModuleMemoryOffset}; -use crate::func_environ::{get_func_name, FuncEnvironment}; +use crate::func_environ::FuncEnvironment; use crate::obj::ModuleTextBuilder; use crate::{ blank_sig, func_signature, indirect_signature, value_type, wasmtime_call_conv, CompiledFunction, CompiledFunctions, FunctionAddressMap, Relocation, RelocationTarget, }; use anyhow::{Context as _, Result}; -use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags, Value}; +use cranelift_codegen::ir::{ + self, ExternalName, Function, InstBuilder, MemFlags, UserExternalName, UserFuncName, Value, +}; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::Context; use cranelift_codegen::{settings, MachReloc, MachTrap}; -use cranelift_codegen::{MachSrcLoc, MachStackMap}; +use cranelift_codegen::{CompiledCode, MachSrcLoc, MachStackMap}; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_frontend::FunctionBuilder; use cranelift_wasm::{ @@ -27,19 +29,28 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::convert::TryFrom; use std::mem; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use wasmtime_environ::{ - AddressMapSection, CompileError, FilePos, FlagValue, FunctionBodyData, FunctionInfo, - InstructionAddressMap, Module, ModuleTranslation, ModuleTypes, PtrSize, StackMapInformation, - Trampoline, TrapCode, TrapEncodingBuilder, TrapInformation, Tunables, VMOffsets, + AddressMapSection, CacheStore, CompileError, FilePos, FlagValue, FunctionBodyData, + FunctionInfo, InstructionAddressMap, Module, ModuleTranslation, ModuleTypes, PtrSize, + StackMapInformation, Trampoline, TrapCode, TrapEncodingBuilder, TrapInformation, Tunables, + VMOffsets, }; #[cfg(feature = "component-model")] mod component; +struct IncrementalCacheContext { + #[cfg(feature = "incremental-cache")] + cache_store: Arc, + num_hits: usize, + num_cached: usize, +} + struct CompilerContext { func_translator: FuncTranslator, codegen_context: Context, + incremental_cache_ctx: Option, } impl Default for CompilerContext { @@ -47,6 +58,7 @@ impl Default for CompilerContext { Self { func_translator: FuncTranslator::new(), codegen_context: Context::new(), + incremental_cache_ctx: None, } } } @@ -57,14 +69,48 @@ pub(crate) struct Compiler { contexts: Mutex>, isa: Box, linkopts: LinkOptions, + cache_store: Option>, +} + +impl Drop for Compiler { + fn drop(&mut self) { + if self.cache_store.is_none() { + return; + } + + let mut num_hits = 0; + let mut num_cached = 0; + for ctx in self.contexts.lock().unwrap().iter() { + if let Some(ref cache_ctx) = ctx.incremental_cache_ctx { + num_hits += cache_ctx.num_hits; + num_cached += cache_ctx.num_cached; + } + } + + let total = num_hits + num_cached; + if num_hits + num_cached > 0 { + log::trace!( + "Incremental compilation cache stats: {}/{} = {}% (hits/lookup)\ncached: {}", + num_hits, + total, + (num_hits as f32) / (total as f32) * 100.0, + num_cached + ); + } + } } impl Compiler { - pub(crate) fn new(isa: Box, linkopts: LinkOptions) -> Compiler { + pub(crate) fn new( + isa: Box, + cache_store: Option>, + linkopts: LinkOptions, + ) -> Compiler { Compiler { contexts: Default::default(), isa, linkopts, + cache_store, } } @@ -75,7 +121,17 @@ impl Compiler { ctx.codegen_context.clear(); ctx }) - .unwrap_or_else(Default::default) + .unwrap_or_else(|| CompilerContext { + #[cfg(feature = "incremental-cache")] + incremental_cache_ctx: self.cache_store.as_ref().map(|cache_store| { + IncrementalCacheContext { + cache_store: cache_store.clone(), + num_hits: 0, + num_cached: 0, + } + }), + ..Default::default() + }) } fn save_context(&self, ctx: CompilerContext) { @@ -141,10 +197,15 @@ impl wasmtime_environ::Compiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + incremental_cache_ctx: mut cache_ctx, } = self.take_context(); - context.func.name = get_func_name(func_index); context.func.signature = func_signature(isa, translation, types, func_index); + context.func.name = UserFuncName::User(UserExternalName { + namespace: 0, + index: func_index.as_u32(), + }); + if tunables.generate_native_debuginfo { context.func.collect_debug_info(); } @@ -210,16 +271,14 @@ impl wasmtime_environ::Compiler for Compiler { &mut func_env, )?; - let mut code_buf: Vec = Vec::new(); - let compiled_code = context - .compile_and_emit(isa, &mut code_buf) - .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + let (_, code_buf) = compile_maybe_cached(&mut context, isa, cache_ctx.as_mut())?; + let compiled_code = context.compiled_code().unwrap(); let func_relocs = compiled_code .buffer .relocs() .into_iter() - .map(mach_reloc_to_reloc) + .map(|item| mach_reloc_to_reloc(&context.func, item)) .collect::>(); let traps = compiled_code @@ -259,6 +318,7 @@ impl wasmtime_environ::Compiler for Compiler { self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx: cache_ctx, }); Ok(Box::new(CompiledFunction { @@ -403,6 +463,74 @@ impl wasmtime_environ::Compiler for Compiler { } } +#[cfg(feature = "incremental-cache")] +mod incremental_cache { + use super::*; + + struct CraneliftCacheStore(Arc); + + impl cranelift_codegen::incremental_cache::CacheKvStore for CraneliftCacheStore { + fn get(&self, key: &[u8]) -> Option> { + self.0.get(key) + } + fn insert(&mut self, key: &[u8], val: Vec) { + self.0.insert(key, val); + } + } + + pub(super) fn compile_maybe_cached<'a>( + context: &'a mut Context, + isa: &dyn TargetIsa, + cache_ctx: Option<&mut IncrementalCacheContext>, + ) -> Result<(&'a CompiledCode, Vec), CompileError> { + let mut code_buf = Vec::new(); + let cache_ctx = match cache_ctx { + Some(ctx) => ctx, + None => { + let compiled_code = + context + .compile_and_emit(isa, &mut code_buf) + .map_err(|error| { + CompileError::Codegen(pretty_error(&error.func, error.inner)) + })?; + return Ok((compiled_code, code_buf)); + } + }; + + let mut cache_store = CraneliftCacheStore(cache_ctx.cache_store.clone()); + let (compiled_code, from_cache) = context + .compile_with_cache(isa, &mut cache_store) + .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + + if from_cache { + cache_ctx.num_hits += 1; + } else { + cache_ctx.num_cached += 1; + } + + code_buf.resize(compiled_code.code_info().total_size as _, 0); + code_buf.copy_from_slice(compiled_code.code_buffer()); + + Ok((compiled_code, code_buf)) + } +} + +#[cfg(feature = "incremental-cache")] +use incremental_cache::*; + +#[cfg(not(feature = "incremental-cache"))] +fn compile_maybe_cached<'a>( + context: &'a mut Context, + isa: &dyn TargetIsa, + _cache_ctx: Option<&mut IncrementalCacheContext>, +) -> Result<(&'a CompiledCode, Vec), CompileError> { + let mut code_buf = Vec::new(); + let compiled_code = context + .compile_and_emit(isa, &mut code_buf) + .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + return Ok((compiled_code, code_buf)); +} + fn to_flag_value(v: &settings::Value) -> FlagValue { match v.kind() { settings::SettingKind::Enum => FlagValue::Enum(v.as_enum().unwrap().into()), @@ -431,9 +559,11 @@ impl Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + incremental_cache_ctx: mut cache_ctx, } = self.take_context(); - context.func = ir::Function::with_name_signature(ExternalName::user(0, 0), host_signature); + // The name doesn't matter here. + context.func = ir::Function::with_name_signature(UserFuncName::default(), host_signature); // This trampoline will load all the parameters from the `values_vec` // that is passed in and then call the real function (also passed @@ -493,10 +623,11 @@ impl Compiler { builder.ins().return_(&[]); builder.finalize(); - let func = self.finish_trampoline(&mut context, isa)?; + let func = self.finish_trampoline(&mut context, cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx: cache_ctx, }); Ok(func) } @@ -541,10 +672,11 @@ impl Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + incremental_cache_ctx: mut cache_ctx, } = self.take_context(); - context.func = - ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wasm_signature); + // The name doesn't matter here. + context.func = ir::Function::with_name_signature(Default::default(), wasm_signature); let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context()); let block0 = builder.create_block(); @@ -570,10 +702,11 @@ impl Compiler { self.wasm_to_host_load_results(ty, &mut builder, values_vec_ptr_val); - let func = self.finish_trampoline(&mut context, isa)?; + let func = self.finish_trampoline(&mut context, cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx: cache_ctx, }); Ok(func) } @@ -668,12 +801,10 @@ impl Compiler { fn finish_trampoline( &self, context: &mut Context, + cache_ctx: Option<&mut IncrementalCacheContext>, isa: &dyn TargetIsa, ) -> Result { - let mut code_buf = Vec::new(); - let compiled_code = context - .compile_and_emit(isa, &mut code_buf) - .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + let (compiled_code, code_buf) = compile_maybe_cached(context, isa, cache_ctx)?; // Processing relocations isn't the hardest thing in the world here but // no trampoline should currently generate a relocation, so assert that @@ -858,14 +989,15 @@ fn collect_address_maps( } } -fn mach_reloc_to_reloc(reloc: &MachReloc) -> Relocation { +fn mach_reloc_to_reloc(func: &Function, reloc: &MachReloc) -> Relocation { let &MachReloc { offset, kind, ref name, addend, } = reloc; - let reloc_target = if let ExternalName::User { namespace, index } = *name { + let reloc_target = if let ExternalName::User(user_func_ref) = *name { + let UserExternalName { namespace, index } = func.params.user_named_funcs()[user_func_ref]; debug_assert_eq!(namespace, 0); RelocationTarget::UserFunc(FuncIndex::from_u32(index)) } else if let ExternalName::LibCall(libcall) = *name { diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index 33bd9d74cd..beb2ff5c42 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -31,10 +31,11 @@ impl ComponentCompiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + mut incremental_cache_ctx, } = self.take_context(); context.func = ir::Function::with_name_signature( - ir::ExternalName::user(0, 0), + ir::UserFuncName::user(0, 0), crate::indirect_signature(isa, ty), ); @@ -149,10 +150,12 @@ impl ComponentCompiler for Compiler { // `values_vec_ptr_val` and then returned. self.wasm_to_host_load_results(ty, &mut builder, values_vec_ptr_val); - let func: CompiledFunction = self.finish_trampoline(&mut context, isa)?; + let func: CompiledFunction = + self.finish_trampoline(&mut context, incremental_cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx, }); Ok(Box::new(func)) } @@ -162,9 +165,10 @@ impl ComponentCompiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + mut incremental_cache_ctx, } = self.take_context(); context.func = ir::Function::with_name_signature( - ir::ExternalName::user(0, 0), + ir::UserFuncName::user(0, 0), crate::indirect_signature(isa, ty), ); let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context()); @@ -177,10 +181,12 @@ impl ComponentCompiler for Compiler { .trap(ir::TrapCode::User(super::ALWAYS_TRAP_CODE)); builder.finalize(); - let func: CompiledFunction = self.finish_trampoline(&mut context, isa)?; + let func: CompiledFunction = + self.finish_trampoline(&mut context, incremental_cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx, }); Ok(Box::new(func)) } @@ -198,10 +204,11 @@ impl ComponentCompiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + mut incremental_cache_ctx, } = self.take_context(); context.func = ir::Function::with_name_signature( - ir::ExternalName::user(0, 0), + ir::UserFuncName::user(0, 0), crate::indirect_signature(isa, ty), ); @@ -213,10 +220,12 @@ impl ComponentCompiler for Compiler { self.translate_transcode(&mut builder, &offsets, transcoder, block0); - let func: CompiledFunction = self.finish_trampoline(&mut context, isa)?; + let func: CompiledFunction = + self.finish_trampoline(&mut context, incremental_cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx, }); Ok(Box::new(func)) } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 7576f423c0..59822f4157 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -21,11 +21,6 @@ use wasmtime_environ::{ }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; -/// Compute an `ir::ExternalName` for a given wasm function index. -pub fn get_func_name(func_index: FuncIndex) -> ir::ExternalName { - ir::ExternalName::user(0, func_index.as_u32()) -} - macro_rules! declare_function_signatures { ( $( @@ -1509,7 +1504,11 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m ) -> WasmResult { let sig = crate::func_signature(self.isa, self.translation, self.types, index); let signature = func.import_signature(sig); - let name = get_func_name(index); + let name = + ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: index.as_u32(), + })); Ok(func.import_function(ir::ExtFuncData { name, signature, diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index 455584b01e..3322ce2003 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -13,6 +13,7 @@ use std::any::Any; use std::borrow::Cow; use std::collections::BTreeMap; use std::fmt; +use std::sync::Arc; use thiserror::Error; /// Information about a function, such as trap information, address map, @@ -71,6 +72,22 @@ pub enum CompileError { DebugInfoNotSupported, } +/// Implementation of an incremental compilation's key/value cache store. +/// +/// In theory, this could just be Cranelift's `CacheKvStore` trait, but it is not as we want to +/// make sure that wasmtime isn't too tied to Cranelift internals (and as a matter of fact, we +/// can't depend on the Cranelift trait here). +pub trait CacheStore: Send + Sync + std::fmt::Debug { + /// Try to retrieve an arbitrary cache key entry, and returns a reference to bytes that were + /// inserted via `Self::insert` before. + fn get(&self, key: &[u8]) -> Option>; + + /// Given an arbitrary key and bytes, stores them in the cache. + /// + /// Returns false when insertion in the cache failed. + fn insert(&self, key: &[u8], value: Vec) -> bool; +} + /// Abstract trait representing the ability to create a `Compiler` below. /// /// This is used in Wasmtime to separate compiler implementations, currently @@ -100,6 +117,10 @@ pub trait CompilerBuilder: Send + Sync + fmt::Debug { /// [`CompilerBuilder::set`] and [`CompilerBuilder::enable`]. fn settings(&self) -> Vec; + /// Enables Cranelift's incremental compilation cache, using the given `CacheStore` + /// implementation. + fn enable_incremental_compilation(&mut self, cache_store: Arc); + /// Builds a new [`Compiler`] object from this configuration. fn build(&self) -> Result>; } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index c0643f039a..18472c9198 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -72,6 +72,9 @@ default = [ # precompiled WebAssembly modules. cranelift = ["dep:wasmtime-cranelift"] +# Enables support for incremental compilation cache to be enabled in `Config`. +incremental-cache = ["wasmtime-cranelift?/incremental-cache"] + # Enables support for the `perf` jitdump profiler jitdump = ["wasmtime-jit/jitdump"] diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 78c483d5f0..8b2a2ac47e 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -14,6 +14,8 @@ use wasmtime_environ::Tunables; use wasmtime_jit::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator, RuntimeMemoryCreator}; +pub use wasmtime_environ::CacheStore; + #[cfg(feature = "pooling-allocator")] pub use wasmtime_runtime::{InstanceLimits, PoolingAllocationStrategy}; @@ -117,6 +119,8 @@ struct CompilerConfig { target: Option, settings: HashMap, flags: HashSet, + #[cfg(compiler)] + cache_store: Option>, } #[cfg(compiler)] @@ -127,6 +131,7 @@ impl CompilerConfig { target: None, settings: HashMap::new(), flags: HashSet::new(), + cache_store: None, } } @@ -227,6 +232,17 @@ impl Config { Ok(self) } + /// Enables the incremental compilation cache in Cranelift, using the provided `CacheStore` + /// backend for storage. + #[cfg(all(feature = "incremental-cache", feature = "cranelift"))] + pub fn enable_incremental_compilation( + &mut self, + cache_store: Arc, + ) -> Result<&mut Self> { + self.compiler_config.cache_store = Some(cache_store); + Ok(self) + } + /// Whether or not to enable support for asynchronous functions in Wasmtime. /// /// When enabled, the config can optionally define host functions with `async`. @@ -1488,6 +1504,10 @@ impl Config { compiler.enable(flag)?; } + if let Some(cache_store) = &self.compiler_config.cache_store { + compiler.enable_incremental_compilation(cache_store.clone()); + } + compiler.build() } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index e1c985208d..a77155532e 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -396,6 +396,7 @@ impl Engine { | "probestack_func_adjusts_sp" // probestack above asserted disabled | "probestack_size_log2" // probestack above asserted disabled | "regalloc" // shouldn't change semantics + | "enable_incremental_compilation_cache_checks" // shouldn't change semantics | "enable_atomics" => return Ok(()), // Everything else is unknown and needs to be added somewhere to diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 7f3ae4a346..40c2003ddc 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true [dependencies] anyhow = { version = "1.0.19" } arbitrary = { version = "1.1.0", features = ["derive"] } -cranelift-codegen = { path = "../cranelift/codegen" } +cranelift-codegen = { path = "../cranelift/codegen", features = ["incremental-cache"] } cranelift-reader = { path = "../cranelift/reader" } cranelift-wasm = { path = "../cranelift/wasm" } cranelift-filetests = { path = "../cranelift/filetests" } @@ -120,3 +120,9 @@ name = "component_api" path = "fuzz_targets/component_api.rs" test = false doc = false + +[[bin]] +name = "cranelift-icache" +path = "fuzz_targets/cranelift-icache.rs" +test = false +doc = false diff --git a/fuzz/README.md b/fuzz/README.md index 47a5f6f9a2..5d8e53b02f 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -30,6 +30,10 @@ At the time of writing, we have the following fuzz targets: * `cranelift-fuzzgen`: Generate a Cranelift function and check that it returns the same results when compiled to the host and when using the Cranelift interpreter; only a subset of Cranelift IR is currently supported. +* `cranelift-icache`: Generate a Cranelift function A, applies a small mutation + to its source, yielding a function A', and checks that A compiled + + incremental compilation generates the same machine code as if A' was compiled + from scratch. * `differential`: Generate a Wasm module and check that Wasmtime returns the same results when run with two different configurations. * `differential_spec`: Generate a Wasm module and check that Wasmtime returns diff --git a/fuzz/fuzz_targets/cranelift-icache.rs b/fuzz/fuzz_targets/cranelift-icache.rs new file mode 100644 index 0000000000..1c6959a78a --- /dev/null +++ b/fuzz/fuzz_targets/cranelift-icache.rs @@ -0,0 +1,148 @@ +#![no_main] + +use cranelift_codegen::{ + cursor::{Cursor, FuncCursor}, + incremental_cache as icache, + ir::{self, immediates::Imm64, ExternalName}, + isa, + settings::{self, Configurable as _}, + Context, +}; +use libfuzzer_sys::fuzz_target; + +use cranelift_fuzzgen::*; +use target_lexicon::Triple; + +fuzz_target!(|func: SingleFunction| { + let mut func = func.0; + + let flags = settings::Flags::new({ + let mut builder = settings::builder(); + // We need llvm ABI extensions for i128 values on x86 + builder.set("enable_llvm_abi_extensions", "true").unwrap(); + builder + }); + + let isa_builder = isa::lookup(Triple::host()) + .map_err(|err| match err { + isa::LookupError::SupportDisabled => { + "support for architecture disabled at compile time" + } + isa::LookupError::Unsupported => "unsupported architecture", + }) + .unwrap(); + + let isa = isa_builder.finish(flags).unwrap(); + + let cache_key_hash = icache::compute_cache_key(&*isa, &mut func); + + let mut context = Context::for_function(func.clone()); + let prev_stencil = match context.compile_stencil(&*isa) { + Ok(stencil) => stencil, + Err(_) => return, + }; + + let (prev_stencil, serialized) = icache::serialize_compiled(prev_stencil); + let serialized = serialized.expect("serialization should work"); + let prev_result = prev_stencil.apply_params(&func.params); + + let new_result = icache::try_finish_recompile(&func, &serialized) + .expect("recompilation should always work for identity"); + + assert_eq!(new_result, prev_result, "MachCompileResult:s don't match"); + + let new_info = new_result.code_info(); + assert_eq!(new_info, prev_result.code_info(), "CodeInfo:s don't match"); + + // If the func has at least one user-defined func ref, change it to match a + // different external function. + let expect_cache_hit = if let Some(user_ext_ref) = + func.stencil.dfg.ext_funcs.values().find_map(|data| { + if let ExternalName::User(user_ext_ref) = &data.name { + Some(user_ext_ref) + } else { + None + } + }) { + let mut prev = func.params.user_named_funcs()[*user_ext_ref].clone(); + prev.index = prev.index.checked_add(1).unwrap_or_else(|| prev.index - 1); + func.params.reset_user_func_name(*user_ext_ref, prev); + true + } else { + // otherwise just randomly change one instruction in the middle and see what happens. + let mut changed = false; + let mut cursor = FuncCursor::new(&mut func); + 'out: while let Some(_block) = cursor.next_block() { + while let Some(inst) = cursor.next_inst() { + // It's impractical to do any replacement at this point. Try to find any + // instruction that returns one int value, and replace it with an iconst. + if cursor.func.dfg.inst_results(inst).len() != 1 { + continue; + } + let out_ty = cursor + .func + .dfg + .value_type(cursor.func.dfg.first_result(inst)); + match out_ty { + ir::types::I32 | ir::types::I64 => {} + _ => continue, + } + + if let ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm, + } = cursor.func.dfg[inst] + { + let imm = imm.bits(); + cursor.func.dfg[inst] = ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm: Imm64::new(imm.checked_add(1).unwrap_or(imm - 1)), + }; + } else { + cursor.func.dfg[inst] = ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm: Imm64::new(42), + }; + } + + changed = true; + break 'out; + } + } + + if !changed { + return; + } + + // We made it so that there shouldn't be a cache hit. + false + }; + + let new_cache_key_hash = icache::compute_cache_key(&*isa, &mut func); + + if expect_cache_hit { + assert!(cache_key_hash == new_cache_key_hash); + } else { + assert!(cache_key_hash != new_cache_key_hash); + } + + context = Context::for_function(func.clone()); + + let after_mutation_result = match context.compile(&*isa) { + Ok(info) => info, + Err(_) => return, + }; + + if expect_cache_hit { + let after_mutation_result_from_cache = icache::try_finish_recompile(&func, &serialized) + .expect("recompilation should always work for identity"); + assert_eq!(*after_mutation_result, after_mutation_result_from_cache); + + let new_info = after_mutation_result_from_cache.code_info(); + assert_eq!( + new_info, + after_mutation_result.code_info(), + "CodeInfo:s don't match" + ); + } +}); diff --git a/tests/all/module.rs b/tests/all/module.rs index 285f959b72..dd5e53f59e 100644 --- a/tests/all/module.rs +++ b/tests/all/module.rs @@ -76,21 +76,21 @@ fn serialize_deterministic() { let p1 = engine.precompile_module(wasm.as_bytes()).unwrap(); let p2 = engine.precompile_module(wasm.as_bytes()).unwrap(); if p1 != p2 { - panic!("precompile_module not determinisitc for:\n{}", wasm); + panic!("precompile_module not deterministic for:\n{}", wasm); } let module1 = Module::new(&engine, wasm).unwrap(); let a1 = module1.serialize().unwrap(); let a2 = module1.serialize().unwrap(); if a1 != a2 { - panic!("Module::serialize not determinisitc for:\n{}", wasm); + panic!("Module::serialize not deterministic for:\n{}", wasm); } let module2 = Module::new(&engine, wasm).unwrap(); let b1 = module2.serialize().unwrap(); let b2 = module2.serialize().unwrap(); if b1 != b2 { - panic!("Module::serialize not determinisitc for:\n{}", wasm); + panic!("Module::serialize not deterministic for:\n{}", wasm); } if a1 != b2 {