diff --git a/Cargo.lock b/Cargo.lock index 226ec05fdf..76caf07c2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -542,6 +542,7 @@ dependencies = [ "miette", "regalloc2", "serde", + "sha2", "smallvec", "souper-ir", "target-lexicon", @@ -2646,6 +2647,9 @@ name = "smallvec" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +dependencies = [ + "serde", +] [[package]] name = "smawk" diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index dbc919cf6d..a0f3eb8136 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -25,6 +25,7 @@ gimli = { version = "0.26.0", default-features = false, features = ["write"], op smallvec = { version = "1.6.1" } regalloc2 = { version = "0.3.2", features = ["checker"] } souper-ir = { version = "2.1.0", optional = true } +sha2 = { version = "0.9.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary # machine code. Integration tests that need external dependencies can be @@ -82,6 +83,14 @@ enable-serde = [ "serde", "cranelift-entity/enable-serde", "regalloc2/enable-serde", + "smallvec/serde" +] + +# Enable the incremental compilation cache for hot-reload use cases. +incremental-cache = [ + "enable-serde", + "bincode", + "sha2" ] # Enable support for the Souper harvester. diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs index eb2a6dfd20..d85f6dc69b 100644 --- a/cranelift/codegen/meta/src/gen_inst.rs +++ b/cranelift/codegen/meta/src/gen_inst.rs @@ -66,7 +66,7 @@ fn gen_formats(formats: &[&InstructionFormat], fmt: &mut Formatter) { /// 16 bytes on 64-bit architectures. If more space is needed to represent an instruction, use a /// `ValueList` to store the additional information out of line. fn gen_instruction_data(formats: &[&InstructionFormat], fmt: &mut Formatter) { - fmt.line("#[derive(Clone, Debug)]"); + fmt.line("#[derive(Clone, Debug, PartialEq, Hash)]"); fmt.line(r#"#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]"#); fmt.line("#[allow(missing_docs)]"); fmt.line("pub enum InstructionData {"); diff --git a/cranelift/codegen/meta/src/shared/settings.rs b/cranelift/codegen/meta/src/shared/settings.rs index cf4473a1cd..9cdfcc19a8 100644 --- a/cranelift/codegen/meta/src/shared/settings.rs +++ b/cranelift/codegen/meta/src/shared/settings.rs @@ -327,5 +327,19 @@ pub(crate) fn define() -> SettingGroup { true, ); + settings.add_bool( + "enable_incremental_compilation_cache_checks", + "Enable additional checks for debugging the incremental compilation cache.", + r#" + Enables additional checks that are useful during development of the incremental + compilation cache. This should be mostly useful for Cranelift hackers, as well as for + helping to debug false incremental cache positives for embedders. + + This option is disabled by default and requires enabling the "incremental-cache" Cargo + feature in cranelift-codegen. + "#, + false, + ); + settings.build() } diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index a9d43d51e3..713d89bec8 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -103,7 +103,7 @@ impl fmt::Display for Reloc { /// The code starts at offset 0 and is followed optionally by relocatable jump tables and copyable /// (raw binary) read-only data. Any padding between sections is always part of the section that /// precedes the boundary between the sections. -#[derive(PartialEq)] +#[derive(Debug, PartialEq)] pub struct CodeInfo { /// Number of bytes in total. pub total_size: CodeOffset, diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index fb7b8bb37d..1003e57337 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -18,7 +18,7 @@ use crate::isa::TargetIsa; use crate::legalizer::simple_legalize; use crate::licm::do_licm; use crate::loop_analysis::LoopAnalysis; -use crate::machinst::CompiledCode; +use crate::machinst::{CompiledCode, CompiledCodeStencil}; use crate::nan_canonicalization::do_nan_canonicalization; use crate::remove_constant_phis::do_remove_constant_phis; use crate::result::{CodegenResult, CompileResult}; @@ -50,7 +50,7 @@ pub struct Context { pub loop_analysis: LoopAnalysis, /// Result of MachBackend compilation, if computed. - compiled_code: Option, + pub(crate) compiled_code: Option, /// Flag: do we want a disassembly with the CompiledCode? pub want_disasm: bool, @@ -109,7 +109,7 @@ impl Context { /// `Vec`. The machine code is not relocated. Instead, any relocations can be obtained /// from `compiled_code()`. /// - /// This function calls `compile` and `emit_to_memory`, taking care to resize `mem` as + /// This function calls `compile`, taking care to resize `mem` as /// needed, so it provides a safe interface. /// /// Returns information about the function's code and read-only data. @@ -126,6 +126,53 @@ impl Context { Ok(compiled_code) } + /// Internally compiles the function into a stencil. + /// + /// Public only for testing and fuzzing purposes. + pub fn compile_stencil(&mut self, isa: &dyn TargetIsa) -> CodegenResult { + let _tt = timing::compile(); + + self.verify_if(isa)?; + + let opt_level = isa.flags().opt_level(); + log::trace!( + "Compiling (opt level {:?}):\n{}", + opt_level, + self.func.display() + ); + + self.compute_cfg(); + if opt_level != OptLevel::None { + self.preopt(isa)?; + } + if isa.flags().enable_nan_canonicalization() { + self.canonicalize_nans(isa)?; + } + + self.legalize(isa)?; + if opt_level != OptLevel::None { + self.compute_domtree(); + self.compute_loop_analysis(); + self.licm(isa)?; + self.simple_gvn(isa)?; + } + + self.compute_domtree(); + self.eliminate_unreachable_code(isa)?; + if opt_level != OptLevel::None { + self.dce(isa)?; + } + + self.remove_constant_phis(isa)?; + + if opt_level != OptLevel::None && isa.flags().enable_alias_analysis() { + self.replace_redundant_loads()?; + self.simple_gvn(isa)?; + } + + isa.compile_function(&self.func, self.want_disasm) + } + /// Compile the function. /// /// Run the function through all the passes necessary to generate code for the target ISA @@ -135,57 +182,13 @@ impl Context { /// Returns information about the function's code and read-only data. pub fn compile(&mut self, isa: &dyn TargetIsa) -> CompileResult<&CompiledCode> { let _tt = timing::compile(); - - let mut inner = || { - self.verify_if(isa)?; - - let opt_level = isa.flags().opt_level(); - log::trace!( - "Compiling (opt level {:?}):\n{}", - opt_level, - self.func.display() - ); - - self.compute_cfg(); - if opt_level != OptLevel::None { - self.preopt(isa)?; - } - if isa.flags().enable_nan_canonicalization() { - self.canonicalize_nans(isa)?; - } - - self.legalize(isa)?; - if opt_level != OptLevel::None { - self.compute_domtree(); - self.compute_loop_analysis(); - self.licm(isa)?; - self.simple_gvn(isa)?; - } - - self.compute_domtree(); - self.eliminate_unreachable_code(isa)?; - if opt_level != OptLevel::None { - self.dce(isa)?; - } - - self.remove_constant_phis(isa)?; - - if opt_level != OptLevel::None && isa.flags().enable_alias_analysis() { - self.replace_redundant_loads()?; - self.simple_gvn(isa)?; - } - - let result = isa.compile_function(&self.func, self.want_disasm)?; - self.compiled_code = Some(result); - Ok(()) - }; - - inner() - .map(|_| self.compiled_code.as_ref().unwrap()) - .map_err(|error| CompileError { - inner: error, - func: &self.func, - }) + let stencil = self.compile_stencil(isa).map_err(|error| CompileError { + inner: error, + func: &self.func, + })?; + Ok(self + .compiled_code + .insert(stencil.apply_params(&self.func.params))) } /// If available, return information about the code layout in the diff --git a/cranelift/codegen/src/cursor.rs b/cranelift/codegen/src/cursor.rs index 2dc8ce7a2b..0a6d74d21b 100644 --- a/cranelift/codegen/src/cursor.rs +++ b/cranelift/codegen/src/cursor.rs @@ -589,7 +589,7 @@ impl<'f> FuncCursor<'f> { /// Use the source location of `inst` for future instructions. pub fn use_srcloc(&mut self, inst: ir::Inst) { - self.srcloc = self.func.srclocs[inst]; + self.srcloc = self.func.srcloc(inst); } /// Create an instruction builder that inserts an instruction at the current position. @@ -612,6 +612,7 @@ impl<'f> Cursor for FuncCursor<'f> { } fn set_srcloc(&mut self, srcloc: ir::SourceLoc) { + self.func.params.ensure_base_srcloc(srcloc); self.srcloc = srcloc; } @@ -658,7 +659,7 @@ impl<'c, 'f> ir::InstInserterBase<'c> for &'c mut FuncCursor<'f> { } self.insert_inst(inst); if !self.srcloc.is_default() { - self.func.srclocs[inst] = self.srcloc; + self.func.set_srcloc(inst, self.srcloc); } &mut self.func.dfg } diff --git a/cranelift/codegen/src/incremental_cache.rs b/cranelift/codegen/src/incremental_cache.rs new file mode 100644 index 0000000000..61702bd776 --- /dev/null +++ b/cranelift/codegen/src/incremental_cache.rs @@ -0,0 +1,254 @@ +//! This module provides a set of primitives that allow implementing an incremental cache on top of +//! Cranelift, making it possible to reuse previous compiled artifacts for functions that have been +//! compiled previously. +//! +//! This set of operation is experimental and can be enabled using the Cargo feature +//! `incremental-cache`. +//! +//! This can bring speedups in different cases: change-code-and-immediately-recompile iterations +//! get faster, modules sharing lots of code can reuse each other's artifacts, etc. +//! +//! The three main primitives are the following: +//! - `compute_cache_key` is used to compute the cache key associated to a `Function`. This is +//! basically the content of the function, modulo a few things the caching system is resilient to. +//! - `serialize_compiled` is used to serialize the result of a compilation, so it can be reused +//! later on by... +//! - `try_finish_recompile`, which reads binary blobs serialized with `serialize_compiled`, +//! re-creating the compilation artifact from those. +//! +//! The `CacheStore` trait and `Context::compile_with_cache` method are provided as +//! high-level, easy-to-use facilities to make use of that cache, and show an example of how to use +//! the above three primitives to form a full incremental caching system. + +use core::fmt; + +use crate::alloc::string::String; +use crate::alloc::vec::Vec; +use crate::ir::function::{FunctionStencil, VersionMarker}; +use crate::ir::Function; +use crate::machinst::{CompiledCode, CompiledCodeStencil}; +use crate::result::CompileResult; +use crate::{isa::TargetIsa, timing}; +use crate::{trace, CompileError, Context}; +use alloc::borrow::{Cow, ToOwned as _}; +use alloc::string::ToString as _; + +impl Context { + /// Compile the function, as in `compile`, but tries to reuse compiled artifacts from former + /// compilations using the provided cache store. + pub fn compile_with_cache( + &mut self, + isa: &dyn TargetIsa, + cache_store: &mut dyn CacheKvStore, + ) -> CompileResult<(&CompiledCode, bool)> { + let cache_key_hash = { + let _tt = timing::try_incremental_cache(); + + let cache_key_hash = compute_cache_key(isa, &mut self.func); + + if let Some(blob) = cache_store.get(&cache_key_hash.0) { + match try_finish_recompile(&self.func, &blob) { + Ok(compiled_code) => { + let info = compiled_code.code_info(); + + if isa.flags().enable_incremental_compilation_cache_checks() { + let actual_result = self.compile(isa)?; + assert_eq!(*actual_result, compiled_code); + assert_eq!(actual_result.code_info(), info); + // no need to set `compiled_code` here, it's set by `compile()`. + return Ok((actual_result, true)); + } + + let compiled_code = self.compiled_code.insert(compiled_code); + return Ok((compiled_code, true)); + } + Err(err) => { + trace!("error when finishing recompilation: {err}"); + } + } + } + + cache_key_hash + }; + + let stencil = self.compile_stencil(isa).map_err(|err| CompileError { + inner: err, + func: &self.func, + })?; + + let stencil = { + let _tt = timing::store_incremental_cache(); + let (stencil, res) = serialize_compiled(stencil); + if let Ok(blob) = res { + cache_store.insert(&cache_key_hash.0, blob); + } + stencil + }; + + let compiled_code = self + .compiled_code + .insert(stencil.apply_params(&self.func.params)); + + Ok((compiled_code, false)) + } +} + +/// Backing storage for an incremental compilation cache, when enabled. +pub trait CacheKvStore { + /// Given a cache key hash, retrieves the associated opaque serialized data. + fn get(&self, key: &[u8]) -> Option>; + + /// Given a new cache key and a serialized blob obtained from `serialize_compiled`, stores it + /// in the cache store. + fn insert(&mut self, key: &[u8], val: Vec); +} + +/// Hashed `CachedKey`, to use as an identifier when looking up whether a function has already been +/// compiled or not. +#[derive(Clone, Hash, PartialEq, Eq)] +pub struct CacheKeyHash([u8; 32]); + +impl std::fmt::Display for CacheKeyHash { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "CacheKeyHash:{:?}", self.0) + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct CachedFunc { + stencil: CompiledCodeStencil, + version_marker: VersionMarker, +} + +/// Key for caching a single function's compilation. +/// +/// If two functions get the same `CacheKey`, then we can reuse the compiled artifacts, modulo some +/// fixups. +/// +/// Note: the key will be invalidated across different versions of cranelift, as the +/// `FunctionStencil` contains a `VersionMarker` itself. +#[derive(Hash)] +struct CacheKey<'a> { + stencil: &'a FunctionStencil, + parameters: CompileParameters, +} + +#[derive(Clone, PartialEq, Hash, serde::Serialize, serde::Deserialize)] +struct CompileParameters { + isa: String, + triple: String, + flags: String, + isa_flags: Vec, +} + +impl CompileParameters { + fn from_isa(isa: &dyn TargetIsa) -> Self { + Self { + isa: isa.name().to_owned(), + triple: isa.triple().to_string(), + flags: isa.flags().to_string(), + isa_flags: isa + .isa_flags() + .into_iter() + .map(|v| v.value_string()) + .collect(), + } + } +} + +impl<'a> CacheKey<'a> { + /// Creates a new cache store key for a function. + /// + /// This is a bit expensive to compute, so it should be cached and reused as much as possible. + fn new(isa: &dyn TargetIsa, f: &'a mut Function) -> Self { + // Make sure the blocks and instructions are sequenced the same way as we might + // have serialized them earlier. This is the symmetric of what's done in + // `try_load`. + f.stencil.layout.full_renumber(); + CacheKey { + stencil: &f.stencil, + parameters: CompileParameters::from_isa(isa), + } + } +} + +/// Compute a cache key, and hash it on your behalf. +/// +/// Since computing the `CacheKey` is a bit expensive, it should be done as least as possible. +pub fn compute_cache_key(isa: &dyn TargetIsa, func: &mut Function) -> CacheKeyHash { + use core::hash::{Hash as _, Hasher}; + use sha2::Digest as _; + + struct Sha256Hasher(sha2::Sha256); + + impl Hasher for Sha256Hasher { + fn finish(&self) -> u64 { + panic!("Sha256Hasher doesn't support finish!"); + } + fn write(&mut self, bytes: &[u8]) { + self.0.update(bytes); + } + } + + let cache_key = CacheKey::new(isa, func); + + let mut hasher = Sha256Hasher(sha2::Sha256::new()); + cache_key.hash(&mut hasher); + let hash: [u8; 32] = hasher.0.finalize().into(); + + CacheKeyHash(hash) +} + +/// Given a function that's been successfully compiled, serialize it to a blob that the caller may +/// store somewhere for future use by `try_finish_recompile`. +/// +/// As this function requires ownership on the `CompiledCodeStencil`, it gives it back at the end +/// of the function call. The value is left untouched. +pub fn serialize_compiled( + result: CompiledCodeStencil, +) -> (CompiledCodeStencil, Result, bincode::Error>) { + let cached = CachedFunc { + stencil: result, + version_marker: VersionMarker, + }; + let result = bincode::serialize(&cached); + (cached.stencil, result) +} + +/// An error returned when recompiling failed. +#[derive(Debug)] +pub enum RecompileError { + /// The version embedded in the cache entry isn't the same as cranelift's current version. + VersionMismatch, + /// An error occurred while deserializing the cache entry. + Deserialize(bincode::Error), +} + +impl fmt::Display for RecompileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RecompileError::VersionMismatch => write!(f, "cranelift version mismatch",), + RecompileError::Deserialize(err) => { + write!(f, "bincode failed during deserialization: {err}") + } + } + } +} + +/// Given a function that's been precompiled and its entry in the caching storage, try to shortcut +/// compilation of the given function. +/// +/// Precondition: the bytes must have retrieved from a cache store entry which hash value +/// is strictly the same as the `Function`'s computed hash retrieved from `compute_cache_key`. +pub fn try_finish_recompile(func: &Function, bytes: &[u8]) -> Result { + match bincode::deserialize::(bytes) { + Ok(result) => { + if result.version_marker != func.stencil.version_marker { + Err(RecompileError::VersionMismatch) + } else { + Ok(result.stencil.apply_params(&func.params)) + } + } + Err(err) => Err(RecompileError::Deserialize(err)), + } +} diff --git a/cranelift/codegen/src/ir/constant.rs b/cranelift/codegen/src/ir/constant.rs index 3cd88d5546..1c540c0c37 100644 --- a/cranelift/codegen/src/ir/constant.rs +++ b/cranelift/codegen/src/ir/constant.rs @@ -10,7 +10,6 @@ use crate::ir::immediates::{IntoBytes, V128Imm}; use crate::ir::Constant; -use crate::HashMap; use alloc::collections::BTreeMap; use alloc::vec::Vec; use core::fmt; @@ -27,7 +26,7 @@ use serde::{Deserialize, Serialize}; /// WebAssembly values, which are [little-endian by design]. /// /// [little-endian by design]: https://github.com/WebAssembly/design/blob/master/Portability.md -#[derive(Clone, Hash, Eq, PartialEq, Debug, Default)] +#[derive(Clone, Hash, Eq, PartialEq, Debug, Default, PartialOrd, Ord)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ConstantData(Vec); @@ -169,16 +168,20 @@ impl FromStr for ConstantData { /// Maintains the mapping between a constant handle (i.e. [`Constant`](crate::ir::Constant)) and /// its constant data (i.e. [`ConstantData`](crate::ir::ConstantData)). -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ConstantPool { /// This mapping maintains the insertion order as long as Constants are created with /// sequentially increasing integers. + /// + /// It is important that, by construction, no entry in that list gets removed. If that ever + /// need to happen, don't forget to update the `Constant` generation scheme. handles_to_values: BTreeMap, - /// This mapping is unordered (no need for lexicographic ordering) but allows us to map - /// constant data back to handles. - values_to_handles: HashMap, + /// Mapping of hashed `ConstantData` to the index into the other hashmap. + /// + /// This allows for deduplication of entries into the `handles_to_values` mapping. + values_to_handles: BTreeMap, } impl ConstantPool { @@ -186,7 +189,7 @@ impl ConstantPool { pub fn new() -> Self { Self { handles_to_values: BTreeMap::new(), - values_to_handles: HashMap::new(), + values_to_handles: BTreeMap::new(), } } @@ -200,13 +203,13 @@ impl ConstantPool { /// data is inserted that is a duplicate of previous constant data, the existing handle will be /// returned. pub fn insert(&mut self, constant_value: ConstantData) -> Constant { - if self.values_to_handles.contains_key(&constant_value) { - *self.values_to_handles.get(&constant_value).unwrap() - } else { - let constant_handle = Constant::new(self.len()); - self.set(constant_handle, constant_value); - constant_handle + if let Some(cst) = self.values_to_handles.get(&constant_value) { + return *cst; } + + let constant_handle = Constant::new(self.len()); + self.set(constant_handle, constant_value); + constant_handle } /// Retrieve the constant data given a handle. @@ -250,7 +253,7 @@ impl ConstantPool { /// Return the combined size of all of the constant values in the pool. pub fn byte_size(&self) -> usize { - self.values_to_handles.keys().map(|c| c.len()).sum() + self.handles_to_values.values().map(|c| c.len()).sum() } } diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs index 65b97cbb71..9173c5fed8 100644 --- a/cranelift/codegen/src/ir/dfg.rs +++ b/cranelift/codegen/src/ir/dfg.rs @@ -4,22 +4,22 @@ use crate::entity::{self, PrimaryMap, SecondaryMap}; use crate::ir; use crate::ir::builder::ReplaceBuilder; use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes}; -use crate::ir::extfunc::ExtFuncData; use crate::ir::instructions::{BranchInfo, CallInfo, InstructionData}; use crate::ir::{types, ConstantData, ConstantPool, Immediate}; use crate::ir::{ - Block, DynamicType, FuncRef, Inst, SigRef, Signature, SourceLoc, Type, Value, - ValueLabelAssignments, ValueList, ValueListPool, + Block, DynamicType, FuncRef, Inst, SigRef, Signature, Type, Value, ValueLabelAssignments, + ValueList, ValueListPool, }; +use crate::ir::{ExtFuncData, RelSourceLoc}; use crate::packed_option::ReservedValue; use crate::write::write_operands; -use crate::HashMap; use core::fmt; use core::iter; use core::mem; use core::ops::{Index, IndexMut}; use core::u16; +use alloc::collections::BTreeMap; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -30,7 +30,7 @@ use serde::{Deserialize, Serialize}; /// The layout of blocks in the function and of instructions in each block is recorded by the /// `Layout` data structure which forms the other half of the function representation. /// -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DataFlowGraph { /// Data about all of the instructions in the function, including opcodes and operands. @@ -76,7 +76,7 @@ pub struct DataFlowGraph { pub ext_funcs: PrimaryMap, /// Saves Value labels. - pub values_labels: Option>, + pub values_labels: Option>, /// Constants used within the function pub constants: ConstantPool, @@ -154,13 +154,13 @@ impl DataFlowGraph { /// Starts collection of debug information. pub fn collect_debug_info(&mut self) { if self.values_labels.is_none() { - self.values_labels = Some(HashMap::new()); + self.values_labels = Some(Default::default()); } } /// Inserts a `ValueLabelAssignments::Alias` for `to_alias` if debug info /// collection is enabled. - pub fn add_value_label_alias(&mut self, to_alias: Value, from: SourceLoc, value: Value) { + pub fn add_value_label_alias(&mut self, to_alias: Value, from: RelSourceLoc, value: Value) { if let Some(values_labels) = self.values_labels.as_mut() { values_labels.insert(to_alias, ir::ValueLabelAssignments::Alias { from, value }); } @@ -435,7 +435,7 @@ impl ValueDef { } /// Internal table storage for extended values. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] enum ValueData { /// Value is defined by an instruction. @@ -457,7 +457,7 @@ enum ValueData { /// ```plain /// | tag:2 | type:14 | num:16 | index:32 | /// ``` -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] struct ValueDataPacked(u64); @@ -1056,7 +1056,7 @@ impl DataFlowGraph { /// Parameters on a basic block are values that dominate everything in the block. All /// branches to this block must provide matching arguments, and the arguments to the entry block must /// match the function arguments. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] struct BlockData { /// List of parameters to this block. diff --git a/cranelift/codegen/src/ir/dynamic_type.rs b/cranelift/codegen/src/ir/dynamic_type.rs index 85589cef67..91b13af98b 100644 --- a/cranelift/codegen/src/ir/dynamic_type.rs +++ b/cranelift/codegen/src/ir/dynamic_type.rs @@ -9,7 +9,7 @@ use crate::ir::Type; use serde::{Deserialize, Serialize}; /// A dynamic type object which has a base vector type and a scaling factor. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DynamicTypeData { /// Base vector type, this is the minimum size of the type. diff --git a/cranelift/codegen/src/ir/entities.rs b/cranelift/codegen/src/ir/entities.rs index 2be7014685..ea113397ad 100644 --- a/cranelift/codegen/src/ir/entities.rs +++ b/cranelift/codegen/src/ir/entities.rs @@ -328,6 +328,12 @@ impl FuncRef { } } +/// A reference to an `UserExternalName`, declared with `Function::declare_imported_user_function`. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UserExternalNameRef(u32); +entity_impl!(UserExternalNameRef, "userextname"); + /// An opaque reference to a function [`Signature`](super::Signature). /// /// `SigRef`s are used to declare a function with diff --git a/cranelift/codegen/src/ir/extfunc.rs b/cranelift/codegen/src/ir/extfunc.rs index 3491849ff9..07bc52c8c1 100644 --- a/cranelift/codegen/src/ir/extfunc.rs +++ b/cranelift/codegen/src/ir/extfunc.rs @@ -14,6 +14,8 @@ use core::str::FromStr; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; +use super::function::FunctionParameters; + /// Function signature. /// /// The function signature describes the types of formal parameters and return values along with @@ -301,7 +303,7 @@ impl FromStr for ArgumentPurpose { /// An external function. /// /// Information about a function that can be called directly with a direct `call` instruction. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ExtFuncData { /// Name of the external function. @@ -324,15 +326,6 @@ pub struct ExtFuncData { pub colocated: bool, } -impl fmt::Display for ExtFuncData { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.colocated { - write!(f, "colocated ")?; - } - write!(f, "{} {}", self.name, self.signature) - } -} - impl ExtFuncData { /// Return an estimate of the distance to the referred-to function symbol. pub fn reloc_distance(&self) -> RelocDistance { @@ -342,6 +335,38 @@ impl ExtFuncData { RelocDistance::Far } } + + /// Returns a displayable version of the `ExtFuncData`, with or without extra context to + /// prettify the output. + pub fn display<'a>( + &'a self, + params: Option<&'a FunctionParameters>, + ) -> DisplayableExtFuncData<'a> { + DisplayableExtFuncData { + ext_func: self, + params, + } + } +} + +/// A displayable `ExtFuncData`, with extra context to prettify the output. +pub struct DisplayableExtFuncData<'a> { + ext_func: &'a ExtFuncData, + params: Option<&'a FunctionParameters>, +} + +impl<'a> fmt::Display for DisplayableExtFuncData<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.ext_func.colocated { + write!(f, "colocated ")?; + } + write!( + f, + "{} {}", + self.ext_func.name.display(self.params), + self.ext_func.signature + ) + } } #[cfg(test)] diff --git a/cranelift/codegen/src/ir/extname.rs b/cranelift/codegen/src/ir/extname.rs index 54c56a4247..64b6d174a5 100644 --- a/cranelift/codegen/src/ir/extname.rs +++ b/cranelift/codegen/src/ir/extname.rs @@ -9,10 +9,108 @@ use core::cmp; use core::fmt::{self, Write}; use core::str::FromStr; +use cranelift_entity::EntityRef as _; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; -const TESTCASE_NAME_LENGTH: usize = 16; +use super::entities::UserExternalNameRef; +use super::function::FunctionParameters; + +pub(crate) const TESTCASE_NAME_LENGTH: usize = 16; + +/// An explicit name for a user-defined function, be it defined in code or in CLIF text. +/// +/// This is used both for naming a function (for debugging purposes) and for declaring external +/// functions. In the latter case, this becomes an `ExternalName`, which gets embedded in +/// relocations later, etc. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub enum UserFuncName { + /// A user-defined name, with semantics left to the user. + User(UserExternalName), + /// A name for a test case, mostly intended for Cranelift testing. + Testcase(TestcaseName), +} + +impl UserFuncName { + /// Creates a new external name from a sequence of bytes. Caller is expected + /// to guarantee bytes are only ascii alphanumeric or `_`. + pub fn testcase>(v: T) -> Self { + Self::Testcase(TestcaseName::new(v)) + } + + /// Create a new external name from a user-defined external function reference. + pub fn user(namespace: u32, index: u32) -> Self { + Self::User(UserExternalName { namespace, index }) + } +} + +impl Default for UserFuncName { + fn default() -> Self { + UserFuncName::User(UserExternalName::default()) + } +} + +impl fmt::Display for UserFuncName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + UserFuncName::User(user) => user.fmt(f), + UserFuncName::Testcase(testcase) => testcase.fmt(f), + } + } +} + +/// An external name in a user-defined symbol table. +/// +/// Cranelift does not interpret these numbers in any way, so they can represent arbitrary values. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UserExternalName { + /// Arbitrary. + pub namespace: u32, + /// Arbitrary. + pub index: u32, +} + +impl fmt::Display for UserExternalName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "u{}:{}", self.namespace, self.index) + } +} + +/// A name for a test case. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct TestcaseName { + /// How many of the bytes in `ascii` are valid? + length: u8, + /// Ascii bytes of the name. + ascii: [u8; TESTCASE_NAME_LENGTH], +} + +impl fmt::Display for TestcaseName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('%')?; + for byte in self.ascii.iter().take(self.length as usize) { + f.write_char(*byte as char)?; + } + Ok(()) + } +} + +impl TestcaseName { + pub(crate) fn new>(v: T) -> Self { + let vec = v.as_ref(); + let len = cmp::min(vec.len(), TESTCASE_NAME_LENGTH); + let mut bytes = [0u8; TESTCASE_NAME_LENGTH]; + bytes[0..len].copy_from_slice(&vec[0..len]); + + Self { + length: len as u8, + ascii: bytes, + } + } +} /// The name of an external is either a reference to a user-defined symbol /// table, or a short sequence of ascii bytes so that test cases do not have @@ -25,31 +123,26 @@ const TESTCASE_NAME_LENGTH: usize = 16; /// External names can also serve as a primitive testing and debugging tool. /// In particular, many `.clif` test files use function names to identify /// functions. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum ExternalName { - /// A name in a user-defined symbol table. Cranelift does not interpret - /// these numbers in any way. - User { - /// Arbitrary. - namespace: u32, - /// Arbitrary. - index: u32, - }, + /// A reference to a name in a user-defined symbol table. + User(UserExternalNameRef), /// A test case function name of up to a hardcoded amount of ascii /// characters. This is not intended to be used outside test cases. - TestCase { - /// How many of the bytes in `ascii` are valid? - length: u8, - /// Ascii bytes of the name. - ascii: [u8; TESTCASE_NAME_LENGTH], - }, + TestCase(TestcaseName), /// A well-known runtime library function. LibCall(LibCall), /// A well-known symbol. KnownSymbol(KnownSymbol), } +impl Default for ExternalName { + fn default() -> Self { + Self::User(UserExternalNameRef::new(0)) + } +} + impl ExternalName { /// Creates a new external name from a sequence of bytes. Caller is expected /// to guarantee bytes are only ascii alphanumeric or `_`. @@ -60,53 +153,56 @@ impl ExternalName { /// # use cranelift_codegen::ir::ExternalName; /// // Create `ExternalName` from a string. /// let name = ExternalName::testcase("hello"); - /// assert_eq!(name.to_string(), "%hello"); + /// assert_eq!(name.display(None).to_string(), "%hello"); /// ``` pub fn testcase>(v: T) -> Self { - let vec = v.as_ref(); - let len = cmp::min(vec.len(), TESTCASE_NAME_LENGTH); - let mut bytes = [0u8; TESTCASE_NAME_LENGTH]; - bytes[0..len].copy_from_slice(&vec[0..len]); - - Self::TestCase { - length: len as u8, - ascii: bytes, - } + Self::TestCase(TestcaseName::new(v)) } - /// Create a new external name from user-provided integer indices. + /// Create a new external name from a user-defined external function reference. /// /// # Examples /// ```rust - /// # use cranelift_codegen::ir::ExternalName; - /// // Create `ExternalName` from integer indices - /// let name = ExternalName::user(123, 456); - /// assert_eq!(name.to_string(), "u123:456"); + /// # use cranelift_codegen::ir::{ExternalName, UserExternalNameRef}; + /// let user_func_ref: UserExternalNameRef = Default::default(); // usually obtained with `Function::declare_imported_user_function()` + /// let name = ExternalName::user(user_func_ref); + /// assert_eq!(name.display(None).to_string(), "userextname0"); /// ``` - pub fn user(namespace: u32, index: u32) -> Self { - Self::User { namespace, index } + pub fn user(func_ref: UserExternalNameRef) -> Self { + Self::User(func_ref) + } + + /// Returns a display for the current `ExternalName`, with extra context to prettify the + /// output. + pub fn display<'a>( + &'a self, + params: Option<&'a FunctionParameters>, + ) -> DisplayableExternalName<'a> { + DisplayableExternalName { name: self, params } } } -impl Default for ExternalName { - fn default() -> Self { - Self::user(0, 0) - } +/// An `ExternalName` that has enough context to be displayed. +pub struct DisplayableExternalName<'a> { + name: &'a ExternalName, + params: Option<&'a FunctionParameters>, } -impl fmt::Display for ExternalName { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Self::User { namespace, index } => write!(f, "u{}:{}", namespace, index), - Self::TestCase { length, ascii } => { - f.write_char('%')?; - for byte in ascii.iter().take(length as usize) { - f.write_char(*byte as char)?; +impl<'a> fmt::Display for DisplayableExternalName<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.name { + ExternalName::User(func_ref) => { + if let Some(params) = self.params { + let name = ¶ms.user_named_funcs()[*func_ref]; + write!(f, "u{}:{}", name.namespace, name.index) + } else { + // Best effort. + write!(f, "{}", *func_ref) } - Ok(()) } - Self::LibCall(lc) => write!(f, "%{}", lc), - Self::KnownSymbol(ks) => write!(f, "%{}", ks), + ExternalName::TestCase(testcase) => testcase.fmt(f), + ExternalName::LibCall(lc) => write!(f, "%{}", lc), + ExternalName::KnownSymbol(ks) => write!(f, "%{}", ks), } } } @@ -133,33 +229,83 @@ impl FromStr for ExternalName { #[cfg(test)] mod tests { use super::ExternalName; - use crate::ir::LibCall; + use crate::ir::{ + entities::UserExternalNameRef, function::FunctionParameters, LibCall, UserExternalName, + }; use alloc::string::ToString; use core::u32; + use cranelift_entity::EntityRef as _; #[test] fn display_testcase() { - assert_eq!(ExternalName::testcase("").to_string(), "%"); - assert_eq!(ExternalName::testcase("x").to_string(), "%x"); - assert_eq!(ExternalName::testcase("x_1").to_string(), "%x_1"); + assert_eq!(ExternalName::testcase("").display(None).to_string(), "%"); + assert_eq!(ExternalName::testcase("x").display(None).to_string(), "%x"); assert_eq!( - ExternalName::testcase("longname12345678").to_string(), + ExternalName::testcase("x_1").display(None).to_string(), + "%x_1" + ); + assert_eq!( + ExternalName::testcase("longname12345678") + .display(None) + .to_string(), "%longname12345678" ); // Constructor will silently drop bytes beyond the 16th assert_eq!( - ExternalName::testcase("longname123456789").to_string(), + ExternalName::testcase("longname123456789") + .display(None) + .to_string(), "%longname12345678" ); } #[test] fn display_user() { - assert_eq!(ExternalName::user(0, 0).to_string(), "u0:0"); - assert_eq!(ExternalName::user(1, 1).to_string(), "u1:1"); assert_eq!( - ExternalName::user(u32::MAX, u32::MAX).to_string(), - "u4294967295:4294967295" + ExternalName::user(UserExternalNameRef::new(0)) + .display(None) + .to_string(), + "userextname0" + ); + assert_eq!( + ExternalName::user(UserExternalNameRef::new(1)) + .display(None) + .to_string(), + "userextname1" + ); + assert_eq!( + ExternalName::user(UserExternalNameRef::new((u32::MAX - 1) as _)) + .display(None) + .to_string(), + "userextname4294967294" + ); + + let mut func_params = FunctionParameters::new(); + + // ref 0 + func_params.ensure_user_func_name(UserExternalName { + namespace: 13, + index: 37, + }); + + // ref 1 + func_params.ensure_user_func_name(UserExternalName { + namespace: 2, + index: 4, + }); + + assert_eq!( + ExternalName::user(UserExternalNameRef::new(0)) + .display(Some(&func_params)) + .to_string(), + "u13:37" + ); + + assert_eq!( + ExternalName::user(UserExternalNameRef::new(1)) + .display(Some(&func_params)) + .to_string(), + "u2:4" ); } @@ -170,7 +316,9 @@ mod tests { Ok(ExternalName::LibCall(LibCall::FloorF32)) ); assert_eq!( - ExternalName::LibCall(LibCall::FloorF32).to_string(), + ExternalName::LibCall(LibCall::FloorF32) + .display(None) + .to_string(), "%FloorF32" ); } diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 856c8f5f0d..5ed3cb553f 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -11,11 +11,12 @@ use crate::ir::{ ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, InstructionData, JumpTable, JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, Type, }; -use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature}; +use crate::ir::{DataFlowGraph, Layout, Signature}; use crate::ir::{DynamicStackSlots, SourceLocs, StackSlots}; use crate::isa::CallConv; use crate::value_label::ValueLabelsRanges; use crate::write::write_function; +use crate::HashMap; #[cfg(feature = "enable-serde")] use alloc::string::String; use core::fmt; @@ -27,9 +28,13 @@ use serde::ser::Serializer; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; +use super::entities::UserExternalNameRef; +use super::extname::UserFuncName; +use super::{RelSourceLoc, SourceLoc, UserExternalName}; + /// A version marker used to ensure that serialized clif ir is never deserialized with a /// different version of Cranelift. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Hash)] pub struct VersionMarker; #[cfg(feature = "enable-serde")] @@ -60,21 +65,99 @@ impl<'de> Deserialize<'de> for VersionMarker { } } -/// -/// Functions can be cloned, but it is not a very fast operation. -/// The clone will have all the same entity numbers as the original. +/// Function parameters used when creating this function, and that will become applied after +/// compilation to materialize the final `CompiledCode`. #[derive(Clone)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] -pub struct Function { +pub struct FunctionParameters { + /// The first `SourceLoc` appearing in the function, serving as a base for every relative + /// source loc in the function. + base_srcloc: Option, + + /// External user-defined function references. + user_named_funcs: PrimaryMap, + + /// Inverted mapping of `user_named_funcs`, to deduplicate internally. + user_ext_name_to_ref: HashMap, +} + +impl FunctionParameters { + /// Creates a new `FunctionParameters` with the given name. + pub fn new() -> Self { + Self { + base_srcloc: None, + user_named_funcs: Default::default(), + user_ext_name_to_ref: Default::default(), + } + } + + /// Returns the base `SourceLoc`. + /// + /// If it was never explicitly set with `ensure_base_srcloc`, will return an invalid + /// `SourceLoc`. + pub fn base_srcloc(&self) -> SourceLoc { + self.base_srcloc.unwrap_or_default() + } + + /// Sets the base `SourceLoc`, if not set yet, and returns the base value. + pub fn ensure_base_srcloc(&mut self, srcloc: SourceLoc) -> SourceLoc { + match self.base_srcloc { + Some(val) => val, + None => { + self.base_srcloc = Some(srcloc); + srcloc + } + } + } + + /// Retrieve a `UserExternalNameRef` for the given name, or add a new one. + /// + /// This method internally deduplicates same `UserExternalName` so they map to the same + /// reference. + pub fn ensure_user_func_name(&mut self, name: UserExternalName) -> UserExternalNameRef { + if let Some(reff) = self.user_ext_name_to_ref.get(&name) { + *reff + } else { + let reff = self.user_named_funcs.push(name.clone()); + self.user_ext_name_to_ref.insert(name, reff); + reff + } + } + + /// Resets an already existing user function name to a new value. + pub fn reset_user_func_name(&mut self, index: UserExternalNameRef, name: UserExternalName) { + if let Some(prev_name) = self.user_named_funcs.get_mut(index) { + self.user_ext_name_to_ref.remove(prev_name); + *prev_name = name.clone(); + self.user_ext_name_to_ref.insert(name, index); + } + } + + /// Returns the internal mapping of `UserExternalNameRef` to `UserExternalName`. + pub fn user_named_funcs(&self) -> &PrimaryMap { + &self.user_named_funcs + } + + fn clear(&mut self) { + self.base_srcloc = None; + self.user_named_funcs.clear(); + self.user_ext_name_to_ref.clear(); + } +} + +/// Function fields needed when compiling a function. +/// +/// Additionally, these fields can be the same for two functions that would be compiled the same +/// way, and finalized by applying `FunctionParameters` onto their `CompiledCodeStencil`. +#[derive(Clone, PartialEq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct FunctionStencil { /// A version marker used to ensure that serialized clif ir is never deserialized with a /// different version of Cranelift. // Note: This must be the first field to ensure that Serde will deserialize it before // attempting to deserialize other fields that are potentially changed between versions. pub version_marker: VersionMarker, - /// Name of this function. Mostly used by `.clif` files. - pub name: ExternalName, - /// Signature of this function. pub signature: Signature, @@ -106,7 +189,7 @@ pub struct Function { /// /// Track the original source location for each instruction. The source locations are not /// interpreted by Cranelift, only preserved. - pub srclocs: SourceLocs, + srclocs: SourceLocs, /// An optional global value which represents an expression evaluating to /// the stack limit for this function. This `GlobalValue` will be @@ -116,28 +199,8 @@ pub struct Function { pub stack_limit: Option, } -impl Function { - /// Create a function with the given name and signature. - pub fn with_name_signature(name: ExternalName, sig: Signature) -> Self { - Self { - version_marker: VersionMarker, - name, - signature: sig, - sized_stack_slots: StackSlots::new(), - dynamic_stack_slots: DynamicStackSlots::new(), - global_values: PrimaryMap::new(), - heaps: PrimaryMap::new(), - tables: PrimaryMap::new(), - jump_tables: PrimaryMap::new(), - dfg: DataFlowGraph::new(), - layout: Layout::new(), - srclocs: SecondaryMap::new(), - stack_limit: None, - } - } - - /// Clear all data structures in this function. - pub fn clear(&mut self) { +impl FunctionStencil { + fn clear(&mut self) { self.signature.clear(CallConv::Fast); self.sized_stack_slots.clear(); self.dynamic_stack_slots.clear(); @@ -151,11 +214,6 @@ impl Function { self.stack_limit = None; } - /// Create a new empty, anonymous function with a Fast calling convention. - pub fn new() -> Self { - Self::with_name_signature(ExternalName::default(), Signature::new(CallConv::Fast)) - } - /// Creates a jump table in the function, to be used by `br_table` instructions. pub fn create_jump_table(&mut self, data: JumpTableData) -> JumpTable { self.jump_tables.push(data) @@ -178,11 +236,6 @@ impl Function { self.dfg.signatures.push(signature) } - /// Declare an external function import. - pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef { - self.dfg.ext_funcs.push(data) - } - /// Declares a global value accessible to the function. pub fn create_global_value(&mut self, data: GlobalValueData) -> GlobalValue { self.global_values.push(data) @@ -218,19 +271,6 @@ impl Function { self.tables.push(data) } - /// Return an object that can display this function with correct ISA-specific annotations. - pub fn display(&self) -> DisplayFunction<'_> { - DisplayFunction(self, Default::default()) - } - - /// Return an object that can display this function with correct ISA-specific annotations. - pub fn display_with<'a>( - &'a self, - annotations: DisplayFunctionAnnotations<'a>, - ) -> DisplayFunction<'a> { - DisplayFunction(self, annotations) - } - /// Find a presumed unique special-purpose function parameter value. /// /// Returns the value of the last `purpose` parameter, or `None` if no such parameter exists. @@ -260,8 +300,8 @@ impl Function { /// Rewrite the branch destination to `new_dest` if the destination matches `old_dest`. /// Does nothing if called with a non-jump or non-branch instruction. /// - /// Unlike [change_branch_destination](Function::change_branch_destination), this method rewrite the destinations of - /// multi-destination branches like `br_table`. + /// Unlike [change_branch_destination](FunctionStencil::change_branch_destination), this method + /// rewrite the destinations of multi-destination branches like `br_table`. pub fn rewrite_branch_destination(&mut self, inst: Inst, old_dest: Block, new_dest: Block) { match self.dfg.analyze_branch(inst) { BranchInfo::SingleDest(dest, ..) => { @@ -356,6 +396,120 @@ impl Function { pub fn fixed_stack_size(&self) -> u32 { self.sized_stack_slots.values().map(|ss| ss.size).sum() } + + /// Returns the list of relative source locations for this function. + pub(crate) fn rel_srclocs(&self) -> &SecondaryMap { + &self.srclocs + } +} + +/// Functions can be cloned, but it is not a very fast operation. +/// The clone will have all the same entity numbers as the original. +#[derive(Clone)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct Function { + /// Name of this function. + /// + /// Mostly used by `.clif` files, only there for debugging / naming purposes. + pub name: UserFuncName, + + /// All the fields required for compiling a function, independently of details irrelevant to + /// compilation and that are stored in the `FunctionParameters` `params` field instead. + pub stencil: FunctionStencil, + + /// All the parameters that can be applied onto the function stencil, that is, that don't + /// matter when caching compilation artifacts. + pub params: FunctionParameters, +} + +impl core::ops::Deref for Function { + type Target = FunctionStencil; + + fn deref(&self) -> &Self::Target { + &self.stencil + } +} + +impl core::ops::DerefMut for Function { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stencil + } +} + +impl Function { + /// Create a function with the given name and signature. + pub fn with_name_signature(name: UserFuncName, sig: Signature) -> Self { + Self { + name, + stencil: FunctionStencil { + version_marker: VersionMarker, + signature: sig, + sized_stack_slots: StackSlots::new(), + dynamic_stack_slots: DynamicStackSlots::new(), + global_values: PrimaryMap::new(), + heaps: PrimaryMap::new(), + tables: PrimaryMap::new(), + jump_tables: PrimaryMap::new(), + dfg: DataFlowGraph::new(), + layout: Layout::new(), + srclocs: SecondaryMap::new(), + stack_limit: None, + }, + params: FunctionParameters::new(), + } + } + + /// Clear all data structures in this function. + pub fn clear(&mut self) { + self.stencil.clear(); + self.params.clear(); + self.name = UserFuncName::default(); + } + + /// Create a new empty, anonymous function with a Fast calling convention. + pub fn new() -> Self { + Self::with_name_signature(Default::default(), Signature::new(CallConv::Fast)) + } + + /// Return an object that can display this function with correct ISA-specific annotations. + pub fn display(&self) -> DisplayFunction<'_> { + DisplayFunction(self, Default::default()) + } + + /// Return an object that can display this function with correct ISA-specific annotations. + pub fn display_with<'a>( + &'a self, + annotations: DisplayFunctionAnnotations<'a>, + ) -> DisplayFunction<'a> { + DisplayFunction(self, annotations) + } + + /// Sets an absolute source location for the given instruction. + /// + /// If no base source location has been set yet, records it at the same time. + pub fn set_srcloc(&mut self, inst: Inst, srcloc: SourceLoc) { + let base = self.params.ensure_base_srcloc(srcloc); + self.stencil.srclocs[inst] = RelSourceLoc::from_base_offset(base, srcloc); + } + + /// Returns an absolute source location for the given instruction. + pub fn srcloc(&self, inst: Inst) -> SourceLoc { + let base = self.params.base_srcloc(); + self.stencil.srclocs[inst].expand(base) + } + + /// Declare a user-defined external function import, to be referenced in `ExtFuncData::User` later. + pub fn declare_imported_user_function( + &mut self, + name: UserExternalName, + ) -> UserExternalNameRef { + self.params.ensure_user_func_name(name) + } + + /// Declare an external function import. + pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef { + self.stencil.dfg.ext_funcs.push(data) + } } /// Additional annotations for function display. diff --git a/cranelift/codegen/src/ir/globalvalue.rs b/cranelift/codegen/src/ir/globalvalue.rs index 8ec39bf0a4..6e26931fed 100644 --- a/cranelift/codegen/src/ir/globalvalue.rs +++ b/cranelift/codegen/src/ir/globalvalue.rs @@ -10,7 +10,7 @@ use core::fmt; use serde::{Deserialize, Serialize}; /// Information about a global value declaration. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum GlobalValueData { /// Value is the address of the VM context struct. @@ -151,7 +151,7 @@ impl fmt::Display for GlobalValueData { "symbol {}{}{}", if colocated { "colocated " } else { "" }, if tls { "tls " } else { "" }, - name + name.display(None) )?; let offset_val: i64 = offset.into(); if offset_val > 0 { diff --git a/cranelift/codegen/src/ir/heap.rs b/cranelift/codegen/src/ir/heap.rs index 91aabccaa2..7d62915af5 100644 --- a/cranelift/codegen/src/ir/heap.rs +++ b/cranelift/codegen/src/ir/heap.rs @@ -8,7 +8,7 @@ use core::fmt; use serde::{Deserialize, Serialize}; /// Information about a heap declaration. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct HeapData { /// The address of the start of the heap's storage. @@ -29,7 +29,7 @@ pub struct HeapData { } /// Style of heap including style-specific information. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum HeapStyle { /// A dynamic heap can be relocated to a different base address when it is grown. diff --git a/cranelift/codegen/src/ir/jumptable.rs b/cranelift/codegen/src/ir/jumptable.rs index bf05169d36..7897dff845 100644 --- a/cranelift/codegen/src/ir/jumptable.rs +++ b/cranelift/codegen/src/ir/jumptable.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; /// Contents of a jump table. /// /// All jump tables use 0-based indexing and are densely populated. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct JumpTableData { // Table entries. diff --git a/cranelift/codegen/src/ir/layout.rs b/cranelift/codegen/src/ir/layout.rs index ec411b4b17..7162c848c5 100644 --- a/cranelift/codegen/src/ir/layout.rs +++ b/cranelift/codegen/src/ir/layout.rs @@ -25,7 +25,7 @@ use core::iter::{IntoIterator, Iterator}; /// While data dependencies are not recorded, instruction ordering does affect control /// dependencies, so part of the semantics of the program are determined by the layout. /// -#[derive(Clone)] +#[derive(Debug, Clone, PartialEq, Hash)] pub struct Layout { /// Linked list nodes for the layout order of blocks Forms a doubly linked list, terminated in /// both ends by `None`. @@ -311,7 +311,7 @@ impl Layout { /// /// This doesn't affect the position of anything, but it gives more room in the internal /// sequence numbers for inserting instructions later. - fn full_renumber(&mut self) { + pub(crate) fn full_renumber(&mut self) { let _tt = timing::layout_renumber(); let mut seq = 0; let mut next_block = self.first_block; @@ -486,7 +486,7 @@ impl Layout { /// A single node in the linked-list of blocks. // Whenever you add new fields here, don't forget to update the custom serializer for `Layout` too. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq, Hash)] struct BlockNode { prev: PackedOption, next: PackedOption, @@ -748,7 +748,7 @@ impl Layout { } } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq, Hash)] struct InstNode { /// The Block containing this instruction, or `None` if the instruction is not yet inserted. block: PackedOption, diff --git a/cranelift/codegen/src/ir/libcall.rs b/cranelift/codegen/src/ir/libcall.rs index 82971f4a22..e7ef344bb3 100644 --- a/cranelift/codegen/src/ir/libcall.rs +++ b/cranelift/codegen/src/ir/libcall.rs @@ -1,7 +1,9 @@ //! Naming well-known routines in the runtime library. -use crate::ir::{types, AbiParam, ExternalName, FuncRef, Function, Opcode, Signature, Type}; -use crate::isa::CallConv; +use crate::{ + ir::{types, AbiParam, ExternalName, FuncRef, Function, Opcode, Signature, Type}, + isa::CallConv, +}; use core::fmt; use core::str::FromStr; #[cfg(feature = "enable-serde")] diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index 15e14660e4..5dc5ad612d 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -38,12 +38,12 @@ pub use crate::ir::dfg::{DataFlowGraph, ValueDef}; pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes}; pub use crate::ir::entities::{ Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, Immediate, Inst, - JumpTable, SigRef, StackSlot, Table, Value, + JumpTable, SigRef, StackSlot, Table, UserExternalNameRef, Value, }; pub use crate::ir::extfunc::{ AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature, }; -pub use crate::ir::extname::ExternalName; +pub use crate::ir::extname::{ExternalName, UserExternalName, UserFuncName}; pub use crate::ir::function::{DisplayFunctionAnnotations, Function}; pub use crate::ir::globalvalue::GlobalValueData; pub use crate::ir::heap::{HeapData, HeapStyle}; @@ -56,6 +56,7 @@ pub use crate::ir::layout::Layout; pub use crate::ir::libcall::{get_probestack_funcref, LibCall}; pub use crate::ir::memflags::{Endianness, MemFlags}; pub use crate::ir::progpoint::{ExpandedProgramPoint, ProgramOrder, ProgramPoint}; +pub use crate::ir::sourceloc::RelSourceLoc; pub use crate::ir::sourceloc::SourceLoc; pub use crate::ir::stackslot::{ DynamicStackSlotData, DynamicStackSlots, StackSlotData, StackSlotKind, StackSlots, @@ -71,7 +72,7 @@ use crate::entity::{entity_impl, PrimaryMap, SecondaryMap}; pub type JumpTables = PrimaryMap; /// Source locations for instructions. -pub type SourceLocs = SecondaryMap; +pub(crate) type SourceLocs = SecondaryMap; /// Marked with a label value. #[derive(Copy, Clone, PartialEq, Eq, Hash)] @@ -80,18 +81,18 @@ pub struct ValueLabel(u32); entity_impl!(ValueLabel, "val"); /// A label of a Value. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ValueLabelStart { /// Source location when it is in effect - pub from: SourceLoc, + pub from: RelSourceLoc, /// The label index. pub label: ValueLabel, } /// Value label assignements: label starts or value aliases. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum ValueLabelAssignments { /// Original value labels assigned at transform. @@ -100,7 +101,7 @@ pub enum ValueLabelAssignments { /// A value alias to original value. Alias { /// Source location when it is in effect - from: SourceLoc, + from: RelSourceLoc, /// The label index. value: Value, diff --git a/cranelift/codegen/src/ir/sourceloc.rs b/cranelift/codegen/src/ir/sourceloc.rs index ccab62f89b..57e7a4160c 100644 --- a/cranelift/codegen/src/ir/sourceloc.rs +++ b/cranelift/codegen/src/ir/sourceloc.rs @@ -51,6 +51,61 @@ impl fmt::Display for SourceLoc { } } +/// Source location relative to another base source location. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct RelSourceLoc(u32); + +impl RelSourceLoc { + /// Create a new relative source location with the given bits. + pub fn new(bits: u32) -> Self { + Self(bits) + } + + /// Creates a new `RelSourceLoc` based on the given base and offset. + /// + /// # Panics + /// + /// Panics if the offset is smaller than the base. + pub fn from_base_offset(base: SourceLoc, offset: SourceLoc) -> Self { + if base.is_default() || offset.is_default() { + Self::default() + } else { + Self(offset.bits().wrapping_sub(base.bits())) + } + } + + /// Expands the relative source location into an absolute one, using the given base. + pub fn expand(&self, base: SourceLoc) -> SourceLoc { + if self.is_default() || base.is_default() { + Default::default() + } else { + SourceLoc::new(self.0.wrapping_add(base.bits())) + } + } + + /// Is this the default relative source location? + pub fn is_default(self) -> bool { + self == Default::default() + } +} + +impl Default for RelSourceLoc { + fn default() -> Self { + Self(!0) + } +} + +impl fmt::Display for RelSourceLoc { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.is_default() { + write!(f, "@-") + } else { + write!(f, "@+{:04x}", self.0) + } + } +} + #[cfg(test)] mod tests { use crate::ir::SourceLoc; diff --git a/cranelift/codegen/src/ir/stackslot.rs b/cranelift/codegen/src/ir/stackslot.rs index e4db80d5d7..aa77a7ac7e 100644 --- a/cranelift/codegen/src/ir/stackslot.rs +++ b/cranelift/codegen/src/ir/stackslot.rs @@ -27,7 +27,7 @@ use serde::{Deserialize, Serialize}; pub type StackSize = u32; /// The kind of a stack slot. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum StackSlotKind { /// An explicit stack slot. This is a chunk of stack memory for use by the `stack_load` @@ -62,7 +62,7 @@ impl fmt::Display for StackSlotKind { } /// Contents of a stack slot. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct StackSlotData { /// The kind of stack slot. @@ -100,7 +100,7 @@ impl fmt::Display for StackSlotData { } /// Contents of a dynamic stack slot. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DynamicStackSlotData { /// The kind of stack slot. diff --git a/cranelift/codegen/src/ir/table.rs b/cranelift/codegen/src/ir/table.rs index 713d1f5df7..6acfb14fa1 100644 --- a/cranelift/codegen/src/ir/table.rs +++ b/cranelift/codegen/src/ir/table.rs @@ -8,7 +8,7 @@ use core::fmt; use serde::{Deserialize, Serialize}; /// Information about a table declaration. -#[derive(Clone)] +#[derive(Clone, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct TableData { /// Global value giving the address of the start of the table. diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index c36e118c24..6ec2358fe4 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -3,7 +3,7 @@ use regalloc2::Allocation; use crate::binemit::{CodeOffset, Reloc, StackMap}; -use crate::ir::types::*; +use crate::ir::{types::*, RelSourceLoc}; use crate::ir::{LibCall, MemFlags, TrapCode}; use crate::isa::aarch64::inst::*; use crate::machinst::{ty_bits, Reg, RegClass, Writable}; @@ -617,7 +617,7 @@ pub struct EmitState { /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. stack_map: Option, /// Current source-code location corresponding to instruction to be emitted. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, } impl MachInstEmitState for EmitState { @@ -626,7 +626,7 @@ impl MachInstEmitState for EmitState { virtual_sp_offset: 0, nominal_sp_to_fp: abi.frame_size() as i64, stack_map: None, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), } } @@ -634,7 +634,7 @@ impl MachInstEmitState for EmitState { self.stack_map = Some(stack_map); } - fn pre_sourceloc(&mut self, srcloc: SourceLoc) { + fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } } @@ -648,7 +648,7 @@ impl EmitState { self.stack_map = None; } - fn cur_srcloc(&self) -> SourceLoc { + fn cur_srcloc(&self) -> RelSourceLoc { self.cur_srcloc } } @@ -954,7 +954,7 @@ impl MachInstEmit for Inst { }; let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1074,7 +1074,7 @@ impl MachInstEmit for Inst { }; let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual store instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1151,7 +1151,7 @@ impl MachInstEmit for Inst { let rt2 = allocs.next(rt2); let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual store instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1183,7 +1183,7 @@ impl MachInstEmit for Inst { let rt2 = allocs.next(rt2.to_reg()); let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1223,7 +1223,7 @@ impl MachInstEmit for Inst { let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1269,7 +1269,7 @@ impl MachInstEmit for Inst { let mem = mem.with_allocs(&mut allocs); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual store instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } @@ -1417,7 +1417,7 @@ impl MachInstEmit for Inst { // again: sink.bind_label(again_label); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } sink.put4(enc_ldaxr(ty, x27wr, x25)); // ldaxr x27, [x25] @@ -1541,7 +1541,7 @@ impl MachInstEmit for Inst { } let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } if op == AtomicRMWLoopOp::Xchg { @@ -1603,7 +1603,7 @@ impl MachInstEmit for Inst { // again: sink.bind_label(again_label); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } // ldaxr x27, [x25] @@ -1630,7 +1630,7 @@ impl MachInstEmit for Inst { sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25] @@ -2662,7 +2662,7 @@ impl MachInstEmit for Inst { let (q, size) = size.enc_size(); let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() && !flags.notrap() { + if !srcloc.is_default() && !flags.notrap() { // Register the offset at which the actual load instruction starts. sink.add_trap(TrapCode::HeapOutOfBounds); } diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 6e45beb66b..241d0f79f3 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -4,7 +4,7 @@ use crate::binemit::{Addend, CodeOffset, Reloc}; use crate::ir::types::{ B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64, }; -use crate::ir::{types, ExternalName, MemFlags, Opcode, SourceLoc, Type}; +use crate::ir::{types, ExternalName, MemFlags, Opcode, Type}; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; @@ -2704,7 +2704,7 @@ impl Inst { &Inst::EmitIsland { needed_space } => format!("emit_island {}", needed_space), &Inst::ElfTlsGetAddr { ref symbol } => { - format!("x0 = elf_tls_get_addr {}", symbol) + format!("x0 = elf_tls_get_addr {}", symbol.display(None)) } &Inst::Unwind { ref inst } => { format!("unwind {:?}", inst) diff --git a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs index f5d86252b3..02c3e9dcf5 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs @@ -70,8 +70,7 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { mod tests { use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ - types, AbiParam, ExternalName, Function, InstBuilder, Signature, StackSlotData, - StackSlotKind, + types, AbiParam, Function, InstBuilder, Signature, StackSlotData, StackSlotKind, }; use crate::isa::{lookup, CallConv}; use crate::settings::{builder, Flags}; @@ -108,8 +107,7 @@ mod tests { } fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + let mut func = Function::with_name_signature(Default::default(), Signature::new(call_conv)); let block0 = func.dfg.make_block(); let mut pos = FuncCursor::new(&mut func); @@ -153,7 +151,7 @@ mod tests { fn create_multi_return_function(call_conv: CallConv) -> Function { let mut sig = Signature::new(call_conv); sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(Default::default(), sig); let block0 = func.dfg.make_block(); let v0 = func.dfg.append_block_param(block0, types::I32); diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index 4d96b80340..264c2d5053 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -7,7 +7,8 @@ use crate::isa::aarch64::settings as aarch64_settings; use crate::isa::unwind::systemv; use crate::isa::{Builder as IsaBuilder, TargetIsa}; use crate::machinst::{ - compile, CompiledCode, MachTextSectionBuilder, Reg, TextSectionBuilder, VCode, + compile, CompiledCode, CompiledCodeStencil, MachTextSectionBuilder, Reg, TextSectionBuilder, + VCode, }; use crate::result::CodegenResult; use crate::settings as shared_settings; @@ -65,7 +66,11 @@ impl AArch64Backend { } impl TargetIsa for AArch64Backend { - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult { + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func, flags.clone())?; @@ -80,7 +85,7 @@ impl TargetIsa for AArch64Backend { log::debug!("disassembly:\n{}", disasm); } - Ok(CompiledCode { + Ok(CompiledCodeStencil { buffer, frame_size, disasm: emit_result.disasm, @@ -204,7 +209,7 @@ mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::types::*; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, JumpTableData, Signature}; + use crate::ir::{AbiParam, Function, InstBuilder, JumpTableData, Signature, UserFuncName}; use crate::isa::CallConv; use crate::settings; use crate::settings::Configurable; @@ -213,7 +218,7 @@ mod test { #[test] fn test_compile_function() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -252,7 +257,7 @@ mod test { #[test] fn test_branch_lowering() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -320,7 +325,7 @@ mod test { #[test] fn test_br_table() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 5904f51b20..f3eb30a7cc 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -49,7 +49,7 @@ use crate::flowgraph; use crate::ir::{self, Function}; #[cfg(feature = "unwind")] use crate::isa::unwind::systemv::RegisterMappingError; -use crate::machinst::{CompiledCode, TextSectionBuilder, UnwindInfoKind}; +use crate::machinst::{CompiledCode, CompiledCodeStencil, TextSectionBuilder, UnwindInfoKind}; use crate::settings; use crate::settings::SetResult; use crate::CodegenResult; @@ -230,7 +230,11 @@ pub trait TargetIsa: fmt::Display + Send + Sync { fn dynamic_vector_bytes(&self, dynamic_ty: ir::Type) -> u32; /// Compile the given function. - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult; + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult; #[cfg(feature = "unwind")] /// Map a regalloc::Reg to its corresponding DWARF register. diff --git a/cranelift/codegen/src/isa/s390x/inst/args.rs b/cranelift/codegen/src/isa/s390x/inst/args.rs index 7a0905641b..d69fe2d60c 100644 --- a/cranelift/codegen/src/isa/s390x/inst/args.rs +++ b/cranelift/codegen/src/isa/s390x/inst/args.rs @@ -326,7 +326,7 @@ impl PrettyPrint for MemArg { &MemArg::Label { target } => target.to_string(), &MemArg::Symbol { ref name, offset, .. - } => format!("{} + {}", name, offset), + } => format!("{} + {}", name.display(None), offset), // Eliminated by `mem_finalize()`. &MemArg::InitialSPOffset { .. } | &MemArg::NominalSPOffset { .. } diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index 62f3736fe1..346fb306e0 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -1,8 +1,8 @@ //! S390x ISA: binary code emission. use crate::binemit::{Reloc, StackMap}; -use crate::ir::MemFlags; -use crate::ir::{SourceLoc, TrapCode}; +use crate::ir::TrapCode; +use crate::ir::{MemFlags, RelSourceLoc}; use crate::isa::s390x::inst::*; use crate::isa::s390x::settings as s390x_settings; use crate::machinst::reg::count_operands; @@ -133,7 +133,7 @@ pub fn mem_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -201,7 +201,7 @@ pub fn mem_rs_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -243,7 +243,7 @@ pub fn mem_imm8_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -281,7 +281,7 @@ pub fn mem_imm16_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -308,7 +308,7 @@ pub fn mem_mem_emit( ) { if add_trap && (dst.can_trap() || src.can_trap()) { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if srcloc != Default::default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -343,7 +343,7 @@ pub fn mem_vrx_emit( if add_trap && mem.can_trap() { let srcloc = state.cur_srcloc(); - if srcloc != SourceLoc::default() { + if !srcloc.is_default() { sink.add_trap(TrapCode::HeapOutOfBounds); } } @@ -1256,7 +1256,7 @@ pub struct EmitState { /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. stack_map: Option, /// Current source-code location corresponding to instruction to be emitted. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, } impl MachInstEmitState for EmitState { @@ -1265,7 +1265,7 @@ impl MachInstEmitState for EmitState { virtual_sp_offset: 0, initial_sp_offset: abi.frame_size() as i64, stack_map: None, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), } } @@ -1273,7 +1273,7 @@ impl MachInstEmitState for EmitState { self.stack_map = Some(stack_map); } - fn pre_sourceloc(&mut self, srcloc: SourceLoc) { + fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } } @@ -1287,7 +1287,7 @@ impl EmitState { self.stack_map = None; } - fn cur_srcloc(&self) -> SourceLoc { + fn cur_srcloc(&self) -> RelSourceLoc { self.cur_srcloc } } diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index 60e148cd8d..d13a69f204 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -2932,10 +2932,12 @@ impl Inst { let link = pretty_print_reg(link.to_reg(), allocs); let tls_symbol = match &info.tls_symbol { None => "".to_string(), - Some(SymbolReloc::TlsGd { name }) => format!(":tls_gdcall:{}", name), + Some(SymbolReloc::TlsGd { name }) => { + format!(":tls_gdcall:{}", name.display(None)) + } _ => unreachable!(), }; - format!("brasl {}, {}{}", link, info.dest, tls_symbol) + format!("brasl {}, {}{}", link, info.dest.display(None), tls_symbol) } &Inst::CallInd { link, ref info, .. } => { let link = pretty_print_reg(link.to_reg(), allocs); @@ -3003,8 +3005,10 @@ impl Inst { let rd = pretty_print_reg(rd.to_reg(), allocs); let tmp = pretty_print_reg(writable_spilltmp_reg().to_reg(), &mut empty_allocs); let symbol = match &**symbol_reloc { - SymbolReloc::Absolute { name, offset } => format!("{} + {}", name, offset), - SymbolReloc::TlsGd { name } => format!("{}@tlsgd", name), + SymbolReloc::Absolute { name, offset } => { + format!("{} + {}", name.display(None), offset) + } + SymbolReloc::TlsGd { name } => format!("{}@tlsgd", name.display(None)), }; format!("bras {}, 12 ; data {} ; lg {}, 0({})", tmp, symbol, rd, tmp) } diff --git a/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs index 152dabe44b..f787a95d02 100644 --- a/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/s390x/inst/unwind/systemv.rs @@ -101,8 +101,7 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { mod tests { use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ - types, AbiParam, ExternalName, Function, InstBuilder, Signature, StackSlotData, - StackSlotKind, + types, AbiParam, Function, InstBuilder, Signature, StackSlotData, StackSlotKind, }; use crate::isa::{lookup, CallConv}; use crate::settings::{builder, Flags}; @@ -139,8 +138,7 @@ mod tests { } fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + let mut func = Function::with_name_signature(Default::default(), Signature::new(call_conv)); let block0 = func.dfg.make_block(); let mut pos = FuncCursor::new(&mut func); @@ -187,7 +185,7 @@ mod tests { ) -> Function { let mut sig = Signature::new(call_conv); sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(Default::default(), sig); let block0 = func.dfg.make_block(); let v0 = func.dfg.append_block_param(block0, types::I32); diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 0ffba418d1..b8587a44ad 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -3,6 +3,7 @@ // Pull in the ISLE generated code. pub mod generated_code; +use crate::ir::ExternalName; // Types that the generated ISLE code uses via `use super::*`. use crate::isa::s390x::abi::{S390xMachineDeps, REG_SAVE_AREA_SIZE}; use crate::isa::s390x::inst::{ diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs index 123209b8f9..e2165c585c 100644 --- a/cranelift/codegen/src/isa/s390x/mod.rs +++ b/cranelift/codegen/src/isa/s390x/mod.rs @@ -7,7 +7,8 @@ use crate::isa::s390x::settings as s390x_settings; use crate::isa::unwind::systemv::RegisterMappingError; use crate::isa::{Builder as IsaBuilder, TargetIsa}; use crate::machinst::{ - compile, CompiledCode, MachTextSectionBuilder, Reg, TextSectionBuilder, VCode, + compile, CompiledCode, CompiledCodeStencil, MachTextSectionBuilder, Reg, TextSectionBuilder, + VCode, }; use crate::result::CodegenResult; use crate::settings as shared_settings; @@ -63,7 +64,11 @@ impl S390xBackend { } impl TargetIsa for S390xBackend { - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult { + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func)?; @@ -78,7 +83,7 @@ impl TargetIsa for S390xBackend { log::debug!("disassembly:\n{}", disasm); } - Ok(CompiledCode { + Ok(CompiledCodeStencil { buffer, frame_size, disasm: emit_result.disasm, @@ -186,7 +191,8 @@ mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::types::*; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; + use crate::ir::UserFuncName; + use crate::ir::{AbiParam, Function, InstBuilder, Signature}; use crate::isa::CallConv; use crate::settings; use crate::settings::Configurable; @@ -195,7 +201,7 @@ mod test { #[test] fn test_compile_function() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -233,7 +239,7 @@ mod test { #[test] fn test_branch_lowering() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 12c5fcf5a6..d0c1137f6f 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -1,7 +1,7 @@ //! Implementation of the standard x64 ABI. -use crate::ir::types::*; -use crate::ir::{self, types, ExternalName, LibCall, MemFlags, Opcode, Signature, TrapCode, Type}; +use crate::ir::{self, types, LibCall, MemFlags, Opcode, Signature, TrapCode, Type}; +use crate::ir::{types::*, ExternalName}; use crate::isa; use crate::isa::{unwind::UnwindInst, x64::inst::*, x64::settings as x64_settings, CallConv}; use crate::machinst::abi_impl::*; diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 087613d52c..61b7f33e36 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -13,9 +13,11 @@ //! -- isa::x64::inst::emit_tests::test_x64_emit use super::*; +use crate::ir::UserExternalNameRef; use crate::isa::x64; use alloc::boxed::Box; use alloc::vec::Vec; +use cranelift_entity::EntityRef as _; impl Inst { fn neg(size: OperandSize, src: Writable) -> Inst { @@ -3395,17 +3397,14 @@ fn test_x64_emit() { // CallKnown insns.push(( Inst::call_known( - ExternalName::User { - namespace: 0, - index: 0, - }, + ExternalName::User(UserExternalNameRef::new(0)), smallvec![], smallvec![], PRegSet::default(), Opcode::Call, ), "E800000000", - "call User { namespace: 0, index: 0 }", + "call User(userextname0)", )); // ======================================================== @@ -3449,38 +3448,29 @@ fn test_x64_emit() { insns.push(( Inst::LoadExtName { dst: Writable::from_reg(r11), - name: Box::new(ExternalName::User { - namespace: 0, - index: 0, - }), + name: Box::new(ExternalName::User(UserExternalNameRef::new(0))), offset: 0, }, "4C8B1D00000000", - "load_ext_name u0:0+0, %r11", + "load_ext_name userextname0+0, %r11", )); insns.push(( Inst::LoadExtName { dst: Writable::from_reg(r11), - name: Box::new(ExternalName::User { - namespace: 0, - index: 0, - }), + name: Box::new(ExternalName::User(UserExternalNameRef::new(0))), offset: 0x12345678, }, "4C8B1D000000004981C378563412", - "load_ext_name u0:0+305419896, %r11", + "load_ext_name userextname0+305419896, %r11", )); insns.push(( Inst::LoadExtName { dst: Writable::from_reg(r11), - name: Box::new(ExternalName::User { - namespace: 0, - index: 0, - }), + name: Box::new(ExternalName::User(UserExternalNameRef::new(0))), offset: -0x12345678, }, "4C8B1D000000004981EB78563412", - "load_ext_name u0:0+-305419896, %r11", + "load_ext_name userextname0+-305419896, %r11", )); // ======================================================== @@ -4690,35 +4680,26 @@ fn test_x64_emit() { insns.push(( Inst::ElfTlsGetAddr { - symbol: ExternalName::User { - namespace: 0, - index: 0, - }, + symbol: ExternalName::User(UserExternalNameRef::new(0)), }, "66488D3D00000000666648E800000000", - "%rax = elf_tls_get_addr User { namespace: 0, index: 0 }", + "%rax = elf_tls_get_addr User(userextname0)", )); insns.push(( Inst::MachOTlsGetAddr { - symbol: ExternalName::User { - namespace: 0, - index: 0, - }, + symbol: ExternalName::User(UserExternalNameRef::new(0)), }, "488B3D00000000FF17", - "%rax = macho_tls_get_addr User { namespace: 0, index: 0 }", + "%rax = macho_tls_get_addr User(userextname0)", )); insns.push(( Inst::CoffTlsGetAddr { - symbol: ExternalName::User { - namespace: 0, - index: 0, - }, + symbol: ExternalName::User(UserExternalNameRef::new(0)), }, "8B050000000065488B0C2558000000488B04C1488D8000000000", - "%rax = coff_tls_get_addr User { namespace: 0, index: 0 }", + "%rax = coff_tls_get_addr User(userextname0)", )); // ======================================================== diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 581a55ef7c..fbe790a0f7 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1,7 +1,7 @@ //! This module defines x86_64-specific machine instruction types. use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; -use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type}; +use crate::ir::{types, ExternalName, Opcode, RelSourceLoc, TrapCode, Type}; use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::inst::regs::pretty_print_reg; use crate::isa::x64::settings as x64_settings; @@ -1624,7 +1624,7 @@ impl PrettyPrint for Inst { format!( "{} {}+{}, {}", ljustify("load_ext_name".into()), - name, + name.display(None), offset, dst, ) @@ -2424,7 +2424,7 @@ pub struct EmitState { /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`. stack_map: Option, /// Current source location. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, } /// Constant state used during emissions of a sequence of instructions. @@ -2465,7 +2465,7 @@ impl MachInstEmitState for EmitState { virtual_sp_offset: 0, nominal_sp_to_fp: abi.frame_size() as i64, stack_map: None, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), } } @@ -2473,7 +2473,7 @@ impl MachInstEmitState for EmitState { self.stack_map = Some(stack_map); } - fn pre_sourceloc(&mut self, srcloc: SourceLoc) { + fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } } diff --git a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs index d3970a575a..335c97a107 100644 --- a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs @@ -97,8 +97,7 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { mod tests { use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ - types, AbiParam, ExternalName, Function, InstBuilder, Signature, StackSlotData, - StackSlotKind, + types, AbiParam, Function, InstBuilder, Signature, StackSlotData, StackSlotKind, }; use crate::isa::{lookup, CallConv}; use crate::settings::{builder, Flags}; @@ -135,8 +134,7 @@ mod tests { } fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + let mut func = Function::with_name_signature(Default::default(), Signature::new(call_conv)); let block0 = func.dfg.make_block(); let mut pos = FuncCursor::new(&mut func); @@ -177,7 +175,7 @@ mod tests { fn create_multi_return_function(call_conv: CallConv) -> Function { let mut sig = Signature::new(call_conv); sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(Default::default(), sig); let block0 = func.dfg.make_block(); let v0 = func.dfg.append_block_param(block0, types::I32); diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 8c7415d88a..1205b661af 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -8,8 +8,10 @@ use crate::ir::{condcodes::IntCC, Function, Type}; use crate::isa::unwind::systemv; use crate::isa::x64::{inst::regs::create_reg_env_systemv, settings as x64_settings}; use crate::isa::Builder as IsaBuilder; -use crate::machinst::Reg; -use crate::machinst::{compile, CompiledCode, MachTextSectionBuilder, TextSectionBuilder, VCode}; +use crate::machinst::{ + compile, CompiledCode, CompiledCodeStencil, MachTextSectionBuilder, Reg, TextSectionBuilder, + VCode, +}; use crate::result::{CodegenError, CodegenResult}; use crate::settings::{self as shared_settings, Flags}; use alloc::{boxed::Box, vec::Vec}; @@ -57,7 +59,11 @@ impl X64Backend { } impl TargetIsa for X64Backend { - fn compile_function(&self, func: &Function, want_disasm: bool) -> CodegenResult { + fn compile_function( + &self, + func: &Function, + want_disasm: bool, + ) -> CodegenResult { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func, flags.clone())?; @@ -72,7 +78,7 @@ impl TargetIsa for X64Backend { log::trace!("disassembly:\n{}", disasm); } - Ok(CompiledCode { + Ok(CompiledCodeStencil { buffer, frame_size, disasm: emit_result.disasm, @@ -201,8 +207,8 @@ fn isa_constructor( mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; - use crate::ir::{types::*, SourceLoc, ValueLabel, ValueLabelStart}; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, JumpTableData, Signature}; + use crate::ir::{types::*, RelSourceLoc, SourceLoc, UserFuncName, ValueLabel, ValueLabelStart}; + use crate::ir::{AbiParam, Function, InstBuilder, JumpTableData, Signature}; use crate::isa::CallConv; use crate::settings; use crate::settings::Configurable; @@ -217,7 +223,7 @@ mod test { /// well do the test here, where we have a backend to use. #[test] fn test_cold_blocks() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); @@ -271,35 +277,35 @@ mod test { pos.func.dfg.values_labels.as_mut().unwrap().insert( v0, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(1), + from: RelSourceLoc::new(1), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v1, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(2), + from: RelSourceLoc::new(2), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v2, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(3), + from: RelSourceLoc::new(3), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v3, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(4), + from: RelSourceLoc::new(4), label: ValueLabel::new(1), }]), ); pos.func.dfg.values_labels.as_mut().unwrap().insert( v4, crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart { - from: SourceLoc::new(5), + from: RelSourceLoc::new(5), label: ValueLabel::new(1), }]), ); @@ -371,7 +377,7 @@ mod test { // expands during emission. #[test] fn br_table() { - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); sig.returns.push(AbiParam::new(I32)); diff --git a/cranelift/codegen/src/legalizer/heap.rs b/cranelift/codegen/src/legalizer/heap.rs index 91ae3da3c7..16d195932a 100644 --- a/cranelift/codegen/src/legalizer/heap.rs +++ b/cranelift/codegen/src/legalizer/heap.rs @@ -7,7 +7,7 @@ use crate::cursor::{Cursor, FuncCursor}; use crate::flowgraph::ControlFlowGraph; use crate::ir::condcodes::IntCC; use crate::ir::immediates::Uimm32; -use crate::ir::{self, InstBuilder}; +use crate::ir::{self, InstBuilder, RelSourceLoc}; use crate::isa::TargetIsa; /// Expand a `heap_addr` instruction according to the definition of the heap. @@ -212,7 +212,9 @@ fn cast_offset_to_pointer_ty( // Add debug value-label alias so that debuginfo can name the extended // value as the address let loc = pos.srcloc(); + let loc = RelSourceLoc::from_base_offset(pos.func.params.base_srcloc(), loc); pos.func + .stencil .dfg .add_value_label_alias(extended_offset, loc, offset); diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 593a8fc1e9..e244fbafc1 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -89,7 +89,7 @@ pub mod write; pub use crate::entity::packed_option; pub use crate::machinst::buffer::{MachCallSite, MachReloc, MachSrcLoc, MachStackMap, MachTrap}; -pub use crate::machinst::TextSectionBuilder; +pub use crate::machinst::{CompiledCode, TextSectionBuilder}; mod alias_analysis; mod bitset; @@ -116,6 +116,9 @@ mod souper_harvest; pub use crate::result::{CodegenError, CodegenResult, CompileError}; +#[cfg(feature = "incremental-cache")] +pub mod incremental_cache; + /// Even when trace logging is disabled, the trace macro has a significant performance cost so we /// disable it by default. #[macro_export] diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs index 32e6874d4a..4d1708dc4b 100644 --- a/cranelift/codegen/src/machinst/blockorder.rs +++ b/cranelift/codegen/src/machinst/blockorder.rs @@ -501,13 +501,14 @@ mod test { use super::*; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::types::*; - use crate::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; + use crate::ir::UserFuncName; + use crate::ir::{AbiParam, Function, InstBuilder, Signature}; use crate::isa::CallConv; fn build_test_func(n_blocks: usize, edges: &[(usize, usize)]) -> Function { assert!(n_blocks > 0); - let name = ExternalName::testcase("test0"); + let name = UserFuncName::testcase("test0"); let mut sig = Signature::new(CallConv::SystemV); sig.params.push(AbiParam::new(I32)); let mut func = Function::with_name_signature(name, sig); diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 084346cc4e..226feaf899 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -141,7 +141,8 @@ //! semantics below (grep for "Preserves execution semantics"). use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; -use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode}; +use crate::ir::function::FunctionParameters; +use crate::ir::{ExternalName, Opcode, RelSourceLoc, SourceLoc, TrapCode}; use crate::isa::unwind::UnwindInst; use crate::machinst::{ BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst, @@ -155,6 +156,42 @@ use std::mem; use std::string::String; use std::vec::Vec; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "enable-serde")] +pub trait CompilePhase { + type MachSrcLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone; + type SourceLocType: for<'a> Deserialize<'a> + Serialize + core::fmt::Debug + PartialEq + Clone; +} + +#[cfg(not(feature = "enable-serde"))] +pub trait CompilePhase { + type MachSrcLocType: core::fmt::Debug + PartialEq + Clone; + type SourceLocType: core::fmt::Debug + PartialEq + Clone; +} + +/// Status of a compiled artifact that needs patching before being used. +/// +/// Only used internally. +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct Stencil; + +/// Status of a compiled artifact ready to use. +#[derive(Clone, Debug, PartialEq)] +pub struct Final; + +impl CompilePhase for Stencil { + type MachSrcLocType = MachSrcLoc; + type SourceLocType = RelSourceLoc; +} + +impl CompilePhase for Final { + type MachSrcLocType = MachSrcLoc; + type SourceLocType = SourceLoc; +} + /// A buffer of output to be produced, fixed up, and then emitted to a CodeSink /// in bulk. /// @@ -174,14 +211,14 @@ pub struct MachBuffer { /// Any call site records referring to this code. call_sites: SmallVec<[MachCallSite; 16]>, /// Any source location mappings referring to this code. - srclocs: SmallVec<[MachSrcLoc; 64]>, + srclocs: SmallVec<[MachSrcLoc; 64]>, /// Any stack maps referring to this code. stack_maps: SmallVec<[MachStackMap; 8]>, /// Any unwind info at a given location. unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, /// The current source location in progress (after `start_srcloc()` and /// before `end_srcloc()`). This is a (start_offset, src_loc) tuple. - cur_srcloc: Option<(CodeOffset, SourceLoc)>, + cur_srcloc: Option<(CodeOffset, RelSourceLoc)>, /// Known label offsets; `UNKNOWN_LABEL_OFFSET` if unknown. label_offsets: SmallVec<[CodeOffset; 16]>, /// Label aliases: when one label points to an unconditional jump, and that @@ -229,23 +266,43 @@ pub struct MachBuffer { constant_labels: SecondaryMap, } +impl MachBufferFinalized { + pub(crate) fn apply_params(self, params: &FunctionParameters) -> MachBufferFinalized { + MachBufferFinalized { + data: self.data, + relocs: self.relocs, + traps: self.traps, + call_sites: self.call_sites, + srclocs: self + .srclocs + .into_iter() + .map(|srcloc| srcloc.apply_params(params)) + .collect(), + stack_maps: self.stack_maps, + unwind_info: self.unwind_info, + } + } +} + /// A `MachBuffer` once emission is completed: holds generated code and records, /// without fixups. This allows the type to be independent of the backend. -pub struct MachBufferFinalized { +#[derive(PartialEq, Debug, Clone)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MachBufferFinalized { /// The buffer contents, as raw bytes. - data: SmallVec<[u8; 1024]>, + pub(crate) data: SmallVec<[u8; 1024]>, /// Any relocations referring to this code. Note that only *external* /// relocations are tracked here; references to labels within the buffer are /// resolved before emission. - relocs: SmallVec<[MachReloc; 16]>, + pub(crate) relocs: SmallVec<[MachReloc; 16]>, /// Any trap records referring to this code. - traps: SmallVec<[MachTrap; 16]>, + pub(crate) traps: SmallVec<[MachTrap; 16]>, /// Any call site records referring to this code. - call_sites: SmallVec<[MachCallSite; 16]>, + pub(crate) call_sites: SmallVec<[MachCallSite; 16]>, /// Any source location mappings referring to this code. - srclocs: SmallVec<[MachSrcLoc; 64]>, + pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>, /// Any stack maps referring to this code. - stack_maps: SmallVec<[MachStackMap; 8]>, + pub(crate) stack_maps: SmallVec<[MachStackMap; 8]>, /// Any unwind info at a given location. pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, } @@ -1211,7 +1268,7 @@ impl MachBuffer { } /// Finish any deferred emissions and/or fixups. - pub fn finish(mut self) -> MachBufferFinalized { + pub fn finish(mut self) -> MachBufferFinalized { let _tt = timing::vcode_emit_finish(); // Do any optimizations on branches at tail of buffer, as if we @@ -1305,7 +1362,7 @@ impl MachBuffer { /// Set the `SourceLoc` for code from this offset until the offset at the /// next call to `end_srcloc()`. - pub fn start_srcloc(&mut self, loc: SourceLoc) { + pub fn start_srcloc(&mut self, loc: RelSourceLoc) { self.cur_srcloc = Some((self.cur_offset(), loc)); } @@ -1351,9 +1408,9 @@ impl MachBuffer { } } -impl MachBufferFinalized { +impl MachBufferFinalized { /// Get a list of source location mapping tuples in sorted-by-start-offset order. - pub fn get_srclocs_sorted(&self) -> &[MachSrcLoc] { + pub fn get_srclocs_sorted(&self) -> &[T::MachSrcLocType] { &self.srclocs[..] } @@ -1437,7 +1494,8 @@ struct MachLabelFixup { } /// A relocation resulting from a compilation. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachReloc { /// The offset at which the relocation applies, *relative to the /// containing section*. @@ -1451,7 +1509,8 @@ pub struct MachReloc { } /// A trap record resulting from a compilation. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachTrap { /// The offset at which the trap instruction occurs, *relative to the /// containing section*. @@ -1461,7 +1520,8 @@ pub struct MachTrap { } /// A call site record resulting from a compilation. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachCallSite { /// The offset of the call's return address, *relative to the containing section*. pub ret_addr: CodeOffset, @@ -1470,8 +1530,9 @@ pub struct MachCallSite { } /// A source-location mapping resulting from a compilation. -#[derive(Clone, Debug)] -pub struct MachSrcLoc { +#[derive(PartialEq, Debug, Clone)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MachSrcLoc { /// The start of the region of code corresponding to a source location. /// This is relative to the start of the function, not to the start of the /// section. @@ -1481,11 +1542,22 @@ pub struct MachSrcLoc { /// section. pub end: CodeOffset, /// The source location. - pub loc: SourceLoc, + pub loc: T::SourceLocType, +} + +impl MachSrcLoc { + fn apply_params(self, params: &FunctionParameters) -> MachSrcLoc { + MachSrcLoc { + start: self.start, + end: self.end, + loc: self.loc.expand(params.base_srcloc()), + } + } } /// Record of stack map metadata: stack offsets containing references. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct MachStackMap { /// The code offset at which this stack map applies. pub offset: CodeOffset, @@ -1599,7 +1671,10 @@ impl TextSectionBuilder for MachTextSectionBuilder { // We use an actual instruction definition to do tests, so we depend on the `arm64` feature here. #[cfg(all(test, feature = "arm64"))] mod test { + use cranelift_entity::EntityRef as _; + use super::*; + use crate::ir::UserExternalNameRef; use crate::isa::aarch64::inst::xreg; use crate::isa::aarch64::inst::{BranchTarget, CondBrKind, EmitInfo, Inst}; use crate::machinst::MachInstEmit; @@ -1982,9 +2057,17 @@ mod test { buf.add_trap(TrapCode::IntegerOverflow); buf.add_trap(TrapCode::IntegerDivisionByZero); buf.add_call_site(Opcode::Call); - buf.add_reloc(Reloc::Abs4, &ExternalName::user(0, 0), 0); + buf.add_reloc( + Reloc::Abs4, + &ExternalName::User(UserExternalNameRef::new(0)), + 0, + ); buf.put1(3); - buf.add_reloc(Reloc::Abs8, &ExternalName::user(1, 1), 1); + buf.add_reloc( + Reloc::Abs8, + &ExternalName::User(UserExternalNameRef::new(1)), + 1, + ); buf.put1(4); let buf = buf.finish(); diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index eab008925f..73e68c5afd 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -11,10 +11,11 @@ use crate::fx::{FxHashMap, FxHashSet}; use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit}; use crate::ir::{ types::{FFLAGS, IFLAGS}, - ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function, - GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Signature, - SourceLoc, Type, Value, ValueDef, ValueLabelAssignments, ValueLabelStart, + ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, Function, GlobalValue, + GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Signature, Type, Value, + ValueDef, ValueLabelAssignments, ValueLabelStart, }; +use crate::ir::{ExternalName, RelSourceLoc}; use crate::machinst::{ non_writable_value_regs, writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, Reg, VCode, VCodeBuilder, VCodeConstant, VCodeConstantData, @@ -813,10 +814,10 @@ impl<'func, I: VCodeInst> Lower<'func, I> { for &arg in self.f.dfg.block_params(block) { self.emit_value_label_marks_for_value(arg); } - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } - fn finish_ir_inst(&mut self, loc: SourceLoc) { + fn finish_ir_inst(&mut self, loc: RelSourceLoc) { self.vcode.set_srcloc(loc); // The VCodeBuilder builds in reverse order (and reverses at // the end), but `ir_insts` is in forward order, so reverse @@ -874,7 +875,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } self.vcode.add_succ(succ, &branch_arg_vregs[..]); } - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } fn collect_branches_and_targets( @@ -974,7 +975,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { self.vcode.add_succ(succ, &branch_arg_vregs[..]); self.emit(I::gen_jump(MachLabel::from_block(succ))); - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } // Original block body. @@ -986,7 +987,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { if bindex.index() == 0 { // Set up the function with arg vreg inits. self.gen_arg_setup(); - self.finish_ir_inst(SourceLoc::default()); + self.finish_ir_inst(Default::default()); } self.finish_bb(); @@ -1104,8 +1105,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } /// Get the source location for a given instruction. - pub fn srcloc(&self, ir_inst: Inst) -> SourceLoc { - self.f.srclocs[ir_inst] + pub fn srcloc(&self, ir_inst: Inst) -> RelSourceLoc { + self.f.rel_srclocs()[ir_inst] } /// Get the number of inputs to the given IR instruction. diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index fc7bb0abfa..a91dd9ef77 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -45,7 +45,8 @@ //! ``` use crate::binemit::{Addend, CodeInfo, CodeOffset, Reloc, StackMap}; -use crate::ir::{DynamicStackSlot, SourceLoc, StackSlot, Type}; +use crate::ir::function::FunctionParameters; +use crate::ir::{DynamicStackSlot, RelSourceLoc, StackSlot, Type}; use crate::result::CodegenResult; use crate::settings::Flags; use crate::value_label::ValueLabelsRanges; @@ -57,6 +58,9 @@ use regalloc2::{Allocation, VReg}; use smallvec::{smallvec, SmallVec}; use std::string::String; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + #[macro_use] pub mod isle; @@ -263,15 +267,17 @@ pub trait MachInstEmitState: Default + Clone + Debug { /// safepoint. fn pre_safepoint(&mut self, _stack_map: StackMap) {} /// Update the emission state to indicate instructions are associated with a - /// particular SourceLoc. - fn pre_sourceloc(&mut self, _srcloc: SourceLoc) {} + /// particular RelSourceLoc. + fn pre_sourceloc(&mut self, _srcloc: RelSourceLoc) {} } /// The result of a `MachBackend::compile_function()` call. Contains machine /// code (as bytes) and a disassembly, if requested. -pub struct CompiledCode { +#[derive(PartialEq, Debug, Clone)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct CompiledCodeBase { /// Machine code. - pub buffer: MachBufferFinalized, + pub buffer: MachBufferFinalized, /// Size of stack frame, in bytes. pub frame_size: u32, /// Disassembly, if requested. @@ -296,7 +302,23 @@ pub struct CompiledCode { pub bb_edges: Vec<(CodeOffset, CodeOffset)>, } -impl CompiledCode { +impl CompiledCodeStencil { + /// Apply function parameters to finalize a stencil into its final form. + pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode { + CompiledCode { + buffer: self.buffer.apply_params(params), + frame_size: self.frame_size, + disasm: self.disasm, + value_labels_ranges: self.value_labels_ranges, + sized_stackslot_offsets: self.sized_stackslot_offsets, + dynamic_stackslot_offsets: self.dynamic_stackslot_offsets, + bb_starts: self.bb_starts, + bb_edges: self.bb_edges, + } + } +} + +impl CompiledCodeBase { /// Get a `CodeInfo` describing section sizes from this compilation result. pub fn code_info(&self) -> CodeInfo { CodeInfo { @@ -310,6 +332,15 @@ impl CompiledCode { } } +/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it. +/// +/// Only used internally, in a transient manner, for the incremental compilation cache. +pub type CompiledCodeStencil = CompiledCodeBase; + +/// `CompiledCode` in its final form (i.e. after `FunctionParameters` have been applied), ready for +/// consumption. +pub type CompiledCode = CompiledCodeBase; + /// An object that can be used to create the text section of an executable. /// /// This primarily handles resolving relative relocations at diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index 1db50be4d1..c6e37b8517 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -19,9 +19,8 @@ use crate::fx::FxHashMap; use crate::fx::FxHashSet; -use crate::ir::{ - self, types, Constant, ConstantData, DynamicStackSlot, LabelValueLoc, SourceLoc, ValueLabel, -}; +use crate::ir::RelSourceLoc; +use crate::ir::{self, types, Constant, ConstantData, DynamicStackSlot, LabelValueLoc, ValueLabel}; use crate::machinst::*; use crate::timing; use crate::trace; @@ -90,7 +89,7 @@ pub struct VCode { /// Source locations for each instruction. (`SourceLoc` is a `u32`, so it is /// reasonable to keep one of these per instruction.) - srclocs: Vec, + srclocs: Vec, /// Entry block. entry: BlockIndex, @@ -261,7 +260,7 @@ pub struct VCodeBuilder { branch_block_arg_succ_start: usize, /// Current source location. - cur_srcloc: SourceLoc, + cur_srcloc: RelSourceLoc, /// Debug-value label in-progress map, keyed by label. For each /// label, we keep disjoint ranges mapping to vregs. We'll flatten @@ -296,7 +295,7 @@ impl VCodeBuilder { succ_start: 0, block_params_start: 0, branch_block_arg_succ_start: 0, - cur_srcloc: SourceLoc::default(), + cur_srcloc: Default::default(), debug_info: FxHashMap::default(), } } @@ -399,7 +398,7 @@ impl VCodeBuilder { } /// Set the current source location. - pub fn set_srcloc(&mut self, srcloc: SourceLoc) { + pub fn set_srcloc(&mut self, srcloc: RelSourceLoc) { self.cur_srcloc = srcloc; } @@ -847,8 +846,8 @@ impl VCode { // Is this the first block? Emit the prologue directly if so. if block == self.entry { trace!(" -> entry block"); - buffer.start_srcloc(SourceLoc::default()); - state.pre_sourceloc(SourceLoc::default()); + buffer.start_srcloc(Default::default()); + state.pre_sourceloc(Default::default()); for inst in &prologue_insts { do_emit(&inst, &[], &mut disasm, &mut buffer, &mut state); } @@ -919,7 +918,7 @@ impl VCode { buffer.start_srcloc(srcloc); cur_srcloc = Some(srcloc); } - state.pre_sourceloc(cur_srcloc.unwrap_or(SourceLoc::default())); + state.pre_sourceloc(cur_srcloc.unwrap_or_default()); // If this is a safepoint, compute a stack map // and pass it to the emit state. diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs index 9c3c7b2262..ef9556d021 100644 --- a/cranelift/codegen/src/settings.rs +++ b/cranelift/codegen/src/settings.rs @@ -548,6 +548,7 @@ probestack_func_adjusts_sp = false enable_jump_tables = true enable_heap_access_spectre_mitigation = true enable_table_access_spectre_mitigation = true +enable_incremental_compilation_cache_checks = false "# ); assert_eq!(f.opt_level(), super::OptLevel::None); diff --git a/cranelift/codegen/src/timing.rs b/cranelift/codegen/src/timing.rs index f21a68fa33..dcefde2da4 100644 --- a/cranelift/codegen/src/timing.rs +++ b/cranelift/codegen/src/timing.rs @@ -52,6 +52,8 @@ define_passes! { verify_flags: "Verify CPU flags", compile: "Compilation passes", + try_incremental_cache: "Try loading from incremental cache", + store_incremental_cache: "Store in incremental cache", flowgraph: "Control flow graph", domtree: "Dominator tree", loop_analysis: "Loop analysis", diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index e3be9b334a..938dfa9059 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -1845,8 +1845,8 @@ mod tests { imm: 0.into(), }); func.layout.append_inst(nullary_with_bad_opcode, block0); - func.layout.append_inst( - func.dfg.make_inst(InstructionData::Jump { + func.stencil.layout.append_inst( + func.stencil.dfg.make_inst(InstructionData::Jump { opcode: Opcode::Jump, destination: block0, args: EntityList::default(), diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs index d0b6ad8132..abdc5992ad 100644 --- a/cranelift/codegen/src/write.rs +++ b/cranelift/codegen/src/write.rs @@ -80,7 +80,12 @@ pub trait FuncWriter { for (fnref, ext_func) in &func.dfg.ext_funcs { if ext_func.signature != SigRef::reserved_value() { any = true; - self.write_entity_definition(w, func, fnref.into(), ext_func)?; + self.write_entity_definition( + w, + func, + fnref.into(), + &ext_func.display(Some(&func.params)), + )?; } } @@ -254,7 +259,7 @@ fn decorate_block( block: Block, ) -> fmt::Result { // Indent all instructions if any srclocs are present. - let indent = if func.srclocs.is_empty() { 4 } else { 36 }; + let indent = if func.rel_srclocs().is_empty() { 4 } else { 36 }; func_w.write_block_header(w, func, block, indent)?; for a in func.dfg.block_params(block).iter().cloned() { @@ -335,7 +340,7 @@ fn write_instruction( let mut s = String::with_capacity(16); // Source location goes first. - let srcloc = func.srclocs[inst]; + let srcloc = func.srcloc(inst); if !srcloc.is_default() { write!(s, "{} ", srcloc)?; } @@ -572,7 +577,7 @@ impl<'a> fmt::Display for DisplayValuesWithDelimiter<'a> { mod tests { use crate::cursor::{Cursor, CursorPosition, FuncCursor}; use crate::ir::types; - use crate::ir::{ExternalName, Function, InstBuilder, StackSlotData, StackSlotKind}; + use crate::ir::{Function, InstBuilder, StackSlotData, StackSlotKind, UserFuncName}; use alloc::string::ToString; #[test] @@ -580,7 +585,7 @@ mod tests { let mut f = Function::new(); assert_eq!(f.to_string(), "function u0:0() fast {\n}\n"); - f.name = ExternalName::testcase("foo"); + f.name = UserFuncName::testcase("foo"); assert_eq!(f.to_string(), "function %foo() fast {\n}\n"); f.create_sized_stack_slot(StackSlotData::new(StackSlotKind::ExplicitSlot, 4)); diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs index d4a057bf4e..4efed334ff 100644 --- a/cranelift/entity/src/list.rs +++ b/cranelift/entity/src/list.rs @@ -62,7 +62,7 @@ use serde::{Deserialize, Serialize}; /// /// The index stored in an `EntityList` points to part 2, the list elements. The value 0 is /// reserved for the empty list which isn't allocated in the vector. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Hash)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct EntityList { index: u32, @@ -90,6 +90,20 @@ pub struct ListPool { free: Vec, } +impl PartialEq for ListPool { + fn eq(&self, other: &Self) -> bool { + // ignore the free list + self.data == other.data + } +} + +impl core::hash::Hash for ListPool { + fn hash(&self, state: &mut H) { + // ignore the free list + self.data.hash(state); + } +} + /// Lists are allocated in sizes that are powers of two, starting from 4. /// Each power of two is assigned a size class number, so the size is `4 << SizeClass`. type SizeClass = u8; diff --git a/cranelift/entity/src/map.rs b/cranelift/entity/src/map.rs index 67cdc11004..332cd061b7 100644 --- a/cranelift/entity/src/map.rs +++ b/cranelift/entity/src/map.rs @@ -23,7 +23,7 @@ use serde::{ /// /// The map does not track if an entry for a key has been inserted or not. Instead it behaves as if /// all keys have a default entry from the beginning. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Hash)] pub struct SecondaryMap where K: EntityRef, diff --git a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif index 2181f9e41e..0dfb996593 100644 --- a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif +++ b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif @@ -14,7 +14,7 @@ block0(v0: i64): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; autiasp ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/call.clif b/cranelift/filetests/filetests/isa/aarch64/call.clif index b1c6d211aa..45807c974d 100644 --- a/cranelift/filetests/filetests/isa/aarch64/call.clif +++ b/cranelift/filetests/filetests/isa/aarch64/call.clif @@ -14,7 +14,7 @@ block0(v0: i64): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; ret @@ -30,7 +30,7 @@ block0(v0: i32): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; ret @@ -54,7 +54,7 @@ block0(v0: i32): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; ldp fp, lr, [sp], #16 ; ret @@ -91,7 +91,7 @@ block0(v0: i8): ; movz x6, #42 ; movz x7, #42 ; strb w15, [sp] -; ldr x15, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; add sp, sp, #16 ; virtual_sp_offset_adjust -16 @@ -140,25 +140,25 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x9, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x9 ; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x11, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x13, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x13 ; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x1, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x1 ; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 @@ -184,25 +184,25 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x9, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x9 ; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x11, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x13, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x13 ; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x1, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x1 ; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 @@ -232,25 +232,25 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x9, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x9 ; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x11, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x13, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x13 ; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x15, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x15 ; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x1, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x1 ; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase { length: 2, ascii: [103, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 2, ascii: [103, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 @@ -283,7 +283,7 @@ block0(v0: i64): ; movz x0, #42 ; movz x2, #42 ; mov x1, x7 -; ldr x10, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x10, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x10 ; ldp fp, lr, [sp], #16 ; ret @@ -315,7 +315,7 @@ block0(v0: i64): ; movz x3, #42 ; movz x0, #42 ; mov x2, x7 -; ldr x10, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x10, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x10 ; ldp fp, lr, [sp], #16 ; ret @@ -347,7 +347,7 @@ block0(v0: i64): ; movz x2, #42 ; movz x0, #42 ; mov x1, x7 -; ldr x10, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x10, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x10 ; ldp fp, lr, [sp], #16 ; ret @@ -388,7 +388,7 @@ block0(v0: i128, v1: i64): ; mov x6, x14 ; str x13, [sp] ; str x15, [sp, #8] -; ldr x7, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x7, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x7 ; add sp, sp, #16 ; virtual_sp_offset_adjust -16 @@ -431,7 +431,7 @@ block0(v0: i128, v1: i64): ; mov x6, x14 ; str x13, [sp] ; str x15, [sp, #8] -; ldr x7, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 49, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x7, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 49, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x7 ; add sp, sp, #16 ; virtual_sp_offset_adjust -16 @@ -477,7 +477,7 @@ block0(v0: i64): ; mov fp, sp ; block0: ; mov x8, x0 -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; mov x0, x8 ; ldp fp, lr, [sp], #16 @@ -496,7 +496,7 @@ block0(v0: i64): ; str x24, [sp, #-16]! ; block0: ; mov x24, x8 -; ldr x5, 8 ; b 12 ; data TestCase { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x5, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x5 ; mov x8, x24 ; ldr x24, [sp], #16 diff --git a/cranelift/filetests/filetests/isa/aarch64/reftypes.clif b/cranelift/filetests/filetests/isa/aarch64/reftypes.clif index 7253ae6c9a..4920258760 100644 --- a/cranelift/filetests/filetests/isa/aarch64/reftypes.clif +++ b/cranelift/filetests/filetests/isa/aarch64/reftypes.clif @@ -69,7 +69,7 @@ block3(v7: r64, v8: r64): ; block0: ; str x1, [sp, #16] ; str x0, [sp, #8] -; ldr x3, 8 ; b 12 ; data TestCase { length: 1, ascii: [102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x3, 8 ; b 12 ; data TestCase(TestcaseName { length: 1, ascii: [102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x3 ; mov x9, sp ; ldr x11, [sp, #8] diff --git a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif index 993d63c3cc..ceb56c34dc 100644 --- a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif +++ b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif @@ -42,7 +42,7 @@ block0(v0: i64): ; subs xzr, sp, x0, UXTX ; b.hs 8 ; udf ; block0: -; ldr x2, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x2, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x2 ; ldp fp, lr, [sp], #16 ; ret @@ -65,7 +65,7 @@ block0(v0: i64): ; subs xzr, sp, x16, UXTX ; b.hs 8 ; udf ; block0: -; ldr x2, 8 ; b 12 ; data TestCase { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x2, 8 ; b 12 ; data TestCase(TestcaseName { length: 3, ascii: [102, 111, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; blr x2 ; ldp fp, lr, [sp], #16 ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif b/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif index b9eecef1bd..ba48ada8d0 100644 --- a/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif +++ b/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif @@ -11,6 +11,6 @@ block0: } ; block0: -; ldr x0, 8 ; b 12 ; data TestCase { length: 9, ascii: [109, 121, 95, 103, 108, 111, 98, 97, 108, 0, 0, 0, 0, 0, 0, 0] } + 0 +; ldr x0, 8 ; b 12 ; data TestCase(TestcaseName { length: 9, ascii: [109, 121, 95, 103, 108, 111, 98, 97, 108, 0, 0, 0, 0, 0, 0, 0] }) + 0 ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif b/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif index d1657b231a..11f87179df 100644 --- a/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif +++ b/cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif @@ -19,7 +19,7 @@ block0(v0: i32): ; stp d8, d9, [sp, #-16]! ; block0: ; mov x25, x0 -; x0 = elf_tls_get_addr u1:0 +; x0 = elf_tls_get_addr userextname0 ; mov x1, x0 ; mov x0, x25 ; ldp d8, d9, [sp], #16 diff --git a/cranelift/filetests/filetests/isa/s390x/struct-arg.clif b/cranelift/filetests/filetests/isa/s390x/struct-arg.clif index cc28db4bc0..c05a17fd61 100644 --- a/cranelift/filetests/filetests/isa/s390x/struct-arg.clif +++ b/cranelift/filetests/filetests/isa/s390x/struct-arg.clif @@ -39,7 +39,7 @@ block0(v0: i64): ; block0: ; mvc 160(63,%r15), 0(%r2) ; la %r2, 160(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r14, %r15, 336(%r15) ; br %r14 @@ -57,7 +57,7 @@ block0(v0: i64, v1: i64): ; block0: ; mvc 160(63,%r15), 0(%r3) ; la %r3, 160(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r14, %r15, 336(%r15) ; br %r14 @@ -91,7 +91,7 @@ block0(v0: i64, v1: i64, v2: i64): ; mvc 416(63,%r15), 0(%r4) ; la %r3, 160(%r15) ; la %r4, 416(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r14, %r15, 592(%r15) ; br %r14 @@ -118,7 +118,7 @@ block0(v0: i64, v1: i64, v2: i64): ; lgr %r2, %r7 ; la %r3, 160(%r15) ; la %r4, 1184(%r15) -; brasl %r14, u0:0 +; brasl %r14, userextname0 ; lmg %r7, %r15, 1304(%r15) ; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif index fa943e746e..e0f598a96f 100644 --- a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif +++ b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif @@ -15,8 +15,8 @@ block0(v0: i32): ; virtual_sp_offset_adjust 160 ; block0: ; larl %r12, %ElfGlobalOffsetTable + 0 -; bras %r1, 12 ; data u1:0@tlsgd ; lg %r2, 0(%r1) -; brasl %r14, %ElfTlsGetOffset:tls_gdcall:u1:0 +; bras %r1, 12 ; data userextname0@tlsgd ; lg %r2, 0(%r1) +; brasl %r14, %ElfTlsGetOffset:tls_gdcall:userextname0 ; ear %r3, %a0 ; sllg %r4, %r3, 32 ; ear %r4, %a1 diff --git a/cranelift/filetests/filetests/isa/x64/struct-arg.clif b/cranelift/filetests/filetests/isa/x64/struct-arg.clif index 5ae2c3fd03..bd877b4cae 100644 --- a/cranelift/filetests/filetests/isa/x64/struct-arg.clif +++ b/cranelift/filetests/filetests/isa/x64/struct-arg.clif @@ -54,7 +54,7 @@ block0(v0: i64): ; movl $64, %edx ; load_ext_name %Memcpy+0, %rcx ; call *%rcx -; call User { namespace: 0, index: 0 } +; call User(userextname0) ; addq %rsp, $64, %rsp ; virtual_sp_offset_adjust -64 ; movq %rbp, %rsp @@ -82,7 +82,7 @@ block0(v0: i64, v1: i64): ; load_ext_name %Memcpy+0, %rcx ; call *%rcx ; movq %r12, %rdi -; call User { namespace: 0, index: 0 } +; call User(userextname0) ; addq %rsp, $64, %rsp ; virtual_sp_offset_adjust -64 ; movq 0(%rsp), %r12 @@ -139,7 +139,7 @@ block0(v0: i64, v1: i64, v2: i64): ; load_ext_name %Memcpy+0, %rcx ; call *%rcx ; movq %r14, %rdi -; call User { namespace: 0, index: 0 } +; call User(userextname0) ; addq %rsp, $192, %rsp ; virtual_sp_offset_adjust -192 ; movq 0(%rsp), %rbx diff --git a/cranelift/filetests/filetests/isa/x64/tls_coff.clif b/cranelift/filetests/filetests/isa/x64/tls_coff.clif index ad05bd5a52..95421d8c78 100644 --- a/cranelift/filetests/filetests/isa/x64/tls_coff.clif +++ b/cranelift/filetests/filetests/isa/x64/tls_coff.clif @@ -14,7 +14,8 @@ block0(v0: i32): ; pushq %rbp ; movq %rsp, %rbp ; block0: -; %rax = coff_tls_get_addr User { namespace: 1, index: 0 } +; %rax = coff_tls_get_addr User(userextname0) ; movq %rbp, %rsp ; popq %rbp ; ret + diff --git a/cranelift/filetests/filetests/isa/x64/tls_elf.clif b/cranelift/filetests/filetests/isa/x64/tls_elf.clif index c7286e159f..cbee527f99 100644 --- a/cranelift/filetests/filetests/isa/x64/tls_elf.clif +++ b/cranelift/filetests/filetests/isa/x64/tls_elf.clif @@ -13,7 +13,7 @@ block0(v0: i32): ; pushq %rbp ; movq %rsp, %rbp ; block0: -; %rax = elf_tls_get_addr User { namespace: 1, index: 0 } +; %rax = elf_tls_get_addr User(userextname0) ; movq %rbp, %rsp ; popq %rbp ; ret diff --git a/cranelift/filetests/filetests/parser/memory.clif b/cranelift/filetests/filetests/parser/memory.clif index ecf872d64f..abe059c0fb 100644 --- a/cranelift/filetests/filetests/parser/memory.clif +++ b/cranelift/filetests/filetests/parser/memory.clif @@ -40,7 +40,7 @@ function %symbol() -> i32 { gv0 = symbol %something ; check: gv0 = symbol %something gv1 = symbol u8:9 - ; check: gv1 = symbol u8:9 + ; check: gv1 = symbol userextname0 block0: v0 = global_value.i32 gv0 ; check: v0 = global_value.i32 gv0 diff --git a/cranelift/filetests/src/function_runner.rs b/cranelift/filetests/src/function_runner.rs index 0502843b6b..b4484c72b4 100644 --- a/cranelift/filetests/src/function_runner.rs +++ b/cranelift/filetests/src/function_runner.rs @@ -78,11 +78,11 @@ impl SingleFunctionCompiler { let mut module = JITModule::new(builder); let mut ctx = module.make_context(); - let name = format!("{}", function.name); + let name = function.name.to_string(); let func_id = module.declare_function(&name, Linkage::Local, &function.signature)?; // Build and declare the trampoline in the module - let trampoline_name = format!("{}", trampoline.name); + let trampoline_name = trampoline.name.to_string(); let trampoline_id = module.declare_function(&trampoline_name, Linkage::Local, &trampoline.signature)?; @@ -260,7 +260,7 @@ fn make_trampoline(signature: &ir::Signature, isa: &dyn TargetIsa) -> Function { wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); // Add the `callee_address` parameter. wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); // Add the `values_vec` parameter. - let mut func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wrapper_sig); + let mut func = ir::Function::with_name_signature(ir::UserFuncName::default(), wrapper_sig); // The trampoline has a single block filled with loads, one call to callee_address, and some loads. let mut builder_context = FunctionBuilderContext::new(); diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs index 644c38e729..7a20d5c6e5 100644 --- a/cranelift/frontend/src/frontend.rs +++ b/cranelift/frontend/src/frontend.rs @@ -10,8 +10,8 @@ use cranelift_codegen::ir::{ types, AbiParam, Block, DataFlowGraph, DynamicStackSlot, DynamicStackSlotData, ExtFuncData, ExternalName, FuncRef, Function, GlobalValue, GlobalValueData, Heap, HeapData, Inst, InstBuilder, InstBuilderBase, InstructionData, JumpTable, JumpTableData, LibCall, MemFlags, - SigRef, Signature, StackSlot, StackSlotData, Type, Value, ValueLabel, ValueLabelAssignments, - ValueLabelStart, + RelSourceLoc, SigRef, Signature, StackSlot, StackSlotData, Type, Value, ValueLabel, + ValueLabelAssignments, ValueLabelStart, }; use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_codegen::packed_option::PackedOption; @@ -112,7 +112,7 @@ impl<'short, 'long> InstBuilderBase<'short> for FuncInstBuilder<'short, 'long> { self.builder.func.dfg.make_inst_results(inst, ctrl_typevar); self.builder.func.layout.append_inst(inst, self.block); if !self.builder.srcloc.is_default() { - self.builder.func.srclocs[inst] = self.builder.srcloc; + self.builder.func.set_srcloc(inst, self.builder.srcloc); } if data.opcode().is_branch() { @@ -469,11 +469,11 @@ impl<'a> FunctionBuilder<'a> { /// /// This will not do anything unless `func.dfg.collect_debug_info` is called first. pub fn set_val_label(&mut self, val: Value, label: ValueLabel) { - if let Some(values_labels) = self.func.dfg.values_labels.as_mut() { - use crate::hash_map::Entry; + if let Some(values_labels) = self.func.stencil.dfg.values_labels.as_mut() { + use alloc::collections::btree_map::Entry; let start = ValueLabelStart { - from: self.srcloc, + from: RelSourceLoc::from_base_offset(self.func.params.base_srcloc(), self.srcloc), label, }; @@ -574,9 +574,12 @@ impl<'a> FunctionBuilder<'a> { // These parameters count as "user" parameters here because they aren't // inserted by the SSABuilder. let user_param_count = &mut self.func_ctx.blocks[block].user_param_count; - for argtyp in &self.func.signature.params { + for argtyp in &self.func.stencil.signature.params { *user_param_count += 1; - self.func.dfg.append_block_param(block, argtyp.value_type); + self.func + .stencil + .dfg + .append_block_param(block, argtyp.value_type); } } @@ -587,9 +590,12 @@ impl<'a> FunctionBuilder<'a> { // These parameters count as "user" parameters here because they aren't // inserted by the SSABuilder. let user_param_count = &mut self.func_ctx.blocks[block].user_param_count; - for argtyp in &self.func.signature.returns { + for argtyp in &self.func.stencil.signature.returns { *user_param_count += 1; - self.func.dfg.append_block_param(block, argtyp.value_type); + self.func + .stencil + .dfg + .append_block_param(block, argtyp.value_type); } } @@ -1037,11 +1043,11 @@ impl<'a> FunctionBuilder<'a> { if let Some(small_type) = size.try_into().ok().and_then(Type::int_with_byte_size) { if let Equal | NotEqual = zero_cc { let mut left_flags = flags; - if size == left_align.get().into() { + if size == left_align.get() as u64 { left_flags.set_aligned(); } let mut right_flags = flags; - if size == right_align.get().into() { + if size == right_align.get() as u64 { right_flags.set_aligned(); } let left_val = self.ins().load(small_type, left_flags, left, 0); @@ -1105,10 +1111,8 @@ mod tests { use alloc::string::ToString; use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::condcodes::IntCC; - use cranelift_codegen::ir::types::*; - use cranelift_codegen::ir::{ - AbiParam, ExternalName, Function, InstBuilder, MemFlags, Signature, Value, - }; + use cranelift_codegen::ir::{types::*, UserFuncName}; + use cranelift_codegen::ir::{AbiParam, Function, InstBuilder, MemFlags, Signature, Value}; use cranelift_codegen::isa::{CallConv, TargetFrontendConfig, TargetIsa}; use cranelift_codegen::settings; use cranelift_codegen::verifier::verify_function; @@ -1120,7 +1124,7 @@ mod tests { sig.params.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1239,7 +1243,7 @@ mod tests { sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1288,7 +1292,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1342,7 +1346,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1399,7 +1403,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1439,7 +1443,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1498,7 +1502,7 @@ block0: sig.returns.push(AbiParam::new(I32)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1711,7 +1715,7 @@ block0: sig.returns.push(AbiParam::new(B1)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1750,7 +1754,7 @@ block0: sig.returns.push(AbiParam::new(F32X4)); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); @@ -1804,7 +1808,7 @@ block0: let sig = Signature::new(CallConv::SystemV); let mut fn_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::testcase("sample"), sig); + let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig); { let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx); diff --git a/cranelift/frontend/src/lib.rs b/cranelift/frontend/src/lib.rs index 73928d6d79..9a07687055 100644 --- a/cranelift/frontend/src/lib.rs +++ b/cranelift/frontend/src/lib.rs @@ -69,7 +69,7 @@ //! //! use cranelift_codegen::entity::EntityRef; //! use cranelift_codegen::ir::types::*; -//! use cranelift_codegen::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; +//! use cranelift_codegen::ir::{AbiParam, UserFuncName, Function, InstBuilder, Signature}; //! use cranelift_codegen::isa::CallConv; //! use cranelift_codegen::settings; //! use cranelift_codegen::verifier::verify_function; @@ -79,7 +79,7 @@ //! sig.returns.push(AbiParam::new(I32)); //! sig.params.push(AbiParam::new(I32)); //! let mut fn_builder_ctx = FunctionBuilderContext::new(); -//! let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); +//! let mut func = Function::with_name_signature(UserFuncName::user(0, 0), sig); //! { //! let mut builder = FunctionBuilder::new(&mut func, &mut fn_builder_ctx); //! @@ -189,9 +189,9 @@ extern crate alloc; extern crate std; #[cfg(not(feature = "std"))] -use hashbrown::{hash_map, HashMap}; +use hashbrown::HashMap; #[cfg(feature = "std")] -use std::collections::{hash_map, HashMap}; +use std::collections::HashMap; pub use crate::frontend::{FunctionBuilder, FunctionBuilderContext}; pub use crate::switch::Switch; diff --git a/cranelift/frontend/src/switch.rs b/cranelift/frontend/src/switch.rs index 3de30d1576..232337474f 100644 --- a/cranelift/frontend/src/switch.rs +++ b/cranelift/frontend/src/switch.rs @@ -14,13 +14,13 @@ type EntryIndex = u128; /// /// ```rust /// # use cranelift_codegen::ir::types::*; -/// # use cranelift_codegen::ir::{ExternalName, Function, Signature, InstBuilder}; +/// # use cranelift_codegen::ir::{UserFuncName, Function, Signature, InstBuilder}; /// # use cranelift_codegen::isa::CallConv; /// # use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Switch}; /// # /// # let mut sig = Signature::new(CallConv::SystemV); /// # let mut fn_builder_ctx = FunctionBuilderContext::new(); -/// # let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); +/// # let mut func = Function::with_name_signature(UserFuncName::user(0, 0), sig); /// # let mut builder = FunctionBuilder::new(&mut func, &mut fn_builder_ctx); /// # /// # let entry = builder.create_block(); diff --git a/cranelift/fuzzgen/src/config.rs b/cranelift/fuzzgen/src/config.rs index 0ea0ae1408..abc3b330aa 100644 --- a/cranelift/fuzzgen/src/config.rs +++ b/cranelift/fuzzgen/src/config.rs @@ -21,12 +21,15 @@ pub struct Config { /// decides to insert more. pub jump_tables_per_function: RangeInclusive, pub jump_table_entries: RangeInclusive, + /// The Switch API specializes either individual blocks or contiguous ranges. /// In `switch_cases` we decide to produce either a single block or a range. /// The size of the range is controlled by `switch_max_range_size`. pub switch_cases: RangeInclusive, pub switch_max_range_size: RangeInclusive, + pub funcrefs_per_function: RangeInclusive, + /// Stack slots. /// The combination of these two determines stack usage per function pub static_stack_slots_per_function: RangeInclusive, @@ -49,6 +52,7 @@ impl Default for Config { switch_cases: 0..=64, // Ranges smaller than 2 don't make sense. switch_max_range_size: 2..=32, + funcrefs_per_function: 0..=8, static_stack_slots_per_function: 0..=8, static_stack_slot_size: 0..=128, } diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs index 68e9af330b..522130d81d 100644 --- a/cranelift/fuzzgen/src/function_generator.rs +++ b/cranelift/fuzzgen/src/function_generator.rs @@ -2,14 +2,14 @@ use crate::codegen::ir::{ArgumentExtension, ArgumentPurpose, ValueList}; use crate::config::Config; use anyhow::Result; use arbitrary::{Arbitrary, Unstructured}; -use cranelift::codegen::ir::types::*; +use cranelift::codegen::ir::{types::*, FuncRef, LibCall, UserExternalName, UserFuncName}; use cranelift::codegen::ir::{ AbiParam, Block, ExternalName, Function, JumpTable, Opcode, Signature, StackSlot, Type, Value, }; use cranelift::codegen::isa::CallConv; use cranelift::frontend::{FunctionBuilder, FunctionBuilderContext, Switch, Variable}; use cranelift::prelude::{ - EntityRef, InstBuilder, IntCC, JumpTableData, StackSlotData, StackSlotKind, + EntityRef, ExtFuncData, InstBuilder, IntCC, JumpTableData, StackSlotData, StackSlotKind, }; use std::collections::HashMap; use std::ops::RangeInclusive; @@ -41,6 +41,25 @@ fn insert_opcode( Ok(()) } +fn insert_call( + fgen: &mut FunctionGenerator, + builder: &mut FunctionBuilder, + opcode: Opcode, + _args: &'static [Type], + _rets: &'static [Type], +) -> Result<()> { + assert_eq!(opcode, Opcode::Call, "only call handled at the moment"); + let (sig, func_ref) = fgen.u.choose(&fgen.func_refs)?.clone(); + + let actuals = fgen.generate_values_for_signature( + builder, + sig.params.iter().map(|abi_param| abi_param.value_type), + )?; + + builder.ins().call(func_ref, &actuals); + Ok(()) +} + fn insert_stack_load( fgen: &mut FunctionGenerator, builder: &mut FunctionBuilder, @@ -369,6 +388,8 @@ const OPCODE_SIGNATURES: &'static [( (Opcode::F64const, &[], &[F64], insert_const), // Bool Consts (Opcode::Bconst, &[], &[B1], insert_const), + // Call + (Opcode::Call, &[], &[], insert_call), ]; pub struct FunctionGenerator<'r, 'data> @@ -380,6 +401,8 @@ where vars: Vec<(Type, Variable)>, blocks: Vec<(Block, BlockSignature)>, jump_tables: Vec, + func_refs: Vec<(Signature, FuncRef)>, + next_func_index: u32, static_stack_slots: Vec, } @@ -394,7 +417,9 @@ where vars: vec![], blocks: vec![], jump_tables: vec![], + func_refs: vec![], static_stack_slots: vec![], + next_func_index: 0, } } @@ -739,6 +764,42 @@ where Ok(()) } + fn generate_funcrefs(&mut self, builder: &mut FunctionBuilder) -> Result<()> { + for _ in 0..self.param(&self.config.funcrefs_per_function)? { + let (ext_name, sig) = if self.u.arbitrary::()? { + let func_index = self.next_func_index; + self.next_func_index = self.next_func_index.wrapping_add(1); + let user_func_ref = builder + .func + .declare_imported_user_function(UserExternalName { + namespace: 0, + index: func_index, + }); + let name = ExternalName::User(user_func_ref); + let signature = self.generate_signature()?; + (name, signature) + } else { + // Use udivi64 as an example of a libcall function. + let mut signature = Signature::new(CallConv::Fast); + signature.params.push(AbiParam::new(I64)); + signature.params.push(AbiParam::new(I64)); + signature.returns.push(AbiParam::new(I64)); + (ExternalName::LibCall(LibCall::UdivI64), signature) + }; + + let sig_ref = builder.import_signature(sig.clone()); + let func_ref = builder.import_function(ExtFuncData { + name: ext_name, + signature: sig_ref, + colocated: self.u.arbitrary()?, + }); + + self.func_refs.push((sig, func_ref)); + } + + Ok(()) + } + fn generate_stack_slots(&mut self, builder: &mut FunctionBuilder) -> Result<()> { for _ in 0..self.param(&self.config.static_stack_slots_per_function)? { let bytes = self.param(&self.config.static_stack_slot_size)? as u32; @@ -865,7 +926,7 @@ where let sig = self.generate_signature()?; let mut fn_builder_ctx = FunctionBuilderContext::new(); - let mut func = Function::with_name_signature(ExternalName::user(0, 1), sig.clone()); + let mut func = Function::with_name_signature(UserFuncName::user(0, 1), sig.clone()); let mut builder = FunctionBuilder::new(&mut func, &mut fn_builder_ctx); @@ -873,6 +934,7 @@ where // Function preamble self.generate_jumptables(&mut builder)?; + self.generate_funcrefs(&mut builder)?; self.generate_stack_slots(&mut builder)?; // Main instruction generation loop diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs index 26151697a2..8dce6e0213 100644 --- a/cranelift/fuzzgen/src/lib.rs +++ b/cranelift/fuzzgen/src/lib.rs @@ -15,6 +15,19 @@ mod function_generator; pub type TestCaseInput = Vec; +/// Simple wrapper to generate a single Cranelift `Function`. +#[derive(Debug)] +pub struct SingleFunction(pub Function); + +impl<'a> Arbitrary<'a> for SingleFunction { + fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { + FuzzGen::new(u) + .generate_func() + .map_err(|_| arbitrary::Error::IncorrectFormat) + .map(Self) + } +} + pub struct TestCase { pub func: Function, /// Generate multiple test inputs for each test case. @@ -171,12 +184,14 @@ where Ok(ctx.func) } - pub fn generate_test(mut self) -> Result { + fn generate_func(&mut self) -> Result { let func = FunctionGenerator::new(&mut self.u, &self.config).generate()?; + self.run_func_passes(func) + } + + pub fn generate_test(mut self) -> Result { + let func = self.generate_func()?; let inputs = self.generate_test_inputs(&func.signature)?; - - let func = self.run_func_passes(func)?; - Ok(TestCase { func, inputs }) } } diff --git a/cranelift/interpreter/src/environment.rs b/cranelift/interpreter/src/environment.rs index 6c7a9a0b26..ab10dc9dba 100644 --- a/cranelift/interpreter/src/environment.rs +++ b/cranelift/interpreter/src/environment.rs @@ -63,18 +63,20 @@ impl<'a> FunctionStore<'a> { /// currently it retrieves the function name as a string and performs string matching. fn get_function_name(func_ref: FuncRef, function: &Function) -> String { function + .stencil .dfg .ext_funcs .get(func_ref) .expect("function to exist") .name + .display(Some(&function.params)) .to_string() } #[cfg(test)] mod tests { use super::*; - use cranelift_codegen::ir::{ExternalName, Signature}; + use cranelift_codegen::ir::{Signature, UserFuncName}; use cranelift_codegen::isa::CallConv; #[test] @@ -95,7 +97,7 @@ mod tests { #[test] fn from() { - let name = ExternalName::testcase("test"); + let name = UserFuncName::testcase("test"); let signature = Signature::new(CallConv::Fast); let func = &Function::with_name_signature(name, signature); let env: FunctionStore = func.into(); diff --git a/cranelift/jit/examples/jit-minimal.rs b/cranelift/jit/examples/jit-minimal.rs index dc24df7c86..f5e70e9b62 100644 --- a/cranelift/jit/examples/jit-minimal.rs +++ b/cranelift/jit/examples/jit-minimal.rs @@ -1,3 +1,4 @@ +use codegen::ir::UserFuncName; use cranelift::prelude::*; use cranelift_codegen::settings::{self, Configurable}; use cranelift_jit::{JITBuilder, JITModule}; @@ -35,7 +36,8 @@ fn main() { .unwrap(); ctx.func.signature = sig_a; - ctx.func.name = ExternalName::user(0, func_a.as_u32()); + ctx.func.name = UserFuncName::user(0, func_a.as_u32()); + { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); let block = bcx.create_block(); @@ -53,7 +55,8 @@ fn main() { module.clear_context(&mut ctx); ctx.func.signature = sig_b; - ctx.func.name = ExternalName::user(0, func_b.as_u32()); + ctx.func.name = UserFuncName::user(0, func_b.as_u32()); + { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); let block = bcx.create_block(); diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index 75e8b2338e..a24d723a40 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -8,7 +8,7 @@ use cranelift_codegen::{binemit::Reloc, CodegenError}; use cranelift_entity::SecondaryMap; use cranelift_module::{ DataContext, DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleCompiledFunction, - ModuleDeclarations, ModuleError, ModuleResult, + ModuleDeclarations, ModuleError, ModuleExtName, ModuleReloc, ModuleResult, }; use log::info; use std::cell::RefCell; @@ -275,9 +275,9 @@ impl JITModule { std::ptr::write(plt_ptr, plt_val); } - fn get_address(&self, name: &ir::ExternalName) -> *const u8 { + fn get_address(&self, name: &ModuleExtName) -> *const u8 { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { let (name, linkage) = if ModuleDeclarations::is_function(name) { if self.hotswap_enabled { return self.get_plt_address(name); @@ -309,12 +309,12 @@ impl JITModule { panic!("can't resolve symbol {}", name); } } - ir::ExternalName::LibCall(ref libcall) => { + ModuleExtName::LibCall(ref libcall) => { let sym = (self.libcall_names)(*libcall); self.lookup_symbol(&sym) .unwrap_or_else(|| panic!("can't resolve libcall {}", sym)) } - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), } } @@ -326,9 +326,9 @@ impl JITModule { unsafe { got_entry.as_ref() }.load(Ordering::SeqCst) } - fn get_got_address(&self, name: &ir::ExternalName) -> NonNull> { + fn get_got_address(&self, name: &ModuleExtName) -> NonNull> { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { if ModuleDeclarations::is_function(name) { let func_id = FuncId::from_name(name); self.function_got_entries[func_id].unwrap() @@ -337,17 +337,17 @@ impl JITModule { self.data_object_got_entries[data_id].unwrap() } } - ir::ExternalName::LibCall(ref libcall) => *self + ModuleExtName::LibCall(ref libcall) => *self .libcall_got_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)), - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), } } - fn get_plt_address(&self, name: &ir::ExternalName) -> *const u8 { + fn get_plt_address(&self, name: &ModuleExtName) -> *const u8 { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { if ModuleDeclarations::is_function(name) { let func_id = FuncId::from_name(name); self.function_plt_entries[func_id] @@ -358,13 +358,13 @@ impl JITModule { unreachable!("PLT relocations can only have functions as target"); } } - ir::ExternalName::LibCall(ref libcall) => self + ModuleExtName::LibCall(ref libcall) => self .libcall_plt_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) .as_ptr() .cast::(), - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), } } @@ -631,12 +631,16 @@ impl Module for JITModule { /// /// TODO: Coalesce redundant decls and signatures. /// TODO: Look into ways to reduce the risk of using a FuncRef in the wrong function. - fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { + fn declare_func_in_func(&mut self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { let decl = self.declarations.get_function_decl(func); let signature = in_func.import_signature(decl.signature.clone()); let colocated = !self.hotswap_enabled && decl.linkage.is_final(); + let user_name_ref = in_func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: func.as_u32(), + }); in_func.import_function(ir::ExtFuncData { - name: ir::ExternalName::user(0, func.as_u32()), + name: ir::ExternalName::user(user_name_ref), signature, colocated, }) @@ -648,24 +652,18 @@ impl Module for JITModule { fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue { let decl = self.declarations.get_data_decl(data); let colocated = !self.hotswap_enabled && decl.linkage.is_final(); + let user_name_ref = func.declare_imported_user_function(ir::UserExternalName { + namespace: 1, + index: data.as_u32(), + }); func.create_global_value(ir::GlobalValueData::Symbol { - name: ir::ExternalName::user(1, data.as_u32()), + name: ir::ExternalName::user(user_name_ref), offset: ir::immediates::Imm64::new(0), colocated, tls: decl.tls, }) } - /// TODO: Same as above. - fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { - ctx.import_function(ir::ExternalName::user(0, func.as_u32())) - } - - /// TODO: Same as above. - fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { - ctx.import_global_value(ir::ExternalName::user(1, data.as_u32())) - } - fn define_function( &mut self, id: FuncId, @@ -681,7 +679,10 @@ impl Module for JITModule { return Err(ModuleError::DuplicateDefinition(decl.name.to_owned())); } - let compiled_code = ctx.compile(self.isa())?; + // work around borrow-checker to allow reuse of ctx below + let _ = ctx.compile(self.isa())?; + let compiled_code = ctx.compiled_code().unwrap(); + let code_size = compiled_code.code_info().total_size; let size = code_size as usize; @@ -696,7 +697,12 @@ impl Module for JITModule { mem.copy_from_slice(compiled_code.code_buffer()); } - let relocs = compiled_code.buffer.relocs().to_vec(); + let relocs = compiled_code + .buffer + .relocs() + .iter() + .map(|reloc| ModuleReloc::from_mach_reloc(reloc, &ctx.func)) + .collect(); self.record_function_for_perf(ptr, size, &decl.name); self.compiled_functions[id] = Some(CompiledBlob { ptr, size, relocs }); @@ -714,16 +720,16 @@ impl Module for JITModule { .unwrap() .perform_relocations( |name| match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { unreachable!("non GOT or PLT relocation in function {} to {}", id, name) } - ir::ExternalName::LibCall(ref libcall) => self + ModuleExtName::LibCall(ref libcall) => self .libcall_plt_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) .as_ptr() .cast::(), - _ => panic!("invalid ExternalName {}", name), + _ => panic!("invalid name"), }, |name| self.get_got_address(name).as_ptr().cast(), |name| self.get_plt_address(name), @@ -738,6 +744,7 @@ impl Module for JITModule { fn define_function_bytes( &mut self, id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult { @@ -771,7 +778,10 @@ impl Module for JITModule { self.compiled_functions[id] = Some(CompiledBlob { ptr, size, - relocs: relocs.to_vec(), + relocs: relocs + .iter() + .map(|reloc| ModuleReloc::from_mach_reloc(reloc, func)) + .collect(), }); if self.isa.flags().is_pic() { @@ -866,6 +876,33 @@ impl Module for JITModule { Ok(()) } + + fn get_name(&self, name: &str) -> Option { + self.declarations().get_name(name) + } + + fn target_config(&self) -> cranelift_codegen::isa::TargetFrontendConfig { + self.isa().frontend_config() + } + + fn make_context(&self) -> cranelift_codegen::Context { + let mut ctx = cranelift_codegen::Context::new(); + ctx.func.signature.call_conv = self.isa().default_call_conv(); + ctx + } + + fn clear_context(&self, ctx: &mut cranelift_codegen::Context) { + ctx.clear(); + ctx.func.signature.call_conv = self.isa().default_call_conv(); + } + + fn make_signature(&self) -> ir::Signature { + ir::Signature::new(self.isa().default_call_conv()) + } + + fn clear_signature(&self, sig: &mut ir::Signature) { + sig.clear(self.isa().default_call_conv()); + } } #[cfg(not(windows))] diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index ee42eb8b0c..1d1d5cc4db 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -1,25 +1,25 @@ use cranelift_codegen::binemit::Reloc; -use cranelift_codegen::ir::ExternalName; -use cranelift_codegen::MachReloc; +use cranelift_module::ModuleExtName; +use cranelift_module::ModuleReloc; use std::convert::TryFrom; #[derive(Clone)] pub(crate) struct CompiledBlob { pub(crate) ptr: *mut u8, pub(crate) size: usize, - pub(crate) relocs: Vec, + pub(crate) relocs: Vec, } impl CompiledBlob { pub(crate) fn perform_relocations( &self, - get_address: impl Fn(&ExternalName) -> *const u8, - get_got_entry: impl Fn(&ExternalName) -> *const u8, - get_plt_entry: impl Fn(&ExternalName) -> *const u8, + get_address: impl Fn(&ModuleExtName) -> *const u8, + get_got_entry: impl Fn(&ModuleExtName) -> *const u8, + get_plt_entry: impl Fn(&ModuleExtName) -> *const u8, ) { use std::ptr::write_unaligned; - for &MachReloc { + for &ModuleReloc { kind, offset, ref name, diff --git a/cranelift/jit/tests/basic.rs b/cranelift/jit/tests/basic.rs index dc95a00e50..64957aa1f7 100644 --- a/cranelift/jit/tests/basic.rs +++ b/cranelift/jit/tests/basic.rs @@ -48,7 +48,7 @@ fn define_simple_function(module: &mut JITModule) -> FuncId { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); @@ -91,7 +91,7 @@ fn switch_error() { call_conv: CallConv::SystemV, }; - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + let mut func = Function::with_name_signature(UserFuncName::default(), sig); let mut func_ctx = FunctionBuilderContext::new(); { @@ -179,7 +179,8 @@ fn libcall_function() { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); + let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); diff --git a/cranelift/module/src/data_context.rs b/cranelift/module/src/data_context.rs index 0c5fd1a1a1..3ec4079968 100644 --- a/cranelift/module/src/data_context.rs +++ b/cranelift/module/src/data_context.rs @@ -3,12 +3,14 @@ use cranelift_codegen::binemit::{Addend, CodeOffset, Reloc}; use cranelift_codegen::entity::PrimaryMap; use cranelift_codegen::ir; -use cranelift_codegen::MachReloc; use std::borrow::ToOwned; use std::boxed::Box; use std::string::String; use std::vec::Vec; +use crate::module::ModuleReloc; +use crate::ModuleExtName; + /// This specifies how data is to be initialized. #[derive(Clone, PartialEq, Eq, Debug)] pub enum Init { @@ -43,9 +45,9 @@ pub struct DataDescription { /// How the data should be initialized. pub init: Init, /// External function declarations. - pub function_decls: PrimaryMap, + pub function_decls: PrimaryMap, /// External data object declarations. - pub data_decls: PrimaryMap, + pub data_decls: PrimaryMap, /// Function addresses to write at specified offsets. pub function_relocs: Vec<(CodeOffset, ir::FuncRef)>, /// Data addresses to write at specified offsets. @@ -59,11 +61,14 @@ pub struct DataDescription { impl DataDescription { /// An iterator over all relocations of the data object. - pub fn all_relocs<'a>(&'a self, pointer_reloc: Reloc) -> impl Iterator + 'a { + pub fn all_relocs<'a>( + &'a self, + pointer_reloc: Reloc, + ) -> impl Iterator + 'a { let func_relocs = self .function_relocs .iter() - .map(move |&(offset, id)| MachReloc { + .map(move |&(offset, id)| ModuleReloc { kind: pointer_reloc, offset, name: self.function_decls[id].clone(), @@ -72,7 +77,7 @@ impl DataDescription { let data_relocs = self .data_relocs .iter() - .map(move |&(offset, id, addend)| MachReloc { + .map(move |&(offset, id, addend)| ModuleReloc { kind: pointer_reloc, offset, name: self.data_decls[id].clone(), @@ -144,7 +149,7 @@ impl DataContext { /// Users of the `Module` API generally should call /// `Module::declare_func_in_data` instead, as it takes care of generating /// the appropriate `ExternalName`. - pub fn import_function(&mut self, name: ir::ExternalName) -> ir::FuncRef { + pub fn import_function(&mut self, name: ModuleExtName) -> ir::FuncRef { self.description.function_decls.push(name) } @@ -155,7 +160,7 @@ impl DataContext { /// Users of the `Module` API generally should call /// `Module::declare_data_in_data` instead, as it takes care of generating /// the appropriate `ExternalName`. - pub fn import_global_value(&mut self, name: ir::ExternalName) -> ir::GlobalValue { + pub fn import_global_value(&mut self, name: ModuleExtName) -> ir::GlobalValue { self.description.data_decls.push(name) } @@ -181,8 +186,9 @@ impl DataContext { #[cfg(test)] mod tests { + use crate::ModuleExtName; + use super::{DataContext, Init}; - use cranelift_codegen::ir; #[test] fn basic_data_context() { @@ -198,11 +204,11 @@ mod tests { data_ctx.define_zeroinit(256); - let _func_a = data_ctx.import_function(ir::ExternalName::user(0, 0)); - let func_b = data_ctx.import_function(ir::ExternalName::user(0, 1)); - let func_c = data_ctx.import_function(ir::ExternalName::user(1, 0)); - let _data_a = data_ctx.import_global_value(ir::ExternalName::user(2, 2)); - let data_b = data_ctx.import_global_value(ir::ExternalName::user(2, 3)); + let _func_a = data_ctx.import_function(ModuleExtName::user(0, 0)); + let func_b = data_ctx.import_function(ModuleExtName::user(0, 1)); + let func_c = data_ctx.import_function(ModuleExtName::user(0, 2)); + let _data_a = data_ctx.import_global_value(ModuleExtName::user(0, 3)); + let data_b = data_ctx.import_global_value(ModuleExtName::user(0, 4)); data_ctx.write_function_addr(8, func_b); data_ctx.write_function_addr(16, func_c); diff --git a/cranelift/module/src/lib.rs b/cranelift/module/src/lib.rs index 5997e14709..4f9bde29d5 100644 --- a/cranelift/module/src/lib.rs +++ b/cranelift/module/src/lib.rs @@ -43,7 +43,7 @@ mod traps; pub use crate::data_context::{DataContext, DataDescription, Init}; pub use crate::module::{ DataId, FuncId, FuncOrDataId, Linkage, Module, ModuleCompiledFunction, ModuleDeclarations, - ModuleError, ModuleResult, + ModuleError, ModuleExtName, ModuleReloc, ModuleResult, }; pub use crate::traps::TrapSite; diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index def84f1018..1312f4880b 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -7,19 +7,57 @@ use super::HashMap; use crate::data_context::DataContext; +use core::fmt::Display; +use cranelift_codegen::binemit::{CodeOffset, Reloc}; use cranelift_codegen::entity::{entity_impl, PrimaryMap}; +use cranelift_codegen::ir::Function; use cranelift_codegen::{binemit, MachReloc}; use cranelift_codegen::{ir, isa, CodegenError, CompileError, Context}; use std::borrow::ToOwned; use std::string::String; +/// A module relocation. +#[derive(Clone)] +pub struct ModuleReloc { + /// The offset at which the relocation applies, *relative to the + /// containing section*. + pub offset: CodeOffset, + /// The kind of relocation. + pub kind: Reloc, + /// The external symbol / name to which this relocation refers. + pub name: ModuleExtName, + /// The addend to add to the symbol value. + pub addend: i64, +} + +impl ModuleReloc { + /// Converts a `MachReloc` produced from a `Function` into a `ModuleReloc`. + pub fn from_mach_reloc(mach_reloc: &MachReloc, func: &Function) -> Self { + let name = match mach_reloc.name { + ir::ExternalName::User(reff) => { + let name = &func.params.user_named_funcs()[reff]; + ModuleExtName::user(name.namespace, name.index) + } + ir::ExternalName::TestCase(_) => unimplemented!(), + ir::ExternalName::LibCall(libcall) => ModuleExtName::LibCall(libcall), + ir::ExternalName::KnownSymbol(ks) => ModuleExtName::KnownSymbol(ks), + }; + Self { + offset: mach_reloc.offset, + kind: mach_reloc.kind, + name, + addend: mach_reloc.addend, + } + } +} + /// A function identifier for use in the `Module` interface. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct FuncId(u32); entity_impl!(FuncId, "funcid"); /// Function identifiers are namespace 0 in `ir::ExternalName` -impl From for ir::ExternalName { +impl From for ModuleExtName { fn from(id: FuncId) -> Self { Self::User { namespace: 0, @@ -30,12 +68,12 @@ impl From for ir::ExternalName { impl FuncId { /// Get the `FuncId` for the function named by `name`. - pub fn from_name(name: &ir::ExternalName) -> FuncId { - if let ir::ExternalName::User { namespace, index } = *name { - debug_assert_eq!(namespace, 0); - FuncId::from_u32(index) + pub fn from_name(name: &ModuleExtName) -> FuncId { + if let ModuleExtName::User { namespace, index } = name { + debug_assert_eq!(*namespace, 0); + FuncId::from_u32(*index) } else { - panic!("unexpected ExternalName kind {}", name) + panic!("unexpected name in DataId::from_name") } } } @@ -46,7 +84,7 @@ pub struct DataId(u32); entity_impl!(DataId, "dataid"); /// Data identifiers are namespace 1 in `ir::ExternalName` -impl From for ir::ExternalName { +impl From for ModuleExtName { fn from(id: DataId) -> Self { Self::User { namespace: 1, @@ -57,12 +95,12 @@ impl From for ir::ExternalName { impl DataId { /// Get the `DataId` for the data object named by `name`. - pub fn from_name(name: &ir::ExternalName) -> DataId { - if let ir::ExternalName::User { namespace, index } = *name { - debug_assert_eq!(namespace, 1); - DataId::from_u32(index) + pub fn from_name(name: &ModuleExtName) -> DataId { + if let ModuleExtName::User { namespace, index } = name { + debug_assert_eq!(*namespace, 1); + DataId::from_u32(*index) } else { - panic!("unexpected ExternalName kind {}", name) + panic!("unexpected name in DataId::from_name") } } } @@ -134,8 +172,8 @@ pub enum FuncOrDataId { Data(DataId), } -/// Mapping to `ir::ExternalName` is trivial based on the `FuncId` and `DataId` mapping. -impl From for ir::ExternalName { +/// Mapping to `ModuleExtName` is trivial based on the `FuncId` and `DataId` mapping. +impl From for ModuleExtName { fn from(id: FuncOrDataId) -> Self { match id { FuncOrDataId::Func(funcid) => Self::from(funcid), @@ -277,6 +315,39 @@ impl DataDeclaration { } } +/// A translated `ExternalName` into something global we can handle. +#[derive(Clone)] +pub enum ModuleExtName { + /// User defined function, converted from `ExternalName::User`. + User { + /// Arbitrary. + namespace: u32, + /// Arbitrary. + index: u32, + }, + /// Call into a library function. + LibCall(ir::LibCall), + /// Symbols known to the linker. + KnownSymbol(ir::KnownSymbol), +} + +impl ModuleExtName { + /// Creates a user-defined external name. + pub fn user(namespace: u32, index: u32) -> Self { + Self::User { namespace, index } + } +} + +impl Display for ModuleExtName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::User { namespace, index } => write!(f, "u{}:{}", namespace, index), + Self::LibCall(lc) => write!(f, "%{}", lc), + Self::KnownSymbol(ks) => write!(f, "{}", ks), + } + } +} + /// This provides a view to the state of a module which allows `ir::ExternalName`s to be translated /// into `FunctionDeclaration`s and `DataDeclaration`s. #[derive(Debug, Default)] @@ -299,11 +370,12 @@ impl ModuleDeclarations { } /// Return whether `name` names a function, rather than a data object. - pub fn is_function(name: &ir::ExternalName) -> bool { - if let ir::ExternalName::User { namespace, .. } = *name { - namespace == 0 - } else { - panic!("unexpected ExternalName kind {}", name) + pub fn is_function(name: &ModuleExtName) -> bool { + match name { + ModuleExtName::User { namespace, .. } => *namespace == 0, + ModuleExtName::LibCall(_) | ModuleExtName::KnownSymbol(_) => { + panic!("unexpected module ext name") + } } } @@ -503,12 +575,16 @@ pub trait Module { /// /// TODO: Coalesce redundant decls and signatures. /// TODO: Look into ways to reduce the risk of using a FuncRef in the wrong function. - fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { - let decl = &self.declarations().functions[func]; - let signature = in_func.import_signature(decl.signature.clone()); + fn declare_func_in_func(&mut self, func_id: FuncId, func: &mut ir::Function) -> ir::FuncRef { + let decl = &self.declarations().functions[func_id]; + let signature = func.import_signature(decl.signature.clone()); + let user_name_ref = func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: func_id.as_u32(), + }); let colocated = decl.linkage.is_final(); - in_func.import_function(ir::ExtFuncData { - name: ir::ExternalName::user(0, func.as_u32()), + func.import_function(ir::ExtFuncData { + name: ir::ExternalName::user(user_name_ref), signature, colocated, }) @@ -520,8 +596,12 @@ pub trait Module { fn declare_data_in_func(&self, data: DataId, func: &mut ir::Function) -> ir::GlobalValue { let decl = &self.declarations().data_objects[data]; let colocated = decl.linkage.is_final(); + let user_name_ref = func.declare_imported_user_function(ir::UserExternalName { + namespace: 1, + index: data.as_u32(), + }); func.create_global_value(ir::GlobalValueData::Symbol { - name: ir::ExternalName::user(1, data.as_u32()), + name: ir::ExternalName::user(user_name_ref), offset: ir::immediates::Imm64::new(0), colocated, tls: decl.tls, @@ -530,12 +610,12 @@ pub trait Module { /// TODO: Same as above. fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { - ctx.import_function(ir::ExternalName::user(0, func.as_u32())) + ctx.import_function(ModuleExtName::user(0, func.as_u32())) } /// TODO: Same as above. fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { - ctx.import_global_value(ir::ExternalName::user(1, data.as_u32())) + ctx.import_global_value(ModuleExtName::user(1, data.as_u32())) } /// Define a function, producing the function body from the given `Context`. @@ -558,7 +638,8 @@ pub trait Module { /// Returns the size of the function's code. fn define_function_bytes( &mut self, - func: FuncId, + func_id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult; @@ -627,7 +708,7 @@ impl Module for &mut M { (**self).declare_anonymous_data(writable, tls) } - fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { + fn declare_func_in_func(&mut self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { (**self).declare_func_in_func(func, in_func) } @@ -635,14 +716,6 @@ impl Module for &mut M { (**self).declare_data_in_func(data, func) } - fn declare_func_in_data(&self, func: FuncId, ctx: &mut DataContext) -> ir::FuncRef { - (**self).declare_func_in_data(func, ctx) - } - - fn declare_data_in_data(&self, data: DataId, ctx: &mut DataContext) -> ir::GlobalValue { - (**self).declare_data_in_data(data, ctx) - } - fn define_function( &mut self, func: FuncId, @@ -653,11 +726,12 @@ impl Module for &mut M { fn define_function_bytes( &mut self, - func: FuncId, + func_id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult { - (**self).define_function_bytes(func, bytes, relocs) + (**self).define_function_bytes(func_id, func, bytes, relocs) } fn define_data(&mut self, data: DataId, data_ctx: &DataContext) -> ModuleResult<()> { diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index 88cca41bbe..6f7a4e0af1 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -10,7 +10,7 @@ use cranelift_codegen::{ }; use cranelift_module::{ DataContext, DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleCompiledFunction, - ModuleDeclarations, ModuleError, ModuleResult, + ModuleDeclarations, ModuleError, ModuleExtName, ModuleReloc, ModuleResult, }; use log::info; use object::write::{ @@ -316,12 +316,18 @@ impl Module for ObjectModule { ctx.compile_and_emit(self.isa(), &mut code)?; - self.define_function_bytes(func_id, &code, ctx.compiled_code().unwrap().buffer.relocs()) + self.define_function_bytes( + func_id, + &ctx.func, + &code, + ctx.compiled_code().unwrap().buffer.relocs(), + ) } fn define_function_bytes( &mut self, func_id: FuncId, + func: &ir::Function, bytes: &[u8], relocs: &[MachReloc], ) -> ModuleResult { @@ -364,7 +370,7 @@ impl Module for ObjectModule { if !relocs.is_empty() { let relocs = relocs .iter() - .map(|record| self.process_reloc(record)) + .map(|record| self.process_reloc(&ModuleReloc::from_mach_reloc(&record, func))) .collect(); self.relocs.push(SymbolRelocs { section, @@ -518,9 +524,9 @@ impl ObjectModule { /// This should only be called during finish because it creates /// symbols for missing libcalls. - fn get_symbol(&mut self, name: &ir::ExternalName) -> SymbolId { + fn get_symbol(&mut self, name: &ModuleExtName) -> SymbolId { match *name { - ir::ExternalName::User { .. } => { + ModuleExtName::User { .. } => { if ModuleDeclarations::is_function(name) { let id = FuncId::from_name(name); self.functions[id].unwrap().0 @@ -529,7 +535,7 @@ impl ObjectModule { self.data_objects[id].unwrap().0 } } - ir::ExternalName::LibCall(ref libcall) => { + ModuleExtName::LibCall(ref libcall) => { let name = (self.libcall_names)(*libcall); if let Some(symbol) = self.object.symbol_id(name.as_bytes()) { symbol @@ -552,7 +558,7 @@ impl ObjectModule { } // These are "magic" names well-known to the linker. // They require special treatment. - ir::ExternalName::KnownSymbol(ref known_symbol) => { + ModuleExtName::KnownSymbol(ref known_symbol) => { if let Some(symbol) = self.known_symbols.get(known_symbol) { *symbol } else { @@ -582,11 +588,10 @@ impl ObjectModule { symbol } } - _ => panic!("invalid ExternalName {}", name), } } - fn process_reloc(&self, record: &MachReloc) -> ObjectRelocRecord { + fn process_reloc(&self, record: &ModuleReloc) -> ObjectRelocRecord { let mut addend = record.addend; let (kind, encoding, size) = match record.kind { Reloc::Abs4 => (RelocationKind::Absolute, RelocationEncoding::Generic, 32), @@ -696,6 +701,7 @@ impl ObjectModule { // FIXME reloc => unimplemented!("{:?}", reloc), }; + ObjectRelocRecord { offset: record.offset, name: record.name.clone(), @@ -762,7 +768,7 @@ struct SymbolRelocs { #[derive(Clone)] struct ObjectRelocRecord { offset: CodeOffset, - name: ir::ExternalName, + name: ModuleExtName, kind: RelocationKind, encoding: RelocationEncoding, size: u8, diff --git a/cranelift/object/tests/basic.rs b/cranelift/object/tests/basic.rs index 9ed9c351a2..d918f55e34 100644 --- a/cranelift/object/tests/basic.rs +++ b/cranelift/object/tests/basic.rs @@ -43,7 +43,7 @@ fn define_simple_function(module: &mut ObjectModule) -> FuncId { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); @@ -82,8 +82,7 @@ fn switch_error() { call_conv: CallConv::SystemV, }; - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); - + let mut func = Function::with_name_signature(UserFuncName::default(), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut func, &mut func_ctx); @@ -166,7 +165,7 @@ fn libcall_function() { .unwrap(); let mut ctx = Context::new(); - ctx.func = Function::with_name_signature(ExternalName::user(0, func_id.as_u32()), sig); + ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig); let mut func_ctx = FunctionBuilderContext::new(); { let mut bcx: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rs index 6b471be0f8..ba2b0dc5eb 100644 --- a/cranelift/reader/src/lexer.rs +++ b/cranelift/reader/src/lexer.rs @@ -47,6 +47,7 @@ pub enum Token<'a> { FuncRef(u32), // fn2 SigRef(u32), // sig2 UserRef(u32), // u345 + UserNameRef(u32), // userextname345 Name(&'a str), // %9arbitrary_alphanum, %x3, %0, %function ... String(&'a str), // "arbitrary quoted string with no escape" ... HexSequence(&'a str), // #89AF @@ -354,6 +355,7 @@ impl<'a> Lexer<'a> { "fn" => Some(Token::FuncRef(number)), "sig" => Some(Token::SigRef(number)), "u" => Some(Token::UserRef(number)), + "userextname" => Some(Token::UserNameRef(number)), _ => None, } } diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs index 78e50b0d88..00e7a5227c 100644 --- a/cranelift/reader/src/parser.rs +++ b/cranelift/reader/src/parser.rs @@ -9,19 +9,19 @@ use crate::sourcemap::SourceMap; use crate::testcommand::TestCommand; use crate::testfile::{Comment, Details, Feature, TestFile}; use cranelift_codegen::data_value::DataValue; -use cranelift_codegen::entity::EntityRef; -use cranelift_codegen::ir; +use cranelift_codegen::entity::{EntityRef, PrimaryMap}; use cranelift_codegen::ir::entities::{AnyEntity, DynamicType}; use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, Uimm32, Uimm64}; use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs}; use cranelift_codegen::ir::types::INVALID; use cranelift_codegen::ir::types::*; +use cranelift_codegen::ir::{self, UserExternalNameRef}; use cranelift_codegen::ir::{ AbiParam, ArgumentExtension, ArgumentPurpose, Block, Constant, ConstantData, DynamicStackSlot, DynamicStackSlotData, DynamicTypeData, ExtFuncData, ExternalName, FuncRef, Function, GlobalValue, GlobalValueData, Heap, HeapData, HeapStyle, JumpTable, JumpTableData, MemFlags, Opcode, SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, Table, TableData, Type, - Value, + UserFuncName, Value, }; use cranelift_codegen::isa::{self, CallConv}; use cranelift_codegen::packed_option::ReservedValue; @@ -225,6 +225,12 @@ pub struct Parser<'a> { /// Comments collected so far. comments: Vec>, + /// Maps inlined external names to a ref value, so they can be declared before parsing the rest + /// of the function later. + /// + /// This maintains backward compatibility with previous ways for declaring external names. + predeclared_external_names: PrimaryMap, + /// Default calling conventions; used when none is specified. default_calling_convention: CallConv, } @@ -508,6 +514,7 @@ impl<'a> Parser<'a> { gathered_comments: Vec::new(), comments: Vec::new(), default_calling_convention: CallConv::Fast, + predeclared_external_names: Default::default(), } } @@ -1282,7 +1289,7 @@ impl<'a> Parser<'a> { let location = self.loc; // function ::= "function" * name signature "{" preamble function-body "}" - let name = self.parse_external_name()?; + let name = self.parse_user_func_name()?; // function ::= "function" name * signature "{" preamble function-body "}" let sig = self.parse_signature()?; @@ -1307,6 +1314,16 @@ impl<'a> Parser<'a> { self.token(); self.claim_gathered_comments(AnyEntity::Function); + // Claim all the declared user-defined function names. + for (user_func_ref, user_external_name) in + std::mem::take(&mut self.predeclared_external_names) + { + let actual_ref = ctx + .function + .declare_imported_user_function(user_external_name); + assert_eq!(user_func_ref, actual_ref); + } + let details = Details { location, comments: self.take_comments(), @@ -1316,18 +1333,17 @@ impl<'a> Parser<'a> { Ok((ctx.function, details)) } - // Parse an external name. + // Parse a user-defined function name // // For example, in a function decl, the parser would be in this state: // // function ::= "function" * name signature { ... } // - fn parse_external_name(&mut self) -> ParseResult { + fn parse_user_func_name(&mut self) -> ParseResult { match self.token() { Some(Token::Name(s)) => { self.consume(); - s.parse() - .map_err(|_| self.error("invalid test case or libcall name")) + Ok(UserFuncName::testcase(s)) } Some(Token::UserRef(namespace)) => { self.consume(); @@ -1336,23 +1352,88 @@ impl<'a> Parser<'a> { self.consume(); match self.token() { Some(Token::Integer(index_str)) => { + self.consume(); let index: u32 = u32::from_str_radix(index_str, 10).map_err(|_| { self.error("the integer given overflows the u32 type") })?; - self.consume(); - Ok(ExternalName::user(namespace, index)) + Ok(UserFuncName::user(namespace, index)) } _ => err!(self.loc, "expected integer"), } } - _ => err!(self.loc, "expected colon"), + _ => { + err!(self.loc, "expected user function name in the form uX:Y") + } } } _ => err!(self.loc, "expected external name"), } } + // Parse an external name. + // + // For example, in a function reference decl, the parser would be in this state: + // + // fn0 = * name signature + // + fn parse_external_name(&mut self) -> ParseResult { + match self.token() { + Some(Token::Name(s)) => { + self.consume(); + s.parse() + .map_err(|_| self.error("invalid test case or libcall name")) + } + + Some(Token::UserNameRef(name_ref)) => { + self.consume(); + Ok(ExternalName::user(UserExternalNameRef::new( + name_ref as usize, + ))) + } + + Some(Token::UserRef(namespace)) => { + self.consume(); + if let Some(Token::Colon) = self.token() { + self.consume(); + match self.token() { + Some(Token::Integer(index_str)) => { + let index: u32 = u32::from_str_radix(index_str, 10).map_err(|_| { + self.error("the integer given overflows the u32 type") + })?; + self.consume(); + + // Deduplicate the reference (O(n), but should be fine for tests), + // to follow `FunctionParameters::declare_imported_user_function`, + // otherwise this will cause ref mismatches when asserted below. + let name_ref = self + .predeclared_external_names + .iter() + .find_map(|(reff, name)| { + if name.index == index && name.namespace == namespace { + Some(reff) + } else { + None + } + }) + .unwrap_or_else(|| { + self.predeclared_external_names + .push(ir::UserExternalName { namespace, index }) + }); + + Ok(ExternalName::user(name_ref)) + } + _ => err!(self.loc, "expected integer"), + } + } else { + err!(self.loc, "expected colon") + } + } + + _ => err!(self.loc, "expected external name"), + } + } + // Parse a function signature. // // signature ::= * "(" [paramlist] ")" ["->" retlist] [callconv] @@ -2249,7 +2330,7 @@ impl<'a> Parser<'a> { .expect("duplicate inst references created"); if !srcloc.is_default() { - ctx.function.srclocs[inst] = srcloc; + ctx.function.set_srcloc(inst, srcloc); } if results.len() != num_results { diff --git a/cranelift/src/bugpoint.rs b/cranelift/src/bugpoint.rs index dcc48245f2..a4673227a1 100644 --- a/cranelift/src/bugpoint.rs +++ b/cranelift/src/bugpoint.rs @@ -810,9 +810,9 @@ fn inst_count(func: &Function) -> usize { } fn resolve_aliases(func: &mut Function) { - for block in func.layout.blocks() { - for inst in func.layout.block_insts(block) { - func.dfg.resolve_aliases_in_arguments(inst); + for block in func.stencil.layout.blocks() { + for inst in func.stencil.layout.block_insts(block) { + func.stencil.dfg.resolve_aliases_in_arguments(inst); } } } diff --git a/cranelift/src/compile.rs b/cranelift/src/compile.rs index af03d8a4fe..2631c00331 100644 --- a/cranelift/src/compile.rs +++ b/cranelift/src/compile.rs @@ -81,6 +81,7 @@ fn handle_module(options: &Options, path: &Path, name: &str, fisa: FlagsOrIsa) - let result = context.compiled_code().unwrap(); print_all( isa, + &context.func.params, &mem, code_info.total_size, options.print, diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rs index c372707b4a..a950d6b98f 100644 --- a/cranelift/src/disasm.rs +++ b/cranelift/src/disasm.rs @@ -1,10 +1,11 @@ use anyhow::Result; use cfg_if::cfg_if; +use cranelift_codegen::ir::function::FunctionParameters; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::{MachReloc, MachStackMap, MachTrap}; use std::fmt::Write; -pub fn print_relocs(relocs: &[MachReloc]) -> String { +fn print_relocs(func_params: &FunctionParameters, relocs: &[MachReloc]) -> String { let mut text = String::new(); for &MachReloc { kind, @@ -16,7 +17,10 @@ pub fn print_relocs(relocs: &[MachReloc]) -> String { writeln!( text, "reloc_external: {} {} {} at {}", - kind, name, addend, offset + kind, + name.display(Some(func_params)), + addend, + offset ) .unwrap(); } @@ -172,6 +176,7 @@ cfg_if! { pub fn print_all( isa: &dyn TargetIsa, + func_params: &FunctionParameters, mem: &[u8], code_size: u32, print: bool, @@ -184,7 +189,7 @@ pub fn print_all( if print { println!( "\n{}\n{}\n{}", - print_relocs(relocs), + print_relocs(func_params, relocs), print_traps(traps), print_stack_maps(stack_maps) ); diff --git a/cranelift/src/wasm.rs b/cranelift/src/wasm.rs index f526e0c8a2..1ccf788f7d 100644 --- a/cranelift/src/wasm.rs +++ b/cranelift/src/wasm.rs @@ -313,6 +313,7 @@ fn handle_module(options: &Options, path: &Path, name: &str, fisa: FlagsOrIsa) - if let Some(total_size) = saved_size { print_all( isa, + &context.func.params, &mem, total_size, options.print, diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index 70833483ae..332269c437 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -16,8 +16,8 @@ use crate::{ use core::convert::TryFrom; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; -use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, InstBuilder}; +use cranelift_codegen::ir::{types::*, UserFuncName}; use cranelift_codegen::isa::{CallConv, TargetFrontendConfig}; use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap}; use cranelift_frontend::FunctionBuilder; @@ -26,11 +26,6 @@ use std::string::String; use std::vec::Vec; use wasmparser::{FuncValidator, FunctionBody, Operator, ValidatorResources, WasmFeatures}; -/// Compute a `ir::ExternalName` for a given wasm function index. -fn get_func_name(func_index: FuncIndex) -> ir::ExternalName { - ir::ExternalName::user(0, func_index.as_u32()) -} - /// A collection of names under which a given entity is exported. pub struct Exportable { /// A wasm entity. @@ -341,7 +336,11 @@ impl<'dummy_environment> FuncEnvironment for DummyFuncEnvironment<'dummy_environ // A real implementation would probably add a `vmctx` argument. // And maybe attempt some signature de-duplication. let signature = func.import_signature(self.vmctx_sig(sigidx)); - let name = get_func_name(index); + let name = + ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: index.as_u32(), + })); Ok(func.import_function(ir::ExtFuncData { name, signature, @@ -849,12 +848,15 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment { DummyFuncEnvironment::new(&self.info, self.expected_reachability.clone()); let func_index = FuncIndex::new(self.get_num_func_imports() + self.info.function_bodies.len()); - let name = get_func_name(func_index); + let sig = func_environ.vmctx_sig(self.get_func_type(func_index)); - let mut func = ir::Function::with_name_signature(name, sig); + let mut func = + ir::Function::with_name_signature(UserFuncName::user(0, func_index.as_u32()), sig); + if self.debug_info { func.collect_debug_info(); } + self.trans .translate_body(&mut validator, body, &mut func, &mut func_environ)?; func diff --git a/cranelift/wasm/src/func_translator.rs b/cranelift/wasm/src/func_translator.rs index 4404324b04..66aecf3b95 100644 --- a/cranelift/wasm/src/func_translator.rs +++ b/cranelift/wasm/src/func_translator.rs @@ -310,7 +310,7 @@ mod tests { let mut ctx = Context::new(); - ctx.func.name = ir::ExternalName::testcase("small1"); + ctx.func.name = ir::UserFuncName::testcase("small1"); ctx.func.signature.params.push(ir::AbiParam::new(I32)); ctx.func.signature.returns.push(ir::AbiParam::new(I32)); @@ -348,7 +348,7 @@ mod tests { let mut ctx = Context::new(); - ctx.func.name = ir::ExternalName::testcase("small2"); + ctx.func.name = ir::UserFuncName::testcase("small2"); ctx.func.signature.params.push(ir::AbiParam::new(I32)); ctx.func.signature.returns.push(ir::AbiParam::new(I32)); @@ -391,7 +391,7 @@ mod tests { let mut ctx = Context::new(); - ctx.func.name = ir::ExternalName::testcase("infloop"); + ctx.func.name = ir::UserFuncName::testcase("infloop"); ctx.func.signature.returns.push(ir::AbiParam::new(I32)); let (body, mut validator) = extract_func(&wasm); diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index 23df5dd712..58f18084d7 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -28,3 +28,4 @@ thiserror = "1.0.4" [features] all-arch = ["cranelift-codegen/all-arch"] component-model = ["wasmtime-environ/component-model"] +incremental-cache = ["cranelift-codegen/incremental-cache"] diff --git a/crates/cranelift/src/builder.rs b/crates/cranelift/src/builder.rs index c810e3498d..4fb2b6369a 100644 --- a/crates/cranelift/src/builder.rs +++ b/crates/cranelift/src/builder.rs @@ -7,12 +7,14 @@ use anyhow::Result; use cranelift_codegen::isa; use cranelift_codegen::settings::{self, Configurable, SetError}; use std::fmt; -use wasmtime_environ::{CompilerBuilder, Setting, SettingKind}; +use std::sync::Arc; +use wasmtime_environ::{CacheStore, CompilerBuilder, Setting, SettingKind}; struct Builder { flags: settings::Builder, isa_flags: isa::Builder, linkopts: LinkOptions, + cache_store: Option>, } #[derive(Clone, Default)] @@ -46,6 +48,7 @@ pub fn builder() -> Box { flags, isa_flags: cranelift_native::builder().expect("host machine is not a supported target"), linkopts: LinkOptions::default(), + cache_store: None, }) } @@ -103,6 +106,7 @@ impl CompilerBuilder for Builder { .finish(settings::Flags::new(self.flags.clone()))?; Ok(Box::new(crate::compiler::Compiler::new( isa, + self.cache_store.clone(), self.linkopts.clone(), ))) } @@ -123,6 +127,13 @@ impl CompilerBuilder for Builder { }) .collect() } + + fn enable_incremental_compilation( + &mut self, + cache_store: Arc, + ) { + self.cache_store = Some(cache_store); + } } impl fmt::Debug for Builder { diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index addb9177bb..3d6829f1cc 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -1,18 +1,20 @@ use crate::builder::LinkOptions; use crate::debug::{DwarfSectionRelocTarget, ModuleMemoryOffset}; -use crate::func_environ::{get_func_name, FuncEnvironment}; +use crate::func_environ::FuncEnvironment; use crate::obj::ModuleTextBuilder; use crate::{ blank_sig, func_signature, indirect_signature, value_type, wasmtime_call_conv, CompiledFunction, CompiledFunctions, FunctionAddressMap, Relocation, RelocationTarget, }; use anyhow::{Context as _, Result}; -use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags, Value}; +use cranelift_codegen::ir::{ + self, ExternalName, Function, InstBuilder, MemFlags, UserExternalName, UserFuncName, Value, +}; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::Context; use cranelift_codegen::{settings, MachReloc, MachTrap}; -use cranelift_codegen::{MachSrcLoc, MachStackMap}; +use cranelift_codegen::{CompiledCode, MachSrcLoc, MachStackMap}; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_frontend::FunctionBuilder; use cranelift_wasm::{ @@ -27,19 +29,28 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::convert::TryFrom; use std::mem; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use wasmtime_environ::{ - AddressMapSection, CompileError, FilePos, FlagValue, FunctionBodyData, FunctionInfo, - InstructionAddressMap, Module, ModuleTranslation, ModuleTypes, PtrSize, StackMapInformation, - Trampoline, TrapCode, TrapEncodingBuilder, TrapInformation, Tunables, VMOffsets, + AddressMapSection, CacheStore, CompileError, FilePos, FlagValue, FunctionBodyData, + FunctionInfo, InstructionAddressMap, Module, ModuleTranslation, ModuleTypes, PtrSize, + StackMapInformation, Trampoline, TrapCode, TrapEncodingBuilder, TrapInformation, Tunables, + VMOffsets, }; #[cfg(feature = "component-model")] mod component; +struct IncrementalCacheContext { + #[cfg(feature = "incremental-cache")] + cache_store: Arc, + num_hits: usize, + num_cached: usize, +} + struct CompilerContext { func_translator: FuncTranslator, codegen_context: Context, + incremental_cache_ctx: Option, } impl Default for CompilerContext { @@ -47,6 +58,7 @@ impl Default for CompilerContext { Self { func_translator: FuncTranslator::new(), codegen_context: Context::new(), + incremental_cache_ctx: None, } } } @@ -57,14 +69,48 @@ pub(crate) struct Compiler { contexts: Mutex>, isa: Box, linkopts: LinkOptions, + cache_store: Option>, +} + +impl Drop for Compiler { + fn drop(&mut self) { + if self.cache_store.is_none() { + return; + } + + let mut num_hits = 0; + let mut num_cached = 0; + for ctx in self.contexts.lock().unwrap().iter() { + if let Some(ref cache_ctx) = ctx.incremental_cache_ctx { + num_hits += cache_ctx.num_hits; + num_cached += cache_ctx.num_cached; + } + } + + let total = num_hits + num_cached; + if num_hits + num_cached > 0 { + log::trace!( + "Incremental compilation cache stats: {}/{} = {}% (hits/lookup)\ncached: {}", + num_hits, + total, + (num_hits as f32) / (total as f32) * 100.0, + num_cached + ); + } + } } impl Compiler { - pub(crate) fn new(isa: Box, linkopts: LinkOptions) -> Compiler { + pub(crate) fn new( + isa: Box, + cache_store: Option>, + linkopts: LinkOptions, + ) -> Compiler { Compiler { contexts: Default::default(), isa, linkopts, + cache_store, } } @@ -75,7 +121,17 @@ impl Compiler { ctx.codegen_context.clear(); ctx }) - .unwrap_or_else(Default::default) + .unwrap_or_else(|| CompilerContext { + #[cfg(feature = "incremental-cache")] + incremental_cache_ctx: self.cache_store.as_ref().map(|cache_store| { + IncrementalCacheContext { + cache_store: cache_store.clone(), + num_hits: 0, + num_cached: 0, + } + }), + ..Default::default() + }) } fn save_context(&self, ctx: CompilerContext) { @@ -141,10 +197,15 @@ impl wasmtime_environ::Compiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + incremental_cache_ctx: mut cache_ctx, } = self.take_context(); - context.func.name = get_func_name(func_index); context.func.signature = func_signature(isa, translation, types, func_index); + context.func.name = UserFuncName::User(UserExternalName { + namespace: 0, + index: func_index.as_u32(), + }); + if tunables.generate_native_debuginfo { context.func.collect_debug_info(); } @@ -210,16 +271,14 @@ impl wasmtime_environ::Compiler for Compiler { &mut func_env, )?; - let mut code_buf: Vec = Vec::new(); - let compiled_code = context - .compile_and_emit(isa, &mut code_buf) - .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + let (_, code_buf) = compile_maybe_cached(&mut context, isa, cache_ctx.as_mut())?; + let compiled_code = context.compiled_code().unwrap(); let func_relocs = compiled_code .buffer .relocs() .into_iter() - .map(mach_reloc_to_reloc) + .map(|item| mach_reloc_to_reloc(&context.func, item)) .collect::>(); let traps = compiled_code @@ -259,6 +318,7 @@ impl wasmtime_environ::Compiler for Compiler { self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx: cache_ctx, }); Ok(Box::new(CompiledFunction { @@ -403,6 +463,74 @@ impl wasmtime_environ::Compiler for Compiler { } } +#[cfg(feature = "incremental-cache")] +mod incremental_cache { + use super::*; + + struct CraneliftCacheStore(Arc); + + impl cranelift_codegen::incremental_cache::CacheKvStore for CraneliftCacheStore { + fn get(&self, key: &[u8]) -> Option> { + self.0.get(key) + } + fn insert(&mut self, key: &[u8], val: Vec) { + self.0.insert(key, val); + } + } + + pub(super) fn compile_maybe_cached<'a>( + context: &'a mut Context, + isa: &dyn TargetIsa, + cache_ctx: Option<&mut IncrementalCacheContext>, + ) -> Result<(&'a CompiledCode, Vec), CompileError> { + let mut code_buf = Vec::new(); + let cache_ctx = match cache_ctx { + Some(ctx) => ctx, + None => { + let compiled_code = + context + .compile_and_emit(isa, &mut code_buf) + .map_err(|error| { + CompileError::Codegen(pretty_error(&error.func, error.inner)) + })?; + return Ok((compiled_code, code_buf)); + } + }; + + let mut cache_store = CraneliftCacheStore(cache_ctx.cache_store.clone()); + let (compiled_code, from_cache) = context + .compile_with_cache(isa, &mut cache_store) + .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + + if from_cache { + cache_ctx.num_hits += 1; + } else { + cache_ctx.num_cached += 1; + } + + code_buf.resize(compiled_code.code_info().total_size as _, 0); + code_buf.copy_from_slice(compiled_code.code_buffer()); + + Ok((compiled_code, code_buf)) + } +} + +#[cfg(feature = "incremental-cache")] +use incremental_cache::*; + +#[cfg(not(feature = "incremental-cache"))] +fn compile_maybe_cached<'a>( + context: &'a mut Context, + isa: &dyn TargetIsa, + _cache_ctx: Option<&mut IncrementalCacheContext>, +) -> Result<(&'a CompiledCode, Vec), CompileError> { + let mut code_buf = Vec::new(); + let compiled_code = context + .compile_and_emit(isa, &mut code_buf) + .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + return Ok((compiled_code, code_buf)); +} + fn to_flag_value(v: &settings::Value) -> FlagValue { match v.kind() { settings::SettingKind::Enum => FlagValue::Enum(v.as_enum().unwrap().into()), @@ -431,9 +559,11 @@ impl Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + incremental_cache_ctx: mut cache_ctx, } = self.take_context(); - context.func = ir::Function::with_name_signature(ExternalName::user(0, 0), host_signature); + // The name doesn't matter here. + context.func = ir::Function::with_name_signature(UserFuncName::default(), host_signature); // This trampoline will load all the parameters from the `values_vec` // that is passed in and then call the real function (also passed @@ -493,10 +623,11 @@ impl Compiler { builder.ins().return_(&[]); builder.finalize(); - let func = self.finish_trampoline(&mut context, isa)?; + let func = self.finish_trampoline(&mut context, cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx: cache_ctx, }); Ok(func) } @@ -541,10 +672,11 @@ impl Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + incremental_cache_ctx: mut cache_ctx, } = self.take_context(); - context.func = - ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wasm_signature); + // The name doesn't matter here. + context.func = ir::Function::with_name_signature(Default::default(), wasm_signature); let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context()); let block0 = builder.create_block(); @@ -570,10 +702,11 @@ impl Compiler { self.wasm_to_host_load_results(ty, &mut builder, values_vec_ptr_val); - let func = self.finish_trampoline(&mut context, isa)?; + let func = self.finish_trampoline(&mut context, cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx: cache_ctx, }); Ok(func) } @@ -668,12 +801,10 @@ impl Compiler { fn finish_trampoline( &self, context: &mut Context, + cache_ctx: Option<&mut IncrementalCacheContext>, isa: &dyn TargetIsa, ) -> Result { - let mut code_buf = Vec::new(); - let compiled_code = context - .compile_and_emit(isa, &mut code_buf) - .map_err(|error| CompileError::Codegen(pretty_error(&error.func, error.inner)))?; + let (compiled_code, code_buf) = compile_maybe_cached(context, isa, cache_ctx)?; // Processing relocations isn't the hardest thing in the world here but // no trampoline should currently generate a relocation, so assert that @@ -858,14 +989,15 @@ fn collect_address_maps( } } -fn mach_reloc_to_reloc(reloc: &MachReloc) -> Relocation { +fn mach_reloc_to_reloc(func: &Function, reloc: &MachReloc) -> Relocation { let &MachReloc { offset, kind, ref name, addend, } = reloc; - let reloc_target = if let ExternalName::User { namespace, index } = *name { + let reloc_target = if let ExternalName::User(user_func_ref) = *name { + let UserExternalName { namespace, index } = func.params.user_named_funcs()[user_func_ref]; debug_assert_eq!(namespace, 0); RelocationTarget::UserFunc(FuncIndex::from_u32(index)) } else if let ExternalName::LibCall(libcall) = *name { diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index 33bd9d74cd..beb2ff5c42 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -31,10 +31,11 @@ impl ComponentCompiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + mut incremental_cache_ctx, } = self.take_context(); context.func = ir::Function::with_name_signature( - ir::ExternalName::user(0, 0), + ir::UserFuncName::user(0, 0), crate::indirect_signature(isa, ty), ); @@ -149,10 +150,12 @@ impl ComponentCompiler for Compiler { // `values_vec_ptr_val` and then returned. self.wasm_to_host_load_results(ty, &mut builder, values_vec_ptr_val); - let func: CompiledFunction = self.finish_trampoline(&mut context, isa)?; + let func: CompiledFunction = + self.finish_trampoline(&mut context, incremental_cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx, }); Ok(Box::new(func)) } @@ -162,9 +165,10 @@ impl ComponentCompiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + mut incremental_cache_ctx, } = self.take_context(); context.func = ir::Function::with_name_signature( - ir::ExternalName::user(0, 0), + ir::UserFuncName::user(0, 0), crate::indirect_signature(isa, ty), ); let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context()); @@ -177,10 +181,12 @@ impl ComponentCompiler for Compiler { .trap(ir::TrapCode::User(super::ALWAYS_TRAP_CODE)); builder.finalize(); - let func: CompiledFunction = self.finish_trampoline(&mut context, isa)?; + let func: CompiledFunction = + self.finish_trampoline(&mut context, incremental_cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx, }); Ok(Box::new(func)) } @@ -198,10 +204,11 @@ impl ComponentCompiler for Compiler { let CompilerContext { mut func_translator, codegen_context: mut context, + mut incremental_cache_ctx, } = self.take_context(); context.func = ir::Function::with_name_signature( - ir::ExternalName::user(0, 0), + ir::UserFuncName::user(0, 0), crate::indirect_signature(isa, ty), ); @@ -213,10 +220,12 @@ impl ComponentCompiler for Compiler { self.translate_transcode(&mut builder, &offsets, transcoder, block0); - let func: CompiledFunction = self.finish_trampoline(&mut context, isa)?; + let func: CompiledFunction = + self.finish_trampoline(&mut context, incremental_cache_ctx.as_mut(), isa)?; self.save_context(CompilerContext { func_translator, codegen_context: context, + incremental_cache_ctx, }); Ok(Box::new(func)) } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 7576f423c0..59822f4157 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -21,11 +21,6 @@ use wasmtime_environ::{ }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; -/// Compute an `ir::ExternalName` for a given wasm function index. -pub fn get_func_name(func_index: FuncIndex) -> ir::ExternalName { - ir::ExternalName::user(0, func_index.as_u32()) -} - macro_rules! declare_function_signatures { ( $( @@ -1509,7 +1504,11 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m ) -> WasmResult { let sig = crate::func_signature(self.isa, self.translation, self.types, index); let signature = func.import_signature(sig); - let name = get_func_name(index); + let name = + ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName { + namespace: 0, + index: index.as_u32(), + })); Ok(func.import_function(ir::ExtFuncData { name, signature, diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index 455584b01e..3322ce2003 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -13,6 +13,7 @@ use std::any::Any; use std::borrow::Cow; use std::collections::BTreeMap; use std::fmt; +use std::sync::Arc; use thiserror::Error; /// Information about a function, such as trap information, address map, @@ -71,6 +72,22 @@ pub enum CompileError { DebugInfoNotSupported, } +/// Implementation of an incremental compilation's key/value cache store. +/// +/// In theory, this could just be Cranelift's `CacheKvStore` trait, but it is not as we want to +/// make sure that wasmtime isn't too tied to Cranelift internals (and as a matter of fact, we +/// can't depend on the Cranelift trait here). +pub trait CacheStore: Send + Sync + std::fmt::Debug { + /// Try to retrieve an arbitrary cache key entry, and returns a reference to bytes that were + /// inserted via `Self::insert` before. + fn get(&self, key: &[u8]) -> Option>; + + /// Given an arbitrary key and bytes, stores them in the cache. + /// + /// Returns false when insertion in the cache failed. + fn insert(&self, key: &[u8], value: Vec) -> bool; +} + /// Abstract trait representing the ability to create a `Compiler` below. /// /// This is used in Wasmtime to separate compiler implementations, currently @@ -100,6 +117,10 @@ pub trait CompilerBuilder: Send + Sync + fmt::Debug { /// [`CompilerBuilder::set`] and [`CompilerBuilder::enable`]. fn settings(&self) -> Vec; + /// Enables Cranelift's incremental compilation cache, using the given `CacheStore` + /// implementation. + fn enable_incremental_compilation(&mut self, cache_store: Arc); + /// Builds a new [`Compiler`] object from this configuration. fn build(&self) -> Result>; } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index c0643f039a..18472c9198 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -72,6 +72,9 @@ default = [ # precompiled WebAssembly modules. cranelift = ["dep:wasmtime-cranelift"] +# Enables support for incremental compilation cache to be enabled in `Config`. +incremental-cache = ["wasmtime-cranelift?/incremental-cache"] + # Enables support for the `perf` jitdump profiler jitdump = ["wasmtime-jit/jitdump"] diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 78c483d5f0..8b2a2ac47e 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -14,6 +14,8 @@ use wasmtime_environ::Tunables; use wasmtime_jit::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator, RuntimeMemoryCreator}; +pub use wasmtime_environ::CacheStore; + #[cfg(feature = "pooling-allocator")] pub use wasmtime_runtime::{InstanceLimits, PoolingAllocationStrategy}; @@ -117,6 +119,8 @@ struct CompilerConfig { target: Option, settings: HashMap, flags: HashSet, + #[cfg(compiler)] + cache_store: Option>, } #[cfg(compiler)] @@ -127,6 +131,7 @@ impl CompilerConfig { target: None, settings: HashMap::new(), flags: HashSet::new(), + cache_store: None, } } @@ -227,6 +232,17 @@ impl Config { Ok(self) } + /// Enables the incremental compilation cache in Cranelift, using the provided `CacheStore` + /// backend for storage. + #[cfg(all(feature = "incremental-cache", feature = "cranelift"))] + pub fn enable_incremental_compilation( + &mut self, + cache_store: Arc, + ) -> Result<&mut Self> { + self.compiler_config.cache_store = Some(cache_store); + Ok(self) + } + /// Whether or not to enable support for asynchronous functions in Wasmtime. /// /// When enabled, the config can optionally define host functions with `async`. @@ -1488,6 +1504,10 @@ impl Config { compiler.enable(flag)?; } + if let Some(cache_store) = &self.compiler_config.cache_store { + compiler.enable_incremental_compilation(cache_store.clone()); + } + compiler.build() } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index e1c985208d..a77155532e 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -396,6 +396,7 @@ impl Engine { | "probestack_func_adjusts_sp" // probestack above asserted disabled | "probestack_size_log2" // probestack above asserted disabled | "regalloc" // shouldn't change semantics + | "enable_incremental_compilation_cache_checks" // shouldn't change semantics | "enable_atomics" => return Ok(()), // Everything else is unknown and needs to be added somewhere to diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 7f3ae4a346..40c2003ddc 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true [dependencies] anyhow = { version = "1.0.19" } arbitrary = { version = "1.1.0", features = ["derive"] } -cranelift-codegen = { path = "../cranelift/codegen" } +cranelift-codegen = { path = "../cranelift/codegen", features = ["incremental-cache"] } cranelift-reader = { path = "../cranelift/reader" } cranelift-wasm = { path = "../cranelift/wasm" } cranelift-filetests = { path = "../cranelift/filetests" } @@ -120,3 +120,9 @@ name = "component_api" path = "fuzz_targets/component_api.rs" test = false doc = false + +[[bin]] +name = "cranelift-icache" +path = "fuzz_targets/cranelift-icache.rs" +test = false +doc = false diff --git a/fuzz/README.md b/fuzz/README.md index 47a5f6f9a2..5d8e53b02f 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -30,6 +30,10 @@ At the time of writing, we have the following fuzz targets: * `cranelift-fuzzgen`: Generate a Cranelift function and check that it returns the same results when compiled to the host and when using the Cranelift interpreter; only a subset of Cranelift IR is currently supported. +* `cranelift-icache`: Generate a Cranelift function A, applies a small mutation + to its source, yielding a function A', and checks that A compiled + + incremental compilation generates the same machine code as if A' was compiled + from scratch. * `differential`: Generate a Wasm module and check that Wasmtime returns the same results when run with two different configurations. * `differential_spec`: Generate a Wasm module and check that Wasmtime returns diff --git a/fuzz/fuzz_targets/cranelift-icache.rs b/fuzz/fuzz_targets/cranelift-icache.rs new file mode 100644 index 0000000000..1c6959a78a --- /dev/null +++ b/fuzz/fuzz_targets/cranelift-icache.rs @@ -0,0 +1,148 @@ +#![no_main] + +use cranelift_codegen::{ + cursor::{Cursor, FuncCursor}, + incremental_cache as icache, + ir::{self, immediates::Imm64, ExternalName}, + isa, + settings::{self, Configurable as _}, + Context, +}; +use libfuzzer_sys::fuzz_target; + +use cranelift_fuzzgen::*; +use target_lexicon::Triple; + +fuzz_target!(|func: SingleFunction| { + let mut func = func.0; + + let flags = settings::Flags::new({ + let mut builder = settings::builder(); + // We need llvm ABI extensions for i128 values on x86 + builder.set("enable_llvm_abi_extensions", "true").unwrap(); + builder + }); + + let isa_builder = isa::lookup(Triple::host()) + .map_err(|err| match err { + isa::LookupError::SupportDisabled => { + "support for architecture disabled at compile time" + } + isa::LookupError::Unsupported => "unsupported architecture", + }) + .unwrap(); + + let isa = isa_builder.finish(flags).unwrap(); + + let cache_key_hash = icache::compute_cache_key(&*isa, &mut func); + + let mut context = Context::for_function(func.clone()); + let prev_stencil = match context.compile_stencil(&*isa) { + Ok(stencil) => stencil, + Err(_) => return, + }; + + let (prev_stencil, serialized) = icache::serialize_compiled(prev_stencil); + let serialized = serialized.expect("serialization should work"); + let prev_result = prev_stencil.apply_params(&func.params); + + let new_result = icache::try_finish_recompile(&func, &serialized) + .expect("recompilation should always work for identity"); + + assert_eq!(new_result, prev_result, "MachCompileResult:s don't match"); + + let new_info = new_result.code_info(); + assert_eq!(new_info, prev_result.code_info(), "CodeInfo:s don't match"); + + // If the func has at least one user-defined func ref, change it to match a + // different external function. + let expect_cache_hit = if let Some(user_ext_ref) = + func.stencil.dfg.ext_funcs.values().find_map(|data| { + if let ExternalName::User(user_ext_ref) = &data.name { + Some(user_ext_ref) + } else { + None + } + }) { + let mut prev = func.params.user_named_funcs()[*user_ext_ref].clone(); + prev.index = prev.index.checked_add(1).unwrap_or_else(|| prev.index - 1); + func.params.reset_user_func_name(*user_ext_ref, prev); + true + } else { + // otherwise just randomly change one instruction in the middle and see what happens. + let mut changed = false; + let mut cursor = FuncCursor::new(&mut func); + 'out: while let Some(_block) = cursor.next_block() { + while let Some(inst) = cursor.next_inst() { + // It's impractical to do any replacement at this point. Try to find any + // instruction that returns one int value, and replace it with an iconst. + if cursor.func.dfg.inst_results(inst).len() != 1 { + continue; + } + let out_ty = cursor + .func + .dfg + .value_type(cursor.func.dfg.first_result(inst)); + match out_ty { + ir::types::I32 | ir::types::I64 => {} + _ => continue, + } + + if let ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm, + } = cursor.func.dfg[inst] + { + let imm = imm.bits(); + cursor.func.dfg[inst] = ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm: Imm64::new(imm.checked_add(1).unwrap_or(imm - 1)), + }; + } else { + cursor.func.dfg[inst] = ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm: Imm64::new(42), + }; + } + + changed = true; + break 'out; + } + } + + if !changed { + return; + } + + // We made it so that there shouldn't be a cache hit. + false + }; + + let new_cache_key_hash = icache::compute_cache_key(&*isa, &mut func); + + if expect_cache_hit { + assert!(cache_key_hash == new_cache_key_hash); + } else { + assert!(cache_key_hash != new_cache_key_hash); + } + + context = Context::for_function(func.clone()); + + let after_mutation_result = match context.compile(&*isa) { + Ok(info) => info, + Err(_) => return, + }; + + if expect_cache_hit { + let after_mutation_result_from_cache = icache::try_finish_recompile(&func, &serialized) + .expect("recompilation should always work for identity"); + assert_eq!(*after_mutation_result, after_mutation_result_from_cache); + + let new_info = after_mutation_result_from_cache.code_info(); + assert_eq!( + new_info, + after_mutation_result.code_info(), + "CodeInfo:s don't match" + ); + } +}); diff --git a/tests/all/module.rs b/tests/all/module.rs index 285f959b72..dd5e53f59e 100644 --- a/tests/all/module.rs +++ b/tests/all/module.rs @@ -76,21 +76,21 @@ fn serialize_deterministic() { let p1 = engine.precompile_module(wasm.as_bytes()).unwrap(); let p2 = engine.precompile_module(wasm.as_bytes()).unwrap(); if p1 != p2 { - panic!("precompile_module not determinisitc for:\n{}", wasm); + panic!("precompile_module not deterministic for:\n{}", wasm); } let module1 = Module::new(&engine, wasm).unwrap(); let a1 = module1.serialize().unwrap(); let a2 = module1.serialize().unwrap(); if a1 != a2 { - panic!("Module::serialize not determinisitc for:\n{}", wasm); + panic!("Module::serialize not deterministic for:\n{}", wasm); } let module2 = Module::new(&engine, wasm).unwrap(); let b1 = module2.serialize().unwrap(); let b2 = module2.serialize().unwrap(); if b1 != b2 { - panic!("Module::serialize not determinisitc for:\n{}", wasm); + panic!("Module::serialize not deterministic for:\n{}", wasm); } if a1 != b2 {