Add support for generating perf maps for simple perf profiling (#6030)

* Add support for generating perf maps for simple perf profiling

* add missing enum entry in C code

* bugfix: use hexa when printing the code region's length too (thanks bjorn3!)

* sanitize file name + use bufwriter

* introduce --profile CLI flag for wasmtime

* Update doc and doc comments for new --profile option

* remove redundant FromStr import

* Apply review feedback: make_line receives a Write impl, report errors

* fix tests?

* better docs

crates/c-api/include/wasmtime/config.h

@@ -80,6 +80,10 @@ enum wasmtime_profiling_strategy_enum { // ProfilingStrategy
   ///
   /// Note that this isn't always enabled at build time.
   WASMTIME_PROFILING_STRATEGY_VTUNE,
+  /// Linux's simple "perfmap" support in `perf` is enabled and when Wasmtime is
+  /// run under `perf` necessary calls will be made to profile generated JIT
+  /// code.
+  WASMTIME_PROFILING_STRATEGY_PERFMAP,
 };
 
 #define WASMTIME_CONFIG_PROP(ret, name, ty) \

crates/c-api/src/config.rs

@@ -36,6 +36,7 @@ pub enum wasmtime_profiling_strategy_t {
     WASMTIME_PROFILING_STRATEGY_NONE,
     WASMTIME_PROFILING_STRATEGY_JITDUMP,
     WASMTIME_PROFILING_STRATEGY_VTUNE,
+    WASMTIME_PROFILING_STRATEGY_PERFMAP,
 }
 
 #[no_mangle]
@@ -157,6 +158,7 @@ pub extern "C" fn wasmtime_config_profiler_set(
         WASMTIME_PROFILING_STRATEGY_NONE => ProfilingStrategy::None,
         WASMTIME_PROFILING_STRATEGY_JITDUMP => ProfilingStrategy::JitDump,
         WASMTIME_PROFILING_STRATEGY_VTUNE => ProfilingStrategy::VTune,
+        WASMTIME_PROFILING_STRATEGY_PERFMAP => ProfilingStrategy::PerfMap,
     });
 }
 

crates/cli-flags/src/lib.rs

@@ -68,18 +68,6 @@ pub const SUPPORTED_WASI_MODULES: &[(&str, &str)] = &[
     ),
 ];
 
-fn pick_profiling_strategy(jitdump: bool, vtune: bool) -> Result<ProfilingStrategy> {
-    Ok(match (jitdump, vtune) {
-        (true, false) => ProfilingStrategy::JitDump,
-        (false, true) => ProfilingStrategy::VTune,
-        (true, true) => {
-            println!("Can't enable --jitdump and --vtune at the same time. Profiling not enabled.");
-            ProfilingStrategy::None
-        }
-        _ => ProfilingStrategy::None,
-    })
-}
-
 fn init_file_per_thread_logger(prefix: &'static str) {
     file_per_thread_logger::initialize(prefix);
 
@@ -142,14 +130,11 @@ pub struct CommonOptions {
     #[clap(long, value_name = "MODULE,MODULE,...", parse(try_from_str = parse_wasi_modules))]
     pub wasi_modules: Option<WasiModules>,
 
+    /// Profiling strategy (valid options are: perfmap, jitdump, vtune)
+    #[clap(long)]
+    pub profile: Option<ProfilingStrategy>,
+
     /// Generate jitdump file (supported on --features=profiling build)
-    #[clap(long, conflicts_with = "vtune")]
-    pub jitdump: bool,
-
-    /// Generate vtune (supported on --features=vtune build)
-    #[clap(long, conflicts_with = "jitdump")]
-    pub vtune: bool,
-
     /// Run optimization passes on translated functions, on by default
     #[clap(short = 'O', long)]
     pub optimize: bool,
@@ -283,7 +268,7 @@ impl CommonOptions {
             .cranelift_debug_verifier(self.enable_cranelift_debug_verifier)
             .debug_info(self.debug_info)
             .cranelift_opt_level(self.opt_level())
-            .profiler(pick_profiling_strategy(self.jitdump, self.vtune)?)
+            .profiler(self.profile.unwrap_or(ProfilingStrategy::None))
             .cranelift_nan_canonicalization(self.enable_cranelift_nan_canonicalization);
 
         self.enable_wasm_features(&mut config);

crates/jit-debug/src/perf_jitdump.rs

@@ -4,7 +4,7 @@
 //!
 //! Usage Example:
 //!     Record
-//!         sudo perf record -k 1 -e instructions:u target/debug/wasmtime -g --jitdump test.wasm
+//!         sudo perf record -k 1 -e instructions:u target/debug/wasmtime -g --profile=jitdump test.wasm
 //!     Combine
 //!         sudo perf inject -v -j -i perf.data -o perf.jit.data
 //!     Report

crates/jit/src/profiling.rs

@@ -11,6 +11,16 @@ cfg_if::cfg_if! {
     }
 }
 
+cfg_if::cfg_if! {
+    if #[cfg(target_os = "linux")] {
+        #[path = "profiling/perfmap_linux.rs"]
+        mod perfmap;
+    } else {
+        #[path = "profiling/perfmap_disabled.rs"]
+        mod perfmap;
+    }
+}
+
 cfg_if::cfg_if! {
     // Note: VTune support is disabled on windows mingw because the ittapi crate doesn't compile
     // there; see also https://github.com/bytecodealliance/wasmtime/pull/4003 for rationale.
@@ -24,6 +34,7 @@ cfg_if::cfg_if! {
 }
 
 pub use jitdump::JitDumpAgent;
+pub use perfmap::PerfMapAgent;
 pub use vtune::VTuneAgent;
 
 /// Common interface for profiling tools.

crates/jit/src/profiling/jitdump_disabled.rs

@@ -8,7 +8,7 @@ pub struct JitDumpAgent {
 }
 
 impl JitDumpAgent {
-    /// Intialize a JitDumpAgent and write out the header
+    /// Intialize a dummy JitDumpAgent that will fail upon instantiation.
     pub fn new() -> Result<Self> {
         if cfg!(feature = "jitdump") {
             bail!("jitdump is not supported on this platform");

crates/jit/src/profiling/jitdump_linux.rs

@@ -4,7 +4,7 @@
 //!
 //! Usage Example:
 //!     Record
-//!         sudo perf record -k 1 -e instructions:u target/debug/wasmtime -g --jitdump test.wasm
+//!         sudo perf record -k 1 -e instructions:u target/debug/wasmtime -g --profile=jitdump test.wasm
 //!     Combine
 //!         sudo perf inject -v -j -i perf.data -o perf.jit.data
 //!     Report

crates/jit/src/profiling/perfmap_disabled.rs

@@ -0,0 +1,28 @@
+use crate::{CompiledModule, ProfilingAgent};
+use anyhow::{bail, Result};
+
+/// Interface for driving the creation of jitdump files
+#[derive(Debug)]
+pub struct PerfMapAgent {
+    _private: (),
+}
+
+impl PerfMapAgent {
+    /// Intialize a dummy PerfMapAgent that will fail upon instantiation.
+    pub fn new() -> Result<Self> {
+        bail!("perfmap support not supported on this platform");
+    }
+}
+
+impl ProfilingAgent for PerfMapAgent {
+    fn module_load(&self, _module: &CompiledModule, _dbg_image: Option<&[u8]>) {}
+    fn load_single_trampoline(
+        &self,
+        _name: &str,
+        _addr: *const u8,
+        _size: usize,
+        __pid: u32,
+        _tid: u32,
+    ) {
+    }
+}

crates/jit/src/profiling/perfmap_linux.rs

@@ -0,0 +1,85 @@
+use crate::{CompiledModule, ProfilingAgent};
+use anyhow::Result;
+use std::io::{self, BufWriter, Write};
+use std::process;
+use std::{fs::File, sync::Mutex};
+use wasmtime_environ::EntityRef as _;
+
+/// Process-wide perf map file. Perf only reads a unique file per process.
+static PERFMAP_FILE: Mutex<Option<File>> = Mutex::new(None);
+
+/// Interface for driving the creation of jitdump files
+pub struct PerfMapAgent;
+
+impl PerfMapAgent {
+    /// Intialize a JitDumpAgent and write out the header.
+    pub fn new() -> Result<Self> {
+        let mut file = PERFMAP_FILE.lock().unwrap();
+        if file.is_none() {
+            let filename = format!("/tmp/perf-{}.map", process::id());
+            *file = Some(File::create(filename)?);
+        }
+        Ok(PerfMapAgent)
+    }
+
+    fn make_line(
+        writer: &mut dyn Write,
+        name: &str,
+        addr: *const u8,
+        len: usize,
+    ) -> io::Result<()> {
+        // Format is documented here: https://github.com/torvalds/linux/blob/master/tools/perf/Documentation/jit-interface.txt
+        // Try our best to sanitize the name, since wasm allows for any utf8 string in there.
+        let sanitized_name = name.replace('\n', "_").replace('\r', "_");
+        write!(writer, "{:x} {:x} {}\n", addr as usize, len, sanitized_name)?;
+        Ok(())
+    }
+}
+
+impl ProfilingAgent for PerfMapAgent {
+    /// Sent when a method is compiled and loaded into memory by the VM.
+    fn module_load(&self, module: &CompiledModule, _dbg_image: Option<&[u8]>) {
+        let mut file = PERFMAP_FILE.lock().unwrap();
+        let file = file.as_mut().unwrap();
+        let mut file = BufWriter::new(file);
+
+        for (idx, func) in module.finished_functions() {
+            let addr = func.as_ptr();
+            let len = func.len();
+            let name = super::debug_name(module, idx);
+            if let Err(err) = Self::make_line(&mut file, &name, addr, len) {
+                eprintln!("Error when writing function info to the perf map file: {err}");
+                return;
+            }
+        }
+
+        // Note: these are the trampolines into exported functions.
+        for (idx, func, len) in module.trampolines() {
+            let (addr, len) = (func as usize as *const u8, len);
+            let name = format!("wasm::trampoline[{}]", idx.index());
+            if let Err(err) = Self::make_line(&mut file, &name, addr, len) {
+                eprintln!("Error when writing export trampoline info to the perf map file: {err}");
+                return;
+            }
+        }
+
+        if let Err(err) = file.flush() {
+            eprintln!("Error when flushing the perf map file buffer: {err}");
+        }
+    }
+
+    fn load_single_trampoline(
+        &self,
+        name: &str,
+        addr: *const u8,
+        size: usize,
+        _pid: u32,
+        _tid: u32,
+    ) {
+        let mut file = PERFMAP_FILE.lock().unwrap();
+        let file = file.as_mut().unwrap();
+        if let Err(err) = Self::make_line(file, name, addr, size) {
+            eprintln!("Error when writing import trampoline info to the perf map file: {err}");
+        }
+    }
+}

crates/jit/src/profiling/vtune.rs

@@ -1,11 +1,11 @@
 //! Adds support for profiling JIT-ed code using VTune. By default, VTune
 //! support is built in to Wasmtime (configure with the `vtune` feature flag).
-//! To enable it at runtime, use the `--vtune` CLI flag.
+//! To enable it at runtime, use the `--profile=vtune` CLI flag.
 //!
 //! ### Profile
 //!
 //! ```ignore
-//! vtune -run-pass-thru=--no-altstack -v -collect hotspots target/debug/wasmtime --vtune test.wasm
+//! vtune -run-pass-thru=--no-altstack -v -collect hotspots target/debug/wasmtime --profile=vtune test.wasm
 //! ```
 //!
 //! Note: `vtune` is a command-line tool for VTune which must [be

crates/jit/src/profiling/vtune_disabled.rs

@@ -8,7 +8,7 @@ pub struct VTuneAgent {
 }
 
 impl VTuneAgent {
-    /// Intialize a VTuneAgent and write out the header
+    /// Intialize a dummy VTuneAgent that will fail upon instantiation.
     pub fn new() -> Result<Self> {
         if cfg!(feature = "vtune") {
             bail!("VTune is not supported on this platform.");

crates/wasmtime/src/config.rs

@@ -6,13 +6,14 @@ use std::collections::{HashMap, HashSet};
 use std::fmt;
 #[cfg(feature = "cache")]
 use std::path::Path;
+use std::str::FromStr;
 use std::sync::Arc;
 use target_lexicon::Architecture;
 use wasmparser::WasmFeatures;
 #[cfg(feature = "cache")]
 use wasmtime_cache::CacheConfig;
 use wasmtime_environ::Tunables;
-use wasmtime_jit::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent};
+use wasmtime_jit::{JitDumpAgent, NullProfilerAgent, PerfMapAgent, ProfilingAgent, VTuneAgent};
 use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator, RuntimeMemoryCreator};
 
 pub use wasmtime_environ::CacheStore;
@@ -221,7 +222,6 @@ impl Config {
     #[cfg(compiler)]
     #[cfg_attr(nightlydoc, doc(cfg(feature = "cranelift")))] // see build.rs
     pub fn target(&mut self, target: &str) -> Result<&mut Self> {
-        use std::str::FromStr;
         self.compiler_config.target =
             Some(target_lexicon::Triple::from_str(target).map_err(|e| anyhow::anyhow!(e))?);
 
@@ -1536,6 +1536,7 @@ impl Config {
 
     pub(crate) fn build_profiler(&self) -> Result<Box<dyn ProfilingAgent>> {
         Ok(match self.profiling_strategy {
+            ProfilingStrategy::PerfMap => Box::new(PerfMapAgent::new()?) as Box<dyn ProfilingAgent>,
             ProfilingStrategy::JitDump => Box::new(JitDumpAgent::new()?) as Box<dyn ProfilingAgent>,
             ProfilingStrategy::VTune => Box::new(VTuneAgent::new()?) as Box<dyn ProfilingAgent>,
             ProfilingStrategy::None => Box::new(NullProfilerAgent),
@@ -1727,11 +1728,14 @@ pub enum OptLevel {
 }
 
 /// Select which profiling technique to support.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum ProfilingStrategy {
     /// No profiler support.
     None,
 
+    /// Collect function name information as the "perf map" file format, used with `perf` on Linux.
+    PerfMap,
+
     /// Collect profiling info for "jitdump" file format, used with `perf` on
     /// Linux.
     JitDump,
@@ -1740,6 +1744,20 @@ pub enum ProfilingStrategy {
     VTune,
 }
 
+impl FromStr for ProfilingStrategy {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s {
+            "none" => Ok(Self::None),
+            "perfmap" => Ok(Self::PerfMap),
+            "jitdump" => Ok(Self::JitDump),
+            "vtune" => Ok(Self::VTune),
+            _ => anyhow::bail!("unknown value for profiling strategy"),
+        }
+    }
+}
+
 /// Select how wasm backtrace detailed information is handled.
 #[derive(Debug, Clone, Copy)]
 pub enum WasmBacktraceDetails {