Test basic DWARF generation (#931)

* Add obj generation with debug info
* Add simple transform check

.github/actions/define-dwarfdump-env/README.md

@@ -0,0 +1,3 @@
+# define-dwarfdump-env
+
+Defines `DWARFDUMP` path executable.

.github/actions/define-dwarfdump-env/action.yml

@@ -0,0 +1,6 @@
+name: 'Set up a DWARFDUMP env'
+description: 'Set up a DWARFDUMP env (see tests/debug/dump.rs)'
+
+runs:
+  using: node12
+  main: 'main.js'

.github/actions/define-dwarfdump-env/main.js

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+
+// On OSX pointing to brew's LLVM location.
+if (process.platform == 'darwin') {
+  console.log("::set-env name=DWARFDUMP::/usr/local/opt/llvm/bin/llvm-dwarfdump");
+}
+
+// On Linux pointing to specific version
+if (process.platform == 'linux') {
+  console.log("::set-env name=DWARFDUMP::/usr/bin/llvm-dwarfdump-9");
+}

.github/workflows/main.yml

@@ -149,6 +149,7 @@ jobs:
     - uses: ./.github/actions/install-rust
       with:
         toolchain: ${{ matrix.rust }}
+    - uses: ./.github/actions/define-dwarfdump-env
 
     - name: Install libclang
       # Note: libclang is pre-installed on the macOS and linux images.
@@ -327,7 +328,7 @@ jobs:
     - run: $CENTOS cargo build --release --manifest-path crates/c-api/Cargo.toml
       shell: bash
     # Test what we just built
-    - run: $CENTOS cargo test --features test_programs --release --all --exclude lightbeam --exclude wasmtime --exclude wasmtime-c-api --exclude wasmtime-fuzzing
+    - run: $CENTOS cargo test --features test_programs --release --all --exclude lightbeam --exclude wasmtime --exclude wasmtime-c-api --exclude wasmtime-fuzzing -- --skip test_debug_dwarf_
       shell: bash
       env:
         RUST_BACKTRACE: 1

Cargo.lock

@@ -627,6 +627,28 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "failure"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
+dependencies = [
+ "backtrace",
+ "failure_derive",
+]
+
+[[package]]
+name = "failure_derive"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
+
 [[package]]
 name = "fake-simd"
 version = "0.1.2"
@@ -649,6 +671,17 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "filecheck"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ded7985594ab426ef685362e5183168eb3b5aacc9f4e26819e8d82d224f33449"
+dependencies = [
+ "failure",
+ "failure_derive",
+ "regex",
+]
+
 [[package]]
 name = "filetime"
 version = "0.2.8"
@@ -1935,6 +1968,7 @@ dependencies = [
  "anyhow",
  "faerie",
  "file-per-thread-logger",
+ "filecheck",
  "libc",
  "more-asserts",
  "pretty_env_logger",

Cargo.toml

@@ -51,6 +51,7 @@ more-asserts = "0.2.1"
 # `cargo test --features test-programs`.
 test-programs = { path = "crates/test-programs" }
 tempfile = "3.1.0"
+filecheck = "0.4.0"
 
 [build-dependencies]
 anyhow = "1.0.19"

crates/debug/src/transform/simulate.rs

@@ -178,7 +178,11 @@ fn generate_vars(
 ) {
     let vmctx_label = get_vmctx_value_label();
 
-    for label in frame_info.value_ranges.keys() {
+    // Normalize order of ValueLabelsRanges keys to have reproducable results.
+    let mut vars = frame_info.value_ranges.keys().collect::<Vec<_>>();
+    vars.sort_by(|a, b| a.index().cmp(&b.index()));
+
+    for label in vars {
         if label.index() == vmctx_label.index() {
             append_vmctx_info(
                 unit,

src/commands/wasm2obj.rs

@@ -1,8 +1,8 @@
 //! The module that implements the `wasmtime wasm2obj` command.
 
+use crate::obj::compile_to_obj;
 use crate::{init_file_per_thread_logger, pick_compilation_strategy, CommonOptions};
-use anyhow::{anyhow, bail, Context as _, Result};
+use anyhow::{anyhow, Context as _, Result};
-use faerie::Artifact;
 use std::{
     fs::File,
     path::{Path, PathBuf},
@@ -10,17 +10,9 @@ use std::{
 };
 use structopt::{clap::AppSettings, StructOpt};
 use target_lexicon::Triple;
-use wasmtime::Strategy;
+use wasmtime_environ::CacheConfig;
-use wasmtime_debug::{emit_debugsections, read_debuginfo};
 #[cfg(feature = "lightbeam")]
 use wasmtime_environ::Lightbeam;
-use wasmtime_environ::{
-    entity::EntityRef, settings, settings::Configurable, wasm::DefinedMemoryIndex,
-    wasm::MemoryIndex, CacheConfig, Compiler, Cranelift, ModuleEnvironment, ModuleMemoryOffset,
-    ModuleVmctxInfo, Tunables, VMOffsets,
-};
-use wasmtime_jit::native;
-use wasmtime_obj::emit_module;
 
 /// The after help text for the `wasm2obj` command.
 pub const WASM2OBJ_AFTER_HELP: &str = "The translation is dependent on the environment chosen.\n\
@@ -78,122 +70,16 @@ impl WasmToObjCommand {
 
         let data = wat::parse_file(&self.module).context("failed to parse module")?;
 
-        let isa_builder = match self.target.as_ref() {
+        let obj = compile_to_obj(
-            Some(target) => native::lookup(target.clone())?,
+            &data,
-            None => native::builder(),
+            self.target.as_ref(),
-        };
+            strategy,
-        let mut flag_builder = settings::builder();
+            self.common.enable_simd,
-
+            self.common.optimize,
-        // There are two possible traps for division, and this way
-        // we get the proper one if code traps.
-        flag_builder.enable("avoid_div_traps").unwrap();
-
-        if self.common.enable_simd {
-            flag_builder.enable("enable_simd").unwrap();
-        }
-
-        if self.common.optimize {
-            flag_builder.set("opt_level", "speed").unwrap();
-        }
-
-        let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-
-        let mut obj = Artifact::new(isa.triple().clone(), self.output.clone());
-
-        // TODO: Expose the tunables as command-line flags.
-        let tunables = Tunables::default();
-
-        let (
-            module,
-            module_translation,
-            lazy_function_body_inputs,
-            lazy_data_initializers,
-            target_config,
-        ) = {
-            let environ = ModuleEnvironment::new(isa.frontend_config(), tunables);
-
-            let translation = environ
-                .translate(&data)
-                .context("failed to translate module")?;
-
-            (
-                translation.module,
-                translation.module_translation.unwrap(),
-                translation.function_body_inputs,
-                translation.data_initializers,
-                translation.target_config,
-            )
-        };
-
-        // TODO: use the traps information
-        let (compilation, relocations, address_transform, value_ranges, stack_slots, _traps) =
-            match strategy {
-                Strategy::Auto | Strategy::Cranelift => Cranelift::compile_module(
-                    &module,
-                    &module_translation,
-                    lazy_function_body_inputs,
-                    &*isa,
             self.common.debug_info,
+            self.output.clone(),
             &cache_config,
-                ),
+        )?;
-                #[cfg(feature = "lightbeam")]
-                Strategy::Lightbeam => Lightbeam::compile_module(
-                    &module,
-                    &module_translation,
-                    lazy_function_body_inputs,
-                    &*isa,
-                    self.common.debug_info,
-                    &cache_config,
-                ),
-                #[cfg(not(feature = "lightbeam"))]
-                Strategy::Lightbeam => bail!("lightbeam support not enabled"),
-                other => bail!("unsupported compilation strategy {:?}", other),
-            }
-            .context("failed to compile module")?;
-
-        if compilation.is_empty() {
-            bail!("no functions were found/compiled");
-        }
-
-        let module_vmctx_info = {
-            let ofs = VMOffsets::new(target_config.pointer_bytes(), &module);
-            ModuleVmctxInfo {
-                memory_offset: if ofs.num_imported_memories > 0 {
-                    ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
-                } else if ofs.num_defined_memories > 0 {
-                    ModuleMemoryOffset::Defined(
-                        ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
-                    )
-                } else {
-                    ModuleMemoryOffset::None
-                },
-                stack_slots,
-            }
-        };
-
-        emit_module(
-            &mut obj,
-            &module,
-            &compilation,
-            &relocations,
-            &lazy_data_initializers,
-            &target_config,
-        )
-        .map_err(|e| anyhow!(e))
-        .context("failed to emit module")?;
-
-        if self.common.debug_info {
-            let debug_data = read_debuginfo(&data);
-            emit_debugsections(
-                &mut obj,
-                &module_vmctx_info,
-                target_config,
-                &debug_data,
-                &address_transform,
-                &value_ranges,
-            )
-            .context("failed to emit debug sections")?;
-        }
 
         // FIXME: Make the format a parameter.
         let file = File::create(Path::new(&self.output)).context("failed to create object file")?;

src/lib.rs

@@ -25,12 +25,15 @@
 )]
 
 pub mod commands;
+mod obj;
 
 use anyhow::{bail, Result};
 use std::path::PathBuf;
 use structopt::StructOpt;
 use wasmtime::{Config, Strategy};
 
+pub use obj::compile_to_obj;
+
 fn pick_compilation_strategy(cranelift: bool, lightbeam: bool) -> Result<Strategy> {
     Ok(match (lightbeam, cranelift) {
         (true, false) => Strategy::Lightbeam,

src/obj.rs

@@ -0,0 +1,144 @@
+use anyhow::{anyhow, bail, Context as _, Result};
+use faerie::Artifact;
+use target_lexicon::Triple;
+use wasmtime::Strategy;
+use wasmtime_debug::{emit_debugsections, read_debuginfo};
+#[cfg(feature = "lightbeam")]
+use wasmtime_environ::Lightbeam;
+use wasmtime_environ::{
+    entity::EntityRef, settings, settings::Configurable, wasm::DefinedMemoryIndex,
+    wasm::MemoryIndex, CacheConfig, Compiler, Cranelift, ModuleEnvironment, ModuleMemoryOffset,
+    ModuleVmctxInfo, Tunables, VMOffsets,
+};
+use wasmtime_jit::native;
+use wasmtime_obj::emit_module;
+
+/// Creates object file from binary wasm data.
+pub fn compile_to_obj(
+    wasm: &[u8],
+    target: Option<&Triple>,
+    strategy: Strategy,
+    enable_simd: bool,
+    optimize: bool,
+    debug_info: bool,
+    artifact_name: String,
+    cache_config: &CacheConfig,
+) -> Result<Artifact> {
+    let isa_builder = match target {
+        Some(target) => native::lookup(target.clone())?,
+        None => native::builder(),
+    };
+    let mut flag_builder = settings::builder();
+
+    // There are two possible traps for division, and this way
+    // we get the proper one if code traps.
+    flag_builder.enable("avoid_div_traps").unwrap();
+
+    if enable_simd {
+        flag_builder.enable("enable_simd").unwrap();
+    }
+
+    if optimize {
+        flag_builder.set("opt_level", "speed").unwrap();
+    }
+
+    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
+
+    let mut obj = Artifact::new(isa.triple().clone(), artifact_name);
+
+    // TODO: Expose the tunables as command-line flags.
+    let tunables = Tunables::default();
+
+    let (
+        module,
+        module_translation,
+        lazy_function_body_inputs,
+        lazy_data_initializers,
+        target_config,
+    ) = {
+        let environ = ModuleEnvironment::new(isa.frontend_config(), tunables);
+
+        let translation = environ
+            .translate(wasm)
+            .context("failed to translate module")?;
+
+        (
+            translation.module,
+            translation.module_translation.unwrap(),
+            translation.function_body_inputs,
+            translation.data_initializers,
+            translation.target_config,
+        )
+    };
+
+    // TODO: use the traps information
+    let (compilation, relocations, address_transform, value_ranges, stack_slots, _traps) =
+        match strategy {
+            Strategy::Auto | Strategy::Cranelift => Cranelift::compile_module(
+                &module,
+                &module_translation,
+                lazy_function_body_inputs,
+                &*isa,
+                debug_info,
+                cache_config,
+            ),
+            #[cfg(feature = "lightbeam")]
+            Strategy::Lightbeam => Lightbeam::compile_module(
+                &module,
+                &module_translation,
+                lazy_function_body_inputs,
+                &*isa,
+                debug_info,
+                cache_config,
+            ),
+            #[cfg(not(feature = "lightbeam"))]
+            Strategy::Lightbeam => bail!("lightbeam support not enabled"),
+            other => bail!("unsupported compilation strategy {:?}", other),
+        }
+        .context("failed to compile module")?;
+
+    if compilation.is_empty() {
+        bail!("no functions were found/compiled");
+    }
+
+    let module_vmctx_info = {
+        let ofs = VMOffsets::new(target_config.pointer_bytes(), &module);
+        ModuleVmctxInfo {
+            memory_offset: if ofs.num_imported_memories > 0 {
+                ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
+            } else if ofs.num_defined_memories > 0 {
+                ModuleMemoryOffset::Defined(
+                    ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
+                )
+            } else {
+                ModuleMemoryOffset::None
+            },
+            stack_slots,
+        }
+    };
+
+    emit_module(
+        &mut obj,
+        &module,
+        &compilation,
+        &relocations,
+        &lazy_data_initializers,
+        &target_config,
+    )
+    .map_err(|e| anyhow!(e))
+    .context("failed to emit module")?;
+
+    if debug_info {
+        let debug_data = read_debuginfo(wasm);
+        emit_debugsections(
+            &mut obj,
+            &module_vmctx_info,
+            target_config,
+            &debug_data,
+            &address_transform,
+            &value_ranges,
+        )
+        .context("failed to emit debug sections")?;
+    }
+    Ok(obj)
+}

tests/debug/dump.rs

@@ -0,0 +1,30 @@
+use anyhow::{bail, Result};
+use std::env;
+use std::process::Command;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[allow(dead_code)]
+pub enum DwarfDumpSection {
+    DebugInfo,
+    DebugLine,
+}
+
+pub fn get_dwarfdump(obj: &str, section: DwarfDumpSection) -> Result<String> {
+    let dwarfdump = env::var("DWARFDUMP").unwrap_or("llvm-dwarfdump".to_string());
+    let section_flag = match section {
+        DwarfDumpSection::DebugInfo => "-debug-info",
+        DwarfDumpSection::DebugLine => "-debug-line",
+    };
+    let output = Command::new(&dwarfdump)
+        .args(&[section_flag, obj])
+        .output()
+        .expect("success");
+    if !output.status.success() {
+        bail!(
+            "failed to execute {}: {}",
+            dwarfdump,
+            String::from_utf8_lossy(&output.stderr),
+        );
+    }
+    Ok(String::from_utf8_lossy(&output.stdout).to_string())
+}

tests/debug/main.rs

@@ -0,0 +1,4 @@
+mod dump;
+mod obj;
+mod simulate;
+mod translate;

tests/debug/obj.rs

@@ -0,0 +1,34 @@
+use anyhow::{Context as _, Result};
+use std::fs::File;
+use std::path::Path;
+use target_lexicon::Triple;
+use wasmtime::Strategy;
+use wasmtime_cli::compile_to_obj;
+use wasmtime_environ::CacheConfig;
+
+pub fn compile_cranelift(
+    wasm: &[u8],
+    target: Option<Triple>,
+    output: impl AsRef<Path>,
+) -> Result<()> {
+    let obj = compile_to_obj(
+        wasm,
+        target.as_ref(),
+        Strategy::Cranelift,
+        false,
+        false,
+        true,
+        output
+            .as_ref()
+            .file_name()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        &CacheConfig::new_cache_disabled(),
+    )?;
+
+    let file = File::create(output).context("failed to create object file")?;
+    obj.write(file).context("failed to write object file")?;
+
+    Ok(())
+}

tests/debug/simulate.rs

@@ -0,0 +1,55 @@
+use super::dump::{get_dwarfdump, DwarfDumpSection};
+use super::obj::compile_cranelift;
+use anyhow::{format_err, Result};
+use filecheck::{CheckerBuilder, NO_VARIABLES};
+use tempfile::NamedTempFile;
+use wat::parse_str;
+
+#[allow(dead_code)]
+fn check_wat(wat: &str) -> Result<()> {
+    let wasm = parse_str(wat)?;
+    let obj_file = NamedTempFile::new()?;
+    let obj_path = obj_file.path().to_str().unwrap();
+    compile_cranelift(&wasm, None, obj_path)?;
+    let dump = get_dwarfdump(obj_path, DwarfDumpSection::DebugInfo)?;
+    let mut builder = CheckerBuilder::new();
+    builder
+        .text(wat)
+        .map_err(|e| format_err!("unable to build checker: {:?}", e))?;
+    let checker = builder.finish();
+    let check = checker
+        .explain(&dump, NO_VARIABLES)
+        .map_err(|e| format_err!("{:?}", e))?;
+    assert!(check.0, "didn't pass check {}", check.1);
+    Ok(())
+}
+
+#[test]
+#[cfg(all(
+    any(target_os = "linux", target_os = "macos"),
+    target_pointer_width = "64"
+))]
+fn test_debug_dwarf_simulate_simple_x86_64() -> Result<()> {
+    check_wat(
+        r#"
+;; check: DW_TAG_compile_unit 
+(module
+;; check: DW_TAG_subprogram 
+;; check: DW_AT_name	("wasm-function[0]")
+;; check:   DW_TAG_formal_parameter
+;; check:     DW_AT_name	("var0")
+;; check:     DW_AT_type
+;; sameln:	            "i32"
+;; check:   DW_TAG_variable
+;; check:     DW_AT_name	("var1")
+;; check:     DW_AT_type
+;; sameln:	            "i32"
+    (func (param i32) (result i32)
+        (local i32)
+        local.get 0
+        local.set 1
+        local.get 1
+    )
+)"#,
+    )
+}

tests/debug/testsuite/fib-wasm.c

@@ -0,0 +1,13 @@
+// Compile with:
+//   clang --target=wasm32 fib-wasm.c -o fib-wasm.wasm -g \
+//     -Wl,--no-entry,--export=fib -nostdlib -fdebug-prefix-map=$PWD=.
+
+int fib(int n) {
+  int i, t, a = 0, b = 1;
+  for (i = 0; i < n; i++) {
+    t = a;
+    a = b;
+    b += t;
+  }
+  return b;
+}

tests/debug/translate.rs

@@ -0,0 +1,56 @@
+use super::dump::{get_dwarfdump, DwarfDumpSection};
+use super::obj::compile_cranelift;
+use anyhow::{format_err, Result};
+use filecheck::{CheckerBuilder, NO_VARIABLES};
+use std::fs::read;
+use tempfile::NamedTempFile;
+
+#[allow(dead_code)]
+fn check_wasm(wasm_path: &str, directives: &str) -> Result<()> {
+    let wasm = read(wasm_path)?;
+    let obj_file = NamedTempFile::new()?;
+    let obj_path = obj_file.path().to_str().unwrap();
+    compile_cranelift(&wasm, None, obj_path)?;
+    let dump = get_dwarfdump(obj_path, DwarfDumpSection::DebugInfo)?;
+    let mut builder = CheckerBuilder::new();
+    builder
+        .text(directives)
+        .map_err(|e| format_err!("unable to build checker: {:?}", e))?;
+    let checker = builder.finish();
+    let check = checker
+        .explain(&dump, NO_VARIABLES)
+        .map_err(|e| format_err!("{:?}", e))?;
+    assert!(check.0, "didn't pass check {}", check.1);
+    Ok(())
+}
+
+#[test]
+#[cfg(all(
+    any(target_os = "linux", target_os = "macos"),
+    target_pointer_width = "64"
+))]
+fn test_debug_dwarf_translate() -> Result<()> {
+    check_wasm(
+        "tests/debug/testsuite/fib-wasm.wasm",
+        r##"
+check: DW_TAG_compile_unit
+# We have "fib" function
+check: DW_TAG_subprogram
+check:      DW_AT_name	("fib")
+# Accepts one parameter
+check:      DW_TAG_formal_parameter
+check:        DW_AT_name	("n")
+check:        DW_AT_decl_line	(5)
+# Has four locals: i, t, a, b
+check:      DW_TAG_variable
+check:        DW_AT_name	("i")
+check:        DW_AT_decl_line	(6)
+check:      DW_TAG_variable
+check:        DW_AT_name	("t")
+check:      DW_TAG_variable
+check:        DW_AT_name	("a")
+check:      DW_TAG_variable
+check:        DW_AT_name	("b")
+    "##,
+    )
+}