Migrating code to object (from faerie) (#1848)

* Using the "object" library everywhere in wasmtime.
* scroll_derive

Cargo.lock

@@ -2345,12 +2345,12 @@ version = "0.17.0"
 dependencies = [
  "anyhow",
  "env_logger",
- "faerie",
  "file-per-thread-logger",
  "filecheck",
  "humantime",
  "libc",
  "more-asserts",
+ "object",
  "pretty_env_logger",
  "rayon",
  "structopt",
@@ -2375,9 +2375,9 @@ name = "wasmtime-debug"
 version = "0.17.0"
 dependencies = [
  "anyhow",
- "faerie",
  "gimli",
  "more-asserts",
+ "object",
  "target-lexicon",
  "thiserror",
  "wasmparser 0.57.0",
@@ -2477,8 +2477,8 @@ name = "wasmtime-obj"
 version = "0.17.0"
 dependencies = [
  "anyhow",
- "faerie",
  "more-asserts",
+ "object",
  "wasmtime-environ",
 ]
 

Cargo.toml

@@ -31,7 +31,7 @@ wasmtime-wast = { path = "crates/wast", version = "0.17.0" }
 wasmtime-wasi = { path = "crates/wasi", version = "0.17.0" }
 wasi-common = { path = "crates/wasi-common", version = "0.17.0" }
 structopt = { version = "0.3.5", features = ["color", "suggestions"] }
-faerie = "0.15.0"
+object = { version = "0.18", default-features = false, features = ["write"] }
 anyhow = "1.0.19"
 target-lexicon = { version = "0.10.0", default-features = false }
 pretty_env_logger = "0.4.0"

crates/debug/Cargo.toml

@@ -14,7 +14,7 @@ edition = "2018"
 [dependencies]
 gimli = "0.21.0"
 wasmparser = "0.57.0"
-faerie = "0.15.0"
+object = { version = "0.18", default-features = false, features = ["write"] }
 wasmtime-environ = { path = "../environ", version = "0.17.0" }
 target-lexicon = { version = "0.10.0", default-features = false }
 anyhow = "1.0"

crates/debug/src/lib.rs

@@ -3,8 +3,10 @@
 #![allow(clippy::cast_ptr_alignment)]
 
 use anyhow::Error;
-use faerie::{Artifact, Decl, SectionKind};
 use more_asserts::assert_gt;
+use object::write::{Object, Relocation, StandardSegment};
+use object::{RelocationEncoding, RelocationKind, SectionKind};
+use std::collections::HashMap;
 use target_lexicon::BinaryFormat;
 use wasmtime_environ::isa::TargetIsa;
 
@@ -16,25 +18,35 @@ mod read_debuginfo;
 mod transform;
 mod write_debuginfo;
 
-pub fn write_debugsections(obj: &mut Artifact, sections: Vec<DwarfSection>) -> Result<(), Error> {
+pub fn write_debugsections(obj: &mut Object, sections: Vec<DwarfSection>) -> Result<(), Error> {
     let (bodies, relocs) = sections
         .into_iter()
         .map(|s| ((s.name.clone(), s.body), (s.name, s.relocs)))
         .unzip::<_, _, Vec<_>, Vec<_>>();
+    let mut ids = HashMap::new();
     for (name, body) in bodies {
-        obj.declare_with(name, Decl::section(SectionKind::Debug), body)?;
+        let segment = obj.segment_name(StandardSegment::Debug).to_vec();
+        let section_id = obj.add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug);
+        ids.insert(name, section_id);
+        obj.append_section_data(section_id, &body, 1);
     }
     for (name, relocs) in relocs {
+        let section_id = *ids.get(&name).unwrap();
         for reloc in relocs {
-            obj.link_with(
+            let target_symbol = if reloc.target.starts_with("_wasm_function") {
-                faerie::Link {
+                obj.symbol_id(reloc.target.as_bytes()).unwrap()
-                    from: &name,
+            } else {
-                    to: &reloc.target,
+                obj.section_symbol(*ids.get(&reloc.target).unwrap())
-                    at: reloc.offset as u64,
+            };
-                },
+            obj.add_relocation(
-                faerie::Reloc::Debug {
+                section_id,
-                    size: reloc.size,
+                Relocation {
-                    addend: reloc.addend,
+                    offset: u64::from(reloc.offset),
+                    size: reloc.size << 3,
+                    kind: RelocationKind::Absolute,
+                    encoding: RelocationEncoding::Generic,
+                    symbol: target_symbol,
+                    addend: i64::from(reloc.addend),
                 },
             )?;
         }
@@ -79,13 +91,14 @@ pub fn write_debugsections_image(
     code_region: (*const u8, usize),
     funcs: &[*const u8],
 ) -> Result<Vec<u8>, Error> {
-    let mut obj = Artifact::new(isa.triple().clone(), String::from("module"));
+    let mut obj = Object::new(BinaryFormat::Elf, isa.triple().architecture);
 
     assert!(!code_region.0.is_null() && code_region.1 > 0);
     assert_gt!(funcs.len(), 0);
 
     let body = unsafe { std::slice::from_raw_parts(code_region.0, code_region.1) };
-    obj.declare_with("all", Decl::function(), body.to_vec())?;
+    let section_id = obj.add_section(vec![], ".text.all".as_bytes().to_vec(), SectionKind::Text);
+    obj.append_section_data(section_id, body, 1);
 
     // Get DWARF sections and patch relocs
     patch_dwarf_sections(&mut sections, funcs);
@@ -93,17 +106,17 @@ pub fn write_debugsections_image(
     write_debugsections(&mut obj, sections)?;
 
     // LLDB is too "magical" about mach-o, generating elf
-    let mut bytes = obj.emit_as(BinaryFormat::Elf)?;
+    let mut bytes = obj.write()?;
     // elf is still missing details...
-    convert_faerie_elf_to_loadable_file(&mut bytes, code_region.0);
+    convert_object_elf_to_loadable_file(&mut bytes, code_region.0);
 
     // let mut file = ::std::fs::File::create(::std::path::Path::new("test.o")).expect("file");
-    // ::std::io::Write::write(&mut file, &bytes).expect("write");
+    // ::std::io::Write::write_all(&mut file, &bytes).expect("write");
 
     Ok(bytes)
 }
 
-fn convert_faerie_elf_to_loadable_file(bytes: &mut Vec<u8>, code_ptr: *const u8) {
+fn convert_object_elf_to_loadable_file(bytes: &mut Vec<u8>, code_ptr: *const u8) {
     use std::ffi::CStr;
     use std::os::raw::c_char;
 
@@ -117,21 +130,24 @@ fn convert_faerie_elf_to_loadable_file(bytes: &mut Vec<u8>, code_ptr: *const u8)
         e_phoff == 0 && e_phnum == 0,
         "program header table is empty"
     );
-    let e_phentsize = unsafe { *(bytes.as_ptr().offset(0x36) as *const u16) };
-    assert_eq!(e_phentsize, 0x38, "size of ph");
     let e_shentsize = unsafe { *(bytes.as_ptr().offset(0x3A) as *const u16) };
     assert_eq!(e_shentsize, 0x40, "size of sh");
 
     let e_shoff = unsafe { *(bytes.as_ptr().offset(0x28) as *const u64) };
     let e_shnum = unsafe { *(bytes.as_ptr().offset(0x3C) as *const u16) };
     let mut shstrtab_off = 0;
+    for i in 0..e_shnum {
+        let off = e_shoff as isize + i as isize * e_shentsize as isize;
+        let sh_type = unsafe { *(bytes.as_ptr().offset(off + 0x4) as *const u32) };
+        if sh_type != /* SHT_SYMTAB */ 3 {
+            continue;
+        }
+        shstrtab_off = unsafe { *(bytes.as_ptr().offset(off + 0x18) as *const u64) };
+    }
     let mut segment = None;
     for i in 0..e_shnum {
         let off = e_shoff as isize + i as isize * e_shentsize as isize;
         let sh_type = unsafe { *(bytes.as_ptr().offset(off + 0x4) as *const u32) };
-        if sh_type == /* SHT_SYMTAB */ 3 {
-            shstrtab_off = unsafe { *(bytes.as_ptr().offset(off + 0x18) as *const u64) };
-        }
         if sh_type != /* SHT_PROGBITS */ 1 {
             continue;
         }
@@ -185,10 +201,11 @@ fn convert_faerie_elf_to_loadable_file(bytes: &mut Vec<u8>, code_ptr: *const u8)
     }
 
     // It is somewhat loadable ELF file at this moment.
-    // Update e_flags, e_phoff and e_phnum.
+    // Update e_flags, e_phoff, e_phentsize and e_phnum.
     unsafe {
         *(bytes.as_ptr().offset(0x10) as *mut u16) = /* ET_DYN */ 3;
         *(bytes.as_ptr().offset(0x20) as *mut u64) = ph_off as u64;
+        *(bytes.as_ptr().offset(0x36) as *mut u16) = 0x38 as u16;
         *(bytes.as_ptr().offset(0x38) as *mut u16) = 1 as u16;
     }
 }

crates/obj/Cargo.toml

@@ -13,7 +13,7 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0"
 wasmtime-environ = { path = "../environ", version = "0.17.0" }
-faerie = "0.15.0"
+object = { version = "0.18", default-features = false, features = ["write"] }
 more-asserts = "0.2.1"
 
 [badges]

crates/obj/src/data_segment.rs

@@ -1,25 +1,37 @@
 use anyhow::Result;
-use faerie::{Artifact, Decl};
+use object::write::{Object, StandardSection, Symbol, SymbolSection};
+use object::{SymbolFlags, SymbolKind, SymbolScope};
 use wasmtime_environ::DataInitializer;
 
 /// Declares data segment symbol
 pub fn declare_data_segment(
-    obj: &mut Artifact,
+    obj: &mut Object,
     _data_initaliazer: &DataInitializer,
     index: usize,
 ) -> Result<()> {
     let name = format!("_memory_{}", index);
-    obj.declare(name, Decl::data())?;
+    let _symbol_id = obj.add_symbol(Symbol {
+        name: name.as_bytes().to_vec(),
+        value: 0,
+        size: 0,
+        kind: SymbolKind::Data,
+        scope: SymbolScope::Linkage,
+        weak: false,
+        section: SymbolSection::Undefined,
+        flags: SymbolFlags::None,
+    });
     Ok(())
 }
 
 /// Emit segment data and initialization location
 pub fn emit_data_segment(
-    obj: &mut Artifact,
+    obj: &mut Object,
     data_initaliazer: &DataInitializer,
     index: usize,
 ) -> Result<()> {
     let name = format!("_memory_{}", index);
-    obj.define(name, Vec::from(data_initaliazer.data))?;
+    let symbol_id = obj.symbol_id(name.as_bytes()).unwrap();
+    let section_id = obj.section_id(StandardSection::Data);
+    obj.add_symbol_data(symbol_id, section_id, data_initaliazer.data, 1);
     Ok(())
 }

crates/obj/src/function.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
-use faerie::{Artifact, Decl, Link};
+use object::write::{Object, Relocation, StandardSection, Symbol, SymbolSection};
+use object::{RelocationEncoding, RelocationKind, SymbolFlags, SymbolKind, SymbolScope};
 use wasmtime_environ::entity::EntityRef;
 use wasmtime_environ::settings;
 use wasmtime_environ::settings::Configurable;
@@ -7,25 +8,43 @@ use wasmtime_environ::{Compilation, Module, RelocationTarget, Relocations};
 
 /// Defines module functions
 pub fn declare_functions(
-    obj: &mut Artifact,
+    obj: &mut Object,
     module: &Module,
     relocations: &Relocations,
 ) -> Result<()> {
     for i in 0..module.local.num_imported_funcs {
         let string_name = format!("_wasm_function_{}", i);
-        obj.declare(string_name, Decl::function_import())?;
+        let _symbol_id = obj.add_symbol(Symbol {
+            name: string_name.as_bytes().to_vec(),
+            value: 0,
+            size: 0,
+            kind: SymbolKind::Text,
+            scope: SymbolScope::Unknown,
+            weak: false,
+            section: SymbolSection::Undefined,
+            flags: SymbolFlags::None,
+        });
     }
     for (i, _function_relocs) in relocations.iter().rev() {
         let func_index = module.local.func_index(i);
         let string_name = format!("_wasm_function_{}", func_index.index());
-        obj.declare(string_name, Decl::function().global())?;
+        let _symbol_id = obj.add_symbol(Symbol {
+            name: string_name.as_bytes().to_vec(),
+            value: 0,
+            size: 0,
+            kind: SymbolKind::Text,
+            scope: SymbolScope::Linkage,
+            weak: false,
+            section: SymbolSection::Undefined,
+            flags: SymbolFlags::None,
+        });
     }
     Ok(())
 }
 
 /// Emits module functions
 pub fn emit_functions(
-    obj: &mut Artifact,
+    obj: &mut Object,
     module: &Module,
     compilation: &Compilation,
     relocations: &Relocations,
@@ -46,22 +65,35 @@ pub fn emit_functions(
         let func_index = module.local.func_index(i);
         let string_name = format!("_wasm_function_{}", func_index.index());
 
-        obj.define(string_name, body.clone())?;
+        let symbol_id = obj.symbol_id(string_name.as_bytes()).unwrap();
+        let section_id = obj.section_id(StandardSection::Text);
+
+        obj.add_symbol_data(symbol_id, section_id, body, 1);
     }
 
     for (i, function_relocs) in relocations.iter() {
         let func_index = module.local.func_index(i);
         let string_name = format!("_wasm_function_{}", func_index.index());
+        let symbol_id = obj.symbol_id(string_name.as_bytes()).unwrap();
+        let (_, section_offset) = obj.symbol_section_and_offset(symbol_id).unwrap();
+        let section_id = obj.section_id(StandardSection::Text);
         for r in function_relocs {
             debug_assert_eq!(r.addend, 0);
             match r.reloc_target {
                 RelocationTarget::UserFunc(target_index) => {
                     let target_name = format!("_wasm_function_{}", target_index.index());
-                    obj.link(Link {
+                    let target_symbol = obj.symbol_id(target_name.as_bytes()).unwrap();
-                        from: &string_name,
+                    obj.add_relocation(
-                        to: &target_name,
+                        section_id,
-                        at: r.offset as u64,
+                        Relocation {
-                    })?;
+                            offset: section_offset + r.offset as u64,
+                            size: 64, // FIXME for all targets
+                            kind: RelocationKind::Absolute,
+                            encoding: RelocationEncoding::Generic,
+                            symbol: target_symbol,
+                            addend: 0,
+                        },
+                    )?;
                 }
                 RelocationTarget::JumpTable(_, _) => {
                     // ignore relocations for jump tables

crates/obj/src/module.rs

@@ -3,24 +3,44 @@ use crate::data_segment::{declare_data_segment, emit_data_segment};
 use crate::function::{declare_functions, emit_functions};
 use crate::table::{declare_table, emit_table};
 use anyhow::Result;
-use faerie::{Artifact, Decl, Link};
+use object::write::{Object, Relocation, StandardSection, Symbol, SymbolSection};
+use object::{RelocationEncoding, RelocationKind, SymbolFlags, SymbolKind, SymbolScope};
 use wasmtime_environ::isa::TargetFrontendConfig;
 use wasmtime_environ::{Compilation, DataInitializer, Module, Relocations};
 
 fn emit_vmcontext_init(
-    obj: &mut Artifact,
+    obj: &mut Object,
     module: &Module,
     target_config: &TargetFrontendConfig,
 ) -> Result<()> {
     let (data, table_relocs) = layout_vmcontext(module, target_config);
-    obj.declare_with("_vmcontext_init", Decl::data().global(), data.to_vec())?;
+    let symbol_id = obj.add_symbol(Symbol {
+        name: "_vmcontext_init".as_bytes().to_vec(),
+        value: 0,
+        size: 0,
+        kind: SymbolKind::Data,
+        scope: SymbolScope::Linkage,
+        weak: false,
+        section: SymbolSection::Undefined,
+        flags: SymbolFlags::None,
+    });
+    let section_id = obj.section_id(StandardSection::Data);
+    let section_offset = obj.add_symbol_data(symbol_id, section_id, &data, 1);
+
     for reloc in table_relocs.iter() {
         let target_name = format!("_table_{}", reloc.index);
-        obj.link(Link {
+        let target_symbol = obj.symbol_id(target_name.as_bytes()).unwrap();
-            from: "_vmcontext_init",
+        obj.add_relocation(
-            to: &target_name,
+            section_id,
-            at: reloc.offset as u64,
+            Relocation {
-        })?;
+                offset: section_offset + reloc.offset as u64,
+                size: 64, // FIXME for all targets
+                kind: RelocationKind::Absolute,
+                encoding: RelocationEncoding::Generic,
+                symbol: target_symbol,
+                addend: 0,
+            },
+        )?;
     }
     Ok(())
 }
@@ -28,7 +48,7 @@ fn emit_vmcontext_init(
 /// Emits a module that has been emitted with the `wasmtime-environ` environment
 /// implementation to a native object file.
 pub fn emit_module(
-    obj: &mut Artifact,
+    obj: &mut Object,
     module: &Module,
     compilation: &Compilation,
     relocations: &Relocations,

crates/obj/src/table.rs

@@ -1,17 +1,29 @@
 use anyhow::Result;
-use faerie::{Artifact, Decl};
+use object::write::{Object, StandardSection, Symbol, SymbolSection};
+use object::{SymbolFlags, SymbolKind, SymbolScope};
 
 /// Declares data segment symbol
-pub fn declare_table(obj: &mut Artifact, index: usize) -> Result<()> {
+pub fn declare_table(obj: &mut Object, index: usize) -> Result<()> {
     let name = format!("_table_{}", index);
-    obj.declare(name, Decl::data())?;
+    let _symbol_id = obj.add_symbol(Symbol {
+        name: name.as_bytes().to_vec(),
+        value: 0,
+        size: 0,
+        kind: SymbolKind::Data,
+        scope: SymbolScope::Linkage,
+        weak: false,
+        section: SymbolSection::Undefined,
+        flags: SymbolFlags::None,
+    });
     Ok(())
 }
 
 /// Emit segment data and initialization location
-pub fn emit_table(obj: &mut Artifact, index: usize) -> Result<()> {
+pub fn emit_table(obj: &mut Object, index: usize) -> Result<()> {
     let name = format!("_table_{}", index);
+    let symbol_id = obj.symbol_id(name.as_bytes()).unwrap();
+    let section_id = obj.section_id(StandardSection::Data);
     // FIXME: We need to initialize table using function symbols
-    obj.define(name, Vec::new())?;
+    obj.add_symbol_data(symbol_id, section_id, &[], 1);
     Ok(())
 }

crates/profiling/Cargo.toml

@@ -16,7 +16,7 @@ cfg-if = "0.1"
 gimli = { version = "0.21.0", optional = true }
 lazy_static = "1.4"
 libc = { version = "0.2.60", default-features = false }
-scroll = { version = "0.10.1", optional = true }
+scroll = { version = "0.10.1", features = ["derive"], optional = true }
 serde = { version = "1.0.99", features = ["derive"] }
 target-lexicon = "0.10.0"
 wasmtime-environ = { path = "../environ", version = "0.17.0" }

src/commands/wasm2obj.rs

@@ -5,6 +5,7 @@ use crate::{init_file_per_thread_logger, pick_compilation_strategy, CommonOption
 use anyhow::{anyhow, Context as _, Result};
 use std::{
     fs::File,
+    io::Write,
     path::{Path, PathBuf},
     str::FromStr,
 };
@@ -75,13 +76,13 @@ impl WasmToObjCommand {
             self.common.enable_simd,
             self.common.opt_level(),
             self.common.debug_info,
-            self.output.clone(),
             &cache_config,
         )?;
 
-        // FIXME: Make the format a parameter.
+        let mut file =
-        let file = File::create(Path::new(&self.output)).context("failed to create object file")?;
+            File::create(Path::new(&self.output)).context("failed to create object file")?;
-        obj.write(file).context("failed to write object file")?;
+        file.write_all(&obj.write()?)
+            .context("failed to write object file")?;
 
         Ok(())
     }

src/obj.rs

@@ -1,5 +1,5 @@
 use anyhow::{anyhow, bail, Context as _, Result};
-use faerie::Artifact;
+use object::write::Object;
 use target_lexicon::Triple;
 use wasmtime::Strategy;
 use wasmtime_debug::{emit_dwarf, read_debuginfo, write_debugsections};
@@ -21,9 +21,8 @@ pub fn compile_to_obj(
     enable_simd: bool,
     opt_level: wasmtime::OptLevel,
     debug_info: bool,
-    artifact_name: String,
     cache_config: &CacheConfig,
-) -> Result<Artifact> {
+) -> Result<Object> {
     let isa_builder = match target {
         Some(target) => native::lookup(target.clone())?,
         None => native::builder(),
@@ -51,7 +50,7 @@ pub fn compile_to_obj(
 
     let isa = isa_builder.finish(settings::Flags::new(flag_builder));
 
-    let mut obj = Artifact::new(isa.triple().clone(), artifact_name);
+    let mut obj = Object::new(isa.triple().binary_format, isa.triple().architecture);
 
     // TODO: Expose the tunables as command-line flags.
     let mut tunables = Tunables::default();

tests/all/debug/obj.rs

@@ -1,5 +1,6 @@
 use anyhow::{Context as _, Result};
 use std::fs::File;
+use std::io::Write;
 use std::path::Path;
 use target_lexicon::Triple;
 use wasmtime::Strategy;
@@ -18,17 +19,12 @@ pub fn compile_cranelift(
         false,
         wasmtime::OptLevel::None,
         true,
-        output
-            .as_ref()
-            .file_name()
-            .unwrap()
-            .to_string_lossy()
-            .to_string(),
         &CacheConfig::new_cache_disabled(),
     )?;
 
-    let file = File::create(output).context("failed to create object file")?;
+    let mut file = File::create(output).context("failed to create object file")?;
-    obj.write(file).context("failed to write object file")?;
+    file.write_all(&obj.write()?)
+        .context("failed to write object file")?;
 
     Ok(())
 }