Refactor some internals of wasmtime-cranelift (#4202)

* Split `wasm_to_host_trampoline` into pieces

In the upcoming component model supoprt for imports my plan is to reuse
some of these pieces but not the entirety of the current
`wasm_to_host_trampoline`. In an effort to make that diff smaller this
commit splits up the function preemptively into pieces to get reused
later.

* Delete unused `for_each_libcall` macros

Came across this when working in the object support for cranelift.

* Refactor some object creation details

This commit refactors some of the internals around creating an object
file in the wasmtime-cranelift integration. The old `ObjectBuilder` is
now named `ModuleTextBuilder` and is only used to create the text
section rather than other sections too. This helps maintain the
invariant that the unwind information section is placed directly after
the text section without having an odd API for doing this.

Additionally the unwind information creation is moved externally from
the `ModuleTextBuilder` to a standalone structure. This separate
structure is currently in use in the component model work I'm doing
although I may change that to using the `ModuleTextBuilder` instead. In
any case it seemed nice to encapsulate all of the unwinding information
into one standalone structure.

Finally, the insertion of native debug information has been refactored
to happen in a new `append_dwarf` method to keep all the dwarf-related
stuff together in one place as much as possible.

* Fix a doctest

* Fix a typo

crates/cranelift/src/compiler.rs

@@ -1,13 +1,13 @@
 use crate::builder::LinkOptions;
-use crate::debug::ModuleMemoryOffset;
+use crate::debug::{DwarfSectionRelocTarget, ModuleMemoryOffset};
 use crate::func_environ::{get_func_name, FuncEnvironment};
-use crate::obj::ObjectBuilder;
+use crate::obj::ModuleTextBuilder;
 use crate::{
     blank_sig, func_signature, indirect_signature, value_type, wasmtime_call_conv,
-    CompiledFunction, FunctionAddressMap, Relocation, RelocationTarget,
+    CompiledFunction, CompiledFunctions, FunctionAddressMap, Relocation, RelocationTarget,
 };
 use anyhow::{Context as _, Result};
-use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags};
+use cranelift_codegen::ir::{self, ExternalName, InstBuilder, MemFlags, Value};
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::print_errors::pretty_error;
 use cranelift_codegen::Context;
@@ -19,10 +19,12 @@ use cranelift_wasm::{
     DefinedFuncIndex, DefinedMemoryIndex, FuncIndex, FuncTranslator, MemoryIndex, SignatureIndex,
     WasmFuncType,
 };
-use object::write::Object;
+use object::write::{Object, StandardSegment, SymbolId};
+use object::{RelocationEncoding, RelocationKind, SectionKind};
 use std::any::Any;
 use std::cmp;
 use std::collections::BTreeMap;
+use std::collections::HashMap;
 use std::convert::TryFrom;
 use std::mem;
 use std::sync::Mutex;
@@ -272,14 +274,14 @@ impl wasmtime_environ::Compiler for Compiler {
         tunables: &Tunables,
         obj: &mut Object<'static>,
     ) -> Result<(PrimaryMap<DefinedFuncIndex, FunctionInfo>, Vec<Trampoline>)> {
-        let funcs: crate::CompiledFunctions = funcs
+        let funcs: CompiledFunctions = funcs
             .into_iter()
             .map(|(_i, f)| *f.downcast().unwrap())
             .collect();
 
-        let mut builder = ObjectBuilder::new(obj, &translation.module, &*self.isa);
+        let mut builder = ModuleTextBuilder::new(obj, &translation.module, &*self.isa);
         if self.linkopts.force_jump_veneers {
-            builder.text.force_veneers();
+            builder.force_veneers();
         }
         let mut addrs = AddressMapSection::default();
         let mut traps = TrapEncodingBuilder::default();
@@ -297,13 +299,7 @@ impl wasmtime_environ::Compiler for Compiler {
             }
             traps.push(range.clone(), &func.traps);
             func_starts.push(range.start);
-            if self.linkopts.padding_between_functions > 0 {
+            builder.append_padding(self.linkopts.padding_between_functions);
-                builder.text.append(
-                    false,
-                    &vec![0; self.linkopts.padding_between_functions],
-                    Some(1),
-                );
-            }
         }
 
         // Build trampolines for every signature that can be used by this module.
@@ -316,40 +312,9 @@ impl wasmtime_environ::Compiler for Compiler {
             trampolines.push(builder.trampoline(*i, &func));
         }
 
-        builder.unwind_info();
+        let symbols = builder.finish()?;
-
-        if tunables.generate_native_debuginfo && funcs.len() > 0 {
-            let ofs = VMOffsets::new(
-                self.isa
-                    .triple()
-                    .architecture
-                    .pointer_width()
-                    .unwrap()
-                    .bytes(),
-                &translation.module,
-            );
-
-            let memory_offset = if ofs.num_imported_memories > 0 {
-                ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
-            } else if ofs.num_defined_memories > 0 {
-                ModuleMemoryOffset::Defined(
-                    ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
-                )
-            } else {
-                ModuleMemoryOffset::None
-            };
-            let dwarf_sections = crate::debug::emit_dwarf(
-                &*self.isa,
-                &translation.debuginfo,
-                &funcs,
-                &memory_offset,
-            )
-            .with_context(|| "failed to emit DWARF debug information")?;
-            builder.dwarf_sections(&dwarf_sections)?;
-        }
-
-        builder.finish()?;
 
+        self.append_dwarf(obj, translation, &funcs, tunables, &symbols)?;
         if tunables.generate_address_map {
             addrs.append_to(obj);
         }
@@ -377,10 +342,9 @@ impl wasmtime_environ::Compiler for Compiler {
         let host_to_wasm = self.host_to_wasm_trampoline(ty)?;
         let wasm_to_host = self.wasm_to_host_trampoline(ty, host_fn)?;
         let module = Module::new();
-        let mut builder = ObjectBuilder::new(obj, &module, &*self.isa);
+        let mut builder = ModuleTextBuilder::new(obj, &module, &*self.isa);
         let a = builder.trampoline(SignatureIndex::new(0), &host_to_wasm);
         let b = builder.trampoline(SignatureIndex::new(1), &wasm_to_host);
-        builder.unwind_info();
         builder.finish()?;
         Ok((a, b))
     }
@@ -508,6 +472,28 @@ impl Compiler {
         Ok(func)
     }
 
+    /// Creates a trampoline for WebAssembly calling into the host where all the
+    /// arguments are spilled to the stack and results are loaded from the
+    /// stack.
+    ///
+    /// This style of trampoline is currently only used with the
+    /// `Func::new`-style created functions in the Wasmtime embedding API. The
+    /// generated trampoline has a function signature appropriate to the `ty`
+    /// specified (e.g. a System-V ABI) and will call a `host_fn` that has a
+    /// type signature of:
+    ///
+    /// ```ignore
+    /// extern "C" fn(*mut VMContext, *mut VMContext, *mut ValRaw, usize)
+    /// ```
+    ///
+    /// where the first two arguments are forwarded from the trampoline
+    /// generated here itself, and the second two arguments are a pointer/length
+    /// into stack-space of this trampoline with storage for both the arguments
+    /// to the function and the results.
+    ///
+    /// Note that `host_fn` is an immediate which is an actual function pointer
+    /// in this process. As such this compiled trampoline is not suitable for
+    /// serialization.
     fn wasm_to_host_trampoline(
         &self,
         ty: &WasmFuncType,
@@ -523,12 +509,6 @@ impl Compiler {
         host_signature.params.push(ir::AbiParam::new(pointer_type));
         host_signature.params.push(ir::AbiParam::new(pointer_type));
 
-        // Compute the size of the values vector. The vmctx and caller vmctx are passed separately.
-        let value_size = mem::size_of::<u128>();
-        let values_vec_len = cmp::max(ty.params().len(), ty.returns().len());
-        let values_vec_byte_size = u32::try_from(value_size * values_vec_len).unwrap();
-        let values_vec_len = u32::try_from(values_vec_len).unwrap();
-
         let CompilerContext {
             mut func_translator,
             codegen_context: mut context,
@@ -537,35 +517,16 @@ impl Compiler {
         context.func =
             ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wasm_signature);
 
-        let ss = context.func.create_stack_slot(ir::StackSlotData::new(
-            ir::StackSlotKind::ExplicitSlot,
-            values_vec_byte_size,
-        ));
-
         let mut builder = FunctionBuilder::new(&mut context.func, func_translator.context());
         let block0 = builder.create_block();
 
-        builder.append_block_params_for_function_params(block0);
+        let (values_vec_ptr_val, values_vec_len) =
-        builder.switch_to_block(block0);
+            self.wasm_to_host_spill_args(ty, &mut builder, block0);
-        builder.seal_block(block0);
-
-        let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0);
-        let mut mflags = MemFlags::trusted();
-        mflags.set_endianness(ir::Endianness::Little);
-        for i in 0..ty.params().len() {
-            let val = builder.func.dfg.block_params(block0)[i + 2];
-            builder
-                .ins()
-                .store(mflags, val, values_vec_ptr_val, (i * value_size) as i32);
-        }
 
         let block_params = builder.func.dfg.block_params(block0);
-        let vmctx_ptr_val = block_params[0];
+        let callee_args = [
-        let caller_vmctx_ptr_val = block_params[1];
+            block_params[0],
-
+            block_params[1],
-        let callee_args = vec![
-            vmctx_ptr_val,
-            caller_vmctx_ptr_val,
             values_vec_ptr_val,
             builder
                 .ins()
@@ -573,12 +534,94 @@ impl Compiler {
         ];
 
         let new_sig = builder.import_signature(host_signature);
-
         let callee_value = builder.ins().iconst(pointer_type, host_fn as i64);
         builder
             .ins()
             .call_indirect(new_sig, callee_value, &callee_args);
 
+        self.wasm_to_host_load_results(ty, &mut builder, values_vec_ptr_val);
+
+        let func = self.finish_trampoline(&mut context, isa)?;
+        self.save_context(CompilerContext {
+            func_translator,
+            codegen_context: context,
+        });
+        Ok(func)
+    }
+
+    /// Used for spilling arguments in wasm-to-host trampolines into the stack
+    /// of the function of `builder` specified.
+    ///
+    /// The `block0` is the entry block of the function and `ty` is the wasm
+    /// signature of the trampoline generated. This function will allocate a
+    /// stack slot suitable for storing both the arguments and return values of
+    /// the function, and then the arguments will all be stored in this block.
+    ///
+    /// The stack slot pointer is returned in addition to the size, in units of
+    /// `ValRaw`, of the stack slot.
+    fn wasm_to_host_spill_args(
+        &self,
+        ty: &WasmFuncType,
+        builder: &mut FunctionBuilder,
+        block0: ir::Block,
+    ) -> (Value, u32) {
+        let isa = &*self.isa;
+        let pointer_type = isa.pointer_type();
+
+        // Compute the size of the values vector.
+        let value_size = mem::size_of::<u128>();
+        let values_vec_len = cmp::max(ty.params().len(), ty.returns().len());
+        let values_vec_byte_size = u32::try_from(value_size * values_vec_len).unwrap();
+        let values_vec_len = u32::try_from(values_vec_len).unwrap();
+
+        let ss = builder.func.create_stack_slot(ir::StackSlotData::new(
+            ir::StackSlotKind::ExplicitSlot,
+            values_vec_byte_size,
+        ));
+
+        builder.append_block_params_for_function_params(block0);
+        builder.switch_to_block(block0);
+        builder.seal_block(block0);
+
+        // Note that loads and stores are unconditionally done in the
+        // little-endian format rather than the host's native-endianness,
+        // despite this load/store being unrelated to execution in wasm itself.
+        // For more details on this see the `ValRaw` type in the
+        // `wasmtime-runtime` crate.
+        let mut mflags = MemFlags::trusted();
+        mflags.set_endianness(ir::Endianness::Little);
+
+        let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0);
+        for i in 0..ty.params().len() {
+            let val = builder.func.dfg.block_params(block0)[i + 2];
+            builder
+                .ins()
+                .store(mflags, val, values_vec_ptr_val, (i * value_size) as i32);
+        }
+        (values_vec_ptr_val, values_vec_len)
+    }
+
+    /// Use for loading the results of a host call from a trampoline's stack
+    /// space.
+    ///
+    /// This is intended to be used with the stack space allocated by
+    /// `wasm_to_host_spill_args` above. This is called after the function call
+    /// is made which will load results from the stack space and then return
+    /// them with the appropriate ABI (e.g. System-V).
+    fn wasm_to_host_load_results(
+        &self,
+        ty: &WasmFuncType,
+        builder: &mut FunctionBuilder,
+        values_vec_ptr_val: Value,
+    ) {
+        let isa = &*self.isa;
+        let value_size = mem::size_of::<u128>();
+
+        // Note that this is little-endian like `wasm_to_host_spill_args` above,
+        // see notes there for more information.
+        let mut mflags = MemFlags::trusted();
+        mflags.set_endianness(ir::Endianness::Little);
+
         let mut results = Vec::new();
         for (i, r) in ty.returns().iter().enumerate() {
             let load = builder.ins().load(
@@ -591,13 +634,6 @@ impl Compiler {
         }
         builder.ins().return_(&results);
         builder.finalize();
-
-        let func = self.finish_trampoline(&mut context, isa)?;
-        self.save_context(CompilerContext {
-            func_translator,
-            codegen_context: context,
-        });
-        Ok(func)
     }
 
     fn finish_trampoline(
@@ -642,6 +678,81 @@ impl Compiler {
             traps: Vec::new(),
         })
     }
+
+    pub fn append_dwarf(
+        &self,
+        obj: &mut Object<'_>,
+        translation: &ModuleTranslation<'_>,
+        funcs: &CompiledFunctions,
+        tunables: &Tunables,
+        func_symbols: &PrimaryMap<DefinedFuncIndex, SymbolId>,
+    ) -> Result<()> {
+        if !tunables.generate_native_debuginfo || funcs.len() == 0 {
+            return Ok(());
+        }
+        let ofs = VMOffsets::new(
+            self.isa
+                .triple()
+                .architecture
+                .pointer_width()
+                .unwrap()
+                .bytes(),
+            &translation.module,
+        );
+
+        let memory_offset = if ofs.num_imported_memories > 0 {
+            ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
+        } else if ofs.num_defined_memories > 0 {
+            ModuleMemoryOffset::Defined(
+                ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
+            )
+        } else {
+            ModuleMemoryOffset::None
+        };
+        let dwarf_sections =
+            crate::debug::emit_dwarf(&*self.isa, &translation.debuginfo, &funcs, &memory_offset)
+                .with_context(|| "failed to emit DWARF debug information")?;
+
+        let (debug_bodies, debug_relocs): (Vec<_>, Vec<_>) = dwarf_sections
+            .iter()
+            .map(|s| ((s.name, &s.body), (s.name, &s.relocs)))
+            .unzip();
+        let mut dwarf_sections_ids = HashMap::new();
+        for (name, body) in debug_bodies {
+            let segment = obj.segment_name(StandardSegment::Debug).to_vec();
+            let section_id = obj.add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug);
+            dwarf_sections_ids.insert(name, section_id);
+            obj.append_section_data(section_id, &body, 1);
+        }
+
+        // Write all debug data relocations.
+        for (name, relocs) in debug_relocs {
+            let section_id = *dwarf_sections_ids.get(name).unwrap();
+            for reloc in relocs {
+                let target_symbol = match reloc.target {
+                    DwarfSectionRelocTarget::Func(index) => {
+                        func_symbols[DefinedFuncIndex::new(index)]
+                    }
+                    DwarfSectionRelocTarget::Section(name) => {
+                        obj.section_symbol(dwarf_sections_ids[name])
+                    }
+                };
+                obj.add_relocation(
+                    section_id,
+                    object::write::Relocation {
+                        offset: u64::from(reloc.offset),
+                        size: reloc.size << 3,
+                        kind: RelocationKind::Absolute,
+                        encoding: RelocationEncoding::Generic,
+                        symbol: target_symbol,
+                        addend: i64::from(reloc.addend),
+                    },
+                )?;
+            }
+        }
+
+        Ok(())
+    }
 }
 
 // Collects an iterator of `InstructionAddressMap` into a `Vec` for insertion

crates/cranelift/src/obj.rs

@@ -13,7 +13,6 @@
 //! function body, the imported wasm function do not. The trampolines symbol
 //! names have format "_trampoline_N", where N is `SignatureIndex`.
 
-use crate::debug::{DwarfSection, DwarfSectionRelocTarget};
 use crate::{CompiledFunction, RelocationTarget};
 use anyhow::Result;
 use cranelift_codegen::isa::{
@@ -23,56 +22,22 @@ use cranelift_codegen::isa::{
 use cranelift_codegen::TextSectionBuilder;
 use gimli::write::{Address, EhFrame, EndianVec, FrameTable, Writer};
 use gimli::RunTimeEndian;
-use object::write::{
+use object::write::{Object, SectionId, StandardSegment, Symbol, SymbolId, SymbolSection};
-    Object, Relocation as ObjectRelocation, SectionId, StandardSegment, Symbol, SymbolId,
+use object::{Architecture, SectionKind, SymbolFlags, SymbolKind, SymbolScope};
-    SymbolSection,
-};
-use object::{
-    Architecture, RelocationEncoding, RelocationKind, SectionKind, SymbolFlags, SymbolKind,
-    SymbolScope,
-};
-use std::collections::HashMap;
 use std::convert::TryFrom;
 use std::ops::Range;
 use wasmtime_environ::obj;
-use wasmtime_environ::{
+use wasmtime_environ::{DefinedFuncIndex, Module, PrimaryMap, SignatureIndex, Trampoline};
-    DefinedFuncIndex, EntityRef, FuncIndex, Module, PrimaryMap, SignatureIndex, Trampoline,
-};
 
 const TEXT_SECTION_NAME: &[u8] = b".text";
 
-/// Iterates through all `LibCall` members and all runtime exported functions.
-#[macro_export]
-macro_rules! for_each_libcall {
-    ($op:ident) => {
-        $op![
-            (UdivI64, wasmtime_i64_udiv),
-            (UdivI64, wasmtime_i64_udiv),
-            (SdivI64, wasmtime_i64_sdiv),
-            (UremI64, wasmtime_i64_urem),
-            (SremI64, wasmtime_i64_srem),
-            (IshlI64, wasmtime_i64_ishl),
-            (UshrI64, wasmtime_i64_ushr),
-            (SshrI64, wasmtime_i64_sshr),
-            (CeilF32, wasmtime_f32_ceil),
-            (FloorF32, wasmtime_f32_floor),
-            (TruncF32, wasmtime_f32_trunc),
-            (NearestF32, wasmtime_f32_nearest),
-            (CeilF64, wasmtime_f64_ceil),
-            (FloorF64, wasmtime_f64_floor),
-            (TruncF64, wasmtime_f64_trunc),
-            (NearestF64, wasmtime_f64_nearest)
-        ];
-    };
-}
-
 /// A helper structure used to assemble the final text section of an exectuable,
 /// plus unwinding information and other related details.
 ///
 /// This builder relies on Cranelift-specific internals but assembles into a
 /// generic `Object` which will get further appended to in a compiler-agnostic
 /// fashion later.
-pub struct ObjectBuilder<'a> {
+pub struct ModuleTextBuilder<'a> {
     /// The target that we're compiling for, used to query target-specific
     /// information as necessary.
     isa: &'a dyn TargetIsa,
@@ -83,49 +48,21 @@ pub struct ObjectBuilder<'a> {
     /// The WebAssembly module we're generating code for.
     module: &'a Module,
 
-    windows_unwind_info_id: Option<SectionId>,
+    text_section: SectionId,
 
-    /// Packed form of windows unwind tables which, if present, will get emitted
+    unwind_info: UnwindInfoBuilder<'a>,
-    /// to a windows-specific unwind info section.
-    windows_unwind_info: Vec<RUNTIME_FUNCTION>,
-
-    systemv_unwind_info_id: Option<SectionId>,
-
-    /// Pending unwinding information for DWARF-based platforms. This is used to
-    /// build a `.eh_frame` lookalike at the very end of object building.
-    systemv_unwind_info: Vec<(u64, &'a systemv::UnwindInfo)>,
 
     /// The corresponding symbol for each function, inserted as they're defined.
     ///
     /// If an index isn't here yet then it hasn't been defined yet.
-    func_symbols: PrimaryMap<FuncIndex, SymbolId>,
+    func_symbols: PrimaryMap<DefinedFuncIndex, SymbolId>,
-
-    /// `object`-crate identifier for the text section.
-    text_section: SectionId,
 
     /// In-progress text section that we're using cranelift's `MachBuffer` to
     /// build to resolve relocations (calls) between functions.
-    pub text: Box<dyn TextSectionBuilder>,
+    text: Box<dyn TextSectionBuilder>,
-
-    /// The unwind info _must_ come directly after the text section. Our FDE's
-    /// instructions are encoded to rely on this placement. We use this `bool`
-    /// for debug assertions to ensure that we get the ordering correct.
-    added_unwind_info: bool,
 }
 
-// This is a mirror of `RUNTIME_FUNCTION` in the Windows API, but defined here
+impl<'a> ModuleTextBuilder<'a> {
-// to ensure everything is always `u32` and to have it available on all
-// platforms. Note that all of these specifiers here are relative to a "base
-// address" which we define as the base of where the text section is eventually
-// loaded.
-#[allow(non_camel_case_types)]
-struct RUNTIME_FUNCTION {
-    begin: u32,
-    end: u32,
-    unwind_address: u32,
-}
-
-impl<'a> ObjectBuilder<'a> {
     pub fn new(obj: &'a mut Object<'static>, module: &'a Module, isa: &'a dyn TargetIsa) -> Self {
         // Entire code (functions and trampolines) will be placed
         // in the ".text" section.
@@ -135,37 +72,15 @@ impl<'a> ObjectBuilder<'a> {
             SectionKind::Text,
         );
 
-        // Create symbols for imports -- needed during linking.
+        let num_defined = module.functions.len() - module.num_imported_funcs;
-        let mut func_symbols = PrimaryMap::with_capacity(module.functions.len());
-        for index in 0..module.num_imported_funcs {
-            let symbol_id = obj.add_symbol(Symbol {
-                name: obj::func_symbol_name(FuncIndex::new(index))
-                    .as_bytes()
-                    .to_vec(),
-                value: 0,
-                size: 0,
-                kind: SymbolKind::Text,
-                scope: SymbolScope::Linkage,
-                weak: false,
-                section: SymbolSection::Undefined,
-                flags: SymbolFlags::None,
-            });
-            func_symbols.push(symbol_id);
-        }
-
         Self {
             isa,
             obj,
             module,
             text_section,
-            func_symbols,
+            func_symbols: PrimaryMap::with_capacity(num_defined),
-            windows_unwind_info_id: None,
+            unwind_info: Default::default(),
-            windows_unwind_info: Vec::new(),
+            text: isa.text_section_builder(num_defined as u32),
-            systemv_unwind_info_id: None,
-            systemv_unwind_info: Vec::new(),
-            text: isa
-                .text_section_builder((module.functions.len() - module.num_imported_funcs) as u32),
-            added_unwind_info: false,
         }
     }
 
@@ -193,37 +108,12 @@ impl<'a> ObjectBuilder<'a> {
             flags: SymbolFlags::None,
         });
 
-        match &func.unwind_info {
+        if let Some(info) = &func.unwind_info {
-            // Windows unwind information is preferred to come after the code
+            self.unwind_info.push(off, body_len, info, |data, align| {
-            // itself. The information is appended here just after the function,
+                self.text.append(false, data, Some(align))
-            // aligned to 4-bytes as required by Windows.
-            //
-            // The location of the unwind info, and the function it describes,
-            // is then recorded in an unwind info table to get embedded into the
-            // object at the end of compilation.
-            Some(UnwindInfo::WindowsX64(info)) => {
-                // Windows prefers Unwind info after the code -- writing it here.
-                let unwind_size = info.emit_size();
-                let mut unwind_info = vec![0; unwind_size];
-                info.emit(&mut unwind_info);
-                let unwind_off = self.text.append(false, &unwind_info, Some(4));
-                self.windows_unwind_info.push(RUNTIME_FUNCTION {
-                    begin: u32::try_from(off).unwrap(),
-                    end: u32::try_from(off + body_len).unwrap(),
-                    unwind_address: u32::try_from(unwind_off).unwrap(),
             });
         }
 
-            // System-V is different enough that we just record the unwinding
-            // information to get processed at a later time.
-            Some(UnwindInfo::SystemV(info)) => {
-                self.systemv_unwind_info.push((off, info));
-            }
-
-            Some(_) => panic!("some unwind info isn't handled here"),
-            None => {}
-        }
-
         for r in func.relocations.iter() {
             match r.reloc_target {
                 // Relocations against user-defined functions means that this is
@@ -272,16 +162,13 @@ impl<'a> ObjectBuilder<'a> {
     ///
     /// This is expected to be called in-order for ascending `index` values.
     pub fn func(&mut self, index: DefinedFuncIndex, func: &'a CompiledFunction) -> Range<u64> {
-        assert!(!self.added_unwind_info);
+        let name = obj::func_symbol_name(self.module.func_index(index));
-        let index = self.module.func_index(index);
-        let name = obj::func_symbol_name(index);
         let (symbol_id, range) = self.append_func(true, name.into_bytes(), func);
         assert_eq!(self.func_symbols.push(symbol_id), index);
         range
     }
 
     pub fn trampoline(&mut self, sig: SignatureIndex, func: &'a CompiledFunction) -> Trampoline {
-        assert!(!self.added_unwind_info);
         let name = obj::trampoline_symbol_name(sig);
         let (_, range) = self.append_func(false, name.into_bytes(), func);
         Trampoline {
@@ -291,117 +178,164 @@ impl<'a> ObjectBuilder<'a> {
         }
     }
 
-    pub fn dwarf_sections(&mut self, sections: &[DwarfSection]) -> Result<()> {
+    /// Forces "veneers" to be used for inter-function calls in the text
-        assert!(
+    /// section which means that in-bounds optimized addresses are never used.
-            self.added_unwind_info,
+    ///
-            "can't add dwarf yet; unwind info must directly follow the text section"
+    /// This is only useful for debugging cranelift itself and typically this
-        );
+    /// option is disabled.
-
+    pub fn force_veneers(&mut self) {
-        // If we have DWARF data, write it in the object file.
+        self.text.force_veneers();
-        let (debug_bodies, debug_relocs): (Vec<_>, Vec<_>) = sections
-            .iter()
-            .map(|s| ((s.name, &s.body), (s.name, &s.relocs)))
-            .unzip();
-        let mut dwarf_sections_ids = HashMap::new();
-        for (name, body) in debug_bodies {
-            let segment = self.obj.segment_name(StandardSegment::Debug).to_vec();
-            let section_id =
-                self.obj
-                    .add_section(segment, name.as_bytes().to_vec(), SectionKind::Debug);
-            dwarf_sections_ids.insert(name, section_id);
-            self.obj.append_section_data(section_id, &body, 1);
     }
 
-        // Write all debug data relocations.
+    /// Appends the specified amount of bytes of padding into the text section.
-        for (name, relocs) in debug_relocs {
+    ///
-            let section_id = *dwarf_sections_ids.get(name).unwrap();
+    /// This is only useful when fuzzing and/or debugging cranelift itself and
-            for reloc in relocs {
+    /// for production scenarios `padding` is 0 and this function does nothing.
-                let target_symbol = match reloc.target {
+    pub fn append_padding(&mut self, padding: usize) {
-                    DwarfSectionRelocTarget::Func(index) => {
+        if padding == 0 {
-                        self.func_symbols[self.module.func_index(DefinedFuncIndex::new(index))]
+            return;
-                    }
-                    DwarfSectionRelocTarget::Section(name) => {
-                        self.obj.section_symbol(dwarf_sections_ids[name])
-                    }
-                };
-                self.obj.add_relocation(
-                    section_id,
-                    ObjectRelocation {
-                        offset: u64::from(reloc.offset),
-                        size: reloc.size << 3,
-                        kind: RelocationKind::Absolute,
-                        encoding: RelocationEncoding::Generic,
-                        symbol: target_symbol,
-                        addend: i64::from(reloc.addend),
-                    },
-                )?;
         }
+        self.text.append(false, &vec![0; padding], Some(1));
     }
 
-        Ok(())
+    /// Indicates that the text section has been written completely and this
-    }
+    /// will finish appending it to the original object.
-
+    ///
-    pub fn unwind_info(&mut self) {
+    /// Note that this will also write out the unwind information sections if
-        assert!(!self.added_unwind_info);
+    /// necessary.
-
+    pub fn finish(mut self) -> Result<PrimaryMap<DefinedFuncIndex, SymbolId>> {
-        if self.windows_unwind_info.len() > 0 {
-            let segment = self.obj.segment_name(StandardSegment::Data).to_vec();
-            self.windows_unwind_info_id = Some(self.obj.add_section(
-                segment,
-                b"_wasmtime_winx64_unwind".to_vec(),
-                SectionKind::ReadOnlyData,
-            ));
-        }
-        if self.systemv_unwind_info.len() > 0 {
-            let segment = self.obj.segment_name(StandardSegment::Data).to_vec();
-            self.systemv_unwind_info_id = Some(self.obj.add_section(
-                segment,
-                b".eh_frame".to_vec(),
-                SectionKind::ReadOnlyData,
-            ));
-        }
-
-        self.added_unwind_info = true;
-    }
-
-    pub fn finish(&mut self) -> Result<()> {
         // Finish up the text section now that we're done adding functions.
         let text = self.text.finish();
         self.obj
             .section_mut(self.text_section)
             .set_data(text, self.isa.code_section_alignment());
 
-        // With all functions added we can also emit the fully-formed unwinding
+        // Append the unwind information for all our functions, if necessary.
-        // information sections.
+        self.unwind_info
-        if self.windows_unwind_info.len() > 0 {
+            .append_section(self.isa, self.obj, self.text_section);
-            self.append_windows_unwind_info();
+
+        Ok(self.func_symbols)
     }
-        if self.systemv_unwind_info.len() > 0 {
+}
-            self.append_systemv_unwind_info();
+
+/// Builder used to create unwind information for a set of functions added to a
+/// text section.
+#[derive(Default)]
+struct UnwindInfoBuilder<'a> {
+    windows_unwind_info: Vec<RUNTIME_FUNCTION>,
+    systemv_unwind_info: Vec<(u64, &'a systemv::UnwindInfo)>,
+}
+
+// This is a mirror of `RUNTIME_FUNCTION` in the Windows API, but defined here
+// to ensure everything is always `u32` and to have it available on all
+// platforms. Note that all of these specifiers here are relative to a "base
+// address" which we define as the base of where the text section is eventually
+// loaded.
+#[allow(non_camel_case_types)]
+struct RUNTIME_FUNCTION {
+    begin: u32,
+    end: u32,
+    unwind_address: u32,
+}
+
+impl<'a> UnwindInfoBuilder<'a> {
+    /// Pushes the unwind information for a function into this builder.
+    ///
+    /// The function being described must be located at `function_offset` within
+    /// the text section itself, and the function's size is specified by
+    /// `function_len`.
+    ///
+    /// The `info` should come from Cranelift itself and this function may
+    /// append more data to the text section in which case the `append_data`
+    /// callback will be invoked. The `append_data` callback receives the data
+    /// to append to the text section as well as the alignment it needs to be
+    /// written at. The return value of `append_data` should be the offset
+    /// within the text section for where the data was written.
+    fn push(
+        &mut self,
+        function_offset: u64,
+        function_len: u64,
+        info: &'a UnwindInfo,
+        append_data: impl FnOnce(&[u8], u32) -> u64,
+    ) {
+        match info {
+            // Windows unwind information is preferred to come after the code
+            // itself. The information is appended here just after the function,
+            // aligned to 4-bytes as required by Windows.
+            //
+            // The location of the unwind info, and the function it describes,
+            // is then recorded in an unwind info table to get embedded into the
+            // object at the end of compilation.
+            UnwindInfo::WindowsX64(info) => {
+                // Windows prefers Unwind info after the code -- writing it here.
+                let unwind_size = info.emit_size();
+                let mut unwind_info = vec![0; unwind_size];
+                info.emit(&mut unwind_info);
+                let unwind_off = append_data(&unwind_info, 4);
+                self.windows_unwind_info.push(RUNTIME_FUNCTION {
+                    begin: u32::try_from(function_offset).unwrap(),
+                    end: u32::try_from(function_offset + function_len).unwrap(),
+                    unwind_address: u32::try_from(unwind_off).unwrap(),
+                });
             }
 
-        Ok(())
+            // System-V is different enough that we just record the unwinding
+            // information to get processed at a later time.
+            UnwindInfo::SystemV(info) => {
+                self.systemv_unwind_info.push((function_offset, info));
+            }
+
+            _ => panic!("some unwind info isn't handled here"),
+        }
+    }
+
+    /// Appends the unwind information section, if any, to the `obj` specified.
+    ///
+    /// This function must be called immediately after the text section was
+    /// added to a builder. The unwind information section must trail the text
+    /// section immediately.
+    ///
+    /// The `text_section`'s section identifier is passed into this function.
+    fn append_section(&self, isa: &dyn TargetIsa, obj: &mut Object<'_>, text_section: SectionId) {
+        // This write will align the text section to a page boundary and then
+        // return the offset at that point. This gives us the full size of the
+        // text section at that point, after alignment.
+        let text_section_size =
+            obj.append_section_data(text_section, &[], isa.code_section_alignment());
+
+        if self.windows_unwind_info.len() > 0 {
+            assert!(self.systemv_unwind_info.len() == 0);
+            let segment = obj.segment_name(StandardSegment::Data).to_vec();
+            let section_id = obj.add_section(
+                segment,
+                b"_wasmtime_winx64_unwind".to_vec(),
+                SectionKind::ReadOnlyData,
+            );
+            self.write_windows_unwind_info(obj, section_id);
+        }
+
+        if self.systemv_unwind_info.len() > 0 {
+            let segment = obj.segment_name(StandardSegment::Data).to_vec();
+            let section_id =
+                obj.add_section(segment, b".eh_frame".to_vec(), SectionKind::ReadOnlyData);
+            self.write_systemv_unwind_info(isa, obj, section_id, text_section_size)
+        }
     }
 
     /// This function appends a nonstandard section to the object which is only
-    /// used during `CodeMemory::allocate_for_object`.
+    /// used during `CodeMemory::publish`.
     ///
     /// This custom section effectively stores a `[RUNTIME_FUNCTION; N]` into
     /// the object file itself. This way registration of unwind info can simply
     /// pass this slice to the OS itself and there's no need to recalculate
     /// anything on the other end of loading a module from a precompiled object.
-    fn append_windows_unwind_info(&mut self) {
+    ///
+    /// Support for reading this is in `crates/jit/src/unwind/winx64.rs`.
+    fn write_windows_unwind_info(&self, obj: &mut Object<'_>, section_id: SectionId) {
         // Currently the binary format supported here only supports
         // little-endian for x86_64, or at least that's all where it's tested.
         // This may need updates for other platforms.
-        assert_eq!(self.obj.architecture(), Architecture::X86_64);
+        assert_eq!(obj.architecture(), Architecture::X86_64);
-
-        let section_id = self.windows_unwind_info_id.unwrap();
-
-        // Page-align the text section so the unwind info can reside on a
-        // separate page that doesn't need executable permissions.
-        self.obj
-            .append_section_data(self.text_section, &[], self.isa.code_section_alignment());
 
         let mut unwind_info = Vec::with_capacity(self.windows_unwind_info.len() * 3 * 4);
         for info in self.windows_unwind_info.iter() {
@@ -409,11 +343,11 @@ impl<'a> ObjectBuilder<'a> {
             unwind_info.extend_from_slice(&info.end.to_le_bytes());
             unwind_info.extend_from_slice(&info.unwind_address.to_le_bytes());
         }
-        self.obj.append_section_data(section_id, &unwind_info, 4);
+        obj.append_section_data(section_id, &unwind_info, 4);
     }
 
     /// This function appends a nonstandard section to the object which is only
-    /// used during `CodeMemory::allocate_for_object`.
+    /// used during `CodeMemory::publish`.
     ///
     /// This will generate a `.eh_frame` section, but not one that can be
     /// naively loaded. The goal of this section is that we can create the
@@ -450,22 +384,20 @@ impl<'a> ObjectBuilder<'a> {
     /// This allows `.eh_frame` to have different virtual memory permissions,
     /// such as being purely read-only instead of read/execute like the code
     /// bits.
-    fn append_systemv_unwind_info(&mut self) {
+    fn write_systemv_unwind_info(
-        let section_id = self.systemv_unwind_info_id.unwrap();
+        &self,
-        let mut cie = self
+        isa: &dyn TargetIsa,
-            .isa
+        obj: &mut Object<'_>,
+        section_id: SectionId,
+        text_section_size: u64,
+    ) {
+        let mut cie = isa
             .create_systemv_cie()
             .expect("must be able to create a CIE for system-v unwind info");
         let mut table = FrameTable::default();
         cie.fde_address_encoding = gimli::constants::DW_EH_PE_pcrel;
         let cie_id = table.add_cie(cie);
 
-        // This write will align the text section to a page boundary
-        // and then return the offset at that point. This gives us the full size
-        // of the text section at that point, after alignment.
-        let text_section_size =
-            self.obj
-                .append_section_data(self.text_section, &[], self.isa.code_section_alignment());
         for (text_section_off, unwind_info) in self.systemv_unwind_info.iter() {
             let backwards_off = text_section_size - text_section_off;
             let actual_offset = -i64::try_from(backwards_off).unwrap();
@@ -476,7 +408,7 @@ impl<'a> ObjectBuilder<'a> {
             let fde = unwind_info.to_fde(Address::Constant(actual_offset as u64));
             table.add_fde(cie_id, fde);
         }
-        let endian = match self.isa.triple().endianness().unwrap() {
+        let endian = match isa.triple().endianness().unwrap() {
             target_lexicon::Endianness::Little => RunTimeEndian::Little,
             target_lexicon::Endianness::Big => RunTimeEndian::Big,
         };
@@ -487,8 +419,7 @@ impl<'a> ObjectBuilder<'a> {
         // a 0 is written at the end of the table for those implementations.
         let mut endian_vec = (eh_frame.0).0;
         endian_vec.write_u32(0).unwrap();
-        self.obj
+        obj.append_section_data(section_id, endian_vec.slice(), 1);
-            .append_section_data(section_id, endian_vec.slice(), 1);
 
         use gimli::constants;
         use gimli::write::Error;

crates/environ/src/lib.rs

@@ -70,28 +70,3 @@ pub const WASM64_MAX_PAGES: u64 = 1 << 48;
 
 /// Version number of this crate.
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-
-/// Iterates through all `LibCall` members and all runtime exported functions.
-#[macro_export]
-macro_rules! for_each_libcall {
-    ($op:ident) => {
-        $op![
-            (UdivI64, wasmtime_i64_udiv),
-            (UdivI64, wasmtime_i64_udiv),
-            (SdivI64, wasmtime_i64_sdiv),
-            (UremI64, wasmtime_i64_urem),
-            (SremI64, wasmtime_i64_srem),
-            (IshlI64, wasmtime_i64_ishl),
-            (UshrI64, wasmtime_i64_ushr),
-            (SshrI64, wasmtime_i64_sshr),
-            (CeilF32, wasmtime_f32_ceil),
-            (FloorF32, wasmtime_f32_floor),
-            (TruncF32, wasmtime_f32_trunc),
-            (NearestF32, wasmtime_f32_nearest),
-            (CeilF64, wasmtime_f64_ceil),
-            (FloorF64, wasmtime_f64_floor),
-            (TruncF64, wasmtime_f64_trunc),
-            (NearestF64, wasmtime_f64_nearest)
-        ];
-    };
-}