Merge pull request #2750 from bjorn3/anon_allocs

Support declaring anonymous functions and data objects

cranelift/jit/src/backend.rs

@@ -180,6 +180,53 @@ impl JITModule {
             .or_else(|| lookup_with_dlsym(name))
     }
 
+    fn new_func_plt_entry(&mut self, id: FuncId, val: *const u8) {
+        let got_entry = self
+            .memory
+            .writable
+            .allocate(
+                std::mem::size_of::<*const u8>(),
+                std::mem::align_of::<*const u8>().try_into().unwrap(),
+            )
+            .unwrap()
+            .cast::<*const u8>();
+        self.function_got_entries[id] = Some(NonNull::new(got_entry).unwrap());
+        unsafe {
+            std::ptr::write(got_entry, val);
+        }
+        let plt_entry = self
+            .memory
+            .code
+            .allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT)
+            .unwrap()
+            .cast::<[u8; 16]>();
+        self.record_function_for_perf(
+            plt_entry as *mut _,
+            std::mem::size_of::<[u8; 16]>(),
+            &format!("{}@plt", self.declarations.get_function_decl(id).name),
+        );
+        self.function_plt_entries[id] = Some(NonNull::new(plt_entry).unwrap());
+        unsafe {
+            Self::write_plt_entry_bytes(plt_entry, got_entry);
+        }
+    }
+
+    fn new_data_got_entry(&mut self, id: DataId, val: *const u8) {
+        let got_entry = self
+            .memory
+            .writable
+            .allocate(
+                std::mem::size_of::<*const u8>(),
+                std::mem::align_of::<*const u8>().try_into().unwrap(),
+            )
+            .unwrap()
+            .cast::<*const u8>();
+        self.data_object_got_entries[id] = Some(NonNull::new(got_entry).unwrap());
+        unsafe {
+            std::ptr::write(got_entry, val);
+        }
+    }
+
     unsafe fn write_plt_entry_bytes(plt_ptr: *mut [u8; 16], got_ptr: *mut *const u8) {
         assert!(
             cfg!(target_arch = "x86_64"),
@@ -494,40 +541,25 @@ impl Module for JITModule {
         linkage: Linkage,
         signature: &ir::Signature,
     ) -> ModuleResult<FuncId> {
-        let (id, _decl) = self
+        let (id, linkage) = self
             .declarations
             .declare_function(name, linkage, signature)?;
         if self.function_got_entries[id].is_none() && self.isa.flags().is_pic() {
-            let got_entry = self
-                .memory
-                .writable
-                .allocate(
-                    std::mem::size_of::<*const u8>(),
-                    std::mem::align_of::<*const u8>().try_into().unwrap(),
-                )
-                .unwrap()
-                .cast::<*const u8>();
-            self.function_got_entries[id] = Some(NonNull::new(got_entry).unwrap());
             // FIXME populate got entries with a null pointer when defined
-            let val = self.lookup_symbol(name).unwrap_or(std::ptr::null());
+            let val = if linkage == Linkage::Import {
-            unsafe {
+                self.lookup_symbol(name).unwrap_or(std::ptr::null())
-                std::ptr::write(got_entry, val);
+            } else {
+                std::ptr::null()
+            };
+            self.new_func_plt_entry(id, val);
         }
-            let plt_entry = self
+        Ok(id)
-                .memory
-                .code
-                .allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT)
-                .unwrap()
-                .cast::<[u8; 16]>();
-            self.record_function_for_perf(
-                plt_entry as *mut _,
-                std::mem::size_of::<[u8; 16]>(),
-                &format!("{}@plt", name),
-            );
-            self.function_plt_entries[id] = Some(NonNull::new(plt_entry).unwrap());
-            unsafe {
-                Self::write_plt_entry_bytes(plt_entry, got_entry);
     }
+
+    fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult<FuncId> {
+        let id = self.declarations.declare_anonymous_function(signature)?;
+        if self.isa.flags().is_pic() {
+            self.new_func_plt_entry(id, std::ptr::null());
         }
         Ok(id)
     }
@@ -540,25 +572,26 @@ impl Module for JITModule {
         tls: bool,
     ) -> ModuleResult<DataId> {
         assert!(!tls, "JIT doesn't yet support TLS");
-        let (id, _decl) = self
+        let (id, linkage) = self
             .declarations
             .declare_data(name, linkage, writable, tls)?;
         if self.data_object_got_entries[id].is_none() && self.isa.flags().is_pic() {
-            let got_entry = self
-                .memory
-                .writable
-                .allocate(
-                    std::mem::size_of::<*const u8>(),
-                    std::mem::align_of::<*const u8>().try_into().unwrap(),
-                )
-                .unwrap()
-                .cast::<*const u8>();
-            self.data_object_got_entries[id] = Some(NonNull::new(got_entry).unwrap());
             // FIXME populate got entries with a null pointer when defined
-            let val = self.lookup_symbol(name).unwrap_or(std::ptr::null());
+            let val = if linkage == Linkage::Import {
-            unsafe {
+                self.lookup_symbol(name).unwrap_or(std::ptr::null())
-                std::ptr::write(got_entry, val);
+            } else {
+                std::ptr::null()
+            };
+            self.new_data_got_entry(id, val);
         }
+        Ok(id)
+    }
+
+    fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult<DataId> {
+        assert!(!tls, "JIT doesn't yet support TLS");
+        let id = self.declarations.declare_anonymous_data(writable, tls)?;
+        if self.isa.flags().is_pic() {
+            self.new_data_got_entry(id, std::ptr::null());
         }
         Ok(id)
     }

cranelift/module/src/module.rs

@@ -268,7 +268,7 @@ impl ModuleDeclarations {
         name: &str,
         linkage: Linkage,
         signature: &ir::Signature,
-    ) -> ModuleResult<(FuncId, &FunctionDeclaration)> {
+    ) -> ModuleResult<(FuncId, Linkage)> {
         // TODO: Can we avoid allocating names so often?
         use super::hash_map::Entry::*;
         match self.names.entry(name.to_owned()) {
@@ -276,7 +276,7 @@ impl ModuleDeclarations {
                 FuncOrDataId::Func(id) => {
                     let existing = &mut self.functions[id];
                     existing.merge(linkage, signature)?;
-                    Ok((id, existing))
+                    Ok((id, existing.linkage))
                 }
                 FuncOrDataId::Data(..) => {
                     Err(ModuleError::IncompatibleDeclaration(name.to_owned()))
@@ -289,11 +289,25 @@ impl ModuleDeclarations {
                     signature: signature.clone(),
                 });
                 entry.insert(FuncOrDataId::Func(id));
-                Ok((id, &self.functions[id]))
+                Ok((id, self.functions[id].linkage))
             }
         }
     }
 
+    /// Declare an anonymous function in this module.
+    pub fn declare_anonymous_function(
+        &mut self,
+        signature: &ir::Signature,
+    ) -> ModuleResult<FuncId> {
+        let id = self.functions.push(FunctionDeclaration {
+            name: String::new(),
+            linkage: Linkage::Local,
+            signature: signature.clone(),
+        });
+        self.functions[id].name = format!(".L{:?}", id);
+        Ok(id)
+    }
+
     /// Declare a data object in this module.
     pub fn declare_data(
         &mut self,
@@ -301,7 +315,7 @@ impl ModuleDeclarations {
         linkage: Linkage,
         writable: bool,
         tls: bool,
-    ) -> ModuleResult<(DataId, &DataDeclaration)> {
+    ) -> ModuleResult<(DataId, Linkage)> {
         // TODO: Can we avoid allocating names so often?
         use super::hash_map::Entry::*;
         match self.names.entry(name.to_owned()) {
@@ -309,7 +323,7 @@ impl ModuleDeclarations {
                 FuncOrDataId::Data(id) => {
                     let existing = &mut self.data_objects[id];
                     existing.merge(linkage, writable, tls);
-                    Ok((id, existing))
+                    Ok((id, existing.linkage))
                 }
 
                 FuncOrDataId::Func(..) => {
@@ -324,10 +338,22 @@ impl ModuleDeclarations {
                     tls,
                 });
                 entry.insert(FuncOrDataId::Data(id));
-                Ok((id, &self.data_objects[id]))
+                Ok((id, self.data_objects[id].linkage))
             }
         }
     }
+
+    /// Declare an anonymous data object in this module.
+    pub fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult<DataId> {
+        let id = self.data_objects.push(DataDeclaration {
+            name: String::new(),
+            linkage: Linkage::Local,
+            writable,
+            tls,
+        });
+        self.data_objects[id].name = format!(".L{:?}", id);
+        Ok(id)
+    }
 }
 
 /// Information about the compiled function.
@@ -411,6 +437,9 @@ pub trait Module {
         signature: &ir::Signature,
     ) -> ModuleResult<FuncId>;
 
+    /// Declare an anonymous function in this module.
+    fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult<FuncId>;
+
     /// Declare a data object in this module.
     fn declare_data(
         &mut self,
@@ -420,6 +449,9 @@ pub trait Module {
         tls: bool,
     ) -> ModuleResult<DataId>;
 
+    /// Declare an anonymous data object in this module.
+    fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult<DataId>;
+
     /// Use this when you're building the IR of a function to reference a function.
     ///
     /// TODO: Coalesce redundant decls and signatures.
@@ -532,6 +564,10 @@ impl<M: Module> Module for &mut M {
         (**self).declare_function(name, linkage, signature)
     }
 
+    fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult<FuncId> {
+        (**self).declare_anonymous_function(signature)
+    }
+
     fn declare_data(
         &mut self,
         name: &str,
@@ -542,6 +578,10 @@ impl<M: Module> Module for &mut M {
         (**self).declare_data(name, linkage, writable, tls)
     }
 
+    fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult<DataId> {
+        (**self).declare_anonymous_data(writable, tls)
+    }
+
     fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef {
         (**self).declare_func_in_func(func, in_func)
     }

cranelift/object/src/backend.rs

@@ -123,6 +123,8 @@ pub struct ObjectModule {
     libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
     function_alignment: u64,
     per_function_section: bool,
+    anon_func_number: u64,
+    anon_data_number: u64,
 }
 
 impl ObjectModule {
@@ -141,6 +143,8 @@ impl ObjectModule {
             libcall_names: builder.libcall_names,
             function_alignment: builder.function_alignment,
             per_function_section: builder.per_function_section,
+            anon_func_number: 0,
+            anon_data_number: 0,
         }
     }
 }
@@ -174,11 +178,11 @@ impl Module for ObjectModule {
     ) -> ModuleResult<FuncId> {
         validate_symbol(name)?;
 
-        let (id, decl) = self
+        let (id, linkage) = self
             .declarations
             .declare_function(name, linkage, signature)?;
 
-        let (scope, weak) = translate_linkage(decl.linkage);
+        let (scope, weak) = translate_linkage(linkage);
 
         if let Some((function, _defined)) = self.functions[id] {
             let symbol = self.object.symbol_mut(function);
@@ -201,6 +205,30 @@ impl Module for ObjectModule {
         Ok(id)
     }
 
+    fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult<FuncId> {
+        // Symbols starting with .L are completely omitted from the symbol table after linking.
+        // Using hexadecimal instead of decimal for slightly smaller symbol names and often slightly
+        // faster linking.
+        let name = format!(".Lfn{:x}", self.anon_func_number);
+        self.anon_func_number += 1;
+
+        let id = self.declarations.declare_anonymous_function(signature)?;
+
+        let symbol_id = self.object.add_symbol(Symbol {
+            name: name.as_bytes().to_vec(),
+            value: 0,
+            size: 0,
+            kind: SymbolKind::Text,
+            scope: SymbolScope::Compilation,
+            weak: false,
+            section: SymbolSection::Undefined,
+            flags: SymbolFlags::None,
+        });
+        self.functions[id] = Some((symbol_id, false));
+
+        Ok(id)
+    }
+
     fn declare_data(
         &mut self,
         name: &str,
@@ -210,16 +238,18 @@ impl Module for ObjectModule {
     ) -> ModuleResult<DataId> {
         validate_symbol(name)?;
 
-        let (id, decl) = self
+        let (id, linkage) = self
             .declarations
             .declare_data(name, linkage, writable, tls)?;
 
-        let kind = if decl.tls {
+        // Merging declarations with conflicting values for tls is not allowed, so it is safe to use
+        // the passed in tls value here.
+        let kind = if tls {
             SymbolKind::Tls
         } else {
             SymbolKind::Data
         };
-        let (scope, weak) = translate_linkage(decl.linkage);
+        let (scope, weak) = translate_linkage(linkage);
 
         if let Some((data, _defined)) = self.data_objects[id] {
             let symbol = self.object.symbol_mut(data);
@@ -243,6 +273,36 @@ impl Module for ObjectModule {
         Ok(id)
     }
 
+    fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult<DataId> {
+        // Symbols starting with .L are completely omitted from the symbol table after linking.
+        // Using hexadecimal instead of decimal for slightly smaller symbol names and often slightly
+        // faster linking.
+        let name = format!(".Ldata{:x}", self.anon_data_number);
+        self.anon_data_number += 1;
+
+        let id = self.declarations.declare_anonymous_data(writable, tls)?;
+
+        let kind = if tls {
+            SymbolKind::Tls
+        } else {
+            SymbolKind::Data
+        };
+
+        let symbol_id = self.object.add_symbol(Symbol {
+            name: name.as_bytes().to_vec(),
+            value: 0,
+            size: 0,
+            kind,
+            scope: SymbolScope::Compilation,
+            weak: false,
+            section: SymbolSection::Undefined,
+            flags: SymbolFlags::None,
+        });
+        self.data_objects[id] = Some((symbol_id, false));
+
+        Ok(id)
+    }
+
     fn define_function(
         &mut self,
         func_id: FuncId,