diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index dd900a29b8..26768ffc74 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -180,6 +180,53 @@ impl JITModule { .or_else(|| lookup_with_dlsym(name)) } + fn new_func_plt_entry(&mut self, id: FuncId, val: *const u8) { + let got_entry = self + .memory + .writable + .allocate( + std::mem::size_of::<*const u8>(), + std::mem::align_of::<*const u8>().try_into().unwrap(), + ) + .unwrap() + .cast::<*const u8>(); + self.function_got_entries[id] = Some(NonNull::new(got_entry).unwrap()); + unsafe { + std::ptr::write(got_entry, val); + } + let plt_entry = self + .memory + .code + .allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT) + .unwrap() + .cast::<[u8; 16]>(); + self.record_function_for_perf( + plt_entry as *mut _, + std::mem::size_of::<[u8; 16]>(), + &format!("{}@plt", self.declarations.get_function_decl(id).name), + ); + self.function_plt_entries[id] = Some(NonNull::new(plt_entry).unwrap()); + unsafe { + Self::write_plt_entry_bytes(plt_entry, got_entry); + } + } + + fn new_data_got_entry(&mut self, id: DataId, val: *const u8) { + let got_entry = self + .memory + .writable + .allocate( + std::mem::size_of::<*const u8>(), + std::mem::align_of::<*const u8>().try_into().unwrap(), + ) + .unwrap() + .cast::<*const u8>(); + self.data_object_got_entries[id] = Some(NonNull::new(got_entry).unwrap()); + unsafe { + std::ptr::write(got_entry, val); + } + } + unsafe fn write_plt_entry_bytes(plt_ptr: *mut [u8; 16], got_ptr: *mut *const u8) { assert!( cfg!(target_arch = "x86_64"), @@ -494,40 +541,25 @@ impl Module for JITModule { linkage: Linkage, signature: &ir::Signature, ) -> ModuleResult { - let (id, _decl) = self + let (id, linkage) = self .declarations .declare_function(name, linkage, signature)?; if self.function_got_entries[id].is_none() && self.isa.flags().is_pic() { - let got_entry = self - .memory - .writable - .allocate( - std::mem::size_of::<*const u8>(), - std::mem::align_of::<*const u8>().try_into().unwrap(), - ) - .unwrap() - .cast::<*const u8>(); - self.function_got_entries[id] = Some(NonNull::new(got_entry).unwrap()); // FIXME populate got entries with a null pointer when defined - let val = self.lookup_symbol(name).unwrap_or(std::ptr::null()); - unsafe { - std::ptr::write(got_entry, val); - } - let plt_entry = self - .memory - .code - .allocate(std::mem::size_of::<[u8; 16]>(), EXECUTABLE_DATA_ALIGNMENT) - .unwrap() - .cast::<[u8; 16]>(); - self.record_function_for_perf( - plt_entry as *mut _, - std::mem::size_of::<[u8; 16]>(), - &format!("{}@plt", name), - ); - self.function_plt_entries[id] = Some(NonNull::new(plt_entry).unwrap()); - unsafe { - Self::write_plt_entry_bytes(plt_entry, got_entry); - } + let val = if linkage == Linkage::Import { + self.lookup_symbol(name).unwrap_or(std::ptr::null()) + } else { + std::ptr::null() + }; + self.new_func_plt_entry(id, val); + } + Ok(id) + } + + fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult { + let id = self.declarations.declare_anonymous_function(signature)?; + if self.isa.flags().is_pic() { + self.new_func_plt_entry(id, std::ptr::null()); } Ok(id) } @@ -540,25 +572,26 @@ impl Module for JITModule { tls: bool, ) -> ModuleResult { assert!(!tls, "JIT doesn't yet support TLS"); - let (id, _decl) = self + let (id, linkage) = self .declarations .declare_data(name, linkage, writable, tls)?; if self.data_object_got_entries[id].is_none() && self.isa.flags().is_pic() { - let got_entry = self - .memory - .writable - .allocate( - std::mem::size_of::<*const u8>(), - std::mem::align_of::<*const u8>().try_into().unwrap(), - ) - .unwrap() - .cast::<*const u8>(); - self.data_object_got_entries[id] = Some(NonNull::new(got_entry).unwrap()); // FIXME populate got entries with a null pointer when defined - let val = self.lookup_symbol(name).unwrap_or(std::ptr::null()); - unsafe { - std::ptr::write(got_entry, val); - } + let val = if linkage == Linkage::Import { + self.lookup_symbol(name).unwrap_or(std::ptr::null()) + } else { + std::ptr::null() + }; + self.new_data_got_entry(id, val); + } + Ok(id) + } + + fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult { + assert!(!tls, "JIT doesn't yet support TLS"); + let id = self.declarations.declare_anonymous_data(writable, tls)?; + if self.isa.flags().is_pic() { + self.new_data_got_entry(id, std::ptr::null()); } Ok(id) } diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index a1145abc04..6047dda103 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -268,7 +268,7 @@ impl ModuleDeclarations { name: &str, linkage: Linkage, signature: &ir::Signature, - ) -> ModuleResult<(FuncId, &FunctionDeclaration)> { + ) -> ModuleResult<(FuncId, Linkage)> { // TODO: Can we avoid allocating names so often? use super::hash_map::Entry::*; match self.names.entry(name.to_owned()) { @@ -276,7 +276,7 @@ impl ModuleDeclarations { FuncOrDataId::Func(id) => { let existing = &mut self.functions[id]; existing.merge(linkage, signature)?; - Ok((id, existing)) + Ok((id, existing.linkage)) } FuncOrDataId::Data(..) => { Err(ModuleError::IncompatibleDeclaration(name.to_owned())) @@ -289,11 +289,25 @@ impl ModuleDeclarations { signature: signature.clone(), }); entry.insert(FuncOrDataId::Func(id)); - Ok((id, &self.functions[id])) + Ok((id, self.functions[id].linkage)) } } } + /// Declare an anonymous function in this module. + pub fn declare_anonymous_function( + &mut self, + signature: &ir::Signature, + ) -> ModuleResult { + let id = self.functions.push(FunctionDeclaration { + name: String::new(), + linkage: Linkage::Local, + signature: signature.clone(), + }); + self.functions[id].name = format!(".L{:?}", id); + Ok(id) + } + /// Declare a data object in this module. pub fn declare_data( &mut self, @@ -301,7 +315,7 @@ impl ModuleDeclarations { linkage: Linkage, writable: bool, tls: bool, - ) -> ModuleResult<(DataId, &DataDeclaration)> { + ) -> ModuleResult<(DataId, Linkage)> { // TODO: Can we avoid allocating names so often? use super::hash_map::Entry::*; match self.names.entry(name.to_owned()) { @@ -309,7 +323,7 @@ impl ModuleDeclarations { FuncOrDataId::Data(id) => { let existing = &mut self.data_objects[id]; existing.merge(linkage, writable, tls); - Ok((id, existing)) + Ok((id, existing.linkage)) } FuncOrDataId::Func(..) => { @@ -324,10 +338,22 @@ impl ModuleDeclarations { tls, }); entry.insert(FuncOrDataId::Data(id)); - Ok((id, &self.data_objects[id])) + Ok((id, self.data_objects[id].linkage)) } } } + + /// Declare an anonymous data object in this module. + pub fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult { + let id = self.data_objects.push(DataDeclaration { + name: String::new(), + linkage: Linkage::Local, + writable, + tls, + }); + self.data_objects[id].name = format!(".L{:?}", id); + Ok(id) + } } /// Information about the compiled function. @@ -411,6 +437,9 @@ pub trait Module { signature: &ir::Signature, ) -> ModuleResult; + /// Declare an anonymous function in this module. + fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult; + /// Declare a data object in this module. fn declare_data( &mut self, @@ -420,6 +449,9 @@ pub trait Module { tls: bool, ) -> ModuleResult; + /// Declare an anonymous data object in this module. + fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult; + /// Use this when you're building the IR of a function to reference a function. /// /// TODO: Coalesce redundant decls and signatures. @@ -532,6 +564,10 @@ impl Module for &mut M { (**self).declare_function(name, linkage, signature) } + fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult { + (**self).declare_anonymous_function(signature) + } + fn declare_data( &mut self, name: &str, @@ -542,6 +578,10 @@ impl Module for &mut M { (**self).declare_data(name, linkage, writable, tls) } + fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult { + (**self).declare_anonymous_data(writable, tls) + } + fn declare_func_in_func(&self, func: FuncId, in_func: &mut ir::Function) -> ir::FuncRef { (**self).declare_func_in_func(func, in_func) } diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index ad3aa331be..abfdcf10c9 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -123,6 +123,8 @@ pub struct ObjectModule { libcall_names: Box String + Send + Sync>, function_alignment: u64, per_function_section: bool, + anon_func_number: u64, + anon_data_number: u64, } impl ObjectModule { @@ -141,6 +143,8 @@ impl ObjectModule { libcall_names: builder.libcall_names, function_alignment: builder.function_alignment, per_function_section: builder.per_function_section, + anon_func_number: 0, + anon_data_number: 0, } } } @@ -174,11 +178,11 @@ impl Module for ObjectModule { ) -> ModuleResult { validate_symbol(name)?; - let (id, decl) = self + let (id, linkage) = self .declarations .declare_function(name, linkage, signature)?; - let (scope, weak) = translate_linkage(decl.linkage); + let (scope, weak) = translate_linkage(linkage); if let Some((function, _defined)) = self.functions[id] { let symbol = self.object.symbol_mut(function); @@ -201,6 +205,30 @@ impl Module for ObjectModule { Ok(id) } + fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult { + // Symbols starting with .L are completely omitted from the symbol table after linking. + // Using hexadecimal instead of decimal for slightly smaller symbol names and often slightly + // faster linking. + let name = format!(".Lfn{:x}", self.anon_func_number); + self.anon_func_number += 1; + + let id = self.declarations.declare_anonymous_function(signature)?; + + let symbol_id = self.object.add_symbol(Symbol { + name: name.as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + self.functions[id] = Some((symbol_id, false)); + + Ok(id) + } + fn declare_data( &mut self, name: &str, @@ -210,16 +238,18 @@ impl Module for ObjectModule { ) -> ModuleResult { validate_symbol(name)?; - let (id, decl) = self + let (id, linkage) = self .declarations .declare_data(name, linkage, writable, tls)?; - let kind = if decl.tls { + // Merging declarations with conflicting values for tls is not allowed, so it is safe to use + // the passed in tls value here. + let kind = if tls { SymbolKind::Tls } else { SymbolKind::Data }; - let (scope, weak) = translate_linkage(decl.linkage); + let (scope, weak) = translate_linkage(linkage); if let Some((data, _defined)) = self.data_objects[id] { let symbol = self.object.symbol_mut(data); @@ -243,6 +273,36 @@ impl Module for ObjectModule { Ok(id) } + fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult { + // Symbols starting with .L are completely omitted from the symbol table after linking. + // Using hexadecimal instead of decimal for slightly smaller symbol names and often slightly + // faster linking. + let name = format!(".Ldata{:x}", self.anon_data_number); + self.anon_data_number += 1; + + let id = self.declarations.declare_anonymous_data(writable, tls)?; + + let kind = if tls { + SymbolKind::Tls + } else { + SymbolKind::Data + }; + + let symbol_id = self.object.add_symbol(Symbol { + name: name.as_bytes().to_vec(), + value: 0, + size: 0, + kind, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + self.data_objects[id] = Some((symbol_id, false)); + + Ok(id) + } + fn define_function( &mut self, func_id: FuncId,