diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index 750eaa2a21..9a1bfe0159 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -59,6 +59,11 @@ pub enum Reloc { /// Set the add immediate field to the low 12 bits of the final address. Does not check for overflow. /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12_NC` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#relocations-for-thread-local-storage) Aarch64TlsGdAddLo12Nc, + + /// s390x TLS GD64 - 64-bit offset of tls_index for GD symbol in GOT + S390xTlsGd64, + /// s390x TLS GDCall - marker to enable optimization of TLS calls + S390xTlsGdCall, } impl fmt::Display for Reloc { @@ -79,6 +84,8 @@ impl fmt::Display for Reloc { Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"), Self::Aarch64TlsGdAdrPage21 => write!(f, "Aarch64TlsGdAdrPage21"), Self::Aarch64TlsGdAddLo12Nc => write!(f, "Aarch64TlsGdAddLo12Nc"), + Self::S390xTlsGd64 => write!(f, "TlsGd64"), + Self::S390xTlsGdCall => write!(f, "TlsGdCall"), } } } diff --git a/cranelift/codegen/src/ir/extname.rs b/cranelift/codegen/src/ir/extname.rs index 362cf8c67e..54c56a4247 100644 --- a/cranelift/codegen/src/ir/extname.rs +++ b/cranelift/codegen/src/ir/extname.rs @@ -4,7 +4,7 @@ //! function. The name of an external declaration doesn't have any meaning to //! Cranelift, which compiles functions independently. -use crate::ir::LibCall; +use crate::ir::{KnownSymbol, LibCall}; use core::cmp; use core::fmt::{self, Write}; use core::str::FromStr; @@ -46,6 +46,8 @@ pub enum ExternalName { }, /// A well-known runtime library function. LibCall(LibCall), + /// A well-known symbol. + KnownSymbol(KnownSymbol), } impl ExternalName { @@ -104,6 +106,7 @@ impl fmt::Display for ExternalName { Ok(()) } Self::LibCall(lc) => write!(f, "%{}", lc), + Self::KnownSymbol(ks) => write!(f, "%{}", ks), } } } @@ -112,11 +115,18 @@ impl FromStr for ExternalName { type Err = (); fn from_str(s: &str) -> Result { - // Try to parse as a libcall name, otherwise it's a test case. - match s.parse() { - Ok(lc) => Ok(Self::LibCall(lc)), - Err(_) => Ok(Self::testcase(s.as_bytes())), + // Try to parse as a known symbol + if let Ok(ks) = s.parse() { + return Ok(Self::KnownSymbol(ks)); } + + // Try to parse as a libcall name + if let Ok(lc) = s.parse() { + return Ok(Self::LibCall(lc)); + } + + // Otherwise its a test case name + Ok(Self::testcase(s.as_bytes())) } } diff --git a/cranelift/codegen/src/ir/known_symbol.rs b/cranelift/codegen/src/ir/known_symbol.rs new file mode 100644 index 0000000000..8120899218 --- /dev/null +++ b/cranelift/codegen/src/ir/known_symbol.rs @@ -0,0 +1,42 @@ +use core::fmt; +use core::str::FromStr; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +/// A well-known symbol. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub enum KnownSymbol { + /// ELF well-known linker symbol _GLOBAL_OFFSET_TABLE_ + ElfGlobalOffsetTable, +} + +impl fmt::Display for KnownSymbol { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, f) + } +} + +impl FromStr for KnownSymbol { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "ElfGlobalOffsetTable" => Ok(Self::ElfGlobalOffsetTable), + _ => Err(()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parsing() { + assert_eq!( + "ElfGlobalOffsetTable".parse(), + Ok(KnownSymbol::ElfGlobalOffsetTable) + ); + } +} diff --git a/cranelift/codegen/src/ir/libcall.rs b/cranelift/codegen/src/ir/libcall.rs index adf343a216..82971f4a22 100644 --- a/cranelift/codegen/src/ir/libcall.rs +++ b/cranelift/codegen/src/ir/libcall.rs @@ -66,6 +66,8 @@ pub enum LibCall { /// Elf __tls_get_addr ElfTlsGetAddr, + /// Elf __tls_get_offset + ElfTlsGetOffset, // When adding a new variant make sure to add it to `all_libcalls` too. } @@ -104,6 +106,7 @@ impl FromStr for LibCall { "Memcmp" => Ok(Self::Memcmp), "ElfTlsGetAddr" => Ok(Self::ElfTlsGetAddr), + "ElfTlsGetOffset" => Ok(Self::ElfTlsGetOffset), _ => Err(()), } } @@ -173,6 +176,7 @@ impl LibCall { Memmove, Memcmp, ElfTlsGetAddr, + ElfTlsGetOffset, ] } @@ -214,7 +218,8 @@ impl LibCall { | LibCall::Memset | LibCall::Memmove | LibCall::Memcmp - | LibCall::ElfTlsGetAddr => unimplemented!(), + | LibCall::ElfTlsGetAddr + | LibCall::ElfTlsGetOffset => unimplemented!(), } sig diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index ac0a3bb44c..15e14660e4 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -15,6 +15,7 @@ mod heap; pub mod immediates; pub mod instructions; pub mod jumptable; +pub(crate) mod known_symbol; pub mod layout; pub(crate) mod libcall; mod memflags; @@ -50,6 +51,7 @@ pub use crate::ir::instructions::{ InstructionData, Opcode, ValueList, ValueListPool, VariableArgs, }; pub use crate::ir::jumptable::JumpTableData; +pub use crate::ir::known_symbol::KnownSymbol; pub use crate::ir::layout::Layout; pub use crate::ir::libcall::{get_probestack_funcref, LibCall}; pub use crate::ir::memflags::{Endianness, MemFlags}; diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 81a700184f..ede3619f84 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -424,6 +424,17 @@ (rd WritableReg) (imm UImm32Shifted)) + ;; Load 32-bit access register into GPR. + (LoadAR + (rd WritableReg) + (ar u8)) + + ;; Insert 32-bit access register into low half of a GPR. + ;; (Identical operation to LoadAR, but considers rd to be use/def.) + (InsertAR + (rd WritableReg) + (ar u8)) + ;; A sign- or zero-extend operation. (Extend (rd WritableReg) @@ -857,11 +868,10 @@ (ridx Reg) (targets VecMachLabel)) - ;; Load an inline symbol reference with RelocDistance::Far. - (LoadExtNameFar + ;; Load an inline symbol reference with relocation. + (LoadSymbolReloc (rd WritableReg) - (name BoxExternalName) - (offset i64)) + (symbol_reloc BoxSymbolReloc)) ;; Load address referenced by `mem` into `rd`. (LoadAddr @@ -903,6 +913,23 @@ (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) (type VecMachLabel extern (enum)) +;; A symbol reference carrying relocation information. +(type SymbolReloc + (enum + ;; Absolute symbol reference (with optional offset). + (Absolute + (name ExternalName) + (offset i64)) + ;; Reference to a TLS symbol in general-dynamic mode. + (TlsGd + (name ExternalName)))) + +;; Boxed version of SymbolReloc to save space. +(type BoxSymbolReloc (primitive BoxSymbolReloc)) +(decl box_symbol_reloc (SymbolReloc) BoxSymbolReloc) +(extern constructor box_symbol_reloc box_symbol_reloc) +(convert SymbolReloc BoxSymbolReloc box_symbol_reloc) + ;; An ALU operation. (type ALUOp (enum @@ -1613,6 +1640,9 @@ (decl memarg_symbol (ExternalName i32 MemFlags) MemArg) (extern constructor memarg_symbol memarg_symbol) +(decl memarg_got () MemArg) +(extern constructor memarg_got memarg_got) + ;; Create a MemArg refering to a stack address formed by ;; adding a base (relative to SP) and an offset. (decl memarg_stack_off (i64 i64) MemArg) @@ -2120,6 +2150,20 @@ (rule (mvc dst src len_minus_one) (SideEffectNoResult.Inst (MInst.Mvc dst src len_minus_one))) +;; Helper for emitting `MInst.LoadAR` instructions. +(decl load_ar (u8) Reg) +(rule (load_ar ar) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.LoadAR dst ar)))) + dst)) + +;; Helper for emitting `MInst.InsertAR` instructions. +(decl insert_ar (Reg u8) Reg) +(rule (insert_ar src ar) + (let ((dst WritableReg (copy_writable_reg $I64 src)) + (_ Unit (emit (MInst.InsertAR dst ar)))) + dst)) + ;; Helper for emitting `MInst.FpuRR` instructions. (decl fpu_rr (Type FPUOp1 Reg) Reg) (rule (fpu_rr ty op src) @@ -2393,12 +2437,11 @@ (_ Unit (emit (MInst.VecReplicateLane size dst src lane_imm)))) dst)) -;; Helper for emitting `MInst.LoadExtNameFar` instructions. -(decl load_ext_name_far (ExternalName i64) Reg) -(rule (load_ext_name_far name offset) +;; Helper for emitting `MInst.LoadSymbolReloc` instructions. +(decl load_symbol_reloc (SymbolReloc) Reg) +(rule (load_symbol_reloc symbol_reloc) (let ((dst WritableReg (temp_writable_reg $I64)) - (boxed_name BoxExternalName (box_external_name name)) - (_ Unit (emit (MInst.LoadExtNameFar dst boxed_name offset)))) + (_ Unit (emit (MInst.LoadSymbolReloc dst symbol_reloc)))) dst)) ;; Helper for emitting `MInst.LoadAddr` instructions. @@ -3405,6 +3448,9 @@ (decl lib_call_info_memcpy () LibCallInfo) (extern constructor lib_call_info_memcpy lib_call_info_memcpy) +(decl lib_call_info_tls_get_offset (SymbolReloc) LibCallInfo) +(extern constructor lib_call_info_tls_get_offset lib_call_info_tls_get_offset) + (decl lib_call_info (LibCallInfo) BoxCallInfo) (extern constructor lib_call_info lib_call_info) diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index 0c17814bd7..4deb69aae1 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -2258,17 +2258,25 @@ impl MachInstEmit for Inst { }; put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits)); } - &Inst::LoadExtNameFar { + &Inst::LoadAR { rd, ar } | &Inst::InsertAR { rd, ar } => { + let rd = allocs.next_writable(rd); + let opcode = 0xb24f; // EAR + put(sink, &enc_rre(opcode, rd.to_reg(), gpr(ar))); + } + &Inst::LoadSymbolReloc { rd, - ref name, - offset, + ref symbol_reloc, } => { let rd = allocs.next_writable(rd); let opcode = 0xa75; // BRAS let reg = writable_spilltmp_reg().to_reg(); put(sink, &enc_ri_b(opcode, reg, 12)); - sink.add_reloc(Reloc::Abs8, name, offset); + let (reloc, name, offset) = match &**symbol_reloc { + SymbolReloc::Absolute { name, offset } => (Reloc::Abs8, name, *offset), + SymbolReloc::TlsGd { name } => (Reloc::S390xTlsGd64, name, 0), + }; + sink.add_reloc(reloc, name, offset); sink.put8(0); let inst = Inst::Load64 { rd, @@ -3198,6 +3206,15 @@ impl MachInstEmit for Inst { &Inst::Call { link, ref info } => { let link = allocs.next_writable(link); + // Add relocation for TLS libcalls to enable linker optimizations. + match &info.tls_symbol { + None => {} + Some(SymbolReloc::TlsGd { name }) => { + sink.add_reloc(Reloc::S390xTlsGdCall, name, 0) + } + _ => unreachable!(), + } + let opcode = 0xc05; // BRASL let reloc = Reloc::S390xPCRel32Dbl; if let Some(s) = state.take_stack_map() { diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs index 79922e6f10..9f593d8510 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs @@ -6828,6 +6828,7 @@ fn test_s390x_binemit() { opcode: Opcode::Call, caller_callconv: CallConv::SystemV, callee_callconv: CallConv::SystemV, + tls_symbol: None, }), }, "C0E500000000", diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index 00c639db1b..e561a688b4 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -29,7 +29,7 @@ mod emit_tests; pub use crate::isa::s390x::lower::isle::generated_code::{ ALUOp, CmpOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuRoundOp, MInst as Inst, RxSBGOp, - ShiftOp, UnaryOp, VecBinaryOp, VecFloatCmpOp, VecIntCmpOp, VecShiftOp, VecUnaryOp, + ShiftOp, SymbolReloc, UnaryOp, VecBinaryOp, VecFloatCmpOp, VecIntCmpOp, VecShiftOp, VecUnaryOp, }; /// Additional information for (direct) Call instructions, left out of line to lower the size of @@ -43,6 +43,7 @@ pub struct CallInfo { pub opcode: Opcode, pub caller_callconv: CallConv, pub callee_callconv: CallConv, + pub tls_symbol: Option, } /// Additional information for CallInd instructions, left out of line to lower the size of the Inst @@ -154,6 +155,8 @@ impl Inst { | Inst::Mov64UImm32Shifted { .. } | Inst::Insert64UImm16Shifted { .. } | Inst::Insert64UImm32Shifted { .. } + | Inst::LoadAR { .. } + | Inst::InsertAR { .. } | Inst::Extend { .. } | Inst::CMov32 { .. } | Inst::CMov64 { .. } @@ -212,7 +215,7 @@ impl Inst { | Inst::Debugtrap | Inst::Trap { .. } | Inst::JTSequence { .. } - | Inst::LoadExtNameFar { .. } + | Inst::LoadSymbolReloc { .. } | Inst::LoadAddr { .. } | Inst::Loop { .. } | Inst::CondBreak { .. } @@ -662,6 +665,12 @@ fn s390x_get_operands VReg>(inst: &Inst, collector: &mut OperandC &Inst::Insert64UImm16Shifted { rd, .. } | &Inst::Insert64UImm32Shifted { rd, .. } => { collector.reg_mod(rd); } + &Inst::LoadAR { rd, .. } => { + collector.reg_def(rd); + } + &Inst::InsertAR { rd, .. } => { + collector.reg_mod(rd); + } &Inst::FpuMove32 { rd, rn } | &Inst::FpuMove64 { rd, rn } => { collector.reg_def(rd); collector.reg_use(rn); @@ -881,7 +890,7 @@ fn s390x_get_operands VReg>(inst: &Inst, collector: &mut OperandC collector.reg_use(ridx); collector.reg_early_def(writable_gpr(1)); } - &Inst::LoadExtNameFar { rd, .. } => { + &Inst::LoadSymbolReloc { rd, .. } => { collector.reg_def(rd); collector.reg_def(writable_gpr(1)); } @@ -1887,6 +1896,10 @@ impl Inst { }; format!("{} {}, {}", op, rd, imm.bits) } + &Inst::LoadAR { rd, ar } | &Inst::InsertAR { rd, ar } => { + let rd = pretty_print_reg(rd.to_reg(), allocs); + format!("ear {}, %a{}", rd, ar) + } &Inst::CMov32 { rd, cond, rm } => { let rd = pretty_print_reg(rd.to_reg(), allocs); let rm = pretty_print_reg(rm, allocs); @@ -2830,7 +2843,12 @@ impl Inst { } &Inst::Call { link, ref info, .. } => { let link = pretty_print_reg(link.to_reg(), allocs); - format!("brasl {}, {}", link, info.dest) + let tls_symbol = match &info.tls_symbol { + None => "".to_string(), + Some(SymbolReloc::TlsGd { name }) => format!(":tls_gdcall:{}", name), + _ => unreachable!(), + }; + format!("brasl {}, {}{}", link, info.dest, tls_symbol) } &Inst::CallInd { link, ref info, .. } => { let link = pretty_print_reg(link.to_reg(), allocs); @@ -2891,17 +2909,17 @@ impl Inst { rtmp, rtmp, rtmp, ridx, rtmp, jt_entries, ) } - &Inst::LoadExtNameFar { + &Inst::LoadSymbolReloc { rd, - ref name, - offset, + ref symbol_reloc, } => { let rd = pretty_print_reg(rd.to_reg(), allocs); let tmp = pretty_print_reg(writable_spilltmp_reg().to_reg(), &mut empty_allocs); - format!( - "bras {}, 12 ; data {} + {} ; lg {}, 0({})", - tmp, name, offset, rd, tmp - ) + let symbol = match &**symbol_reloc { + SymbolReloc::Absolute { name, offset } => format!("{} + {}", name, offset), + SymbolReloc::TlsGd { name } => format!("{}@tlsgd", name), + }; + format!("bras {}, 12 ; data {} ; lg {}, 0({})", tmp, symbol, rd, tmp) } &Inst::LoadAddr { rd, ref mem } => { let rd = pretty_print_reg(rd.to_reg(), allocs); diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle index 85ed948fa5..495ea33a24 100644 --- a/cranelift/codegen/src/isa/s390x/lower.isle +++ b/cranelift/codegen/src/isa/s390x/lower.isle @@ -2388,7 +2388,7 @@ ;; Load the address of a function, general case. (rule (lower (func_addr (func_ref_data _ name _))) - (load_ext_name_far name 0)) + (load_symbol_reloc (SymbolReloc.Absolute name 0))) ;;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2401,7 +2401,34 @@ ;; Load the address of a symbol, general case. (rule (lower (symbol_value (symbol_value_data name _ offset))) - (load_ext_name_far name offset)) + (load_symbol_reloc (SymbolReloc.Absolute name offset))) + + +;;;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Load the address of a TLS symbol (ELF general-dynamic model). +(rule (lower (tls_value (symbol_value_data name _ 0))) + (if (tls_model_is_elf_gd)) + (let ((symbol SymbolReloc (SymbolReloc.TlsGd name)) + (got Reg (load_addr (memarg_got))) + (got_offset Reg (load_symbol_reloc symbol)) + (tls_offset Reg (lib_call_tls_get_offset got got_offset symbol))) + (add_reg $I64 tls_offset (thread_pointer)))) + +;; Helper to perform a call to the __tls_get_offset library routine. +(decl lib_call_tls_get_offset (Reg Reg SymbolReloc) Reg) +(rule (lib_call_tls_get_offset got got_offset symbol) + (let ((libcall LibCallInfo (lib_call_info_tls_get_offset symbol)) + (_ Unit (lib_accumulate_outgoing_args_size libcall)) + (_ Unit (emit_mov $I64 (writable_gpr 12) got)) + (_ Unit (emit_mov $I64 (writable_gpr 2) got_offset)) + (_ Unit (emit_side_effect (lib_call libcall)))) + (copy_reg $I64 (writable_gpr 2)))) + +;; Helper to extract the current thread pointer from %a0/%a1. +(decl thread_pointer () Reg) +(rule (thread_pointer) + (insert_ar (lshl_imm $I64 (load_ar 0) 32) 1)) ;;;; Rules for `load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -3811,7 +3838,7 @@ (let ((abi ABISig (abi_sig sig_ref)) (_ Unit (abi_accumulate_outgoing_args_size abi)) (_ InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args)) - (target Reg (load_ext_name_far name 0)) + (target Reg (load_symbol_reloc (SymbolReloc.Absolute name 0))) (_ InstOutput (side_effect (abi_call_ind abi target (Opcode.Call))))) (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new)))) diff --git a/cranelift/codegen/src/isa/s390x/lower.rs b/cranelift/codegen/src/isa/s390x/lower.rs index e59fbda137..c5567c5460 100644 --- a/cranelift/codegen/src/isa/s390x/lower.rs +++ b/cranelift/codegen/src/isa/s390x/lower.rs @@ -189,6 +189,7 @@ impl LowerBackend for S390xBackend { | Opcode::StackAddr | Opcode::FuncAddr | Opcode::SymbolValue + | Opcode::TlsValue | Opcode::GetFramePointer | Opcode::GetStackPointer | Opcode::GetReturnAddress => { @@ -200,7 +201,6 @@ impl LowerBackend for S390xBackend { } Opcode::ConstAddr - | Opcode::TlsValue | Opcode::GetPinnedReg | Opcode::SetPinnedReg | Opcode::Vsplit diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index d40510aaca..a2bfae76f9 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -7,7 +7,7 @@ pub mod generated_code; use crate::isa::s390x::abi::{S390xMachineDeps, REG_SAVE_AREA_SIZE}; use crate::isa::s390x::inst::{ gpr, stack_reg, writable_gpr, zero_reg, CallIndInfo, CallInfo, Cond, Inst as MInst, MemArg, - MemArgPair, UImm12, UImm16Shifted, UImm32Shifted, + MemArgPair, SymbolReloc, UImm12, UImm16Shifted, UImm32Shifted, }; use crate::isa::s390x::settings::Flags as IsaFlags; use crate::machinst::isle::*; @@ -16,7 +16,7 @@ use crate::settings::Flags; use crate::{ ir::{ condcodes::*, immediates::*, types::*, AtomicRmwOp, Endianness, Inst, InstructionData, - LibCall, MemFlags, Opcode, TrapCode, Value, ValueList, + KnownSymbol, LibCall, MemFlags, Opcode, TrapCode, Value, ValueList, }, isa::unwind::UnwindInst, isa::CallConv, @@ -34,12 +34,14 @@ use target_lexicon::Triple; /// Information describing a library call to be emitted. pub struct LibCallInfo { libcall: LibCall, + tls_symbol: Option, } type BoxCallInfo = Box; type BoxCallIndInfo = Box; type VecMachLabel = Vec; type BoxExternalName = Box; +type BoxSymbolReloc = Box; type VecMInst = Vec; type VecMInstBuilder = Cell>; @@ -117,6 +119,7 @@ where opcode: *opcode, caller_callconv: self.lower_ctx.abi().call_conv(), callee_callconv: abi.call_conv(), + tls_symbol: None, }) } @@ -136,6 +139,14 @@ where fn lib_call_info_memcpy(&mut self) -> LibCallInfo { LibCallInfo { libcall: LibCall::Memcpy, + tls_symbol: None, + } + } + + fn lib_call_info_tls_get_offset(&mut self, tls_symbol: &SymbolReloc) -> LibCallInfo { + LibCallInfo { + libcall: LibCall::ElfTlsGetOffset, + tls_symbol: Some(tls_symbol.clone()), } } @@ -156,6 +167,7 @@ where smallvec![gpr(2), gpr(3), gpr(4)], smallvec![writable_gpr(2)], ), + LibCall::ElfTlsGetOffset => (smallvec![gpr(2), gpr(12)], smallvec![writable_gpr(2)]), _ => unreachable!(), }; @@ -173,9 +185,15 @@ where opcode: Opcode::Call, caller_callconv, callee_callconv, + tls_symbol: info.tls_symbol.clone(), }) } + #[inline] + fn box_symbol_reloc(&mut self, symbol_reloc: &SymbolReloc) -> BoxSymbolReloc { + Box::new(symbol_reloc.clone()) + } + #[inline] fn allow_div_traps(&mut self, _: Type) -> Option<()> { if !self.flags.avoid_div_traps() { @@ -707,6 +725,15 @@ where } } + #[inline] + fn memarg_got(&mut self) -> MemArg { + MemArg::Symbol { + name: Box::new(ExternalName::KnownSymbol(KnownSymbol::ElfGlobalOffsetTable)), + offset: 0, + flags: MemFlags::trusted(), + } + } + #[inline] fn memarg_symbol_offset_sum(&mut self, off1: i64, off2: i64) -> Option { let off = i32::try_from(off1 + off2).ok()?; diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 6d6768bf19..d137a02061 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -15,6 +15,7 @@ pub use crate::isa::unwind::UnwindInst; pub use crate::machinst::{ ABIArg, ABIArgSlot, ABISig, InputSourceInst, RealReg, Reg, RelocDistance, Writable, }; +pub use crate::settings::TlsModel; pub type Unit = (); pub type ValueSlice = (ValueList, usize); @@ -633,6 +634,15 @@ macro_rules! isle_prelude_methods { } } + #[inline] + fn tls_model_is_elf_gd(&mut self) -> Option<()> { + if self.flags.tls_model() == TlsModel::ElfGd { + Some(()) + } else { + None + } + } + #[inline] fn func_ref_data(&mut self, func_ref: FuncRef) -> (SigRef, ExternalName, RelocDistance) { let funcdata = &self.lower_ctx.dfg().ext_funcs[func_ref]; diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 4b95834ee9..cdb47de449 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -766,6 +766,9 @@ (decl avoid_div_traps () Type) (extern extractor avoid_div_traps avoid_div_traps) +(decl pure tls_model_is_elf_gd () Unit) +(extern constructor tls_model_is_elf_gd tls_model_is_elf_gd) + ;;;; Helpers for accessing instruction data ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Accessor for `FuncRef`. diff --git a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif new file mode 100644 index 0000000000..fa943e746e --- /dev/null +++ b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif @@ -0,0 +1,26 @@ +test compile precise-output +set tls_model=elf_gd +target s390x + +function u0:0(i32) -> i64 { +gv0 = symbol colocated tls u1:0 + +block0(v0: i32): + v1 = global_value.i64 gv0 + return v1 +} + +; stmg %r12, %r15, 96(%r15) +; aghi %r15, -160 +; virtual_sp_offset_adjust 160 +; block0: +; larl %r12, %ElfGlobalOffsetTable + 0 +; bras %r1, 12 ; data u1:0@tlsgd ; lg %r2, 0(%r1) +; brasl %r14, %ElfTlsGetOffset:tls_gdcall:u1:0 +; ear %r3, %a0 +; sllg %r4, %r3, 32 +; ear %r4, %a1 +; agr %r2, %r4 +; lmg %r12, %r15, 256(%r15) +; br %r14 + diff --git a/cranelift/module/src/lib.rs b/cranelift/module/src/lib.rs index 5b307b60a9..36ed9abd54 100644 --- a/cranelift/module/src/lib.rs +++ b/cranelift/module/src/lib.rs @@ -78,5 +78,6 @@ pub fn default_libcall_names() -> Box String + Send + Syn ir::LibCall::Memcmp => "memcmp".to_owned(), ir::LibCall::ElfTlsGetAddr => "__tls_get_addr".to_owned(), + ir::LibCall::ElfTlsGetOffset => "__tls_get_offset".to_owned(), }) } diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index 803c60832c..8237e934fd 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -121,6 +121,7 @@ pub struct ObjectModule { relocs: Vec, libcalls: HashMap, libcall_names: Box String + Send + Sync>, + known_symbols: HashMap, function_alignment: u64, per_function_section: bool, anon_func_number: u64, @@ -141,6 +142,7 @@ impl ObjectModule { relocs: Vec::new(), libcalls: HashMap::new(), libcall_names: builder.libcall_names, + known_symbols: HashMap::new(), function_alignment: builder.function_alignment, per_function_section: builder.per_function_section, anon_func_number: 0, @@ -547,6 +549,28 @@ impl ObjectModule { symbol } } + // These are "magic" names well-known to the linker. + // They require special treatment. + ir::ExternalName::KnownSymbol(ref known_symbol) => { + if let Some(symbol) = self.known_symbols.get(known_symbol) { + *symbol + } else { + let symbol = match known_symbol { + ir::KnownSymbol::ElfGlobalOffsetTable => self.object.add_symbol(Symbol { + name: "_GLOBAL_OFFSET_TABLE_".as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Unknown, + weak: false, + section: SymbolSection::Undefined, + flags: SymbolFlags::None, + }), + }; + self.known_symbols.insert(*known_symbol, symbol); + symbol + } + } _ => panic!("invalid ExternalName {}", name), } }