s390x: Implement tls_value (#4616)

Implement the tls_value for s390 in the ELF general-dynamic mode.

Notable differences to the x86_64 implementation are:
- We use a __tls_get_offset libcall instead of __tls_get_addr.
- The current thread pointer (stored in a pair of access registers)
  needs to be added to the result of __tls_get_offset.
- __tls_get_offset has a variant ABI that requires the address of
  the GOT (global offset table) is passed in %r12.

This means we need a new libcall entries for __tls_get_offset.
In addition, we also need a way to access _GLOBAL_OFFSET_TABLE_.
The latter is a "magic" symbol with a well-known name defined
by the ABI and recognized by the linker.  This patch introduces
a new ExternalName::KnownSymbol variant to support such names
(originally due to @afonso360).

We also need to emit a relocation on a symbol placed in a
constant pool, as well as an extra relocation on the call
to __tls_get_offset required for TLS linker optimization.

Needed by the cg_clif frontend.

cranelift/codegen/src/binemit/mod.rs

@@ -59,6 +59,11 @@ pub enum Reloc {
     /// Set the add immediate field to the low 12 bits of the final address. Does not check for overflow.
     /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12_NC` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#relocations-for-thread-local-storage)
     Aarch64TlsGdAddLo12Nc,
+
+    /// s390x TLS GD64 - 64-bit offset of tls_index for GD symbol in GOT
+    S390xTlsGd64,
+    /// s390x TLS GDCall - marker to enable optimization of TLS calls
+    S390xTlsGdCall,
 }
 
 impl fmt::Display for Reloc {
@@ -79,6 +84,8 @@ impl fmt::Display for Reloc {
             Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
             Self::Aarch64TlsGdAdrPage21 => write!(f, "Aarch64TlsGdAdrPage21"),
             Self::Aarch64TlsGdAddLo12Nc => write!(f, "Aarch64TlsGdAddLo12Nc"),
+            Self::S390xTlsGd64 => write!(f, "TlsGd64"),
+            Self::S390xTlsGdCall => write!(f, "TlsGdCall"),
         }
     }
 }

cranelift/codegen/src/ir/extname.rs

@@ -4,7 +4,7 @@
 //! function. The name of an external declaration doesn't have any meaning to
 //! Cranelift, which compiles functions independently.
 
-use crate::ir::LibCall;
+use crate::ir::{KnownSymbol, LibCall};
 use core::cmp;
 use core::fmt::{self, Write};
 use core::str::FromStr;
@@ -46,6 +46,8 @@ pub enum ExternalName {
     },
     /// A well-known runtime library function.
     LibCall(LibCall),
+    /// A well-known symbol.
+    KnownSymbol(KnownSymbol),
 }
 
 impl ExternalName {
@@ -104,6 +106,7 @@ impl fmt::Display for ExternalName {
                 Ok(())
             }
             Self::LibCall(lc) => write!(f, "%{}", lc),
+            Self::KnownSymbol(ks) => write!(f, "%{}", ks),
         }
     }
 }
@@ -112,11 +115,18 @@ impl FromStr for ExternalName {
     type Err = ();
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        // Try to parse as a libcall name, otherwise it's a test case.
+        // Try to parse as a known symbol
-        match s.parse() {
+        if let Ok(ks) = s.parse() {
-            Ok(lc) => Ok(Self::LibCall(lc)),
+            return Ok(Self::KnownSymbol(ks));
-            Err(_) => Ok(Self::testcase(s.as_bytes())),
         }
+
+        // Try to parse as a libcall name
+        if let Ok(lc) = s.parse() {
+            return Ok(Self::LibCall(lc));
+        }
+
+        // Otherwise its a test case name
+        Ok(Self::testcase(s.as_bytes()))
     }
 }
 

cranelift/codegen/src/ir/known_symbol.rs

@@ -0,0 +1,42 @@
+use core::fmt;
+use core::str::FromStr;
+#[cfg(feature = "enable-serde")]
+use serde::{Deserialize, Serialize};
+
+/// A well-known symbol.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub enum KnownSymbol {
+    /// ELF well-known linker symbol _GLOBAL_OFFSET_TABLE_
+    ElfGlobalOffsetTable,
+}
+
+impl fmt::Display for KnownSymbol {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Debug::fmt(self, f)
+    }
+}
+
+impl FromStr for KnownSymbol {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "ElfGlobalOffsetTable" => Ok(Self::ElfGlobalOffsetTable),
+            _ => Err(()),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parsing() {
+        assert_eq!(
+            "ElfGlobalOffsetTable".parse(),
+            Ok(KnownSymbol::ElfGlobalOffsetTable)
+        );
+    }
+}

cranelift/codegen/src/ir/libcall.rs

@@ -66,6 +66,8 @@ pub enum LibCall {
 
     /// Elf __tls_get_addr
     ElfTlsGetAddr,
+    /// Elf __tls_get_offset
+    ElfTlsGetOffset,
     // When adding a new variant make sure to add it to `all_libcalls` too.
 }
 
@@ -104,6 +106,7 @@ impl FromStr for LibCall {
             "Memcmp" => Ok(Self::Memcmp),
 
             "ElfTlsGetAddr" => Ok(Self::ElfTlsGetAddr),
+            "ElfTlsGetOffset" => Ok(Self::ElfTlsGetOffset),
             _ => Err(()),
         }
     }
@@ -173,6 +176,7 @@ impl LibCall {
             Memmove,
             Memcmp,
             ElfTlsGetAddr,
+            ElfTlsGetOffset,
         ]
     }
 
@@ -214,7 +218,8 @@ impl LibCall {
             | LibCall::Memset
             | LibCall::Memmove
             | LibCall::Memcmp
-            | LibCall::ElfTlsGetAddr => unimplemented!(),
+            | LibCall::ElfTlsGetAddr
+            | LibCall::ElfTlsGetOffset => unimplemented!(),
         }
 
         sig

cranelift/codegen/src/ir/mod.rs

@@ -15,6 +15,7 @@ mod heap;
 pub mod immediates;
 pub mod instructions;
 pub mod jumptable;
+pub(crate) mod known_symbol;
 pub mod layout;
 pub(crate) mod libcall;
 mod memflags;
@@ -50,6 +51,7 @@ pub use crate::ir::instructions::{
     InstructionData, Opcode, ValueList, ValueListPool, VariableArgs,
 };
 pub use crate::ir::jumptable::JumpTableData;
+pub use crate::ir::known_symbol::KnownSymbol;
 pub use crate::ir::layout::Layout;
 pub use crate::ir::libcall::{get_probestack_funcref, LibCall};
 pub use crate::ir::memflags::{Endianness, MemFlags};

cranelift/codegen/src/isa/s390x/inst.isle

@@ -424,6 +424,17 @@
       (rd WritableReg)
       (imm UImm32Shifted))
 
+    ;; Load 32-bit access register into GPR.
+    (LoadAR
+      (rd WritableReg)
+      (ar u8))
+
+    ;; Insert 32-bit access register into low half of a GPR.
+    ;; (Identical operation to LoadAR, but considers rd to be use/def.)
+    (InsertAR
+      (rd WritableReg)
+      (ar u8))
+
     ;; A sign- or zero-extend operation.
     (Extend
       (rd WritableReg)
@@ -857,11 +868,10 @@
       (ridx Reg)
       (targets VecMachLabel))
 
-    ;; Load an inline symbol reference with RelocDistance::Far.
+    ;; Load an inline symbol reference with relocation.
-    (LoadExtNameFar
+    (LoadSymbolReloc
       (rd WritableReg)
-      (name BoxExternalName)
+      (symbol_reloc BoxSymbolReloc))
-      (offset i64))
 
     ;; Load address referenced by `mem` into `rd`.
     (LoadAddr
@@ -903,6 +913,23 @@
 (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
 (type VecMachLabel extern (enum))
 
+;; A symbol reference carrying relocation information.
+(type SymbolReloc
+  (enum
+    ;; Absolute symbol reference (with optional offset).
+    (Absolute
+      (name ExternalName)
+      (offset i64))
+    ;; Reference to a TLS symbol in general-dynamic mode.
+    (TlsGd
+      (name ExternalName))))
+
+;; Boxed version of SymbolReloc to save space.
+(type BoxSymbolReloc (primitive BoxSymbolReloc))
+(decl box_symbol_reloc (SymbolReloc) BoxSymbolReloc)
+(extern constructor box_symbol_reloc box_symbol_reloc)
+(convert SymbolReloc BoxSymbolReloc box_symbol_reloc)
+
 ;; An ALU operation.
 (type ALUOp
   (enum
@@ -1613,6 +1640,9 @@
 (decl memarg_symbol (ExternalName i32 MemFlags) MemArg)
 (extern constructor memarg_symbol memarg_symbol)
 
+(decl memarg_got () MemArg)
+(extern constructor memarg_got memarg_got)
+
 ;; Create a MemArg refering to a stack address formed by
 ;; adding a base (relative to SP) and an offset.
 (decl memarg_stack_off (i64 i64) MemArg)
@@ -2120,6 +2150,20 @@
 (rule (mvc dst src len_minus_one)
       (SideEffectNoResult.Inst (MInst.Mvc dst src len_minus_one)))
 
+;; Helper for emitting `MInst.LoadAR` instructions.
+(decl load_ar (u8) Reg)
+(rule (load_ar ar)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadAR dst ar))))
+        dst))
+
+;; Helper for emitting `MInst.InsertAR` instructions.
+(decl insert_ar (Reg u8) Reg)
+(rule (insert_ar src ar)
+      (let ((dst WritableReg (copy_writable_reg $I64 src))
+            (_ Unit (emit (MInst.InsertAR dst ar))))
+        dst))
+
 ;; Helper for emitting `MInst.FpuRR` instructions.
 (decl fpu_rr (Type FPUOp1 Reg) Reg)
 (rule (fpu_rr ty op src)
@@ -2393,12 +2437,11 @@
             (_ Unit (emit (MInst.VecReplicateLane size dst src lane_imm))))
         dst))
 
-;; Helper for emitting `MInst.LoadExtNameFar` instructions.
+;; Helper for emitting `MInst.LoadSymbolReloc` instructions.
-(decl load_ext_name_far (ExternalName i64) Reg)
+(decl load_symbol_reloc (SymbolReloc) Reg)
-(rule (load_ext_name_far name offset)
+(rule (load_symbol_reloc symbol_reloc)
       (let ((dst WritableReg (temp_writable_reg $I64))
-            (boxed_name BoxExternalName (box_external_name name))
+            (_ Unit (emit (MInst.LoadSymbolReloc dst symbol_reloc))))
-            (_ Unit (emit (MInst.LoadExtNameFar dst boxed_name offset))))
         dst))
 
 ;; Helper for emitting `MInst.LoadAddr` instructions.
@@ -3405,6 +3448,9 @@
 (decl lib_call_info_memcpy () LibCallInfo)
 (extern constructor lib_call_info_memcpy lib_call_info_memcpy)
 
+(decl lib_call_info_tls_get_offset (SymbolReloc) LibCallInfo)
+(extern constructor lib_call_info_tls_get_offset lib_call_info_tls_get_offset)
+
 (decl lib_call_info (LibCallInfo) BoxCallInfo)
 (extern constructor lib_call_info lib_call_info)
 

cranelift/codegen/src/isa/s390x/inst/emit.rs

@@ -2258,17 +2258,25 @@ impl MachInstEmit for Inst {
                 };
                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
             }
-            &Inst::LoadExtNameFar {
+            &Inst::LoadAR { rd, ar } | &Inst::InsertAR { rd, ar } => {
+                let rd = allocs.next_writable(rd);
+                let opcode = 0xb24f; // EAR
+                put(sink, &enc_rre(opcode, rd.to_reg(), gpr(ar)));
+            }
+            &Inst::LoadSymbolReloc {
                 rd,
-                ref name,
+                ref symbol_reloc,
-                offset,
             } => {
                 let rd = allocs.next_writable(rd);
 
                 let opcode = 0xa75; // BRAS
                 let reg = writable_spilltmp_reg().to_reg();
                 put(sink, &enc_ri_b(opcode, reg, 12));
-                sink.add_reloc(Reloc::Abs8, name, offset);
+                let (reloc, name, offset) = match &**symbol_reloc {
+                    SymbolReloc::Absolute { name, offset } => (Reloc::Abs8, name, *offset),
+                    SymbolReloc::TlsGd { name } => (Reloc::S390xTlsGd64, name, 0),
+                };
+                sink.add_reloc(reloc, name, offset);
                 sink.put8(0);
                 let inst = Inst::Load64 {
                     rd,
@@ -3198,6 +3206,15 @@ impl MachInstEmit for Inst {
             &Inst::Call { link, ref info } => {
                 let link = allocs.next_writable(link);
 
+                // Add relocation for TLS libcalls to enable linker optimizations.
+                match &info.tls_symbol {
+                    None => {}
+                    Some(SymbolReloc::TlsGd { name }) => {
+                        sink.add_reloc(Reloc::S390xTlsGdCall, name, 0)
+                    }
+                    _ => unreachable!(),
+                }
+
                 let opcode = 0xc05; // BRASL
                 let reloc = Reloc::S390xPCRel32Dbl;
                 if let Some(s) = state.take_stack_map() {

cranelift/codegen/src/isa/s390x/inst/emit_tests.rs

@@ -6828,6 +6828,7 @@ fn test_s390x_binemit() {
                 opcode: Opcode::Call,
                 caller_callconv: CallConv::SystemV,
                 callee_callconv: CallConv::SystemV,
+                tls_symbol: None,
             }),
         },
         "C0E500000000",

cranelift/codegen/src/isa/s390x/inst/mod.rs

@@ -29,7 +29,7 @@ mod emit_tests;
 
 pub use crate::isa::s390x::lower::isle::generated_code::{
     ALUOp, CmpOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuRoundOp, MInst as Inst, RxSBGOp,
-    ShiftOp, UnaryOp, VecBinaryOp, VecFloatCmpOp, VecIntCmpOp, VecShiftOp, VecUnaryOp,
+    ShiftOp, SymbolReloc, UnaryOp, VecBinaryOp, VecFloatCmpOp, VecIntCmpOp, VecShiftOp, VecUnaryOp,
 };
 
 /// Additional information for (direct) Call instructions, left out of line to lower the size of
@@ -43,6 +43,7 @@ pub struct CallInfo {
     pub opcode: Opcode,
     pub caller_callconv: CallConv,
     pub callee_callconv: CallConv,
+    pub tls_symbol: Option<SymbolReloc>,
 }
 
 /// Additional information for CallInd instructions, left out of line to lower the size of the Inst
@@ -154,6 +155,8 @@ impl Inst {
             | Inst::Mov64UImm32Shifted { .. }
             | Inst::Insert64UImm16Shifted { .. }
             | Inst::Insert64UImm32Shifted { .. }
+            | Inst::LoadAR { .. }
+            | Inst::InsertAR { .. }
             | Inst::Extend { .. }
             | Inst::CMov32 { .. }
             | Inst::CMov64 { .. }
@@ -212,7 +215,7 @@ impl Inst {
             | Inst::Debugtrap
             | Inst::Trap { .. }
             | Inst::JTSequence { .. }
-            | Inst::LoadExtNameFar { .. }
+            | Inst::LoadSymbolReloc { .. }
             | Inst::LoadAddr { .. }
             | Inst::Loop { .. }
             | Inst::CondBreak { .. }
@@ -662,6 +665,12 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
         &Inst::Insert64UImm16Shifted { rd, .. } | &Inst::Insert64UImm32Shifted { rd, .. } => {
             collector.reg_mod(rd);
         }
+        &Inst::LoadAR { rd, .. } => {
+            collector.reg_def(rd);
+        }
+        &Inst::InsertAR { rd, .. } => {
+            collector.reg_mod(rd);
+        }
         &Inst::FpuMove32 { rd, rn } | &Inst::FpuMove64 { rd, rn } => {
             collector.reg_def(rd);
             collector.reg_use(rn);
@@ -881,7 +890,7 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
             collector.reg_use(ridx);
             collector.reg_early_def(writable_gpr(1));
         }
-        &Inst::LoadExtNameFar { rd, .. } => {
+        &Inst::LoadSymbolReloc { rd, .. } => {
             collector.reg_def(rd);
             collector.reg_def(writable_gpr(1));
         }
@@ -1887,6 +1896,10 @@ impl Inst {
                 };
                 format!("{} {}, {}", op, rd, imm.bits)
             }
+            &Inst::LoadAR { rd, ar } | &Inst::InsertAR { rd, ar } => {
+                let rd = pretty_print_reg(rd.to_reg(), allocs);
+                format!("ear {}, %a{}", rd, ar)
+            }
             &Inst::CMov32 { rd, cond, rm } => {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);
                 let rm = pretty_print_reg(rm, allocs);
@@ -2830,7 +2843,12 @@ impl Inst {
             }
             &Inst::Call { link, ref info, .. } => {
                 let link = pretty_print_reg(link.to_reg(), allocs);
-                format!("brasl {}, {}", link, info.dest)
+                let tls_symbol = match &info.tls_symbol {
+                    None => "".to_string(),
+                    Some(SymbolReloc::TlsGd { name }) => format!(":tls_gdcall:{}", name),
+                    _ => unreachable!(),
+                };
+                format!("brasl {}, {}{}", link, info.dest, tls_symbol)
             }
             &Inst::CallInd { link, ref info, .. } => {
                 let link = pretty_print_reg(link.to_reg(), allocs);
@@ -2891,17 +2909,17 @@ impl Inst {
                     rtmp, rtmp, rtmp, ridx, rtmp, jt_entries,
                 )
             }
-            &Inst::LoadExtNameFar {
+            &Inst::LoadSymbolReloc {
                 rd,
-                ref name,
+                ref symbol_reloc,
-                offset,
             } => {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);
                 let tmp = pretty_print_reg(writable_spilltmp_reg().to_reg(), &mut empty_allocs);
-                format!(
+                let symbol = match &**symbol_reloc {
-                    "bras {}, 12 ; data {} + {} ; lg {}, 0({})",
+                    SymbolReloc::Absolute { name, offset } => format!("{} + {}", name, offset),
-                    tmp, name, offset, rd, tmp
+                    SymbolReloc::TlsGd { name } => format!("{}@tlsgd", name),
-                )
+                };
+                format!("bras {}, 12 ; data {} ; lg {}, 0({})", tmp, symbol, rd, tmp)
             }
             &Inst::LoadAddr { rd, ref mem } => {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);

cranelift/codegen/src/isa/s390x/lower.isle

@@ -2388,7 +2388,7 @@
 
 ;; Load the address of a function, general case.
 (rule (lower (func_addr (func_ref_data _ name _)))
-      (load_ext_name_far name 0))
+      (load_symbol_reloc (SymbolReloc.Absolute name 0)))
 
 
 ;;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2401,7 +2401,34 @@
 
 ;; Load the address of a symbol, general case.
 (rule (lower (symbol_value (symbol_value_data name _ offset)))
-      (load_ext_name_far name offset))
+      (load_symbol_reloc (SymbolReloc.Absolute name offset)))
+
+
+;;;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Load the address of a TLS symbol (ELF general-dynamic model).
+(rule (lower (tls_value (symbol_value_data name _ 0)))
+      (if (tls_model_is_elf_gd))
+      (let ((symbol SymbolReloc (SymbolReloc.TlsGd name))
+            (got Reg (load_addr (memarg_got)))
+            (got_offset Reg (load_symbol_reloc symbol))
+            (tls_offset Reg (lib_call_tls_get_offset got got_offset symbol)))
+        (add_reg $I64 tls_offset (thread_pointer))))
+
+;; Helper to perform a call to the __tls_get_offset library routine.
+(decl lib_call_tls_get_offset (Reg Reg SymbolReloc) Reg)
+(rule (lib_call_tls_get_offset got got_offset symbol)
+      (let ((libcall LibCallInfo (lib_call_info_tls_get_offset symbol))
+            (_ Unit (lib_accumulate_outgoing_args_size libcall))
+            (_ Unit (emit_mov $I64 (writable_gpr 12) got))
+            (_ Unit (emit_mov $I64 (writable_gpr 2) got_offset))
+            (_ Unit (emit_side_effect (lib_call libcall))))
+        (copy_reg $I64 (writable_gpr 2))))
+
+;; Helper to extract the current thread pointer from %a0/%a1.
+(decl thread_pointer () Reg)
+(rule (thread_pointer)
+      (insert_ar (lshl_imm $I64 (load_ar 0) 32) 1))
 
 
 ;;;; Rules for `load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3811,7 +3838,7 @@
       (let ((abi ABISig (abi_sig sig_ref))
             (_ Unit (abi_accumulate_outgoing_args_size abi))
             (_ InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args))
-            (target Reg (load_ext_name_far name 0))
+            (target Reg (load_symbol_reloc (SymbolReloc.Absolute name 0)))
             (_ InstOutput (side_effect (abi_call_ind abi target (Opcode.Call)))))
         (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new))))
 

cranelift/codegen/src/isa/s390x/lower.rs

@@ -189,6 +189,7 @@ impl LowerBackend for S390xBackend {
             | Opcode::StackAddr
             | Opcode::FuncAddr
             | Opcode::SymbolValue
+            | Opcode::TlsValue
             | Opcode::GetFramePointer
             | Opcode::GetStackPointer
             | Opcode::GetReturnAddress => {
@@ -200,7 +201,6 @@ impl LowerBackend for S390xBackend {
             }
 
             Opcode::ConstAddr
-            | Opcode::TlsValue
             | Opcode::GetPinnedReg
             | Opcode::SetPinnedReg
             | Opcode::Vsplit

cranelift/codegen/src/isa/s390x/lower/isle.rs

@@ -7,7 +7,7 @@ pub mod generated_code;
 use crate::isa::s390x::abi::{S390xMachineDeps, REG_SAVE_AREA_SIZE};
 use crate::isa::s390x::inst::{
     gpr, stack_reg, writable_gpr, zero_reg, CallIndInfo, CallInfo, Cond, Inst as MInst, MemArg,
-    MemArgPair, UImm12, UImm16Shifted, UImm32Shifted,
+    MemArgPair, SymbolReloc, UImm12, UImm16Shifted, UImm32Shifted,
 };
 use crate::isa::s390x::settings::Flags as IsaFlags;
 use crate::machinst::isle::*;
@@ -16,7 +16,7 @@ use crate::settings::Flags;
 use crate::{
     ir::{
         condcodes::*, immediates::*, types::*, AtomicRmwOp, Endianness, Inst, InstructionData,
-        LibCall, MemFlags, Opcode, TrapCode, Value, ValueList,
+        KnownSymbol, LibCall, MemFlags, Opcode, TrapCode, Value, ValueList,
     },
     isa::unwind::UnwindInst,
     isa::CallConv,
@@ -34,12 +34,14 @@ use target_lexicon::Triple;
 /// Information describing a library call to be emitted.
 pub struct LibCallInfo {
     libcall: LibCall,
+    tls_symbol: Option<SymbolReloc>,
 }
 
 type BoxCallInfo = Box<CallInfo>;
 type BoxCallIndInfo = Box<CallIndInfo>;
 type VecMachLabel = Vec<MachLabel>;
 type BoxExternalName = Box<ExternalName>;
+type BoxSymbolReloc = Box<SymbolReloc>;
 type VecMInst = Vec<MInst>;
 type VecMInstBuilder = Cell<Vec<MInst>>;
 
@@ -117,6 +119,7 @@ where
             opcode: *opcode,
             caller_callconv: self.lower_ctx.abi().call_conv(),
             callee_callconv: abi.call_conv(),
+            tls_symbol: None,
         })
     }
 
@@ -136,6 +139,14 @@ where
     fn lib_call_info_memcpy(&mut self) -> LibCallInfo {
         LibCallInfo {
             libcall: LibCall::Memcpy,
+            tls_symbol: None,
+        }
+    }
+
+    fn lib_call_info_tls_get_offset(&mut self, tls_symbol: &SymbolReloc) -> LibCallInfo {
+        LibCallInfo {
+            libcall: LibCall::ElfTlsGetOffset,
+            tls_symbol: Some(tls_symbol.clone()),
         }
     }
 
@@ -156,6 +167,7 @@ where
                 smallvec![gpr(2), gpr(3), gpr(4)],
                 smallvec![writable_gpr(2)],
             ),
+            LibCall::ElfTlsGetOffset => (smallvec![gpr(2), gpr(12)], smallvec![writable_gpr(2)]),
             _ => unreachable!(),
         };
 
@@ -173,9 +185,15 @@ where
             opcode: Opcode::Call,
             caller_callconv,
             callee_callconv,
+            tls_symbol: info.tls_symbol.clone(),
         })
     }
 
+    #[inline]
+    fn box_symbol_reloc(&mut self, symbol_reloc: &SymbolReloc) -> BoxSymbolReloc {
+        Box::new(symbol_reloc.clone())
+    }
+
     #[inline]
     fn allow_div_traps(&mut self, _: Type) -> Option<()> {
         if !self.flags.avoid_div_traps() {
@@ -707,6 +725,15 @@ where
         }
     }
 
+    #[inline]
+    fn memarg_got(&mut self) -> MemArg {
+        MemArg::Symbol {
+            name: Box::new(ExternalName::KnownSymbol(KnownSymbol::ElfGlobalOffsetTable)),
+            offset: 0,
+            flags: MemFlags::trusted(),
+        }
+    }
+
     #[inline]
     fn memarg_symbol_offset_sum(&mut self, off1: i64, off2: i64) -> Option<i32> {
         let off = i32::try_from(off1 + off2).ok()?;

cranelift/codegen/src/machinst/isle.rs

@@ -15,6 +15,7 @@ pub use crate::isa::unwind::UnwindInst;
 pub use crate::machinst::{
     ABIArg, ABIArgSlot, ABISig, InputSourceInst, RealReg, Reg, RelocDistance, Writable,
 };
+pub use crate::settings::TlsModel;
 
 pub type Unit = ();
 pub type ValueSlice = (ValueList, usize);
@@ -633,6 +634,15 @@ macro_rules! isle_prelude_methods {
             }
         }
 
+        #[inline]
+        fn tls_model_is_elf_gd(&mut self) -> Option<()> {
+            if self.flags.tls_model() == TlsModel::ElfGd {
+                Some(())
+            } else {
+                None
+            }
+        }
+
         #[inline]
         fn func_ref_data(&mut self, func_ref: FuncRef) -> (SigRef, ExternalName, RelocDistance) {
             let funcdata = &self.lower_ctx.dfg().ext_funcs[func_ref];

cranelift/codegen/src/prelude.isle

@@ -766,6 +766,9 @@
 (decl avoid_div_traps () Type)
 (extern extractor avoid_div_traps avoid_div_traps)
 
+(decl pure tls_model_is_elf_gd () Unit)
+(extern constructor tls_model_is_elf_gd tls_model_is_elf_gd)
+
 ;;;; Helpers for accessing instruction data ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Accessor for `FuncRef`.

cranelift/filetests/filetests/isa/s390x/tls_elf.clif

@@ -0,0 +1,26 @@
+test compile precise-output
+set tls_model=elf_gd
+target s390x
+
+function u0:0(i32) -> i64 {
+gv0 = symbol colocated tls u1:0
+
+block0(v0: i32):
+    v1 = global_value.i64 gv0
+    return v1
+}
+
+;   stmg %r12, %r15, 96(%r15)
+;   aghi %r15, -160
+;   virtual_sp_offset_adjust 160
+; block0:
+;   larl %r12, %ElfGlobalOffsetTable + 0
+;   bras %r1, 12 ; data u1:0@tlsgd ; lg %r2, 0(%r1)
+;   brasl %r14, %ElfTlsGetOffset:tls_gdcall:u1:0
+;   ear %r3, %a0
+;   sllg %r4, %r3, 32
+;   ear %r4, %a1
+;   agr %r2, %r4
+;   lmg %r12, %r15, 256(%r15)
+;   br %r14
+

cranelift/module/src/lib.rs

@@ -78,5 +78,6 @@ pub fn default_libcall_names() -> Box<dyn Fn(ir::LibCall) -> String + Send + Syn
         ir::LibCall::Memcmp => "memcmp".to_owned(),
 
         ir::LibCall::ElfTlsGetAddr => "__tls_get_addr".to_owned(),
+        ir::LibCall::ElfTlsGetOffset => "__tls_get_offset".to_owned(),
     })
 }

cranelift/object/src/backend.rs

@@ -121,6 +121,7 @@ pub struct ObjectModule {
     relocs: Vec<SymbolRelocs>,
     libcalls: HashMap<ir::LibCall, SymbolId>,
     libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
+    known_symbols: HashMap<ir::KnownSymbol, SymbolId>,
     function_alignment: u64,
     per_function_section: bool,
     anon_func_number: u64,
@@ -141,6 +142,7 @@ impl ObjectModule {
             relocs: Vec::new(),
             libcalls: HashMap::new(),
             libcall_names: builder.libcall_names,
+            known_symbols: HashMap::new(),
             function_alignment: builder.function_alignment,
             per_function_section: builder.per_function_section,
             anon_func_number: 0,
@@ -547,6 +549,28 @@ impl ObjectModule {
                     symbol
                 }
             }
+            // These are "magic" names well-known to the linker.
+            // They require special treatment.
+            ir::ExternalName::KnownSymbol(ref known_symbol) => {
+                if let Some(symbol) = self.known_symbols.get(known_symbol) {
+                    *symbol
+                } else {
+                    let symbol = match known_symbol {
+                        ir::KnownSymbol::ElfGlobalOffsetTable => self.object.add_symbol(Symbol {
+                            name: "_GLOBAL_OFFSET_TABLE_".as_bytes().to_vec(),
+                            value: 0,
+                            size: 0,
+                            kind: SymbolKind::Data,
+                            scope: SymbolScope::Unknown,
+                            weak: false,
+                            section: SymbolSection::Undefined,
+                            flags: SymbolFlags::None,
+                        }),
+                    };
+                    self.known_symbols.insert(*known_symbol, symbol);
+                    symbol
+                }
+            }
             _ => panic!("invalid ExternalName {}", name),
         }
     }