diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index 96a12b4e23..33bc8f6414 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -66,6 +66,17 @@ pub enum Reloc { /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12_NC` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#relocations-for-thread-local-storage) Aarch64TlsGdAddLo12Nc, + /// AArch64 GOT Page + /// Set the immediate value of an ADRP to bits 32:12 of X; check that –232 <= X < 232 + /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations) + Aarch64AdrGotPage21, + + /// AArch64 GOT Low bits + + /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0 + /// This is equivalent to `R_AARCH64_LD64_GOT_LO12_NC` (312) in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations) + Aarch64Ld64GotLo12Nc, + /// procedure call. /// call symbol /// expands to the following assembly and relocation: @@ -100,6 +111,8 @@ impl fmt::Display for Reloc { Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"), Self::Aarch64TlsGdAdrPage21 => write!(f, "Aarch64TlsGdAdrPage21"), Self::Aarch64TlsGdAddLo12Nc => write!(f, "Aarch64TlsGdAddLo12Nc"), + Self::Aarch64AdrGotPage21 => write!(f, "Aarch64AdrGotPage21"), + Self::Aarch64Ld64GotLo12Nc => write!(f, "Aarch64AdrGotLo12Nc"), Self::S390xTlsGd64 => write!(f, "TlsGd64"), Self::S390xTlsGdCall => write!(f, "TlsGdCall"), } diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 6e088c4ce4..b8bf2ef480 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -852,6 +852,11 @@ (rd WritableReg) ;; Offset in range -2^20 .. 2^20. (off i32)) + + ;; Compute the address (using a PC-relative offset) of a 4KB page. + (Adrp + (rd WritableReg) + (off i32)) ;; Raw 32-bit word, used for inline constants and jump-table entries. (Word4 diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 94ba0d5584..90a5dbd936 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -334,11 +334,21 @@ pub(crate) fn enc_br(rn: Reg) -> u32 { 0b1101011_0000_11111_000000_00000_00000 | (machreg_to_gpr(rn) << 5) } -pub(crate) fn enc_adr(off: i32, rd: Writable) -> u32 { +pub(crate) fn enc_adr_inst(opcode: u32, off: i32, rd: Writable) -> u32 { let off = u32::try_from(off).unwrap(); let immlo = off & 3; let immhi = (off >> 2) & ((1 << 19) - 1); - (0b00010000 << 24) | (immlo << 29) | (immhi << 5) | machreg_to_gpr(rd.to_reg()) + opcode | (immlo << 29) | (immhi << 5) | machreg_to_gpr(rd.to_reg()) +} + +pub(crate) fn enc_adr(off: i32, rd: Writable) -> u32 { + let opcode = 0b00010000 << 24; + enc_adr_inst(opcode, off, rd) +} + +pub(crate) fn enc_adrp(off: i32, rd: Writable) -> u32 { + let opcode = 0b10010000 << 24; + enc_adr_inst(opcode, off, rd) } fn enc_csel(rd: Writable, rn: Reg, rm: Reg, cond: Cond, op: u32, o2: u32) -> u32 { @@ -3143,6 +3153,12 @@ impl MachInstEmit for Inst { assert!(off < (1 << 20)); sink.put4(enc_adr(off, rd)); } + &Inst::Adrp { rd, off } => { + let rd = allocs.next_writable(rd); + assert!(off > -(1 << 20)); + assert!(off < (1 << 20)); + sink.put4(enc_adrp(off, rd)); + } &Inst::Word4 { data } => { sink.put4(data); } @@ -3250,20 +3266,52 @@ impl MachInstEmit for Inst { offset, } => { let rd = allocs.next_writable(rd); - let inst = Inst::ULoad64 { - rd, - mem: AMode::Label { - label: MemLabel::PCRel(8), - }, - flags: MemFlags::trusted(), - }; - inst.emit(&[], sink, emit_info, state); - let inst = Inst::Jump { - dest: BranchTarget::ResolvedOffset(12), - }; - inst.emit(&[], sink, emit_info, state); - sink.add_reloc(Reloc::Abs8, name, offset); - sink.put8(0); + + if emit_info.0.is_pic() { + // See this CE Example for the variations of this with and without BTI & PAUTH + // https://godbolt.org/z/ncqjbbvvn + // + // Emit the following code: + // adrp rd, :got:X + // ldr rd, [rd, :got_lo12:X] + + // adrp rd, symbol + sink.add_reloc(Reloc::Aarch64AdrGotPage21, name, 0); + let inst = Inst::Adrp { rd, off: 0 }; + inst.emit(&[], sink, emit_info, state); + + // ldr rd, [rd, :got_lo12:X] + sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, name, 0); + let inst = Inst::ULoad64 { + rd, + mem: AMode::reg(rd.to_reg()), + flags: MemFlags::trusted(), + }; + inst.emit(&[], sink, emit_info, state); + } else { + // With absolute offsets we set up a load from a preallocated space, and then jump + // over it. + // + // Emit the following code: + // ldr rd, #8 + // b #0x10 + // <8 byte space> + + let inst = Inst::ULoad64 { + rd, + mem: AMode::Label { + label: MemLabel::PCRel(8), + }, + flags: MemFlags::trusted(), + }; + inst.emit(&[], sink, emit_info, state); + let inst = Inst::Jump { + dest: BranchTarget::ResolvedOffset(12), + }; + inst.emit(&[], sink, emit_info, state); + sink.add_reloc(Reloc::Abs8, name, offset); + sink.put8(0); + } } &Inst::LoadAddr { rd, ref mem } => { let rd = allocs.next_writable(rd); @@ -3395,7 +3443,8 @@ impl MachInstEmit for Inst { // adrp x0,