aarch64: Support GOT Relative relocations in PIC mode

cranelift/codegen/src/binemit/mod.rs

@@ -66,6 +66,17 @@ pub enum Reloc {
     /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12_NC` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#relocations-for-thread-local-storage)
     Aarch64TlsGdAddLo12Nc,
 
+    /// AArch64 GOT Page
+    /// Set the immediate value of an ADRP to bits 32:12 of X; check that –232 <= X < 232
+    /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
+    Aarch64AdrGotPage21,
+
+    /// AArch64 GOT Low bits
+
+    /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0
+    /// This is equivalent to `R_AARCH64_LD64_GOT_LO12_NC` (312) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
+    Aarch64Ld64GotLo12Nc,
+
     /// procedure call.
     /// call symbol
     /// expands to the following assembly and relocation:
@@ -100,6 +111,8 @@ impl fmt::Display for Reloc {
             Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
             Self::Aarch64TlsGdAdrPage21 => write!(f, "Aarch64TlsGdAdrPage21"),
             Self::Aarch64TlsGdAddLo12Nc => write!(f, "Aarch64TlsGdAddLo12Nc"),
+            Self::Aarch64AdrGotPage21 => write!(f, "Aarch64AdrGotPage21"),
+            Self::Aarch64Ld64GotLo12Nc => write!(f, "Aarch64AdrGotLo12Nc"),
             Self::S390xTlsGd64 => write!(f, "TlsGd64"),
             Self::S390xTlsGdCall => write!(f, "TlsGdCall"),
         }

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -852,6 +852,11 @@
         (rd WritableReg)
         ;; Offset in range -2^20 .. 2^20.
         (off i32))
+
+       ;; Compute the address (using a PC-relative offset) of a 4KB page.
+       (Adrp
+        (rd WritableReg)
+        (off i32))
 
        ;; Raw 32-bit word, used for inline constants and jump-table entries.
        (Word4

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -334,11 +334,21 @@ pub(crate) fn enc_br(rn: Reg) -> u32 {
     0b1101011_0000_11111_000000_00000_00000 | (machreg_to_gpr(rn) << 5)
 }
 
-pub(crate) fn enc_adr(off: i32, rd: Writable<Reg>) -> u32 {
+pub(crate) fn enc_adr_inst(opcode: u32, off: i32, rd: Writable<Reg>) -> u32 {
     let off = u32::try_from(off).unwrap();
     let immlo = off & 3;
     let immhi = (off >> 2) & ((1 << 19) - 1);
-    (0b00010000 << 24) | (immlo << 29) | (immhi << 5) | machreg_to_gpr(rd.to_reg())
+    opcode | (immlo << 29) | (immhi << 5) | machreg_to_gpr(rd.to_reg())
+}
+
+pub(crate) fn enc_adr(off: i32, rd: Writable<Reg>) -> u32 {
+    let opcode = 0b00010000 << 24;
+    enc_adr_inst(opcode, off, rd)
+}
+
+pub(crate) fn enc_adrp(off: i32, rd: Writable<Reg>) -> u32 {
+    let opcode = 0b10010000 << 24;
+    enc_adr_inst(opcode, off, rd)
 }
 
 fn enc_csel(rd: Writable<Reg>, rn: Reg, rm: Reg, cond: Cond, op: u32, o2: u32) -> u32 {
@@ -3143,6 +3153,12 @@ impl MachInstEmit for Inst {
                 assert!(off < (1 << 20));
                 sink.put4(enc_adr(off, rd));
             }
+            &Inst::Adrp { rd, off } => {
+                let rd = allocs.next_writable(rd);
+                assert!(off > -(1 << 20));
+                assert!(off < (1 << 20));
+                sink.put4(enc_adrp(off, rd));
+            }
             &Inst::Word4 { data } => {
                 sink.put4(data);
             }
@@ -3250,20 +3266,52 @@ impl MachInstEmit for Inst {
                 offset,
             } => {
                 let rd = allocs.next_writable(rd);
-                let inst = Inst::ULoad64 {
-                    rd,
-                    mem: AMode::Label {
-                        label: MemLabel::PCRel(8),
-                    },
-                    flags: MemFlags::trusted(),
-                };
-                inst.emit(&[], sink, emit_info, state);
-                let inst = Inst::Jump {
-                    dest: BranchTarget::ResolvedOffset(12),
-                };
-                inst.emit(&[], sink, emit_info, state);
-                sink.add_reloc(Reloc::Abs8, name, offset);
-                sink.put8(0);
+
+                if emit_info.0.is_pic() {
+                    // See this CE Example for the variations of this with and without BTI & PAUTH
+                    // https://godbolt.org/z/ncqjbbvvn
+                    //
+                    // Emit the following code:
+                    //   adrp    rd, :got:X
+                    //   ldr     rd, [rd, :got_lo12:X]
+
+                    // adrp rd, symbol
+                    sink.add_reloc(Reloc::Aarch64AdrGotPage21, name, 0);
+                    let inst = Inst::Adrp { rd, off: 0 };
+                    inst.emit(&[], sink, emit_info, state);
+
+                    // ldr rd, [rd, :got_lo12:X]
+                    sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, name, 0);
+                    let inst = Inst::ULoad64 {
+                        rd,
+                        mem: AMode::reg(rd.to_reg()),
+                        flags: MemFlags::trusted(),
+                    };
+                    inst.emit(&[], sink, emit_info, state);
+                } else {
+                    // With absolute offsets we set up a load from a preallocated space, and then jump
+                    // over it.
+                    //
+                    // Emit the following code:
+                    //   ldr     rd, #8
+                    //   b       #0x10
+                    //   <8 byte space>
+
+                    let inst = Inst::ULoad64 {
+                        rd,
+                        mem: AMode::Label {
+                            label: MemLabel::PCRel(8),
+                        },
+                        flags: MemFlags::trusted(),
+                    };
+                    inst.emit(&[], sink, emit_info, state);
+                    let inst = Inst::Jump {
+                        dest: BranchTarget::ResolvedOffset(12),
+                    };
+                    inst.emit(&[], sink, emit_info, state);
+                    sink.add_reloc(Reloc::Abs8, name, offset);
+                    sink.put8(0);
+                }
             }
             &Inst::LoadAddr { rd, ref mem } => {
                 let rd = allocs.next_writable(rd);
@@ -3395,7 +3443,8 @@ impl MachInstEmit for Inst {
 
                 // adrp x0, <label>
                 sink.add_reloc(Reloc::Aarch64TlsGdAdrPage21, symbol, 0);
-                sink.put4(0x90000000);
+                let inst = Inst::Adrp { rd, off: 0 };
+                inst.emit(&[], sink, emit_info, state);
 
                 // add x0, x0, <label>
                 sink.add_reloc(Reloc::Aarch64TlsGdAddLo12Nc, symbol, 0);

cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs

@@ -6073,6 +6073,24 @@ fn test_aarch64_binemit() {
         "adr x15, pc+1048572",
     ));
 
+    insns.push((
+        Inst::Adrp {
+            rd: writable_xreg(8),
+            off: 0,
+        },
+        "08000090",
+        "adrp x8, pc+0",
+    ));
+
+    insns.push((
+        Inst::Adrp {
+            rd: writable_xreg(3),
+            off: 16,
+        },
+        "83000090",
+        "adrp x3, pc+65536",
+    ));
+
     insns.push((
         Inst::FpuMove64 {
             rd: writable_vreg(8),

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -1080,7 +1080,7 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
             }
             CondBrKind::Cond(_) => {}
         },
-        &Inst::Adr { rd, .. } => {
+        &Inst::Adr { rd, .. } | &Inst::Adrp { rd, .. } => {
             collector.reg_def(rd);
         }
         &Inst::Word4 { .. } | &Inst::Word8 { .. } => {}
@@ -2745,6 +2745,12 @@ impl Inst {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);
                 format!("adr {}, pc+{}", rd, off)
             }
+            &Inst::Adrp { rd, off } => {
+                let rd = pretty_print_reg(rd.to_reg(), allocs);
+                // This instruction addresses 4KiB pages, so multiply it by the page size.
+                let byte_offset = off * 4096;
+                format!("adrp {}, pc+{}", rd, byte_offset)
+            }
             &Inst::Word4 { data } => format!("data.i32 {}", data),
             &Inst::Word8 { data } => format!("data.i64 {}", data),
             &Inst::JTSequence {
@@ -2789,7 +2795,7 @@ impl Inst {
                 offset,
             } => {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);
-                format!("ldr {}, 8 ; b 12 ; data {:?} + {}", rd, name, offset)
+                format!("load_ext_name {rd}, {name:?}+{offset}")
             }
             &Inst::LoadAddr { rd, ref mem } => {
                 // TODO: we really should find a better way to avoid duplication of

cranelift/filetests/filetests/isa/aarch64/bti.clif

@@ -102,7 +102,7 @@ block0(v0: i64):
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
 ; block0:
-;   ldr x3, 8 ; b 12 ; data TestCase(%g) + 0
+;   load_ext_name x3, TestCase(%g)+0
 ;   blr x3
 ;   ldp fp, lr, [sp], #16
 ;   ret

cranelift/filetests/filetests/isa/aarch64/call-pauth.clif

@@ -14,7 +14,7 @@ block0(v0: i64):
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
 ; block0:
-;   ldr x3, 8 ; b 12 ; data TestCase(%g) + 0
+;   load_ext_name x3, TestCase(%g)+0
 ;   blr x3
 ;   ldp fp, lr, [sp], #16
 ;   autiasp ; ret