1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
15 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
16 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
17 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
18 #include "llvm/Object/ELFObjectFile.h"
19 #include "llvm/Support/Endian.h"
20
21 #include "DefineExternalSectionStartAndEndSymbols.h"
22 #include "EHFrameSupportImpl.h"
23 #include "ELFLinkGraphBuilder.h"
24 #include "JITLinkGeneric.h"
25
26 #define DEBUG_TYPE "jitlink"
27
28 using namespace llvm;
29 using namespace llvm::jitlink;
30
31 namespace {
32
33 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
34 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
35
36 class TLSInfoTableManager_ELF_x86_64
37 : public TableManager<TLSInfoTableManager_ELF_x86_64> {
38 public:
39 static const uint8_t TLSInfoEntryContent[16];
40
getSectionName()41 static StringRef getSectionName() { return ELFTLSInfoSectionName; }
42
visitEdge(LinkGraph & G,Block * B,Edge & E)43 bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
44 if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
45 LLVM_DEBUG({
46 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
47 << formatv("{0:x}", B->getFixupAddress(E)) << " ("
48 << formatv("{0:x}", B->getAddress()) << " + "
49 << formatv("{0:x}", E.getOffset()) << ")\n";
50 });
51 E.setKind(x86_64::Delta32);
52 E.setTarget(getEntryForTarget(G, E.getTarget()));
53 return true;
54 }
55 return false;
56 }
57
createEntry(LinkGraph & G,Symbol & Target)58 Symbol &createEntry(LinkGraph &G, Symbol &Target) {
59 // the TLS Info entry's key value will be written by the fixTLVSectionByName
60 // pass, so create mutable content.
61 auto &TLSInfoEntry = G.createMutableContentBlock(
62 getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
63 orc::ExecutorAddr(), 8, 0);
64 TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
65 return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
66 }
67
68 private:
getTLSInfoSection(LinkGraph & G)69 Section &getTLSInfoSection(LinkGraph &G) {
70 if (!TLSInfoTable)
71 TLSInfoTable =
72 &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read);
73 return *TLSInfoTable;
74 }
75
getTLSInfoEntryContent() const76 ArrayRef<char> getTLSInfoEntryContent() const {
77 return {reinterpret_cast<const char *>(TLSInfoEntryContent),
78 sizeof(TLSInfoEntryContent)};
79 }
80
81 Section *TLSInfoTable = nullptr;
82 };
83
84 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
86 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/
87 };
88
buildTables_ELF_x86_64(LinkGraph & G)89 Error buildTables_ELF_x86_64(LinkGraph &G) {
90 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
91
92 x86_64::GOTTableManager GOT;
93 x86_64::PLTTableManager PLT(GOT);
94 TLSInfoTableManager_ELF_x86_64 TLSInfo;
95 visitExistingEdges(G, GOT, PLT, TLSInfo);
96 return Error::success();
97 }
98 } // namespace
99
100 namespace llvm {
101 namespace jitlink {
102
103 // This should become a template as the ELFFile is so a lot of this could become
104 // generic
105 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
106 private:
107 using ELFT = object::ELF64LE;
108
109 enum ELFX86RelocationKind : Edge::Kind {
110 Branch32 = Edge::FirstRelocation,
111 Pointer32Signed,
112 Pointer64,
113 PCRel32,
114 PCRel32GOTLoad,
115 PCRel32GOTLoadRelaxable,
116 PCRel32REXGOTLoadRelaxable,
117 PCRel32TLV,
118 PCRel64GOT,
119 GOTOFF64,
120 GOT64,
121 Delta64,
122 };
123
getRelocationKind(const uint32_t Type)124 static Expected<ELFX86RelocationKind> getRelocationKind(const uint32_t Type) {
125 switch (Type) {
126 case ELF::R_X86_64_32S:
127 return ELFX86RelocationKind::Pointer32Signed;
128 case ELF::R_X86_64_PC32:
129 return ELFX86RelocationKind::PCRel32;
130 case ELF::R_X86_64_PC64:
131 case ELF::R_X86_64_GOTPC64:
132 return ELFX86RelocationKind::Delta64;
133 case ELF::R_X86_64_64:
134 return ELFX86RelocationKind::Pointer64;
135 case ELF::R_X86_64_GOTPCREL:
136 return ELFX86RelocationKind::PCRel32GOTLoad;
137 case ELF::R_X86_64_GOTPCRELX:
138 return ELFX86RelocationKind::PCRel32GOTLoadRelaxable;
139 case ELF::R_X86_64_REX_GOTPCRELX:
140 return ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable;
141 case ELF::R_X86_64_GOTPCREL64:
142 return ELFX86RelocationKind::PCRel64GOT;
143 case ELF::R_X86_64_GOT64:
144 return ELFX86RelocationKind::GOT64;
145 case ELF::R_X86_64_GOTOFF64:
146 return ELFX86RelocationKind::GOTOFF64;
147 case ELF::R_X86_64_PLT32:
148 return ELFX86RelocationKind::Branch32;
149 case ELF::R_X86_64_TLSGD:
150 return ELFX86RelocationKind::PCRel32TLV;
151 }
152 return make_error<JITLinkError>(
153 "Unsupported x86-64 relocation type " + formatv("{0:d}: ", Type) +
154 object::getELFRelocationTypeName(ELF::EM_X86_64, Type));
155 }
156
addRelocations()157 Error addRelocations() override {
158 LLVM_DEBUG(dbgs() << "Processing relocations:\n");
159
160 using Base = ELFLinkGraphBuilder<ELFT>;
161 using Self = ELFLinkGraphBuilder_x86_64;
162 for (const auto &RelSect : Base::Sections) {
163 // Validate the section to read relocation entries from.
164 if (RelSect.sh_type == ELF::SHT_REL)
165 return make_error<StringError>(
166 "No SHT_REL in valid x64 ELF object files",
167 inconvertibleErrorCode());
168
169 if (Error Err = Base::forEachRelaRelocation(RelSect, this,
170 &Self::addSingleRelocation))
171 return Err;
172 }
173
174 return Error::success();
175 }
176
addSingleRelocation(const typename ELFT::Rela & Rel,const typename ELFT::Shdr & FixupSection,Block & BlockToFix)177 Error addSingleRelocation(const typename ELFT::Rela &Rel,
178 const typename ELFT::Shdr &FixupSection,
179 Block &BlockToFix) {
180 using Base = ELFLinkGraphBuilder<ELFT>;
181
182 uint32_t SymbolIndex = Rel.getSymbol(false);
183 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
184 if (!ObjSymbol)
185 return ObjSymbol.takeError();
186
187 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
188 if (!GraphSymbol)
189 return make_error<StringError>(
190 formatv("Could not find symbol at given index, did you add it to "
191 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
192 SymbolIndex, (*ObjSymbol)->st_shndx,
193 Base::GraphSymbols.size()),
194 inconvertibleErrorCode());
195
196 // Validate the relocation kind.
197 auto ELFRelocKind = getRelocationKind(Rel.getType(false));
198 if (!ELFRelocKind)
199 return ELFRelocKind.takeError();
200
201 int64_t Addend = Rel.r_addend;
202 Edge::Kind Kind = Edge::Invalid;
203 switch (*ELFRelocKind) {
204 case PCRel32:
205 Kind = x86_64::Delta32;
206 break;
207 case Delta64:
208 Kind = x86_64::Delta64;
209 break;
210 case Pointer32Signed:
211 Kind = x86_64::Pointer32Signed;
212 break;
213 case Pointer64:
214 Kind = x86_64::Pointer64;
215 break;
216 case PCRel32GOTLoad: {
217 Kind = x86_64::RequestGOTAndTransformToDelta32;
218 break;
219 }
220 case PCRel32REXGOTLoadRelaxable: {
221 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
222 Addend = 0;
223 break;
224 }
225 case PCRel32TLV: {
226 Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
227 break;
228 }
229 case PCRel32GOTLoadRelaxable: {
230 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
231 Addend = 0;
232 break;
233 }
234 case PCRel64GOT: {
235 Kind = x86_64::RequestGOTAndTransformToDelta64;
236 break;
237 }
238 case GOT64: {
239 Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
240 break;
241 }
242 case GOTOFF64: {
243 Kind = x86_64::Delta64FromGOT;
244 break;
245 }
246 case Branch32: {
247 Kind = x86_64::BranchPCRel32;
248 // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
249 // adjust the addend by '+4' to compensate.
250 Addend += 4;
251 break;
252 }
253 }
254
255 auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;
256 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
257 Edge GE(Kind, Offset, *GraphSymbol, Addend);
258 LLVM_DEBUG({
259 dbgs() << " ";
260 printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));
261 dbgs() << "\n";
262 });
263
264 BlockToFix.addEdge(std::move(GE));
265 return Error::success();
266 }
267
268 public:
ELFLinkGraphBuilder_x86_64(StringRef FileName,const object::ELFFile<object::ELF64LE> & Obj)269 ELFLinkGraphBuilder_x86_64(StringRef FileName,
270 const object::ELFFile<object::ELF64LE> &Obj)
271 : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName,
272 x86_64::getEdgeKindName) {}
273 };
274
275 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
276 friend class JITLinker<ELFJITLinker_x86_64>;
277
278 public:
ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,std::unique_ptr<LinkGraph> G,PassConfiguration PassConfig)279 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
280 std::unique_ptr<LinkGraph> G,
281 PassConfiguration PassConfig)
282 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
283 getPassConfig().PostAllocationPasses.push_back(
284 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
285 }
286
287 private:
288 Symbol *GOTSymbol = nullptr;
289
getOrCreateGOTSymbol(LinkGraph & G)290 Error getOrCreateGOTSymbol(LinkGraph &G) {
291 auto DefineExternalGOTSymbolIfPresent =
292 createDefineExternalSectionStartAndEndSymbolsPass(
293 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
294 if (Sym.getName() == ELFGOTSymbolName)
295 if (auto *GOTSection = G.findSectionByName(
296 x86_64::GOTTableManager::getSectionName())) {
297 GOTSymbol = &Sym;
298 return {*GOTSection, true};
299 }
300 return {};
301 });
302
303 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
304 // external.
305 if (auto Err = DefineExternalGOTSymbolIfPresent(G))
306 return Err;
307
308 // If we succeeded then we're done.
309 if (GOTSymbol)
310 return Error::success();
311
312 // Otherwise look for a GOT section: If it already has a start symbol we'll
313 // record it, otherwise we'll create our own.
314 // If there's a GOT section but we didn't find an external GOT symbol...
315 if (auto *GOTSection =
316 G.findSectionByName(x86_64::GOTTableManager::getSectionName())) {
317
318 // Check for an existing defined symbol.
319 for (auto *Sym : GOTSection->symbols())
320 if (Sym->getName() == ELFGOTSymbolName) {
321 GOTSymbol = Sym;
322 return Error::success();
323 }
324
325 // If there's no defined symbol then create one.
326 SectionRange SR(*GOTSection);
327 if (SR.empty())
328 GOTSymbol =
329 &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
330 Linkage::Strong, Scope::Local, true);
331 else
332 GOTSymbol =
333 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
334 Linkage::Strong, Scope::Local, false, true);
335 }
336
337 return Error::success();
338 }
339
applyFixup(LinkGraph & G,Block & B,const Edge & E) const340 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
341 return x86_64::applyFixup(G, B, E, GOTSymbol);
342 }
343 };
344
345 Expected<std::unique_ptr<LinkGraph>>
createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer)346 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
347 LLVM_DEBUG({
348 dbgs() << "Building jitlink graph for new input "
349 << ObjectBuffer.getBufferIdentifier() << "...\n";
350 });
351
352 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
353 if (!ELFObj)
354 return ELFObj.takeError();
355
356 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
357 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
358 ELFObjFile.getELFFile())
359 .buildGraph();
360 }
361
362 static SectionRangeSymbolDesc
identifyELFSectionStartAndEndSymbols(LinkGraph & G,Symbol & Sym)363 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
364 constexpr StringRef StartSymbolPrefix = "__start";
365 constexpr StringRef EndSymbolPrefix = "__end";
366
367 auto SymName = Sym.getName();
368 if (SymName.startswith(StartSymbolPrefix)) {
369 if (auto *Sec =
370 G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
371 return {*Sec, true};
372 } else if (SymName.startswith(EndSymbolPrefix)) {
373 if (auto *Sec =
374 G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
375 return {*Sec, false};
376 }
377 return {};
378 }
379
link_ELF_x86_64(std::unique_ptr<LinkGraph> G,std::unique_ptr<JITLinkContext> Ctx)380 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
381 std::unique_ptr<JITLinkContext> Ctx) {
382 PassConfiguration Config;
383
384 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
385
386 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
387 Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
388 ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,
389 x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));
390 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
391
392 // Construct a JITLinker and run the link function.
393 // Add a mark-live pass.
394 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
395 Config.PrePrunePasses.push_back(std::move(MarkLive));
396 else
397 Config.PrePrunePasses.push_back(markAllSymbolsLive);
398
399 // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
400 Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);
401
402 // Resolve any external section start / end symbols.
403 Config.PostAllocationPasses.push_back(
404 createDefineExternalSectionStartAndEndSymbolsPass(
405 identifyELFSectionStartAndEndSymbols));
406
407 // Add GOT/Stubs optimizer pass.
408 Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);
409 }
410
411 if (auto Err = Ctx->modifyPassConfig(*G, Config))
412 return Ctx->notifyFailed(std::move(Err));
413
414 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
415 }
416 } // end namespace jitlink
417 } // end namespace llvm
418