1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * sorttable.c: Sort the kernel's table
4 *
5 * Added ORC unwind tables sort support and other updates:
6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
7 * Shile Zhang <[email protected]>
8 *
9 * Copyright 2011 - 2012 Cavium, Inc.
10 *
11 * Based on code taken from recortmcount.c which is:
12 *
13 * Copyright 2009 John F. Reiser <[email protected]>. All rights reserved.
14 *
15 * Restructured to fit Linux format, as well as other updates:
16 * Copyright 2010 Steven Rostedt <[email protected]>, Red Hat Inc.
17 */
18
19 /*
20 * Strategy: alter the vmlinux file in-place.
21 */
22
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #include <sys/stat.h>
26 #include <getopt.h>
27 #include <elf.h>
28 #include <fcntl.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <unistd.h>
33 #include <errno.h>
34 #include <pthread.h>
35
36 #include <tools/be_byteshift.h>
37 #include <tools/le_byteshift.h>
38
39 #ifndef EM_ARCOMPACT
40 #define EM_ARCOMPACT 93
41 #endif
42
43 #ifndef EM_XTENSA
44 #define EM_XTENSA 94
45 #endif
46
47 #ifndef EM_AARCH64
48 #define EM_AARCH64 183
49 #endif
50
51 #ifndef EM_MICROBLAZE
52 #define EM_MICROBLAZE 189
53 #endif
54
55 #ifndef EM_ARCV2
56 #define EM_ARCV2 195
57 #endif
58
59 #ifndef EM_RISCV
60 #define EM_RISCV 243
61 #endif
62
63 #ifndef EM_LOONGARCH
64 #define EM_LOONGARCH 258
65 #endif
66
67 typedef union {
68 Elf32_Ehdr e32;
69 Elf64_Ehdr e64;
70 } Elf_Ehdr;
71
72 typedef union {
73 Elf32_Shdr e32;
74 Elf64_Shdr e64;
75 } Elf_Shdr;
76
77 typedef union {
78 Elf32_Sym e32;
79 Elf64_Sym e64;
80 } Elf_Sym;
81
82 static uint32_t (*r)(const uint32_t *);
83 static uint16_t (*r2)(const uint16_t *);
84 static uint64_t (*r8)(const uint64_t *);
85 static void (*w)(uint32_t, uint32_t *);
86 typedef void (*table_sort_t)(char *, int);
87
88 static struct elf_funcs {
89 int (*compare_extable)(const void *a, const void *b);
90 uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr);
91 uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr);
92 uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr);
93 uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr);
94 uint64_t (*shdr_addr)(Elf_Shdr *shdr);
95 uint64_t (*shdr_offset)(Elf_Shdr *shdr);
96 uint64_t (*shdr_size)(Elf_Shdr *shdr);
97 uint64_t (*shdr_entsize)(Elf_Shdr *shdr);
98 uint32_t (*shdr_link)(Elf_Shdr *shdr);
99 uint32_t (*shdr_name)(Elf_Shdr *shdr);
100 uint32_t (*shdr_type)(Elf_Shdr *shdr);
101 uint8_t (*sym_type)(Elf_Sym *sym);
102 uint32_t (*sym_name)(Elf_Sym *sym);
103 uint64_t (*sym_value)(Elf_Sym *sym);
104 uint16_t (*sym_shndx)(Elf_Sym *sym);
105 } e;
106
ehdr64_shoff(Elf_Ehdr * ehdr)107 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr)
108 {
109 return r8(&ehdr->e64.e_shoff);
110 }
111
ehdr32_shoff(Elf_Ehdr * ehdr)112 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr)
113 {
114 return r(&ehdr->e32.e_shoff);
115 }
116
ehdr_shoff(Elf_Ehdr * ehdr)117 static uint64_t ehdr_shoff(Elf_Ehdr *ehdr)
118 {
119 return e.ehdr_shoff(ehdr);
120 }
121
122 #define EHDR_HALF(fn_name) \
123 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr) \
124 { \
125 return r2(&ehdr->e64.e_##fn_name); \
126 } \
127 \
128 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr) \
129 { \
130 return r2(&ehdr->e32.e_##fn_name); \
131 } \
132 \
133 static uint16_t ehdr_##fn_name(Elf_Ehdr *ehdr) \
134 { \
135 return e.ehdr_##fn_name(ehdr); \
136 }
137
138 EHDR_HALF(shentsize)
EHDR_HALF(shstrndx)139 EHDR_HALF(shstrndx)
140 EHDR_HALF(shnum)
141
142 #define SHDR_WORD(fn_name) \
143 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \
144 { \
145 return r(&shdr->e64.sh_##fn_name); \
146 } \
147 \
148 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \
149 { \
150 return r(&shdr->e32.sh_##fn_name); \
151 } \
152 \
153 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \
154 { \
155 return e.shdr_##fn_name(shdr); \
156 }
157
158 #define SHDR_ADDR(fn_name) \
159 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr) \
160 { \
161 return r8(&shdr->e64.sh_##fn_name); \
162 } \
163 \
164 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr) \
165 { \
166 return r(&shdr->e32.sh_##fn_name); \
167 } \
168 \
169 static uint64_t shdr_##fn_name(Elf_Shdr *shdr) \
170 { \
171 return e.shdr_##fn_name(shdr); \
172 }
173
174 #define SHDR_WORD(fn_name) \
175 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \
176 { \
177 return r(&shdr->e64.sh_##fn_name); \
178 } \
179 \
180 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \
181 { \
182 return r(&shdr->e32.sh_##fn_name); \
183 } \
184 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \
185 { \
186 return e.shdr_##fn_name(shdr); \
187 }
188
189 SHDR_ADDR(addr)
190 SHDR_ADDR(offset)
191 SHDR_ADDR(size)
192 SHDR_ADDR(entsize)
193
194 SHDR_WORD(link)
195 SHDR_WORD(name)
196 SHDR_WORD(type)
197
198 #define SYM_ADDR(fn_name) \
199 static uint64_t sym64_##fn_name(Elf_Sym *sym) \
200 { \
201 return r8(&sym->e64.st_##fn_name); \
202 } \
203 \
204 static uint64_t sym32_##fn_name(Elf_Sym *sym) \
205 { \
206 return r(&sym->e32.st_##fn_name); \
207 } \
208 \
209 static uint64_t sym_##fn_name(Elf_Sym *sym) \
210 { \
211 return e.sym_##fn_name(sym); \
212 }
213
214 #define SYM_WORD(fn_name) \
215 static uint32_t sym64_##fn_name(Elf_Sym *sym) \
216 { \
217 return r(&sym->e64.st_##fn_name); \
218 } \
219 \
220 static uint32_t sym32_##fn_name(Elf_Sym *sym) \
221 { \
222 return r(&sym->e32.st_##fn_name); \
223 } \
224 \
225 static uint32_t sym_##fn_name(Elf_Sym *sym) \
226 { \
227 return e.sym_##fn_name(sym); \
228 }
229
230 #define SYM_HALF(fn_name) \
231 static uint16_t sym64_##fn_name(Elf_Sym *sym) \
232 { \
233 return r2(&sym->e64.st_##fn_name); \
234 } \
235 \
236 static uint16_t sym32_##fn_name(Elf_Sym *sym) \
237 { \
238 return r2(&sym->e32.st_##fn_name); \
239 } \
240 \
241 static uint16_t sym_##fn_name(Elf_Sym *sym) \
242 { \
243 return e.sym_##fn_name(sym); \
244 }
245
246 static uint8_t sym64_type(Elf_Sym *sym)
247 {
248 return ELF64_ST_TYPE(sym->e64.st_info);
249 }
250
sym32_type(Elf_Sym * sym)251 static uint8_t sym32_type(Elf_Sym *sym)
252 {
253 return ELF32_ST_TYPE(sym->e32.st_info);
254 }
255
sym_type(Elf_Sym * sym)256 static uint8_t sym_type(Elf_Sym *sym)
257 {
258 return e.sym_type(sym);
259 }
260
261 SYM_ADDR(value)
SYM_WORD(name)262 SYM_WORD(name)
263 SYM_HALF(shndx)
264
265 /*
266 * Get the whole file as a programming convenience in order to avoid
267 * malloc+lseek+read+free of many pieces. If successful, then mmap
268 * avoids copying unused pieces; else just read the whole file.
269 * Open for both read and write.
270 */
271 static void *mmap_file(char const *fname, size_t *size)
272 {
273 int fd;
274 struct stat sb;
275 void *addr = NULL;
276
277 fd = open(fname, O_RDWR);
278 if (fd < 0) {
279 perror(fname);
280 return NULL;
281 }
282 if (fstat(fd, &sb) < 0) {
283 perror(fname);
284 goto out;
285 }
286 if (!S_ISREG(sb.st_mode)) {
287 fprintf(stderr, "not a regular file: %s\n", fname);
288 goto out;
289 }
290
291 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
292 if (addr == MAP_FAILED) {
293 fprintf(stderr, "Could not mmap file: %s\n", fname);
294 goto out;
295 }
296
297 *size = sb.st_size;
298
299 out:
300 close(fd);
301 return addr;
302 }
303
rbe(const uint32_t * x)304 static uint32_t rbe(const uint32_t *x)
305 {
306 return get_unaligned_be32(x);
307 }
308
r2be(const uint16_t * x)309 static uint16_t r2be(const uint16_t *x)
310 {
311 return get_unaligned_be16(x);
312 }
313
r8be(const uint64_t * x)314 static uint64_t r8be(const uint64_t *x)
315 {
316 return get_unaligned_be64(x);
317 }
318
rle(const uint32_t * x)319 static uint32_t rle(const uint32_t *x)
320 {
321 return get_unaligned_le32(x);
322 }
323
r2le(const uint16_t * x)324 static uint16_t r2le(const uint16_t *x)
325 {
326 return get_unaligned_le16(x);
327 }
328
r8le(const uint64_t * x)329 static uint64_t r8le(const uint64_t *x)
330 {
331 return get_unaligned_le64(x);
332 }
333
wbe(uint32_t val,uint32_t * x)334 static void wbe(uint32_t val, uint32_t *x)
335 {
336 put_unaligned_be32(val, x);
337 }
338
wle(uint32_t val,uint32_t * x)339 static void wle(uint32_t val, uint32_t *x)
340 {
341 put_unaligned_le32(val, x);
342 }
343
344 /*
345 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of
346 * the way to -256..-1, to avoid conflicting with real section
347 * indices.
348 */
349 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1))
350
is_shndx_special(unsigned int i)351 static inline int is_shndx_special(unsigned int i)
352 {
353 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE;
354 }
355
356 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */
get_secindex(unsigned int shndx,unsigned int sym_offs,const Elf32_Word * symtab_shndx_start)357 static inline unsigned int get_secindex(unsigned int shndx,
358 unsigned int sym_offs,
359 const Elf32_Word *symtab_shndx_start)
360 {
361 if (is_shndx_special(shndx))
362 return SPECIAL(shndx);
363 if (shndx != SHN_XINDEX)
364 return shndx;
365 return r(&symtab_shndx_start[sym_offs]);
366 }
367
compare_extable_32(const void * a,const void * b)368 static int compare_extable_32(const void *a, const void *b)
369 {
370 Elf32_Addr av = r(a);
371 Elf32_Addr bv = r(b);
372
373 if (av < bv)
374 return -1;
375 return av > bv;
376 }
377
compare_extable_64(const void * a,const void * b)378 static int compare_extable_64(const void *a, const void *b)
379 {
380 Elf64_Addr av = r8(a);
381 Elf64_Addr bv = r8(b);
382
383 if (av < bv)
384 return -1;
385 return av > bv;
386 }
387
compare_extable(const void * a,const void * b)388 static int compare_extable(const void *a, const void *b)
389 {
390 return e.compare_extable(a, b);
391 }
392
get_index(void * start,int entsize,int index)393 static inline void *get_index(void *start, int entsize, int index)
394 {
395 return start + (entsize * index);
396 }
397
398 static int extable_ent_size;
399 static int long_size;
400
401
402 #ifdef UNWINDER_ORC_ENABLED
403 /* ORC unwinder only support X86_64 */
404 #include <asm/orc_types.h>
405
406 #define ERRSTR_MAXSZ 256
407
408 static char g_err[ERRSTR_MAXSZ];
409 static int *g_orc_ip_table;
410 static struct orc_entry *g_orc_table;
411
412 static pthread_t orc_sort_thread;
413
orc_ip(const int * ip)414 static inline unsigned long orc_ip(const int *ip)
415 {
416 return (unsigned long)ip + *ip;
417 }
418
orc_sort_cmp(const void * _a,const void * _b)419 static int orc_sort_cmp(const void *_a, const void *_b)
420 {
421 struct orc_entry *orc_a, *orc_b;
422 const int *a = g_orc_ip_table + *(int *)_a;
423 const int *b = g_orc_ip_table + *(int *)_b;
424 unsigned long a_val = orc_ip(a);
425 unsigned long b_val = orc_ip(b);
426
427 if (a_val > b_val)
428 return 1;
429 if (a_val < b_val)
430 return -1;
431
432 /*
433 * The "weak" section terminator entries need to always be on the left
434 * to ensure the lookup code skips them in favor of real entries.
435 * These terminator entries exist to handle any gaps created by
436 * whitelisted .o files which didn't get objtool generation.
437 */
438 orc_a = g_orc_table + (a - g_orc_ip_table);
439 orc_b = g_orc_table + (b - g_orc_ip_table);
440 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED)
441 return 0;
442 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
443 }
444
sort_orctable(void * arg)445 static void *sort_orctable(void *arg)
446 {
447 int i;
448 int *idxs = NULL;
449 int *tmp_orc_ip_table = NULL;
450 struct orc_entry *tmp_orc_table = NULL;
451 unsigned int *orc_ip_size = (unsigned int *)arg;
452 unsigned int num_entries = *orc_ip_size / sizeof(int);
453 unsigned int orc_size = num_entries * sizeof(struct orc_entry);
454
455 idxs = (int *)malloc(*orc_ip_size);
456 if (!idxs) {
457 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
458 strerror(errno));
459 pthread_exit(g_err);
460 }
461
462 tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
463 if (!tmp_orc_ip_table) {
464 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
465 strerror(errno));
466 pthread_exit(g_err);
467 }
468
469 tmp_orc_table = (struct orc_entry *)malloc(orc_size);
470 if (!tmp_orc_table) {
471 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
472 strerror(errno));
473 pthread_exit(g_err);
474 }
475
476 /* initialize indices array, convert ip_table to absolute address */
477 for (i = 0; i < num_entries; i++) {
478 idxs[i] = i;
479 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
480 }
481 memcpy(tmp_orc_table, g_orc_table, orc_size);
482
483 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
484
485 for (i = 0; i < num_entries; i++) {
486 if (idxs[i] == i)
487 continue;
488
489 /* convert back to relative address */
490 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
491 g_orc_table[i] = tmp_orc_table[idxs[i]];
492 }
493
494 free(idxs);
495 free(tmp_orc_ip_table);
496 free(tmp_orc_table);
497 pthread_exit(NULL);
498 }
499 #endif
500
501 #ifdef MCOUNT_SORT_ENABLED
502 static pthread_t mcount_sort_thread;
503
504 struct elf_mcount_loc {
505 Elf_Ehdr *ehdr;
506 Elf_Shdr *init_data_sec;
507 uint64_t start_mcount_loc;
508 uint64_t stop_mcount_loc;
509 };
510
511 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
sort_mcount_loc(void * arg)512 static void *sort_mcount_loc(void *arg)
513 {
514 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg;
515 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec)
516 + shdr_offset(emloc->init_data_sec);
517 uint64_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc;
518 unsigned char *start_loc = (void *)emloc->ehdr + offset;
519
520 qsort(start_loc, count/long_size, long_size, compare_extable);
521 return NULL;
522 }
523
524 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */
get_mcount_loc(struct elf_mcount_loc * emloc,Elf_Shdr * symtab_sec,const char * strtab)525 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec,
526 const char *strtab)
527 {
528 Elf_Sym *sym, *end_sym;
529 int symentsize = shdr_entsize(symtab_sec);
530 int found = 0;
531
532 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec);
533 end_sym = (void *)sym + shdr_size(symtab_sec);
534
535 while (sym < end_sym) {
536 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) {
537 emloc->start_mcount_loc = sym_value(sym);
538 if (++found == 2)
539 break;
540 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) {
541 emloc->stop_mcount_loc = sym_value(sym);
542 if (++found == 2)
543 break;
544 }
545 sym = (void *)sym + symentsize;
546 }
547
548 if (!emloc->start_mcount_loc) {
549 fprintf(stderr, "get start_mcount_loc error!");
550 return;
551 }
552
553 if (!emloc->stop_mcount_loc) {
554 fprintf(stderr, "get stop_mcount_loc error!");
555 return;
556 }
557 }
558 #endif
559
do_sort(Elf_Ehdr * ehdr,char const * const fname,table_sort_t custom_sort)560 static int do_sort(Elf_Ehdr *ehdr,
561 char const *const fname,
562 table_sort_t custom_sort)
563 {
564 int rc = -1;
565 Elf_Shdr *shdr_start;
566 Elf_Shdr *strtab_sec = NULL;
567 Elf_Shdr *symtab_sec = NULL;
568 Elf_Shdr *extab_sec = NULL;
569 Elf_Shdr *string_sec;
570 Elf_Sym *sym;
571 const Elf_Sym *symtab;
572 Elf32_Word *symtab_shndx = NULL;
573 Elf_Sym *sort_needed_sym = NULL;
574 Elf_Shdr *sort_needed_sec;
575 uint32_t *sort_needed_loc;
576 void *sym_start;
577 void *sym_end;
578 const char *secstrings;
579 const char *strtab;
580 char *extab_image;
581 int sort_need_index;
582 int symentsize;
583 int shentsize;
584 int idx;
585 int i;
586 unsigned int shnum;
587 unsigned int shstrndx;
588 #ifdef MCOUNT_SORT_ENABLED
589 struct elf_mcount_loc mstruct = {0};
590 #endif
591 #ifdef UNWINDER_ORC_ENABLED
592 unsigned int orc_ip_size = 0;
593 unsigned int orc_size = 0;
594 unsigned int orc_num_entries = 0;
595 #endif
596
597 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
598 shentsize = ehdr_shentsize(ehdr);
599
600 shstrndx = ehdr_shstrndx(ehdr);
601 if (shstrndx == SHN_XINDEX)
602 shstrndx = shdr_link(shdr_start);
603 string_sec = get_index(shdr_start, shentsize, shstrndx);
604 secstrings = (const char *)ehdr + shdr_offset(string_sec);
605
606 shnum = ehdr_shnum(ehdr);
607 if (shnum == SHN_UNDEF)
608 shnum = shdr_size(shdr_start);
609
610 for (i = 0; i < shnum; i++) {
611 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
612
613 idx = shdr_name(shdr);
614 if (!strcmp(secstrings + idx, "__ex_table"))
615 extab_sec = shdr;
616 if (!strcmp(secstrings + idx, ".symtab"))
617 symtab_sec = shdr;
618 if (!strcmp(secstrings + idx, ".strtab"))
619 strtab_sec = shdr;
620
621 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX)
622 symtab_shndx = (Elf32_Word *)((const char *)ehdr +
623 shdr_offset(shdr));
624
625 #ifdef MCOUNT_SORT_ENABLED
626 /* locate the .init.data section in vmlinux */
627 if (!strcmp(secstrings + idx, ".init.data"))
628 mstruct.init_data_sec = shdr;
629 #endif
630
631 #ifdef UNWINDER_ORC_ENABLED
632 /* locate the ORC unwind tables */
633 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
634 orc_ip_size = shdr_size(shdr);
635 g_orc_ip_table = (int *)((void *)ehdr +
636 shdr_offset(shdr));
637 }
638 if (!strcmp(secstrings + idx, ".orc_unwind")) {
639 orc_size = shdr_size(shdr);
640 g_orc_table = (struct orc_entry *)((void *)ehdr +
641 shdr_offset(shdr));
642 }
643 #endif
644 } /* for loop */
645
646 #ifdef UNWINDER_ORC_ENABLED
647 if (!g_orc_ip_table || !g_orc_table) {
648 fprintf(stderr,
649 "incomplete ORC unwind tables in file: %s\n", fname);
650 goto out;
651 }
652
653 orc_num_entries = orc_ip_size / sizeof(int);
654 if (orc_ip_size % sizeof(int) != 0 ||
655 orc_size % sizeof(struct orc_entry) != 0 ||
656 orc_num_entries != orc_size / sizeof(struct orc_entry)) {
657 fprintf(stderr,
658 "inconsistent ORC unwind table entries in file: %s\n",
659 fname);
660 goto out;
661 }
662
663 /* create thread to sort ORC unwind tables concurrently */
664 if (pthread_create(&orc_sort_thread, NULL,
665 sort_orctable, &orc_ip_size)) {
666 fprintf(stderr,
667 "pthread_create orc_sort_thread failed '%s': %s\n",
668 strerror(errno), fname);
669 goto out;
670 }
671 #endif
672 if (!extab_sec) {
673 fprintf(stderr, "no __ex_table in file: %s\n", fname);
674 goto out;
675 }
676
677 if (!symtab_sec) {
678 fprintf(stderr, "no .symtab in file: %s\n", fname);
679 goto out;
680 }
681
682 if (!strtab_sec) {
683 fprintf(stderr, "no .strtab in file: %s\n", fname);
684 goto out;
685 }
686
687 extab_image = (void *)ehdr + shdr_offset(extab_sec);
688 strtab = (const char *)ehdr + shdr_offset(strtab_sec);
689 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec));
690
691 #ifdef MCOUNT_SORT_ENABLED
692 mstruct.ehdr = ehdr;
693 get_mcount_loc(&mstruct, symtab_sec, strtab);
694
695 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) {
696 fprintf(stderr,
697 "incomplete mcount's sort in file: %s\n",
698 fname);
699 goto out;
700 }
701
702 /* create thread to sort mcount_loc concurrently */
703 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) {
704 fprintf(stderr,
705 "pthread_create mcount_sort_thread failed '%s': %s\n",
706 strerror(errno), fname);
707 goto out;
708 }
709 #endif
710
711 if (custom_sort) {
712 custom_sort(extab_image, shdr_size(extab_sec));
713 } else {
714 int num_entries = shdr_size(extab_sec) / extable_ent_size;
715 qsort(extab_image, num_entries,
716 extable_ent_size, compare_extable);
717 }
718
719 /* find the flag main_extable_sort_needed */
720 sym_start = (void *)ehdr + shdr_offset(symtab_sec);
721 sym_end = sym_start + shdr_size(symtab_sec);
722 symentsize = shdr_entsize(symtab_sec);
723
724 for (sym = sym_start; (void *)sym + symentsize < sym_end;
725 sym = (void *)sym + symentsize) {
726 if (sym_type(sym) != STT_OBJECT)
727 continue;
728 if (!strcmp(strtab + sym_name(sym),
729 "main_extable_sort_needed")) {
730 sort_needed_sym = sym;
731 break;
732 }
733 }
734
735 if (!sort_needed_sym) {
736 fprintf(stderr,
737 "no main_extable_sort_needed symbol in file: %s\n",
738 fname);
739 goto out;
740 }
741
742 sort_need_index = get_secindex(sym_shndx(sym),
743 ((void *)sort_needed_sym - (void *)symtab) / symentsize,
744 symtab_shndx);
745 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index);
746 sort_needed_loc = (void *)ehdr +
747 shdr_offset(sort_needed_sec) +
748 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec);
749
750 /* extable has been sorted, clear the flag */
751 w(0, sort_needed_loc);
752 rc = 0;
753
754 out:
755 #ifdef UNWINDER_ORC_ENABLED
756 if (orc_sort_thread) {
757 void *retval = NULL;
758 /* wait for ORC tables sort done */
759 rc = pthread_join(orc_sort_thread, &retval);
760 if (rc) {
761 fprintf(stderr,
762 "pthread_join failed '%s': %s\n",
763 strerror(errno), fname);
764 } else if (retval) {
765 rc = -1;
766 fprintf(stderr,
767 "failed to sort ORC tables '%s': %s\n",
768 (char *)retval, fname);
769 }
770 }
771 #endif
772
773 #ifdef MCOUNT_SORT_ENABLED
774 if (mcount_sort_thread) {
775 void *retval = NULL;
776 /* wait for mcount sort done */
777 rc = pthread_join(mcount_sort_thread, &retval);
778 if (rc) {
779 fprintf(stderr,
780 "pthread_join failed '%s': %s\n",
781 strerror(errno), fname);
782 } else if (retval) {
783 rc = -1;
784 fprintf(stderr,
785 "failed to sort mcount '%s': %s\n",
786 (char *)retval, fname);
787 }
788 }
789 #endif
790 return rc;
791 }
792
compare_relative_table(const void * a,const void * b)793 static int compare_relative_table(const void *a, const void *b)
794 {
795 int32_t av = (int32_t)r(a);
796 int32_t bv = (int32_t)r(b);
797
798 if (av < bv)
799 return -1;
800 if (av > bv)
801 return 1;
802 return 0;
803 }
804
sort_relative_table(char * extab_image,int image_size)805 static void sort_relative_table(char *extab_image, int image_size)
806 {
807 int i = 0;
808
809 /*
810 * Do the same thing the runtime sort does, first normalize to
811 * being relative to the start of the section.
812 */
813 while (i < image_size) {
814 uint32_t *loc = (uint32_t *)(extab_image + i);
815 w(r(loc) + i, loc);
816 i += 4;
817 }
818
819 qsort(extab_image, image_size / 8, 8, compare_relative_table);
820
821 /* Now denormalize. */
822 i = 0;
823 while (i < image_size) {
824 uint32_t *loc = (uint32_t *)(extab_image + i);
825 w(r(loc) - i, loc);
826 i += 4;
827 }
828 }
829
sort_relative_table_with_data(char * extab_image,int image_size)830 static void sort_relative_table_with_data(char *extab_image, int image_size)
831 {
832 int i = 0;
833
834 while (i < image_size) {
835 uint32_t *loc = (uint32_t *)(extab_image + i);
836
837 w(r(loc) + i, loc);
838 w(r(loc + 1) + i + 4, loc + 1);
839 /* Don't touch the fixup type or data */
840
841 i += sizeof(uint32_t) * 3;
842 }
843
844 qsort(extab_image, image_size / 12, 12, compare_relative_table);
845
846 i = 0;
847 while (i < image_size) {
848 uint32_t *loc = (uint32_t *)(extab_image + i);
849
850 w(r(loc) - i, loc);
851 w(r(loc + 1) - (i + 4), loc + 1);
852 /* Don't touch the fixup type or data */
853
854 i += sizeof(uint32_t) * 3;
855 }
856 }
857
do_file(char const * const fname,void * addr)858 static int do_file(char const *const fname, void *addr)
859 {
860 Elf_Ehdr *ehdr = addr;
861 table_sort_t custom_sort = NULL;
862
863 switch (ehdr->e32.e_ident[EI_DATA]) {
864 case ELFDATA2LSB:
865 r = rle;
866 r2 = r2le;
867 r8 = r8le;
868 w = wle;
869 break;
870 case ELFDATA2MSB:
871 r = rbe;
872 r2 = r2be;
873 r8 = r8be;
874 w = wbe;
875 break;
876 default:
877 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n",
878 ehdr->e32.e_ident[EI_DATA], fname);
879 return -1;
880 }
881
882 if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 ||
883 (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) ||
884 ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) {
885 fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname);
886 return -1;
887 }
888
889 switch (r2(&ehdr->e32.e_machine)) {
890 case EM_386:
891 case EM_AARCH64:
892 case EM_LOONGARCH:
893 case EM_RISCV:
894 case EM_S390:
895 case EM_X86_64:
896 custom_sort = sort_relative_table_with_data;
897 break;
898 case EM_PARISC:
899 case EM_PPC:
900 case EM_PPC64:
901 custom_sort = sort_relative_table;
902 break;
903 case EM_ARCOMPACT:
904 case EM_ARCV2:
905 case EM_ARM:
906 case EM_MICROBLAZE:
907 case EM_MIPS:
908 case EM_XTENSA:
909 break;
910 default:
911 fprintf(stderr, "unrecognized e_machine %d %s\n",
912 r2(&ehdr->e32.e_machine), fname);
913 return -1;
914 }
915
916 switch (ehdr->e32.e_ident[EI_CLASS]) {
917 case ELFCLASS32: {
918 struct elf_funcs efuncs = {
919 .compare_extable = compare_extable_32,
920 .ehdr_shoff = ehdr32_shoff,
921 .ehdr_shentsize = ehdr32_shentsize,
922 .ehdr_shstrndx = ehdr32_shstrndx,
923 .ehdr_shnum = ehdr32_shnum,
924 .shdr_addr = shdr32_addr,
925 .shdr_offset = shdr32_offset,
926 .shdr_link = shdr32_link,
927 .shdr_size = shdr32_size,
928 .shdr_name = shdr32_name,
929 .shdr_type = shdr32_type,
930 .shdr_entsize = shdr32_entsize,
931 .sym_type = sym32_type,
932 .sym_name = sym32_name,
933 .sym_value = sym32_value,
934 .sym_shndx = sym32_shndx,
935 };
936
937 e = efuncs;
938 long_size = 4;
939 extable_ent_size = 8;
940
941 if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) ||
942 r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) {
943 fprintf(stderr,
944 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname);
945 return -1;
946 }
947
948 }
949 break;
950 case ELFCLASS64: {
951 struct elf_funcs efuncs = {
952 .compare_extable = compare_extable_64,
953 .ehdr_shoff = ehdr64_shoff,
954 .ehdr_shentsize = ehdr64_shentsize,
955 .ehdr_shstrndx = ehdr64_shstrndx,
956 .ehdr_shnum = ehdr64_shnum,
957 .shdr_addr = shdr64_addr,
958 .shdr_offset = shdr64_offset,
959 .shdr_link = shdr64_link,
960 .shdr_size = shdr64_size,
961 .shdr_name = shdr64_name,
962 .shdr_type = shdr64_type,
963 .shdr_entsize = shdr64_entsize,
964 .sym_type = sym64_type,
965 .sym_name = sym64_name,
966 .sym_value = sym64_value,
967 .sym_shndx = sym64_shndx,
968 };
969
970 e = efuncs;
971 long_size = 8;
972 extable_ent_size = 16;
973
974 if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) ||
975 r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) {
976 fprintf(stderr,
977 "unrecognized ET_EXEC/ET_DYN file: %s\n",
978 fname);
979 return -1;
980 }
981
982 }
983 break;
984 default:
985 fprintf(stderr, "unrecognized ELF class %d %s\n",
986 ehdr->e32.e_ident[EI_CLASS], fname);
987 return -1;
988 }
989
990 return do_sort(ehdr, fname, custom_sort);
991 }
992
main(int argc,char * argv[])993 int main(int argc, char *argv[])
994 {
995 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */
996 size_t size = 0;
997 void *addr = NULL;
998
999 if (argc < 2) {
1000 fprintf(stderr, "usage: sorttable vmlinux...\n");
1001 return 0;
1002 }
1003
1004 /* Process each file in turn, allowing deep failure. */
1005 for (i = 1; i < argc; i++) {
1006 addr = mmap_file(argv[i], &size);
1007 if (!addr) {
1008 ++n_error;
1009 continue;
1010 }
1011
1012 if (do_file(argv[i], addr))
1013 ++n_error;
1014
1015 munmap(addr, size);
1016 }
1017
1018 return !!n_error;
1019 }
1020