Lines Matching +full:kconfig +full:- +full:ext
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
71 /* single .BTF.ext data section */
81 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
127 /* section's .BTF.ext data */
156 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
187 free(linker->filename); in bpf_linker__free()
189 if (linker->elf) in bpf_linker__free()
190 elf_end(linker->elf); in bpf_linker__free()
192 if (linker->fd >= 0) in bpf_linker__free()
193 close(linker->fd); in bpf_linker__free()
195 strset__free(linker->strtab_strs); in bpf_linker__free()
197 btf__free(linker->btf); in bpf_linker__free()
198 btf_ext__free(linker->btf_ext); in bpf_linker__free()
200 for (i = 1; i < linker->sec_cnt; i++) { in bpf_linker__free()
201 struct dst_sec *sec = &linker->secs[i]; in bpf_linker__free()
203 free(sec->sec_name); in bpf_linker__free()
204 free(sec->raw_data); in bpf_linker__free()
205 free(sec->sec_vars); in bpf_linker__free()
207 free(sec->func_info.recs); in bpf_linker__free()
208 free(sec->line_info.recs); in bpf_linker__free()
209 free(sec->core_relo_info.recs); in bpf_linker__free()
211 free(linker->secs); in bpf_linker__free()
213 free(linker->glob_syms); in bpf_linker__free()
234 linker->fd = -1; in bpf_linker__new()
244 return errno = -err, NULL; in bpf_linker__new()
249 struct dst_sec *secs = linker->secs, *sec; in add_dst_sec()
250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; in add_dst_sec()
257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); in add_dst_sec()
259 linker->secs = secs; in add_dst_sec()
260 linker->sec_cnt = new_cnt; in add_dst_sec()
262 sec = &linker->secs[new_cnt - 1]; in add_dst_sec()
263 sec->id = new_cnt - 1; in add_dst_sec()
264 sec->sec_name = strdup(sec_name); in add_dst_sec()
265 if (!sec->sec_name) in add_dst_sec()
273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; in add_new_sym()
275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym); in add_new_sym()
277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); in add_new_sym()
284 symtab->raw_data = syms; in add_new_sym()
285 symtab->sec_sz += sizeof(*sym); in add_new_sym()
286 symtab->shdr->sh_size += sizeof(*sym); in add_new_sym()
287 symtab->data->d_size += sizeof(*sym); in add_new_sym()
301 linker->filename = strdup(file); in init_output_elf()
302 if (!linker->filename) in init_output_elf()
303 return -ENOMEM; in init_output_elf()
305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); in init_output_elf()
306 if (linker->fd < 0) { in init_output_elf()
307 err = -errno; in init_output_elf()
312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); in init_output_elf()
313 if (!linker->elf) { in init_output_elf()
315 return -EINVAL; in init_output_elf()
319 linker->elf_hdr = elf64_newehdr(linker->elf); in init_output_elf()
320 if (!linker->elf_hdr) { in init_output_elf()
322 return -EINVAL; in init_output_elf()
325 linker->elf_hdr->e_machine = EM_BPF; in init_output_elf()
326 linker->elf_hdr->e_type = ET_REL; in init_output_elf()
328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; in init_output_elf()
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; in init_output_elf()
337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); in init_output_elf()
338 if (libbpf_get_error(linker->strtab_strs)) in init_output_elf()
339 return libbpf_get_error(linker->strtab_strs); in init_output_elf()
343 return -ENOMEM; in init_output_elf()
345 sec->scn = elf_newscn(linker->elf); in init_output_elf()
346 if (!sec->scn) { in init_output_elf()
348 return -EINVAL; in init_output_elf()
351 sec->shdr = elf64_getshdr(sec->scn); in init_output_elf()
352 if (!sec->shdr) in init_output_elf()
353 return -EINVAL; in init_output_elf()
355 sec->data = elf_newdata(sec->scn); in init_output_elf()
356 if (!sec->data) { in init_output_elf()
358 return -EINVAL; in init_output_elf()
361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); in init_output_elf()
365 sec->sec_idx = elf_ndxscn(sec->scn); in init_output_elf()
366 linker->elf_hdr->e_shstrndx = sec->sec_idx; in init_output_elf()
367 linker->strtab_sec_idx = sec->sec_idx; in init_output_elf()
369 sec->shdr->sh_name = str_off; in init_output_elf()
370 sec->shdr->sh_type = SHT_STRTAB; in init_output_elf()
371 sec->shdr->sh_flags = SHF_STRINGS; in init_output_elf()
372 sec->shdr->sh_offset = 0; in init_output_elf()
373 sec->shdr->sh_link = 0; in init_output_elf()
374 sec->shdr->sh_info = 0; in init_output_elf()
375 sec->shdr->sh_addralign = 1; in init_output_elf()
376 sec->shdr->sh_size = sec->sec_sz = 0; in init_output_elf()
377 sec->shdr->sh_entsize = 0; in init_output_elf()
382 return -ENOMEM; in init_output_elf()
384 sec->scn = elf_newscn(linker->elf); in init_output_elf()
385 if (!sec->scn) { in init_output_elf()
387 return -EINVAL; in init_output_elf()
390 sec->shdr = elf64_getshdr(sec->scn); in init_output_elf()
391 if (!sec->shdr) in init_output_elf()
392 return -EINVAL; in init_output_elf()
394 sec->data = elf_newdata(sec->scn); in init_output_elf()
395 if (!sec->data) { in init_output_elf()
397 return -EINVAL; in init_output_elf()
400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); in init_output_elf()
404 sec->sec_idx = elf_ndxscn(sec->scn); in init_output_elf()
405 linker->symtab_sec_idx = sec->sec_idx; in init_output_elf()
407 sec->shdr->sh_name = str_off; in init_output_elf()
408 sec->shdr->sh_type = SHT_SYMTAB; in init_output_elf()
409 sec->shdr->sh_flags = 0; in init_output_elf()
410 sec->shdr->sh_offset = 0; in init_output_elf()
411 sec->shdr->sh_link = linker->strtab_sec_idx; in init_output_elf()
415 sec->shdr->sh_info = 0; in init_output_elf()
416 sec->shdr->sh_addralign = 8; in init_output_elf()
417 sec->shdr->sh_entsize = sizeof(Elf64_Sym); in init_output_elf()
420 linker->btf = btf__new_empty(); in init_output_elf()
421 err = libbpf_get_error(linker->btf); in init_output_elf()
425 /* add the special all-zero symbol */ in init_output_elf()
428 return -EINVAL; in init_output_elf()
430 init_sym->st_name = 0; in init_output_elf()
431 init_sym->st_info = 0; in init_output_elf()
432 init_sym->st_other = 0; in init_output_elf()
433 init_sym->st_shndx = SHN_UNDEF; in init_output_elf()
434 init_sym->st_value = 0; in init_output_elf()
435 init_sym->st_size = 0; in init_output_elf()
447 return libbpf_err(-EINVAL); in bpf_linker__add_file()
449 if (!linker->elf) in bpf_linker__add_file()
450 return libbpf_err(-EINVAL); in bpf_linker__add_file()
476 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; in is_dwarf_sec_name()
481 Elf64_Shdr *shdr = sec->shdr; in is_ignored_sec()
482 const char *name = sec->sec_name; in is_ignored_sec()
485 if (shdr->sh_type == SHT_STRTAB) in is_ignored_sec()
489 if (shdr->sh_type == SHT_LLVM_ADDRSIG) in is_ignored_sec()
493 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && in is_ignored_sec()
494 strcmp(sec->sec_name, ".text") == 0) in is_ignored_sec()
498 if (is_dwarf_sec_name(sec->sec_name)) in is_ignored_sec()
501 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { in is_ignored_sec()
502 name += sizeof(".rel") - 1; in is_ignored_sec()
507 /* .BTF and .BTF.ext don't need relocations */ in is_ignored_sec()
518 struct src_sec *secs = obj->secs, *sec; in add_src_sec()
519 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; in add_src_sec()
526 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); in add_src_sec()
528 obj->secs = secs; in add_src_sec()
529 obj->sec_cnt = new_cnt; in add_src_sec()
531 sec = &obj->secs[new_cnt - 1]; in add_src_sec()
532 sec->id = new_cnt - 1; in add_src_sec()
533 sec->sec_name = sec_name; in add_src_sec()
558 obj->filename = filename; in linker_load_obj_file()
560 obj->fd = open(filename, O_RDONLY | O_CLOEXEC); in linker_load_obj_file()
561 if (obj->fd < 0) { in linker_load_obj_file()
562 err = -errno; in linker_load_obj_file()
566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); in linker_load_obj_file()
567 if (!obj->elf) { in linker_load_obj_file()
568 err = -errno; in linker_load_obj_file()
573 /* Sanity check ELF file high-level properties */ in linker_load_obj_file()
574 ehdr = elf64_getehdr(obj->elf); in linker_load_obj_file()
576 err = -errno; in linker_load_obj_file()
580 if (ehdr->e_ident[EI_DATA] != host_endianness) { in linker_load_obj_file()
581 err = -EOPNOTSUPP; in linker_load_obj_file()
585 if (ehdr->e_type != ET_REL in linker_load_obj_file()
586 || ehdr->e_machine != EM_BPF in linker_load_obj_file()
587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { in linker_load_obj_file()
588 err = -EOPNOTSUPP; in linker_load_obj_file()
593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { in linker_load_obj_file()
594 err = -errno; in linker_load_obj_file()
600 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { in linker_load_obj_file()
606 err = -errno; in linker_load_obj_file()
612 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); in linker_load_obj_file()
614 err = -errno; in linker_load_obj_file()
622 err = -errno; in linker_load_obj_file()
630 return -ENOMEM; in linker_load_obj_file()
632 sec->scn = scn; in linker_load_obj_file()
633 sec->shdr = shdr; in linker_load_obj_file()
634 sec->data = data; in linker_load_obj_file()
635 sec->sec_idx = elf_ndxscn(scn); in linker_load_obj_file()
638 sec->skipped = true; in linker_load_obj_file()
642 switch (shdr->sh_type) { in linker_load_obj_file()
644 if (obj->symtab_sec_idx) { in linker_load_obj_file()
645 err = -EOPNOTSUPP; in linker_load_obj_file()
649 obj->symtab_sec_idx = sec_idx; in linker_load_obj_file()
656 obj->btf = btf__new(data->d_buf, shdr->sh_size); in linker_load_obj_file()
657 err = libbpf_get_error(obj->btf); in linker_load_obj_file()
662 sec->skipped = true; in linker_load_obj_file()
666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); in linker_load_obj_file()
667 err = libbpf_get_error(obj->btf_ext); in linker_load_obj_file()
669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err); in linker_load_obj_file()
672 sec->skipped = true; in linker_load_obj_file()
687 err = -EINVAL; in linker_load_obj_file()
705 if (!obj->symtab_sec_idx) { in linker_sanity_check_elf()
706 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); in linker_sanity_check_elf()
707 return -EINVAL; in linker_sanity_check_elf()
709 if (!obj->shstrs_sec_idx) { in linker_sanity_check_elf()
710 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); in linker_sanity_check_elf()
711 return -EINVAL; in linker_sanity_check_elf()
714 for (i = 1; i < obj->sec_cnt; i++) { in linker_sanity_check_elf()
715 sec = &obj->secs[i]; in linker_sanity_check_elf()
717 if (sec->sec_name[0] == '\0') { in linker_sanity_check_elf()
718 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); in linker_sanity_check_elf()
719 return -EINVAL; in linker_sanity_check_elf()
722 if (is_dwarf_sec_name(sec->sec_name)) in linker_sanity_check_elf()
725 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) { in linker_sanity_check_elf()
726 pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n", in linker_sanity_check_elf()
727 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, in linker_sanity_check_elf()
728 obj->filename); in linker_sanity_check_elf()
729 return -EINVAL; in linker_sanity_check_elf()
731 if (sec->shdr->sh_addralign != sec->data->d_align) { in linker_sanity_check_elf()
733 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, in linker_sanity_check_elf()
734 (long long unsigned)sec->data->d_align, obj->filename); in linker_sanity_check_elf()
735 return -EINVAL; in linker_sanity_check_elf()
738 if (sec->shdr->sh_size != sec->data->d_size) { in linker_sanity_check_elf()
740 sec->sec_idx, (long long unsigned)sec->shdr->sh_size, in linker_sanity_check_elf()
741 (long long unsigned)sec->data->d_size, obj->filename); in linker_sanity_check_elf()
742 return -EINVAL; in linker_sanity_check_elf()
745 switch (sec->shdr->sh_type) { in linker_sanity_check_elf()
754 if (sec->shdr->sh_flags & SHF_EXECINSTR) { in linker_sanity_check_elf()
755 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) { in linker_sanity_check_elf()
757 sec->sec_idx, (long long unsigned)sec->shdr->sh_size, in linker_sanity_check_elf()
758 obj->filename); in linker_sanity_check_elf()
759 return -EINVAL; in linker_sanity_check_elf()
774 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); in linker_sanity_check_elf()
775 return -EINVAL; in linker_sanity_check_elf()
788 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) in linker_sanity_check_elf_symtab()
789 return -EINVAL; in linker_sanity_check_elf_symtab()
790 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) in linker_sanity_check_elf_symtab()
791 return -EINVAL; in linker_sanity_check_elf_symtab()
793 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { in linker_sanity_check_elf_symtab()
795 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); in linker_sanity_check_elf_symtab()
796 return -EINVAL; in linker_sanity_check_elf_symtab()
798 link_sec = &obj->secs[sec->shdr->sh_link]; in linker_sanity_check_elf_symtab()
799 if (link_sec->shdr->sh_type != SHT_STRTAB) { in linker_sanity_check_elf_symtab()
801 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); in linker_sanity_check_elf_symtab()
802 return -EINVAL; in linker_sanity_check_elf_symtab()
805 n = sec->shdr->sh_size / sec->shdr->sh_entsize; in linker_sanity_check_elf_symtab()
806 sym = sec->data->d_buf; in linker_sanity_check_elf_symtab()
808 int sym_type = ELF64_ST_TYPE(sym->st_info); in linker_sanity_check_elf_symtab()
809 int sym_bind = ELF64_ST_BIND(sym->st_info); in linker_sanity_check_elf_symtab()
810 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other); in linker_sanity_check_elf_symtab()
813 if (sym->st_name != 0 || sym->st_info != 0 in linker_sanity_check_elf_symtab()
814 || sym->st_other != 0 || sym->st_shndx != 0 in linker_sanity_check_elf_symtab()
815 || sym->st_value != 0 || sym->st_size != 0) { in linker_sanity_check_elf_symtab()
816 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename); in linker_sanity_check_elf_symtab()
817 return -EINVAL; in linker_sanity_check_elf_symtab()
823 i, sec->sec_idx, sym_bind); in linker_sanity_check_elf_symtab()
824 return -EINVAL; in linker_sanity_check_elf_symtab()
828 i, sec->sec_idx, sym_vis); in linker_sanity_check_elf_symtab()
829 return -EINVAL; in linker_sanity_check_elf_symtab()
831 if (sym->st_shndx == 0) { in linker_sanity_check_elf_symtab()
833 || sym->st_value != 0 || sym->st_size != 0) { in linker_sanity_check_elf_symtab()
835 i, obj->filename); in linker_sanity_check_elf_symtab()
837 return -EINVAL; in linker_sanity_check_elf_symtab()
841 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) { in linker_sanity_check_elf_symtab()
843 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); in linker_sanity_check_elf_symtab()
844 return -EINVAL; in linker_sanity_check_elf_symtab()
847 if (sym->st_value != 0) in linker_sanity_check_elf_symtab()
848 return -EINVAL; in linker_sanity_check_elf_symtab()
862 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) in linker_sanity_check_elf_relos()
863 return -EINVAL; in linker_sanity_check_elf_relos()
864 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) in linker_sanity_check_elf_relos()
865 return -EINVAL; in linker_sanity_check_elf_relos()
868 if (sec->shdr->sh_link != obj->symtab_sec_idx) { in linker_sanity_check_elf_relos()
870 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); in linker_sanity_check_elf_relos()
871 return -EINVAL; in linker_sanity_check_elf_relos()
875 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { in linker_sanity_check_elf_relos()
877 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); in linker_sanity_check_elf_relos()
878 return -EINVAL; in linker_sanity_check_elf_relos()
880 link_sec = &obj->secs[sec->shdr->sh_info]; in linker_sanity_check_elf_relos()
882 /* .rel<secname> -> <secname> pattern is followed */ in linker_sanity_check_elf_relos()
883 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 in linker_sanity_check_elf_relos()
884 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { in linker_sanity_check_elf_relos()
886 sec->sec_idx, obj->filename); in linker_sanity_check_elf_relos()
887 return -EINVAL; in linker_sanity_check_elf_relos()
891 if (link_sec->skipped) in linker_sanity_check_elf_relos()
895 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) { in linker_sanity_check_elf_relos()
897 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); in linker_sanity_check_elf_relos()
898 return -EINVAL; in linker_sanity_check_elf_relos()
902 n = sec->shdr->sh_size / sec->shdr->sh_entsize; in linker_sanity_check_elf_relos()
903 relo = sec->data->d_buf; in linker_sanity_check_elf_relos()
904 sym_sec = &obj->secs[obj->symtab_sec_idx]; in linker_sanity_check_elf_relos()
906 size_t sym_idx = ELF64_R_SYM(relo->r_info); in linker_sanity_check_elf_relos()
907 size_t sym_type = ELF64_R_TYPE(relo->r_info); in linker_sanity_check_elf_relos()
912 i, sec->sec_idx, sym_type, obj->filename); in linker_sanity_check_elf_relos()
913 return -EINVAL; in linker_sanity_check_elf_relos()
916 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { in linker_sanity_check_elf_relos()
918 i, sec->sec_idx, sym_idx, obj->filename); in linker_sanity_check_elf_relos()
919 return -EINVAL; in linker_sanity_check_elf_relos()
922 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { in linker_sanity_check_elf_relos()
923 if (relo->r_offset % sizeof(struct bpf_insn) != 0) { in linker_sanity_check_elf_relos()
925 i, sec->sec_idx, sym_idx, obj->filename); in linker_sanity_check_elf_relos()
926 return -EINVAL; in linker_sanity_check_elf_relos()
939 return -EINVAL; in check_btf_type_id()
952 return -EINVAL; in check_btf_str_off()
962 if (!obj->btf) in linker_sanity_check_btf()
965 n = btf__type_cnt(obj->btf); in linker_sanity_check_btf()
967 t = btf_type_by_id(obj->btf, i); in linker_sanity_check_btf()
969 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf); in linker_sanity_check_btf()
970 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf); in linker_sanity_check_btf()
982 if (!obj->btf_ext) in linker_sanity_check_btf_ext()
985 /* can't use .BTF.ext without .BTF */ in linker_sanity_check_btf_ext()
986 if (!obj->btf) in linker_sanity_check_btf_ext()
987 return -EINVAL; in linker_sanity_check_btf_ext()
989 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); in linker_sanity_check_btf_ext()
990 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); in linker_sanity_check_btf_ext()
1004 dst_sec->sec_sz = 0; in init_sec()
1005 dst_sec->sec_idx = 0; in init_sec()
1006 dst_sec->ephemeral = src_sec->ephemeral; in init_sec()
1009 if (src_sec->ephemeral) in init_sec()
1012 scn = elf_newscn(linker->elf); in init_sec()
1014 return -ENOMEM; in init_sec()
1017 return -ENOMEM; in init_sec()
1020 return -ENOMEM; in init_sec()
1022 dst_sec->scn = scn; in init_sec()
1023 dst_sec->shdr = shdr; in init_sec()
1024 dst_sec->data = data; in init_sec()
1025 dst_sec->sec_idx = elf_ndxscn(scn); in init_sec()
1027 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); in init_sec()
1031 shdr->sh_name = name_off; in init_sec()
1032 shdr->sh_type = src_sec->shdr->sh_type; in init_sec()
1033 shdr->sh_flags = src_sec->shdr->sh_flags; in init_sec()
1034 shdr->sh_size = 0; in init_sec()
1039 shdr->sh_link = 0; in init_sec()
1040 shdr->sh_info = 0; in init_sec()
1041 shdr->sh_addralign = src_sec->shdr->sh_addralign; in init_sec()
1042 shdr->sh_entsize = src_sec->shdr->sh_entsize; in init_sec()
1044 data->d_type = src_sec->data->d_type; in init_sec()
1045 data->d_size = 0; in init_sec()
1046 data->d_buf = NULL; in init_sec()
1047 data->d_align = src_sec->data->d_align; in init_sec()
1048 data->d_off = 0; in init_sec()
1058 for (i = 1; i < linker->sec_cnt; i++) { in find_dst_sec_by_name()
1059 sec = &linker->secs[i]; in find_dst_sec_by_name()
1061 if (strcmp(sec->sec_name, sec_name) == 0) in find_dst_sec_by_name()
1070 if (dst->ephemeral || src->ephemeral) in secs_match()
1073 if (dst->shdr->sh_type != src->shdr->sh_type) { in secs_match()
1074 pr_warn("sec %s types mismatch\n", dst->sec_name); in secs_match()
1077 if (dst->shdr->sh_flags != src->shdr->sh_flags) { in secs_match()
1078 pr_warn("sec %s flags mismatch\n", dst->sec_name); in secs_match()
1081 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { in secs_match()
1082 pr_warn("sec %s entsize mismatch\n", dst->sec_name); in secs_match()
1091 if (dst_sec->sec_sz != src_sec->shdr->sh_size) in sec_content_is_same()
1093 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) in sec_content_is_same()
1108 if (src->ephemeral) in extend_sec()
1112 * ephemeral) and non-externs (map definitions). So it's possible that in extend_sec()
1113 * it has to be "upgraded" from ephemeral to non-ephemeral when the in extend_sec()
1114 * first non-ephemeral entity appears. In such case, we add ELF in extend_sec()
1117 if (dst->ephemeral) { in extend_sec()
1123 dst_align = dst->shdr->sh_addralign; in extend_sec()
1124 src_align = src->shdr->sh_addralign; in extend_sec()
1130 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; in extend_sec()
1132 /* no need to re-align final size */ in extend_sec()
1133 dst_final_sz = dst_align_sz + src->shdr->sh_size; in extend_sec()
1135 if (src->shdr->sh_type != SHT_NOBITS) { in extend_sec()
1136 tmp = realloc(dst->raw_data, dst_final_sz); in extend_sec()
1138 * 1. When dst->raw_data is NULL it returns: in extend_sec()
1140 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL, in extend_sec()
1145 * dst->raw_data would be freed again in bpf_linker__free(). in extend_sec()
1150 return -ENOMEM; in extend_sec()
1151 dst->raw_data = tmp; in extend_sec()
1154 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); in extend_sec()
1156 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); in extend_sec()
1159 dst->sec_sz = dst_final_sz; in extend_sec()
1160 dst->shdr->sh_size = dst_final_sz; in extend_sec()
1161 dst->data->d_size = dst_final_sz; in extend_sec()
1163 dst->shdr->sh_addralign = dst_align; in extend_sec()
1164 dst->data->d_align = dst_align; in extend_sec()
1166 src->dst_off = dst_align_sz; in extend_sec()
1173 if (!sec || sec->skipped) in is_data_sec()
1175 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ in is_data_sec()
1176 if (sec->ephemeral) in is_data_sec()
1178 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; in is_data_sec()
1183 if (!sec || sec->skipped || sec->ephemeral) in is_relo_sec()
1185 return sec->shdr->sh_type == SHT_REL; in is_relo_sec()
1192 for (i = 1; i < obj->sec_cnt; i++) { in linker_append_sec_data()
1196 src_sec = &obj->secs[i]; in linker_append_sec_data()
1200 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); in linker_append_sec_data()
1202 dst_sec = add_dst_sec(linker, src_sec->sec_name); in linker_append_sec_data()
1204 return -ENOMEM; in linker_append_sec_data()
1207 pr_warn("failed to init section '%s'\n", src_sec->sec_name); in linker_append_sec_data()
1212 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); in linker_append_sec_data()
1213 return -1; in linker_append_sec_data()
1217 if (strcmp(src_sec->sec_name, "license") == 0 in linker_append_sec_data()
1218 || strcmp(src_sec->sec_name, "version") == 0) { in linker_append_sec_data()
1220 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); in linker_append_sec_data()
1221 return -EINVAL; in linker_append_sec_data()
1223 src_sec->skipped = true; in linker_append_sec_data()
1224 src_sec->dst_id = dst_sec->id; in linker_append_sec_data()
1230 src_sec->dst_id = dst_sec->id; in linker_append_sec_data()
1242 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; in linker_append_elf_syms()
1243 Elf64_Sym *sym = symtab->data->d_buf; in linker_append_elf_syms()
1244 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err; in linker_append_elf_syms()
1245 int str_sec_idx = symtab->shdr->sh_link; in linker_append_elf_syms()
1248 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); in linker_append_elf_syms()
1249 if (!obj->sym_map) in linker_append_elf_syms()
1250 return -ENOMEM; in linker_append_elf_syms()
1253 /* We already validated all-zero symbol #0 and we already in linker_append_elf_syms()
1259 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); in linker_append_elf_syms()
1261 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); in linker_append_elf_syms()
1262 return -EINVAL; in linker_append_elf_syms()
1275 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; in get_sym_by_idx()
1276 Elf64_Sym *syms = symtab->raw_data; in get_sym_by_idx()
1287 for (i = 0; i < linker->glob_sym_cnt; i++) { in find_glob_sym()
1288 glob_sym = &linker->glob_syms[i]; in find_glob_sym()
1289 name = strset__data(linker->strtab_strs) + glob_sym->name_off; in find_glob_sym()
1302 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1, in add_glob_sym()
1303 sizeof(*linker->glob_syms)); in add_glob_sym()
1307 sym = &syms[linker->glob_sym_cnt]; in add_glob_sym()
1309 sym->var_idx = -1; in add_glob_sym()
1311 linker->glob_syms = syms; in add_glob_sym()
1312 linker->glob_sym_cnt++; in add_glob_sym()
1333 n1 = btf__str_by_offset(btf1, t1->name_off); in glob_sym_btf_matches()
1334 n2 = btf__str_by_offset(btf2, t2->name_off); in glob_sym_btf_matches()
1373 n1 = btf__str_by_offset(btf1, t1->name_off); in glob_sym_btf_matches()
1374 n2 = btf__str_by_offset(btf2, t2->name_off); in glob_sym_btf_matches()
1394 if (t1->size != t2->size) { in glob_sym_btf_matches()
1396 sym_name, btf_kind_str(t1), n1, t1->size, t2->size); in glob_sym_btf_matches()
1406 id1 = t1->type; in glob_sym_btf_matches()
1407 id2 = t2->type; in glob_sym_btf_matches()
1411 id1 = btf_array(t1)->type; in glob_sym_btf_matches()
1412 id2 = btf_array(t2)->type; in glob_sym_btf_matches()
1423 id1 = t1->type; in glob_sym_btf_matches()
1424 id2 = t2->type; in glob_sym_btf_matches()
1428 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC; in glob_sym_btf_matches()
1429 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC; in glob_sym_btf_matches()
1435 id1 = t1->type; in glob_sym_btf_matches()
1436 id2 = t2->type; in glob_sym_btf_matches()
1455 n1 = btf__str_by_offset(btf1, m1->name_off); in glob_sym_btf_matches()
1456 n2 = btf__str_by_offset(btf2, m2->name_off); in glob_sym_btf_matches()
1462 if (m1->offset != m2->offset) { in glob_sym_btf_matches()
1467 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) in glob_sym_btf_matches()
1487 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) in glob_sym_btf_matches()
1492 id1 = t1->type; in glob_sym_btf_matches()
1493 id2 = t2->type; in glob_sym_btf_matches()
1521 if (main_def->map_type != extra_def->map_type) { in map_defs_match()
1527 if (main_def->key_size != extra_def->key_size) { in map_defs_match()
1531 if (!!main_def->key_type_id != !!extra_def->key_type_id) { in map_defs_match()
1535 if ((main_def->parts & MAP_DEF_KEY_TYPE) in map_defs_match()
1537 main_btf, main_def->key_type_id, in map_defs_match()
1538 extra_btf, extra_def->key_type_id)) { in map_defs_match()
1544 if (main_def->value_size != extra_def->value_size) { in map_defs_match()
1548 if (!!main_def->value_type_id != !!extra_def->value_type_id) { in map_defs_match()
1552 if ((main_def->parts & MAP_DEF_VALUE_TYPE) in map_defs_match()
1554 main_btf, main_def->value_type_id, in map_defs_match()
1555 extra_btf, extra_def->value_type_id)) { in map_defs_match()
1560 if (main_def->max_entries != extra_def->max_entries) { in map_defs_match()
1564 if (main_def->map_flags != extra_def->map_flags) { in map_defs_match()
1568 if (main_def->numa_node != extra_def->numa_node) { in map_defs_match()
1572 if (main_def->pinning != extra_def->pinning) { in map_defs_match()
1577 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) { in map_defs_match()
1582 if (main_def->parts & MAP_DEF_INNER_MAP) { in map_defs_match()
1608 t = btf__type_by_id(obj->btf, btf_id); in glob_map_defs_match()
1613 t = skip_mods_and_typedefs(obj->btf, t->type, NULL); in glob_map_defs_match()
1615 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def); in glob_map_defs_match()
1621 /* re-parse existing map definition */ in glob_map_defs_match()
1622 t = btf__type_by_id(linker->btf, glob_sym->btf_id); in glob_map_defs_match()
1623 t = skip_mods_and_typedefs(linker->btf, t->type, NULL); in glob_map_defs_match()
1624 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def); in glob_map_defs_match()
1635 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def, in glob_map_defs_match()
1636 obj->btf, &src_def, &src_inner_def); in glob_map_defs_match()
1648 if (!glob_sym->btf_id || !btf_id) { in glob_syms_match()
1653 src_t = btf__type_by_id(obj->btf, btf_id); in glob_syms_match()
1661 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0) in glob_syms_match()
1665 linker->btf, glob_sym->btf_id, obj->btf, btf_id)) in glob_syms_match()
1673 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC) in btf_is_non_static()
1685 if (!obj->btf) { in find_glob_sym_btf()
1686 pr_warn("failed to find BTF info for object '%s'\n", obj->filename); in find_glob_sym_btf()
1687 return -EINVAL; in find_glob_sym_btf()
1690 n = btf__type_cnt(obj->btf); in find_glob_sym_btf()
1692 t = btf__type_by_id(obj->btf, i); in find_glob_sym_btf()
1698 name = btf__str_by_offset(obj->btf, t->name_off); in find_glob_sym_btf()
1712 t = btf__type_by_id(obj->btf, vi->type); in find_glob_sym_btf()
1713 name = btf__str_by_offset(obj->btf, t->name_off); in find_glob_sym_btf()
1717 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC) in find_glob_sym_btf()
1722 if (btf_id && btf_id != vi->type) { in find_glob_sym_btf()
1724 sym_name, btf_id, vi->type); in find_glob_sym_btf()
1725 return -EINVAL; in find_glob_sym_btf()
1729 *out_btf_id = vi->type; in find_glob_sym_btf()
1735 /* free-floating extern or global FUNC */ in find_glob_sym_btf()
1743 return -ENOENT; in find_glob_sym_btf()
1751 for (i = 1; i < obj->sec_cnt; i++) { in find_src_sec_by_name()
1752 sec = &obj->secs[i]; in find_src_sec_by_name()
1754 if (strcmp(sec->sec_name, sec_name) == 0) in find_src_sec_by_name()
1774 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; in complete_extern_btf_info()
1778 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0); in complete_extern_btf_info()
1781 src_t = btf_type_by_id(src_btf, src_t->type); in complete_extern_btf_info()
1782 dst_t = btf_type_by_id(dst_btf, dst_t->type); in complete_extern_btf_info()
1798 if (!src_p->name_off) in complete_extern_btf_info()
1802 s = btf__str_by_offset(src_btf, src_p->name_off); in complete_extern_btf_info()
1806 dst_p->name_off = off; in complete_extern_btf_info()
1813 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info)); in sym_update_bind()
1818 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type); in sym_update_type()
1826 sym->st_other &= ~0x03; in sym_update_visibility()
1827 sym->st_other |= sym_vis; in sym_update_visibility()
1842 sym_type = ELF64_ST_TYPE(sym->st_info); in linker_append_elf_sym()
1843 sym_bind = ELF64_ST_BIND(sym->st_info); in linker_append_elf_sym()
1844 sym_vis = ELF64_ST_VISIBILITY(sym->st_other); in linker_append_elf_sym()
1845 sym_is_extern = sym->st_shndx == SHN_UNDEF; in linker_append_elf_sym()
1848 if (!obj->btf) { in linker_append_elf_sym()
1850 return -ENOTSUP; in linker_append_elf_sym()
1852 } else if (sym->st_shndx < SHN_LORESERVE) { in linker_append_elf_sym()
1853 src_sec = &obj->secs[sym->st_shndx]; in linker_append_elf_sym()
1854 if (src_sec->skipped) in linker_append_elf_sym()
1856 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_elf_sym()
1859 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) { in linker_append_elf_sym()
1860 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx; in linker_append_elf_sym()
1877 t = btf__type_by_id(obj->btf, btf_sec_id); in linker_append_elf_sym()
1878 sec_name = btf__str_by_offset(obj->btf, t->name_off); in linker_append_elf_sym()
1890 return -ENOENT; in linker_append_elf_sym()
1892 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_elf_sym()
1902 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; in linker_append_elf_sym()
1904 /* If both symbols are non-externs, at least one of in linker_append_elf_sym()
1908 if (!sym_is_extern && !glob_sym->is_extern in linker_append_elf_sym()
1909 && !glob_sym->is_weak && sym_bind != STB_WEAK) { in linker_append_elf_sym()
1910 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n", in linker_append_elf_sym()
1911 src_sym_idx, sym_name, obj->filename); in linker_append_elf_sym()
1912 return -EINVAL; in linker_append_elf_sym()
1916 return -EINVAL; in linker_append_elf_sym()
1918 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx); in linker_append_elf_sym()
1921 * well; this way a mix of weak and non-weak extern in linker_append_elf_sym()
1930 glob_sym->is_weak = false; in linker_append_elf_sym()
1933 /* Non-default visibility is "contaminating", with stricter in linker_append_elf_sym()
1939 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other)) in linker_append_elf_sym()
1952 * strong-strong conflict. We also already tightened binding in linker_append_elf_sym()
1955 if (!glob_sym->is_extern && sym_bind == STB_WEAK) in linker_append_elf_sym()
1958 /* At this point, new symbol is strong non-extern, in linker_append_elf_sym()
1963 dst_sym->st_shndx = dst_sec->sec_idx; in linker_append_elf_sym()
1964 dst_sym->st_value = src_sec->dst_off + sym->st_value; in linker_append_elf_sym()
1965 dst_sym->st_size = sym->st_size; in linker_append_elf_sym()
1967 /* see comment below about dst_sec->id vs dst_sec->sec_idx */ in linker_append_elf_sym()
1968 glob_sym->sec_id = dst_sec->id; in linker_append_elf_sym()
1969 glob_sym->is_extern = false; in linker_append_elf_sym()
1971 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id, in linker_append_elf_sym()
1972 obj->btf, btf_id)) in linker_append_elf_sym()
1973 return -EINVAL; in linker_append_elf_sym()
1976 glob_sym->underlying_btf_id = 0; in linker_append_elf_sym()
1978 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; in linker_append_elf_sym()
1983 name_off = strset__add_str(linker->strtab_strs, sym_name); in linker_append_elf_sym()
1989 return -ENOMEM; in linker_append_elf_sym()
1991 dst_sym->st_name = name_off; in linker_append_elf_sym()
1992 dst_sym->st_info = sym->st_info; in linker_append_elf_sym()
1993 dst_sym->st_other = sym->st_other; in linker_append_elf_sym()
1994 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; in linker_append_elf_sym()
1995 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; in linker_append_elf_sym()
1996 dst_sym->st_size = sym->st_size; in linker_append_elf_sym()
1998 obj->sym_map[src_sym_idx] = dst_sym_idx; in linker_append_elf_sym()
2001 dst_sec->sec_sym_idx = dst_sym_idx; in linker_append_elf_sym()
2002 dst_sym->st_value = 0; in linker_append_elf_sym()
2008 return -ENOMEM; in linker_append_elf_sym()
2010 glob_sym->sym_idx = dst_sym_idx; in linker_append_elf_sym()
2011 /* we use dst_sec->id (and not dst_sec->sec_idx), because in linker_append_elf_sym()
2012 * ephemeral sections (.kconfig, .ksyms, etc) don't have in linker_append_elf_sym()
2015 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0. in linker_append_elf_sym()
2017 glob_sym->sec_id = dst_sec ? dst_sec->id : 0; in linker_append_elf_sym()
2018 glob_sym->name_off = name_off; in linker_append_elf_sym()
2020 glob_sym->btf_id = 0; in linker_append_elf_sym()
2021 glob_sym->is_extern = sym_is_extern; in linker_append_elf_sym()
2022 glob_sym->is_weak = sym_bind == STB_WEAK; in linker_append_elf_sym()
2030 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; in linker_append_elf_relos()
2033 for (i = 1; i < obj->sec_cnt; i++) { in linker_append_elf_relos()
2039 src_sec = &obj->secs[i]; in linker_append_elf_relos()
2043 /* shdr->sh_info points to relocatable section */ in linker_append_elf_relos()
2044 src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; in linker_append_elf_relos()
2045 if (src_linked_sec->skipped) in linker_append_elf_relos()
2048 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); in linker_append_elf_relos()
2050 dst_sec = add_dst_sec(linker, src_sec->sec_name); in linker_append_elf_relos()
2052 return -ENOMEM; in linker_append_elf_relos()
2055 pr_warn("failed to init section '%s'\n", src_sec->sec_name); in linker_append_elf_relos()
2059 pr_warn("sections %s are not compatible\n", src_sec->sec_name); in linker_append_elf_relos()
2060 return -1; in linker_append_elf_relos()
2063 /* shdr->sh_link points to SYMTAB */ in linker_append_elf_relos()
2064 dst_sec->shdr->sh_link = linker->symtab_sec_idx; in linker_append_elf_relos()
2066 /* shdr->sh_info points to relocated section */ in linker_append_elf_relos()
2067 dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; in linker_append_elf_relos()
2068 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; in linker_append_elf_relos()
2070 src_sec->dst_id = dst_sec->id; in linker_append_elf_relos()
2075 src_rel = src_sec->data->d_buf; in linker_append_elf_relos()
2076 dst_rel = dst_sec->raw_data + src_sec->dst_off; in linker_append_elf_relos()
2077 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; in linker_append_elf_relos()
2082 src_sym_idx = ELF64_R_SYM(src_rel->r_info); in linker_append_elf_relos()
2083 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; in linker_append_elf_relos()
2085 dst_sym_idx = obj->sym_map[src_sym_idx]; in linker_append_elf_relos()
2086 dst_rel->r_offset += src_linked_sec->dst_off; in linker_append_elf_relos()
2087 sym_type = ELF64_R_TYPE(src_rel->r_info); in linker_append_elf_relos()
2088 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); in linker_append_elf_relos()
2090 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { in linker_append_elf_relos()
2091 struct src_sec *sec = &obj->secs[src_sym->st_shndx]; in linker_append_elf_relos()
2094 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { in linker_append_elf_relos()
2104 insn = dst_linked_sec->raw_data + dst_rel->r_offset; in linker_append_elf_relos()
2105 if (insn->code == (BPF_JMP | BPF_CALL)) in linker_append_elf_relos()
2106 insn->imm += sec->dst_off / sizeof(struct bpf_insn); in linker_append_elf_relos()
2108 insn->imm += sec->dst_off; in linker_append_elf_relos()
2110 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); in linker_append_elf_relos()
2111 return -EINVAL; in linker_append_elf_relos()
2124 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; in find_sym_by_name()
2125 Elf64_Sym *sym = symtab->data->d_buf; in find_sym_by_name()
2126 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; in find_sym_by_name()
2127 int str_sec_idx = symtab->shdr->sh_link; in find_sym_by_name()
2131 if (sym->st_shndx != sec_idx) in find_sym_by_name()
2133 if (ELF64_ST_TYPE(sym->st_info) != sym_type) in find_sym_by_name()
2136 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); in find_sym_by_name()
2155 if (!obj->btf) in linker_fixup_btf()
2158 n = btf__type_cnt(obj->btf); in linker_fixup_btf()
2163 t = btf_type_by_id(obj->btf, i); in linker_fixup_btf()
2167 sec_name = btf__str_by_offset(obj->btf, t->name_off); in linker_fixup_btf()
2171 if (sec->shdr) in linker_fixup_btf()
2172 t->size = sec->shdr->sh_size; in linker_fixup_btf()
2175 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used in linker_fixup_btf()
2179 * sections, we pre-create "section shells" to be able in linker_fixup_btf()
2180 * to keep track of extra per-section metadata later in linker_fixup_btf()
2189 * matching non-extern VAR/FUNC in other sections. in linker_fixup_btf()
2203 return -ENOMEM; in linker_fixup_btf()
2205 sec->ephemeral = true; in linker_fixup_btf()
2206 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ in linker_fixup_btf()
2210 sec->sec_type_id = i; in linker_fixup_btf()
2215 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); in linker_fixup_btf()
2216 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off); in linker_fixup_btf()
2217 int var_linkage = btf_var(vt)->linkage; in linker_fixup_btf()
2224 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); in linker_fixup_btf()
2227 return -ENOENT; in linker_fixup_btf()
2230 vi->offset = sym->st_value; in linker_fixup_btf()
2245 return -EINVAL; in remap_type_id()
2259 if (!obj->btf) in linker_append_btf()
2262 start_id = btf__type_cnt(linker->btf); in linker_append_btf()
2263 n = btf__type_cnt(obj->btf); in linker_append_btf()
2265 obj->btf_type_map = calloc(n + 1, sizeof(int)); in linker_append_btf()
2266 if (!obj->btf_type_map) in linker_append_btf()
2267 return -ENOMEM; in linker_append_btf()
2272 t = btf__type_by_id(obj->btf, i); in linker_append_btf()
2280 name = btf__str_by_offset(obj->btf, t->name_off); in linker_append_btf()
2292 * to strong symbol or weak got upgraded to non-weak in linker_append_btf()
2294 if (glob_sym->underlying_btf_id == 0) in linker_append_btf()
2295 glob_sym->underlying_btf_id = -t->type; in linker_append_btf()
2301 if (glob_sym->btf_id) { in linker_append_btf()
2303 obj->btf_type_map[i] = glob_sym->btf_id; in linker_append_btf()
2308 id = btf__add_type(linker->btf, obj->btf, t); in linker_append_btf()
2310 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); in linker_append_btf()
2314 obj->btf_type_map[i] = id; in linker_append_btf()
2318 glob_sym->btf_id = id; in linker_append_btf()
2319 glob_sym->underlying_btf_id = -t->type; in linker_append_btf()
2324 n = btf__type_cnt(linker->btf); in linker_append_btf()
2326 struct btf_type *dst_t = btf_type_by_id(linker->btf, i); in linker_append_btf()
2328 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map)) in linker_append_btf()
2329 return -EINVAL; in linker_append_btf()
2335 for (i = 0; i < linker->glob_sym_cnt; i++) { in linker_append_btf()
2336 struct glob_sym *glob_sym = &linker->glob_syms[i]; in linker_append_btf()
2339 if (glob_sym->underlying_btf_id >= 0) in linker_append_btf()
2342 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id]; in linker_append_btf()
2344 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id); in linker_append_btf()
2345 glob_t->type = glob_sym->underlying_btf_id; in linker_append_btf()
2349 for (i = 1; i < obj->sec_cnt; i++) { in linker_append_btf()
2355 src_sec = &obj->secs[i]; in linker_append_btf()
2356 if (!src_sec->sec_type_id || src_sec->skipped) in linker_append_btf()
2358 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf()
2364 * file otherwise has no read-only static variables in in linker_append_btf()
2368 dst_sec->has_btf = true; in linker_append_btf()
2370 t = btf__type_by_id(obj->btf, src_sec->sec_type_id); in linker_append_btf()
2374 void *sec_vars = dst_sec->sec_vars; in linker_append_btf()
2375 int new_id = obj->btf_type_map[src_var->type]; in linker_append_btf()
2378 t = btf_type_by_id(linker->btf, new_id); in linker_append_btf()
2380 name = btf__str_by_offset(linker->btf, t->name_off); in linker_append_btf()
2382 if (glob_sym->sec_id != dst_sec->id) { in linker_append_btf()
2384 name, glob_sym->sec_id, dst_sec->id); in linker_append_btf()
2385 return -EINVAL; in linker_append_btf()
2394 if (glob_sym && glob_sym->var_idx >= 0) { in linker_append_btf()
2397 dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; in linker_append_btf()
2400 * re-calculate and update it in sec_var. in linker_append_btf()
2402 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id); in linker_append_btf()
2406 return -EINVAL; in linker_append_btf()
2408 dst_var->size = sz; in linker_append_btf()
2413 dst_sec->sec_var_cnt + 1, in linker_append_btf()
2414 sizeof(*dst_sec->sec_vars)); in linker_append_btf()
2416 return -ENOMEM; in linker_append_btf()
2418 dst_sec->sec_vars = sec_vars; in linker_append_btf()
2419 dst_sec->sec_var_cnt++; in linker_append_btf()
2421 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; in linker_append_btf()
2422 dst_var->type = obj->btf_type_map[src_var->type]; in linker_append_btf()
2423 dst_var->size = src_var->size; in linker_append_btf()
2424 dst_var->offset = src_sec->dst_off + src_var->offset; in linker_append_btf()
2427 glob_sym->var_idx = dst_sec->sec_var_cnt - 1; in linker_append_btf()
2438 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); in add_btf_ext_rec()
2441 ext_data->recs = tmp; in add_btf_ext_rec()
2443 tmp += ext_data->rec_cnt * ext_data->rec_sz; in add_btf_ext_rec()
2444 memcpy(tmp, src_rec, ext_data->rec_sz); in add_btf_ext_rec()
2446 ext_data->rec_cnt++; in add_btf_ext_rec()
2459 if (!obj->btf_ext) in linker_append_btf_ext()
2462 rec_sz = obj->btf_ext->func_info.rec_size; in linker_append_btf_ext()
2463 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { in linker_append_btf_ext()
2466 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); in linker_append_btf_ext()
2469 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); in linker_append_btf_ext()
2470 return -EINVAL; in linker_append_btf_ext()
2472 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf_ext()
2474 if (dst_sec->func_info.rec_sz == 0) in linker_append_btf_ext()
2475 dst_sec->func_info.rec_sz = rec_sz; in linker_append_btf_ext()
2476 if (dst_sec->func_info.rec_sz != rec_sz) { in linker_append_btf_ext()
2477 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); in linker_append_btf_ext()
2478 return -EINVAL; in linker_append_btf_ext()
2481 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { in linker_append_btf_ext()
2482 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); in linker_append_btf_ext()
2484 return -ENOMEM; in linker_append_btf_ext()
2486 dst_rec->insn_off += src_sec->dst_off; in linker_append_btf_ext()
2487 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; in linker_append_btf_ext()
2491 rec_sz = obj->btf_ext->line_info.rec_size; in linker_append_btf_ext()
2492 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { in linker_append_btf_ext()
2495 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); in linker_append_btf_ext()
2498 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); in linker_append_btf_ext()
2499 return -EINVAL; in linker_append_btf_ext()
2501 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf_ext()
2503 if (dst_sec->line_info.rec_sz == 0) in linker_append_btf_ext()
2504 dst_sec->line_info.rec_sz = rec_sz; in linker_append_btf_ext()
2505 if (dst_sec->line_info.rec_sz != rec_sz) { in linker_append_btf_ext()
2506 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); in linker_append_btf_ext()
2507 return -EINVAL; in linker_append_btf_ext()
2510 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { in linker_append_btf_ext()
2511 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); in linker_append_btf_ext()
2513 return -ENOMEM; in linker_append_btf_ext()
2515 dst_rec->insn_off += src_sec->dst_off; in linker_append_btf_ext()
2517 s = btf__str_by_offset(obj->btf, src_rec->file_name_off); in linker_append_btf_ext()
2518 str_off = btf__add_str(linker->btf, s); in linker_append_btf_ext()
2520 return -ENOMEM; in linker_append_btf_ext()
2521 dst_rec->file_name_off = str_off; in linker_append_btf_ext()
2523 s = btf__str_by_offset(obj->btf, src_rec->line_off); in linker_append_btf_ext()
2524 str_off = btf__add_str(linker->btf, s); in linker_append_btf_ext()
2526 return -ENOMEM; in linker_append_btf_ext()
2527 dst_rec->line_off = str_off; in linker_append_btf_ext()
2529 /* dst_rec->line_col is fine */ in linker_append_btf_ext()
2533 rec_sz = obj->btf_ext->core_relo_info.rec_size; in linker_append_btf_ext()
2534 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { in linker_append_btf_ext()
2537 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); in linker_append_btf_ext()
2540 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); in linker_append_btf_ext()
2541 return -EINVAL; in linker_append_btf_ext()
2543 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf_ext()
2545 if (dst_sec->core_relo_info.rec_sz == 0) in linker_append_btf_ext()
2546 dst_sec->core_relo_info.rec_sz = rec_sz; in linker_append_btf_ext()
2547 if (dst_sec->core_relo_info.rec_sz != rec_sz) { in linker_append_btf_ext()
2548 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); in linker_append_btf_ext()
2549 return -EINVAL; in linker_append_btf_ext()
2552 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { in linker_append_btf_ext()
2553 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); in linker_append_btf_ext()
2555 return -ENOMEM; in linker_append_btf_ext()
2557 dst_rec->insn_off += src_sec->dst_off; in linker_append_btf_ext()
2558 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; in linker_append_btf_ext()
2560 s = btf__str_by_offset(obj->btf, src_rec->access_str_off); in linker_append_btf_ext()
2561 str_off = btf__add_str(linker->btf, s); in linker_append_btf_ext()
2563 return -ENOMEM; in linker_append_btf_ext()
2564 dst_rec->access_str_off = str_off; in linker_append_btf_ext()
2566 /* dst_rec->kind is fine */ in linker_append_btf_ext()
2580 if (!linker->elf) in bpf_linker__finalize()
2581 return libbpf_err(-EINVAL); in bpf_linker__finalize()
2588 strs_sz = strset__data_size(linker->strtab_strs); in bpf_linker__finalize()
2589 strs = strset__data(linker->strtab_strs); in bpf_linker__finalize()
2591 sec = &linker->secs[linker->strtab_sec_idx]; in bpf_linker__finalize()
2592 sec->data->d_align = 1; in bpf_linker__finalize()
2593 sec->data->d_off = 0LL; in bpf_linker__finalize()
2594 sec->data->d_buf = (void *)strs; in bpf_linker__finalize()
2595 sec->data->d_type = ELF_T_BYTE; in bpf_linker__finalize()
2596 sec->data->d_size = strs_sz; in bpf_linker__finalize()
2597 sec->shdr->sh_size = strs_sz; in bpf_linker__finalize()
2599 for (i = 1; i < linker->sec_cnt; i++) { in bpf_linker__finalize()
2600 sec = &linker->secs[i]; in bpf_linker__finalize()
2603 if (sec->sec_idx == linker->strtab_sec_idx) in bpf_linker__finalize()
2606 /* special ephemeral sections (.ksyms, .kconfig, etc) */ in bpf_linker__finalize()
2607 if (!sec->scn) in bpf_linker__finalize()
2610 sec->data->d_buf = sec->raw_data; in bpf_linker__finalize()
2614 if (elf_update(linker->elf, ELF_C_NULL) < 0) { in bpf_linker__finalize()
2615 err = -errno; in bpf_linker__finalize()
2621 if (elf_update(linker->elf, ELF_C_WRITE) < 0) { in bpf_linker__finalize()
2622 err = -errno; in bpf_linker__finalize()
2627 elf_end(linker->elf); in bpf_linker__finalize()
2628 close(linker->fd); in bpf_linker__finalize()
2630 linker->elf = NULL; in bpf_linker__finalize()
2631 linker->fd = -1; in bpf_linker__finalize()
2644 name_off = strset__add_str(linker->strtab_strs, sec_name); in emit_elf_data_sec()
2648 scn = elf_newscn(linker->elf); in emit_elf_data_sec()
2650 return -ENOMEM; in emit_elf_data_sec()
2653 return -ENOMEM; in emit_elf_data_sec()
2656 return -EINVAL; in emit_elf_data_sec()
2658 shdr->sh_name = name_off; in emit_elf_data_sec()
2659 shdr->sh_type = SHT_PROGBITS; in emit_elf_data_sec()
2660 shdr->sh_flags = 0; in emit_elf_data_sec()
2661 shdr->sh_size = raw_sz; in emit_elf_data_sec()
2662 shdr->sh_link = 0; in emit_elf_data_sec()
2663 shdr->sh_info = 0; in emit_elf_data_sec()
2664 shdr->sh_addralign = align; in emit_elf_data_sec()
2665 shdr->sh_entsize = 0; in emit_elf_data_sec()
2667 data->d_type = ELF_T_BYTE; in emit_elf_data_sec()
2668 data->d_size = raw_sz; in emit_elf_data_sec()
2669 data->d_buf = (void *)raw_data; in emit_elf_data_sec()
2670 data->d_align = align; in emit_elf_data_sec()
2671 data->d_off = 0; in emit_elf_data_sec()
2679 struct btf *btf = linker->btf; in finalize_btf()
2685 if (btf__type_cnt(linker->btf) == 1) in finalize_btf()
2688 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf()
2689 struct dst_sec *sec = &linker->secs[i]; in finalize_btf()
2691 if (!sec->has_btf) in finalize_btf()
2694 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); in finalize_btf()
2697 sec->sec_name, id); in finalize_btf()
2701 for (j = 0; j < sec->sec_var_cnt; j++) { in finalize_btf()
2702 struct btf_var_secinfo *vi = &sec->sec_vars[j]; in finalize_btf()
2704 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) in finalize_btf()
2705 return -EINVAL; in finalize_btf()
2711 pr_warn(".BTF.ext generation failed: %d\n", err); in finalize_btf()
2715 opts.btf_ext = linker->btf_ext; in finalize_btf()
2716 err = btf__dedup(linker->btf, &opts); in finalize_btf()
2723 raw_data = btf__raw_data(linker->btf, &raw_sz); in finalize_btf()
2725 return -ENOMEM; in finalize_btf()
2733 /* Emit .BTF.ext section */ in finalize_btf()
2734 if (linker->btf_ext) { in finalize_btf()
2735 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz); in finalize_btf()
2737 return -ENOMEM; in finalize_btf()
2741 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err); in finalize_btf()
2757 if (!sec_data->rec_cnt) in emit_btf_ext_data()
2760 str_off = btf__add_str(linker->btf, sec_name); in emit_btf_ext_data()
2762 return -ENOMEM; in emit_btf_ext_data()
2765 sec_info->sec_name_off = str_off; in emit_btf_ext_data()
2766 sec_info->num_info = sec_data->rec_cnt; in emit_btf_ext_data()
2769 sz = sec_data->rec_cnt * sec_data->rec_sz; in emit_btf_ext_data()
2770 memcpy(cur, sec_data->recs, sz); in emit_btf_ext_data()
2773 return cur - output; in emit_btf_ext_data()
2784 /* validate that all sections have the same .BTF.ext record sizes in finalize_btf_ext()
2788 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2789 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2791 if (sec->func_info.rec_cnt) { in finalize_btf_ext()
2793 func_rec_sz = sec->func_info.rec_sz; in finalize_btf_ext()
2794 if (func_rec_sz != sec->func_info.rec_sz) { in finalize_btf_ext()
2796 func_rec_sz, sec->func_info.rec_sz); in finalize_btf_ext()
2797 return -EINVAL; in finalize_btf_ext()
2800 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; in finalize_btf_ext()
2802 if (sec->line_info.rec_cnt) { in finalize_btf_ext()
2804 line_rec_sz = sec->line_info.rec_sz; in finalize_btf_ext()
2805 if (line_rec_sz != sec->line_info.rec_sz) { in finalize_btf_ext()
2807 line_rec_sz, sec->line_info.rec_sz); in finalize_btf_ext()
2808 return -EINVAL; in finalize_btf_ext()
2811 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; in finalize_btf_ext()
2813 if (sec->core_relo_info.rec_cnt) { in finalize_btf_ext()
2815 core_relo_rec_sz = sec->core_relo_info.rec_sz; in finalize_btf_ext()
2816 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { in finalize_btf_ext()
2818 core_relo_rec_sz, sec->core_relo_info.rec_sz); in finalize_btf_ext()
2819 return -EINVAL; in finalize_btf_ext()
2822 … core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; in finalize_btf_ext()
2845 return -ENOMEM; in finalize_btf_ext()
2848 hdr->magic = BTF_MAGIC; in finalize_btf_ext()
2849 hdr->version = BTF_VERSION; in finalize_btf_ext()
2850 hdr->flags = 0; in finalize_btf_ext()
2851 hdr->hdr_len = sizeof(struct btf_ext_header); in finalize_btf_ext()
2855 hdr->func_info_off = 0; in finalize_btf_ext()
2856 hdr->func_info_len = funcs_sz; in finalize_btf_ext()
2857 hdr->line_info_off = funcs_sz; in finalize_btf_ext()
2858 hdr->line_info_len = lines_sz; in finalize_btf_ext()
2859 hdr->core_relo_off = funcs_sz + lines_sz; in finalize_btf_ext()
2860 hdr->core_relo_len = core_relos_sz; in finalize_btf_ext()
2866 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2867 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2869 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); in finalize_btf_ext()
2883 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2884 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2886 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); in finalize_btf_ext()
2900 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2901 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2903 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); in finalize_btf_ext()
2913 linker->btf_ext = btf_ext__new(data, total_sz); in finalize_btf_ext()
2914 err = libbpf_get_error(linker->btf_ext); in finalize_btf_ext()
2916 linker->btf_ext = NULL; in finalize_btf_ext()
2917 pr_warn("failed to parse final .BTF.ext data: %d\n", err); in finalize_btf_ext()